diff --git a/packages/app/src/browser-automation/handler.test.ts b/packages/app/src/browser-automation/handler.test.ts index 718894b7c..5b49b5f45 100644 --- a/packages/app/src/browser-automation/handler.test.ts +++ b/packages/app/src/browser-automation/handler.test.ts @@ -1,5 +1,6 @@ -import { beforeEach, describe, expect, test, vi } from "vitest"; +import { beforeEach, describe, expect, test } from "vitest"; import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; +import { createJSONStorage, type StateStorage } from "zustand/middleware"; import { mountBrowserAutomationHandler } from "./handler"; import type { DesktopHostBridge } from "@/desktop/host"; import { useBrowserStore } from "@/stores/browser-store"; @@ -8,20 +9,6 @@ import { useWorkspaceLayoutStore, } from "@/stores/workspace-layout-store"; -vi.mock("expo-router", () => ({ - router: { - navigate: vi.fn(), - }, -})); - -vi.mock("@react-native-async-storage/async-storage", () => ({ - default: { - getItem: vi.fn(async () => null), - setItem: vi.fn(async () => undefined), - removeItem: vi.fn(async () => undefined), - }, -})); - type BrowserAutomationExecuteRequest = Extract< SessionOutboundMessage, { type: "browser.automation.execute.request" } @@ -30,9 +17,28 @@ type BrowserAutomationExecuteResponse = Extract< SessionInboundMessage, { type: "browser.automation.execute.response" } >; +type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"]; + +class FakeStateStorage implements StateStorage { + private readonly values = new Map(); + + public getItem = (key: string): string | null => this.values.get(key) ?? null; + + public setItem = (key: string, value: string): void => { + this.values.set(key, value); + }; + + public removeItem = (key: string): void => { + this.values.delete(key); + }; + + public clear(): void { + this.values.clear(); + } +} class FakeDaemonClient { - public sentResponses: BrowserAutomationExecuteResponse[] = []; + public readonly sentResponses: BrowserAutomationExecuteResponse[] = []; private handler: ((request: BrowserAutomationExecuteRequest) => void) | null = null; public on( @@ -55,6 +61,97 @@ class FakeDaemonClient { public receive(nextRequest: BrowserAutomationExecuteRequest): void { this.handler?.(nextRequest); } + + public payloadAt(index: number): BrowserAutomationResponsePayload { + const response = this.sentResponses[index]; + if (!response) { + throw new Error(`Missing browser automation response at index ${index}`); + } + return response.payload; + } +} + +class FakeBrowserBridge { + public readonly executedRequests: BrowserAutomationExecuteRequest[] = []; + public readonly registeredWorkspaceBrowsers: Array<{ browserId: string; workspaceId: string }> = + []; + public readonly activeWorkspaceBrowsers: Array<{ + browserId: string | null; + workspaceId: string; + }> = []; + public response: BrowserAutomationResponsePayload | null = null; + public thrownError: unknown = null; + + public executeAutomationCommand = async ( + request: BrowserAutomationExecuteRequest, + ): Promise => { + this.executedRequests.push(request); + if (this.thrownError) { + throw this.thrownError; + } + return this.response ?? currentListTabsPayload(request.requestId); + }; + + public registerWorkspaceBrowser = async (input: { + browserId: string; + workspaceId: string; + }): Promise => { + this.registeredWorkspaceBrowsers.push(input); + }; + + public setWorkspaceActiveBrowser = async (input: { + browserId: string | null; + workspaceId: string; + }): Promise => { + this.activeWorkspaceBrowsers.push(input); + }; +} + +class FakeResidentBrowser { + public readonly ensuredWebviews: Array<{ browserId: string; url: string }> = []; + + public ensure = (input: { browserId: string; url: string }): HTMLElement | null => { + this.ensuredWebviews.push(input); + return null; + }; +} + +class BrowserAutomationHandlerHarness { + public readonly client = new FakeDaemonClient(); + public readonly browser = new FakeBrowserBridge(); + public readonly resident = new FakeResidentBrowser(); + private unsubscribe: (() => void) | null = null; + + public mount( + input: { + serverId?: string; + host?: DesktopHostBridge | null; + registrationWaitTimeoutMs?: number; + registrationPollIntervalMs?: number; + } = {}, + ): void { + this.unsubscribe = mountBrowserAutomationHandler({ + client: this.client, + ...(input.serverId ? { serverId: input.serverId } : {}), + getHost: () => (input.host === undefined ? { browser: this.browser } : input.host), + ensureResidentBrowserWebview: this.resident.ensure, + ...(input.registrationWaitTimeoutMs !== undefined + ? { registrationWaitTimeoutMs: input.registrationWaitTimeoutMs } + : {}), + ...(input.registrationPollIntervalMs !== undefined + ? { registrationPollIntervalMs: input.registrationPollIntervalMs } + : {}), + }); + } + + public unmount(): void { + this.unsubscribe?.(); + this.unsubscribe = null; + } + + public receive(request: BrowserAutomationExecuteRequest): void { + this.client.receive(request); + } } function browserAutomationRequest(): BrowserAutomationExecuteRequest { @@ -78,36 +175,28 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest { }; } -function emptyListTabsPayload(requestId = "req-new:list_tabs") { +function emptyListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload { return { requestId, - ok: true as const, + ok: true, result: { - command: "list_tabs" as const, + command: "list_tabs", tabs: [], }, }; } -function currentListTabsPayload(requestId = "req-new:list_tabs") { +function currentListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload { return { requestId, - ok: true as const, + ok: true, result: { - command: "list_tabs" as const, + command: "list_tabs", tabs: currentBrowserTabs(), }, }; } -function flushPromises(): Promise { - return new Promise((resolve) => setTimeout(resolve, 0)); -} - -function waitForAsyncWork(): Promise { - return new Promise((resolve) => setTimeout(resolve, 20)); -} - function currentBrowserTabs() { return Object.values(useBrowserStore.getState().browsersById).map((browser) => ({ browserId: browser.browserId, @@ -119,152 +208,170 @@ function currentBrowserTabs() { })); } +function newTabResultFrom(payload: BrowserAutomationResponsePayload) { + expect(payload).toMatchObject({ + requestId: "req-new", + ok: true, + result: { command: "new_tab", workspaceId: "wks_workspace_a", url: "https://example.com" }, + }); + if (!payload.ok || payload.result.command !== "new_tab") { + throw new Error("Expected browser_new_tab success payload"); + } + return payload.result; +} + +function workspaceBrowserTabs(workspaceKey: string, browserId: string) { + return useWorkspaceLayoutStore + .getState() + .getWorkspaceTabs(workspaceKey) + .filter((tab) => tab.target.kind === "browser" && tab.target.browserId === browserId); +} + +function flushAsyncWork(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +function waitForRegistrationTimeout(): Promise { + return new Promise((resolve) => setTimeout(resolve, 20)); +} + +const browserAutomationStorage = new FakeStateStorage(); +useBrowserStore.persist.setOptions({ + storage: createJSONStorage(() => browserAutomationStorage), +}); +useWorkspaceLayoutStore.persist.setOptions({ + storage: createJSONStorage(() => browserAutomationStorage), +}); + describe("mountBrowserAutomationHandler", () => { beforeEach(() => { + browserAutomationStorage.clear(); useBrowserStore.setState({ browsersById: {} }); useWorkspaceLayoutStore.setState({ layoutByWorkspace: {} }); }); - test("creates a focused workspace browser tab for browser_new_tab", async () => { - const client = new FakeDaemonClient(); - const setWorkspaceActiveBrowser = vi.fn(async () => undefined); - const registerWorkspaceBrowser = vi.fn(async () => undefined); - const ensureResidentBrowserWebview = vi.fn(); - const executeAutomationCommand = vi.fn(async () => currentListTabsPayload()); + test("browser_new_tab creates and focuses a workspace browser tab", async () => { + const browser = new BrowserAutomationHandlerHarness(); const workspaceKey = buildWorkspaceTabPersistenceKey({ serverId: "server-1", workspaceId: "wks_workspace_a", }); - if (!workspaceKey) throw new Error("expected workspace key"); - const focusedTabId = useWorkspaceLayoutStore + if (!workspaceKey) { + throw new Error("Expected workspace key"); + } + const previousFocusedTabId = useWorkspaceLayoutStore .getState() .openTabFocused(workspaceKey, { kind: "draft", draftId: "human-draft" }); - if (!focusedTabId) throw new Error("expected focused tab"); - mountBrowserAutomationHandler({ - client, - serverId: "server-1", - getHost: () => - ({ - browser: { - executeAutomationCommand, - registerWorkspaceBrowser, - setWorkspaceActiveBrowser, - }, - }) satisfies DesktopHostBridge, - ensureResidentBrowserWebview, - }); + browser.mount({ serverId: "server-1" }); - client.receive(browserNewTabRequest()); - await flushPromises(); + browser.receive(browserNewTabRequest()); + await flushAsyncWork(); - const payload = client.sentResponses[0]?.payload; - expect(payload?.ok).toBe(true); - if (!payload?.ok) throw new Error("expected success"); - expect(payload.result.command).toBe("new_tab"); - if (payload.result.command !== "new_tab") throw new Error("expected new_tab result"); - const browserId = payload.result.browserId; - expect(payload.result.url).toBe("https://example.com"); - const browserTab = useWorkspaceLayoutStore - .getState() - .getWorkspaceTabs(workspaceKey) - .find((tab) => tab.target.kind === "browser" && tab.target.browserId === browserId); - expect(browserTab).toBeDefined(); + const result = newTabResultFrom(browser.client.payloadAt(0)); + const openedTabs = workspaceBrowserTabs(workspaceKey, result.browserId); const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey]; - expect(layout?.root.kind).toBe("pane"); - if (layout?.root.kind !== "pane") throw new Error("expected root pane"); - expect(layout.root.pane.focusedTabId).toBe(browserTab?.tabId); - expect(layout.root.pane.focusedTabId).not.toBe(focusedTabId); - expect(registerWorkspaceBrowser).toHaveBeenCalledWith({ - browserId, - workspaceId: "wks_workspace_a", - }); - expect(setWorkspaceActiveBrowser).toHaveBeenCalledWith({ - browserId, - workspaceId: "wks_workspace_a", - }); - expect(ensureResidentBrowserWebview).toHaveBeenCalledWith({ - browserId, - url: "https://example.com", - }); - expect(executeAutomationCommand).toHaveBeenCalledTimes(1); + expect(openedTabs).toEqual([ + expect.objectContaining({ + target: { kind: "browser", browserId: result.browserId }, + }), + ]); + expect(layout?.root).toEqual( + expect.objectContaining({ + kind: "pane", + pane: expect.objectContaining({ focusedTabId: openedTabs[0]?.tabId }), + }), + ); + expect(openedTabs[0]?.tabId).not.toBe(previousFocusedTabId); + expect(browser.browser.registeredWorkspaceBrowsers).toEqual([ + { browserId: result.browserId, workspaceId: "wks_workspace_a" }, + ]); + expect(browser.browser.activeWorkspaceBrowsers).toEqual([ + { browserId: result.browserId, workspaceId: "wks_workspace_a" }, + ]); + expect(browser.resident.ensuredWebviews).toEqual([ + { browserId: result.browserId, url: "https://example.com" }, + ]); + expect(browser.browser.executedRequests).toEqual([ + { + type: "browser.automation.execute.request", + requestId: "req-new:list_tabs", + agentId: "agent-1", + workspaceId: "wks_workspace_a", + command: { command: "list_tabs", args: {} }, + }, + ]); }); - test("returns browser_timeout when the resident webview does not register", async () => { - const client = new FakeDaemonClient(); - const ensureResidentBrowserWebview = vi.fn(); - const executeAutomationCommand = vi.fn(async () => emptyListTabsPayload()); - mountBrowserAutomationHandler({ - client, + test("browser_new_tab returns a retryable timeout when the resident webview does not register", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.browser.response = emptyListTabsPayload(); + browser.mount({ serverId: "server-1", - getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge, - ensureResidentBrowserWebview, registrationWaitTimeoutMs: 1, registrationPollIntervalMs: 1, }); - client.receive(browserNewTabRequest()); - await waitForAsyncWork(); + browser.receive(browserNewTabRequest()); + await waitForRegistrationTimeout(); - expect(client.sentResponses[0]?.payload).toMatchObject({ - requestId: "req-new", - ok: false, - error: { - code: "browser_timeout", - retryable: true, + expect(browser.client.sentResponses).toEqual([ + { + type: "browser.automation.execute.response", + payload: { + requestId: "req-new", + ok: false, + error: { + code: "browser_timeout", + message: expect.stringContaining("Timed out waiting for browser tab"), + retryable: true, + }, + }, }, - }); - expect(ensureResidentBrowserWebview).toHaveBeenCalledWith( + ]); + expect(browser.resident.ensuredWebviews).toEqual([ expect.objectContaining({ url: "https://example.com" }), - ); - expect(client.sentResponses[0]?.payload).not.toMatchObject({ - ok: true, - result: { command: "new_tab" }, - }); + ]); }); - test("wraps browser_new_tab registration bridge errors in a response", async () => { - const client = new FakeDaemonClient(); - const executeAutomationCommand = vi.fn(async () => { - throw new Error("IPC registration check failed"); - }); - mountBrowserAutomationHandler({ - client, - serverId: "server-1", - getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge, - }); + test("browser_new_tab wraps registration bridge errors in a response", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.browser.thrownError = new Error("IPC registration check failed"); + browser.mount({ serverId: "server-1" }); - client.receive(browserNewTabRequest()); - await flushPromises(); + browser.receive(browserNewTabRequest()); + await flushAsyncWork(); - expect(client.sentResponses[0]?.payload).toEqual({ - requestId: "req-new", - ok: false, - error: { - code: "browser_unknown_error", - message: "IPC registration check failed", - retryable: false, + expect(browser.client.sentResponses).toEqual([ + { + type: "browser.automation.execute.response", + payload: { + requestId: "req-new", + ok: false, + error: { + code: "browser_unknown_error", + message: "IPC registration check failed", + retryable: false, + }, + }, }, - }); + ]); }); - test("sends a success response from the desktop bridge", async () => { - const client = new FakeDaemonClient(); - const executeAutomationCommand = vi.fn(async () => ({ - requestId: "req-1", - ok: true as const, - result: { command: "list_tabs" as const, tabs: [] }, - })); + test("non-new-tab requests send the desktop bridge response", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.browser.response = { + requestId: "desktop-req", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }; + browser.mount(); + const request = browserAutomationRequest(); - mountBrowserAutomationHandler({ - client, - getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge, - }); + browser.receive(request); + await flushAsyncWork(); - client.receive(browserAutomationRequest()); - await flushPromises(); - - expect(executeAutomationCommand).toHaveBeenCalledWith(browserAutomationRequest()); - expect(client.sentResponses).toEqual([ + expect(browser.browser.executedRequests).toEqual([request]); + expect(browser.client.sentResponses).toEqual([ { type: "browser.automation.execute.response", payload: { @@ -276,14 +383,14 @@ describe("mountBrowserAutomationHandler", () => { ]); }); - test("missing bridge sends browser_unsupported", async () => { - const client = new FakeDaemonClient(); - mountBrowserAutomationHandler({ client, getHost: () => null }); + test("missing desktop bridge sends browser_unsupported", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.mount({ host: null }); - client.receive(browserAutomationRequest()); - await flushPromises(); + browser.receive(browserAutomationRequest()); + await flushAsyncWork(); - expect(client.sentResponses).toEqual([ + expect(browser.client.sentResponses).toEqual([ { type: "browser.automation.execute.response", payload: { @@ -300,80 +407,68 @@ describe("mountBrowserAutomationHandler", () => { }); test("typed bridge errors become failure responses", async () => { - const client = new FakeDaemonClient(); - mountBrowserAutomationHandler({ - client, - getHost: () => ({ - browser: { - executeAutomationCommand: async () => { - throw { - code: "browser_tab_not_found", - message: "Browser tab browser-1 was not found.", - retryable: false, - }; + const browser = new BrowserAutomationHandlerHarness(); + browser.browser.thrownError = { + code: "browser_tab_not_found", + message: "Browser tab browser-1 was not found.", + retryable: false, + }; + browser.mount(); + + browser.receive(browserAutomationRequest()); + await flushAsyncWork(); + + expect(browser.client.sentResponses).toEqual([ + { + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: false, + error: { + code: "browser_tab_not_found", + message: "Browser tab browser-1 was not found.", + retryable: false, }, }, - }), - }); - - client.receive(browserAutomationRequest()); - await flushPromises(); - - expect(client.sentResponses[0]?.payload).toEqual({ - requestId: "req-1", - ok: false, - error: { - code: "browser_tab_not_found", - message: "Browser tab browser-1 was not found.", - retryable: false, }, - }); + ]); }); test("unimplemented preload IPC reports browser_unsupported", async () => { - const client = new FakeDaemonClient(); - mountBrowserAutomationHandler({ - client, - getHost: () => ({ - browser: { - executeAutomationCommand: async () => { - throw new Error('No handler registered for "paseo:browser:execute-automation-command"'); + const browser = new BrowserAutomationHandlerHarness(); + browser.browser.thrownError = new Error( + 'No handler registered for "paseo:browser:execute-automation-command"', + ); + browser.mount(); + + browser.receive(browserAutomationRequest()); + await flushAsyncWork(); + + expect(browser.client.sentResponses).toEqual([ + { + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: false, + error: { + code: "browser_unsupported", + message: "Desktop browser automation is not implemented by this desktop build yet.", + retryable: false, }, }, - }), - }); - - client.receive(browserAutomationRequest()); - await flushPromises(); - - expect(client.sentResponses[0]?.payload).toEqual({ - requestId: "req-1", - ok: false, - error: { - code: "browser_unsupported", - message: "Desktop browser automation is not implemented by this desktop build yet.", - retryable: false, }, - }); + ]); }); - test("unsubscribe stops handling requests", async () => { - const client = new FakeDaemonClient(); - const executeAutomationCommand = vi.fn(async () => ({ - requestId: "req-1", - ok: true as const, - result: { command: "list_tabs" as const, tabs: [] }, - })); - const unsubscribe = mountBrowserAutomationHandler({ - client, - getHost: () => ({ browser: { executeAutomationCommand } }), - }); + test("unsubscribe stops handling browser automation requests", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.mount(); - unsubscribe(); - client.receive(browserAutomationRequest()); - await flushPromises(); + browser.unmount(); + browser.receive(browserAutomationRequest()); + await flushAsyncWork(); - expect(executeAutomationCommand).not.toHaveBeenCalled(); - expect(client.sentResponses).toEqual([]); + expect(browser.browser.executedRequests).toEqual([]); + expect(browser.client.sentResponses).toEqual([]); }); }); diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index 0b2acae09..f53e2deba 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -1,18 +1,33 @@ -import { describe, expect, test } from "vitest"; +import { resolve as resolvePath } from "node:path"; + +import { describe, expect, test, vi } from "vitest"; +import type { + BrowserAutomationCommand, + BrowserAutomationConsoleLogEntry, + BrowserAutomationCookieEntry, + BrowserAutomationExecuteRequest, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; import type { BrowserRegistry, TabContents, TabImage } from "./service.js"; import { executeAutomationCommand } from "./service.js"; const BROWSER_A = "11111111-1111-4111-8111-111111111111"; const BROWSER_B = "22222222-2222-4222-8222-222222222222"; +const WORKSPACE_A = "workspace-a"; +const WORKSPACE_B = "workspace-b"; class FakeImage implements TabImage { + public constructor( + private readonly bytes = new Uint8Array([137, 80, 78, 71, 1, 2, 3]), + private readonly size = { width: 640, height: 480 }, + ) {} + public toPNG(): Uint8Array { - return new Uint8Array([137, 80, 78, 71]); + return this.bytes; } public getSize(): { width: number; height: number } { - return { width: 10, height: 5 }; + return this.size; } } @@ -21,18 +36,43 @@ class FakeTab implements TabContents { public readonly scripts: string[] = []; public readonly actions: string[] = []; public readonly capturedViewports: Array<{ stayHidden?: boolean }> = []; + public readonly debugCommands: Array<{ command: string; params?: Record }> = []; + public readonly pdfOptions: Record[] = []; + public readonly downloads: Array<{ url: string; fileName?: string }> = []; + public destroyed = false; public bodyText = ""; public snapshotElements: unknown[] = []; + public actionScriptResult: unknown = true; + public networkEntries: unknown[] = []; + public storageState: unknown = { localStorage: [], sessionStorage: [] }; + public viewport = { width: 1024, height: 768, deviceScaleFactor: 2 }; + public consoleMessages: BrowserAutomationConsoleLogEntry[] = []; + public cookies: BrowserAutomationCookieEntry[] = []; + public captureNeverPaints = false; + public pdfBytes = new Uint8Array([37, 80, 68, 70]); + public downloadResult = { + filePath: "/workspace/downloads/file.txt", + totalBytes: 42, + state: "completed", + }; + public layoutMetrics = { + cssLayoutViewport: { clientWidth: 390, clientHeight: 844 }, + cssContentSize: { width: 390, height: 1200 }, + }; + public fullPageScreenshotData = "fullPagePng"; + public documentNodeId = 1; + public queriedNodeId = 2; + public backgroundThrottlingAllowed = true; public constructor( public readonly id: number, - private readonly url: string, + private readonly initialUrl: string, private readonly title: string, ) {} public getURL(): string { - return this.loadedUrls.at(-1) ?? this.url; + return this.loadedUrls.at(-1) ?? this.initialUrl; } public getTitle(): string { @@ -60,10 +100,19 @@ class FakeTab implements TabContents { if (code.includes("document.body.innerText")) { return this.bodyText; } - if (code.includes("querySelectorAll")) { + if (code.includes("CANDIDATE_SELECTOR")) { return JSON.stringify(this.snapshotElements); } - return true; + if (code.includes("performance.getEntriesByType")) { + return JSON.stringify(this.networkEntries); + } + if (code.includes("localStorage") && code.includes("sessionStorage")) { + return JSON.stringify(this.storageState); + } + if (code.includes("window.innerWidth")) { + return JSON.stringify(this.viewport); + } + return this.actionScriptResult; } public async loadURL(url: string): Promise { @@ -84,6 +133,10 @@ class FakeTab implements TabContents { public async capturePage(options?: { stayHidden?: boolean }): Promise { this.capturedViewports.push(options ?? {}); + this.actions.push("capture"); + if (this.captureNeverPaints) { + return new Promise(() => {}); + } return new FakeImage(); } @@ -92,23 +145,69 @@ class FakeTab implements TabContents { } public isBackgroundThrottlingAllowed(): boolean { - return true; + return this.backgroundThrottlingAllowed; } public setBackgroundThrottling(allowed: boolean): void { + this.backgroundThrottlingAllowed = allowed; this.actions.push(`background:${allowed}`); } + + public getConsoleMessages(): BrowserAutomationConsoleLogEntry[] { + return this.consoleMessages; + } + + public async getCookies(_url: string): Promise { + return this.cookies; + } + + public async sendDebugCommand( + command: string, + params?: Record, + ): Promise { + this.debugCommands.push({ command, ...(params ? { params } : {}) }); + if (command === "Page.getLayoutMetrics") { + return this.layoutMetrics; + } + if (command === "Page.captureScreenshot") { + return { data: this.fullPageScreenshotData }; + } + if (command === "DOM.getDocument") { + return { root: { nodeId: this.documentNodeId } }; + } + if (command === "DOM.querySelector") { + return { nodeId: this.queriedNodeId }; + } + return {}; + } + + public async printToPDF(options?: Record): Promise { + this.pdfOptions.push(options ?? {}); + return this.pdfBytes; + } + + public async downloadURL(input: { url: string; fileName?: string }): Promise<{ + filePath: string; + totalBytes?: number; + state: string; + }> { + this.downloads.push(input); + return this.downloadResult; + } } class FakeRegistry implements BrowserRegistry { private readonly tabs = new Map(); - - public activeBrowserId: string | null = null; + private readonly activeBrowserIdsByWorkspace = new Map(); public register(browserId: string, workspaceId: string, tab: FakeTab): void { this.tabs.set(browserId, { workspaceId, tab }); } + public setActiveBrowser(workspaceId: string, browserId: string): void { + this.activeBrowserIdsByWorkspace.set(workspaceId, browserId); + } + public listRegisteredBrowserIds(): string[] { return Array.from(this.tabs.keys()); } @@ -127,36 +226,168 @@ class FakeRegistry implements BrowserRegistry { return this.tabs.get(browserId)?.workspaceId ?? null; } - public getWorkspaceActiveBrowserId(): string | null { - return this.activeBrowserId; + public getWorkspaceActiveBrowserId(workspaceId: string): string | null { + return this.activeBrowserIdsByWorkspace.get(workspaceId) ?? null; } } -function pageRequest(command: { command: "page_info"; args: { browserId: string } }) { +class BrowserAutomationHarness { + public readonly registry = new FakeRegistry(); + public readonly snapshotEngine = new BrowserSnapshotEngine(); + public readonly tab = new FakeTab(1, "https://a.test/form", "Fixture"); + + public constructor() { + this.registry.register(BROWSER_A, WORKSPACE_A, this.tab); + this.registry.setActiveBrowser(WORKSPACE_A, BROWSER_A); + } + + public async execute( + command: BrowserAutomationCommand, + input: { requestId?: string; workspaceId?: string; cwd?: string } = {}, + ) { + return executeAutomationCommand(automationRequest(command, input), this.registry, { + snapshotEngine: this.snapshotEngine, + }); + } + + public async snapshot() { + return this.execute({ + command: "snapshot", + args: { browserId: BROWSER_A }, + }); + } +} + +function automationRequest( + command: BrowserAutomationCommand, + input: { requestId?: string; workspaceId?: string; cwd?: string } = {}, +): BrowserAutomationExecuteRequest { + const workspaceFields: { workspaceId?: string } = {}; + if (input.workspaceId === undefined) { + workspaceFields.workspaceId = WORKSPACE_A; + } else if (input.workspaceId) { + workspaceFields.workspaceId = input.workspaceId; + } return { - type: "browser.automation.execute.request" as const, - requestId: "req-page", - workspaceId: "workspace-a", + type: "browser.automation.execute.request", + requestId: input.requestId ?? `req-${command.command}`, + ...(input.cwd ? { cwd: input.cwd } : {}), + ...workspaceFields, command, }; } +function formElements() { + return [ + { + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + { + role: "checkbox", + tagName: "input", + text: "Agree", + selector: "#agree", + attributes: { id: "agree", type: "checkbox" }, + }, + { + role: "combobox", + tagName: "select", + text: "Country", + selector: "#country", + attributes: { id: "country" }, + }, + { + role: "button", + tagName: "button", + text: "Source", + selector: "#source", + attributes: { id: "source" }, + }, + { + role: "button", + tagName: "button", + text: "Target", + selector: "#target", + attributes: { id: "target" }, + }, + ]; +} + +function containsScript(tab: FakeTab, ...parts: string[]): boolean { + return tab.scripts.some((script) => parts.every((part) => script.includes(part))); +} + +function requireSnapshotRefs(result: Awaited>) { + expect(result).toEqual({ + requestId: "req-snapshot", + ok: true, + result: { + command: "snapshot", + browserId: BROWSER_A, + workspaceId: WORKSPACE_A, + url: "https://a.test/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + { + ref: "@e2", + role: "checkbox", + tagName: "input", + text: "Agree", + selector: "#agree", + attributes: { id: "agree", type: "checkbox" }, + }, + { + ref: "@e3", + role: "combobox", + tagName: "select", + text: "Country", + selector: "#country", + attributes: { id: "country" }, + }, + { + ref: "@e4", + role: "button", + tagName: "button", + text: "Source", + selector: "#source", + attributes: { id: "source" }, + }, + { + ref: "@e5", + role: "button", + tagName: "button", + text: "Target", + selector: "#target", + attributes: { id: "target" }, + }, + ], + }, + }); +} + describe("executeAutomationCommand", () => { - test("list tabs reports workspace ownership and active tab information", () => { - const tabA = new FakeTab(1, "https://a.test", "A"); - const tabB = new FakeTab(2, "https://b.test", "B"); + test("list tabs without a workspace reports every live registered tab", () => { const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tabA); - registry.register(BROWSER_B, "workspace-b", tabB); - registry.activeBrowserId = BROWSER_A; + registry.register(BROWSER_A, WORKSPACE_A, new FakeTab(1, "https://a.test", "A")); + registry.register(BROWSER_B, WORKSPACE_B, new FakeTab(2, "https://b.test", "B")); const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-list", - workspaceId: "workspace-a", - command: { command: "list_tabs", args: {} }, - }, + automationRequest( + { command: "list_tabs", args: {} }, + { requestId: "req-list", workspaceId: "" }, + ), registry, ); @@ -168,7 +399,52 @@ describe("executeAutomationCommand", () => { tabs: [ { browserId: BROWSER_A, - workspaceId: "workspace-a", + workspaceId: WORKSPACE_A, + url: "https://a.test", + title: "A", + isActive: false, + isLoading: false, + canGoBack: true, + canGoForward: false, + }, + { + browserId: BROWSER_B, + workspaceId: WORKSPACE_B, + url: "https://b.test", + title: "B", + isActive: false, + isLoading: false, + canGoBack: true, + canGoForward: false, + }, + ], + }, + }); + }); + + test("list tabs filters destroyed tabs out of the visible tab list", () => { + const liveTab = new FakeTab(1, "https://a.test", "A"); + const destroyedTab = new FakeTab(2, "https://dead.test", "Dead"); + destroyedTab.destroyed = true; + const registry = new FakeRegistry(); + registry.register(BROWSER_A, WORKSPACE_A, liveTab); + registry.register(BROWSER_B, WORKSPACE_A, destroyedTab); + registry.setActiveBrowser(WORKSPACE_A, BROWSER_A); + + const result = executeAutomationCommand( + automationRequest({ command: "list_tabs", args: {} }, { requestId: "req-live" }), + registry, + ); + + expect(result).toEqual({ + requestId: "req-live", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: BROWSER_A, + workspaceId: WORKSPACE_A, url: "https://a.test", title: "A", isActive: true, @@ -181,14 +457,28 @@ describe("executeAutomationCommand", () => { }); }); + test("list tabs returns an empty list when the workspace has no registered tabs", () => { + const result = executeAutomationCommand( + automationRequest({ command: "list_tabs", args: {} }, { requestId: "req-empty" }), + new FakeRegistry(), + ); + + expect(result).toEqual({ + requestId: "req-empty", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }); + }); + test("page info reads the explicit browser id from command args", () => { - const tab = new FakeTab(1, "https://a.test", "A"); - const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tab); + const browser = new BrowserAutomationHarness(); const result = executeAutomationCommand( - pageRequest({ command: "page_info", args: { browserId: BROWSER_A } }), - registry, + automationRequest( + { command: "page_info", args: { browserId: BROWSER_A } }, + { requestId: "req-page" }, + ), + browser.registry, ); expect(result).toEqual({ @@ -198,10 +488,10 @@ describe("executeAutomationCommand", () => { command: "page_info", tab: { browserId: BROWSER_A, - workspaceId: "workspace-a", - url: "https://a.test", - title: "A", - isActive: false, + workspaceId: WORKSPACE_A, + url: "https://a.test/form", + title: "Fixture", + isActive: true, isLoading: false, canGoBack: true, canGoForward: false, @@ -212,10 +502,13 @@ describe("executeAutomationCommand", () => { test("page info returns tab not found for an id in another workspace", () => { const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-b", new FakeTab(1, "https://a.test", "A")); + registry.register(BROWSER_A, WORKSPACE_B, new FakeTab(1, "https://a.test", "A")); const result = executeAutomationCommand( - pageRequest({ command: "page_info", args: { browserId: BROWSER_A } }), + automationRequest( + { command: "page_info", args: { browserId: BROWSER_A } }, + { requestId: "req-page" }, + ), registry, ); @@ -234,10 +527,13 @@ describe("executeAutomationCommand", () => { const tab = new FakeTab(1, "https://a.test", "A"); tab.destroyed = true; const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tab); + registry.register(BROWSER_A, WORKSPACE_A, tab); const result = executeAutomationCommand( - pageRequest({ command: "page_info", args: { browserId: BROWSER_A } }), + automationRequest( + { command: "page_info", args: { browserId: BROWSER_A } }, + { requestId: "req-page" }, + ), registry, ); @@ -253,8 +549,8 @@ describe("executeAutomationCommand", () => { }); test("snapshot and click use refs from the same explicit tab", async () => { - const tab = new FakeTab(1, "https://a.test/form", "Form"); - tab.snapshotElements = [ + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = [ { role: "button", tagName: "button", @@ -263,30 +559,12 @@ describe("executeAutomationCommand", () => { attributes: { id: "submit" }, }, ]; - const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tab); - const snapshotEngine = new BrowserSnapshotEngine(); - const snapshot = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-snapshot", - workspaceId: "workspace-a", - command: { command: "snapshot", args: { browserId: BROWSER_A } }, - }, - registry, - { snapshotEngine }, - ); - const click = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-click", - workspaceId: "workspace-a", - command: { command: "click", args: { browserId: BROWSER_A, ref: "@e1" } }, - }, - registry, - { snapshotEngine }, - ); + const snapshot = await browser.snapshot(); + const click = await browser.execute({ + command: "click", + args: { browserId: BROWSER_A, ref: "@e1" }, + }); expect(snapshot).toEqual({ requestId: "req-snapshot", @@ -294,9 +572,9 @@ describe("executeAutomationCommand", () => { result: { command: "snapshot", browserId: BROWSER_A, - workspaceId: "workspace-a", + workspaceId: WORKSPACE_A, url: "https://a.test/form", - title: "Form", + title: "Fixture", elements: [ { ref: "@e1", @@ -314,26 +592,141 @@ describe("executeAutomationCommand", () => { ok: true, result: { command: "click", browserId: BROWSER_A, ref: "@e1" }, }); + expect(containsScript(browser.tab, "#submit", ".click()")).toBe(true); + }); + + test("set background writes the requested color into the explicit page", async () => { + const browser = new BrowserAutomationHarness(); + + const result = await browser.execute({ + command: "set_background", + args: { browserId: BROWSER_A, color: "red" }, + }); + + expect(result).toEqual({ + requestId: "req-set_background", + ok: true, + result: { command: "set_background", browserId: BROWSER_A, color: "red" }, + }); + expect(containsScript(browser.tab, "document.body.style.background", "red")).toBe(true); + }); + + test.each([ + { + name: "fill updates a ref from the latest snapshot", + command: { command: "fill", args: { browserId: BROWSER_A, ref: "@e1", value: "Ada" } }, + result: { command: "fill", browserId: BROWSER_A, ref: "@e1" }, + scriptParts: ["#name", "Ada"], + }, + { + name: "focus focuses a ref from the latest snapshot", + command: { command: "focus", args: { browserId: BROWSER_A, ref: "@e1" } }, + result: { command: "focus", browserId: BROWSER_A, ref: "@e1" }, + scriptParts: ["#name", "focus"], + }, + { + name: "clear clears a ref from the latest snapshot", + command: { command: "clear", args: { browserId: BROWSER_A, ref: "@e1" } }, + result: { command: "clear", browserId: BROWSER_A, ref: "@e1" }, + scriptParts: ["#name", "deleteContent"], + }, + { + name: "check sets the requested checked state on a ref", + command: { command: "check", args: { browserId: BROWSER_A, ref: "@e2", checked: false } }, + result: { command: "check", browserId: BROWSER_A, ref: "@e2", checked: false }, + scriptParts: ["#agree", "nextChecked = false"], + }, + { + name: "select sets the requested value on a ref", + command: { command: "select", args: { browserId: BROWSER_A, ref: "@e3", value: "us" } }, + result: { command: "select", browserId: BROWSER_A, ref: "@e3", value: "us" }, + scriptParts: ["#country", "us"], + }, + { + name: "hover dispatches hover events to a ref", + command: { command: "hover", args: { browserId: BROWSER_A, ref: "@e4" } }, + result: { command: "hover", browserId: BROWSER_A, ref: "@e4" }, + scriptParts: ["#source", "mouseover"], + }, + { + name: "drag dispatches drag events between refs", + command: { + command: "drag", + args: { browserId: BROWSER_A, sourceRef: "@e4", targetRef: "@e5" }, + }, + result: { command: "drag", browserId: BROWSER_A, sourceRef: "@e4", targetRef: "@e5" }, + scriptParts: ["#source", "#target", "dragstart"], + }, + ] as const)("$name", async ({ command, result, scriptParts }) => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + const action = await browser.execute(command); + + expect(action).toEqual({ + requestId: `req-${command.command}`, + ok: true, + result, + }); + expect(containsScript(browser.tab, ...scriptParts)).toBe(true); + }); + + test("refs become stale after navigation changes the tab URL", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + await browser.execute({ + command: "navigate", + args: { browserId: BROWSER_A, url: "https://a.test/next" }, + }); + const click = await browser.execute({ + command: "click", + args: { browserId: BROWSER_A, ref: "@e1" }, + }); + + expect(click).toEqual({ + requestId: "req-click", + ok: false, + error: { + code: "browser_stale_ref", + message: "Browser element reference @e1 is stale. Take a new snapshot and try again.", + retryable: false, + }, + }); + }); + + test("refs become stale when the same URL no longer contains the element", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + browser.tab.actionScriptResult = false; + const fill = await browser.execute({ + command: "fill", + args: { browserId: BROWSER_A, ref: "@e1", value: "Ada" }, + }); + + expect(fill).toEqual({ + requestId: "req-fill", + ok: false, + error: { + code: "browser_stale_ref", + message: "Browser element reference @e1 is stale. Take a new snapshot and try again.", + retryable: false, + }, + }); }); test("wait resolves when the explicit tab contains the requested text", async () => { - const tab = new FakeTab(1, "https://a.test", "A"); - tab.bodyText = "Ready"; - const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tab); + const browser = new BrowserAutomationHarness(); + browser.tab.bodyText = "Ready"; - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-wait", - workspaceId: "workspace-a", - command: { - command: "wait", - args: { browserId: BROWSER_A, text: "Ready", timeoutMs: 100 }, - }, - }, - registry, - ); + const result = await browser.execute({ + command: "wait", + args: { browserId: BROWSER_A, text: "Ready", timeoutMs: 100 }, + }); expect(result).toEqual({ requestId: "req-wait", @@ -342,63 +735,117 @@ describe("executeAutomationCommand", () => { }); }); - test("navigate loads the requested URL in the explicit tab", async () => { - const tab = new FakeTab(1, "https://a.test", "A"); - const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tab); + test("wait returns a retryable timeout when text never appears", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.bodyText = "Still loading"; - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-navigate", - workspaceId: "workspace-a", - command: { - command: "navigate", - args: { browserId: BROWSER_A, url: "https://example.com/next" }, - }, + const result = await browser.execute({ + command: "wait", + args: { browserId: BROWSER_A, text: "Ready", timeoutMs: 1 }, + }); + + expect(result).toEqual({ + requestId: "req-wait", + ok: false, + error: { + code: "browser_timeout", + message: "Timed out waiting for browser text: Ready", + retryable: true, }, - registry, - ); + }); + }); + + test.each([ + { + name: "type writes text into a ref", + command: { command: "type", args: { browserId: BROWSER_A, ref: "@e1", text: "Ada" } }, + result: { command: "type", browserId: BROWSER_A, ref: "@e1" }, + scriptParts: ["#name", "Ada"], + needsSnapshot: true, + }, + { + name: "keypress dispatches a key to the focused element when ref is omitted", + command: { command: "keypress", args: { browserId: BROWSER_A, key: "Enter" } }, + result: { command: "keypress", browserId: BROWSER_A, key: "Enter" }, + scriptParts: ["document.activeElement", "Enter"], + needsSnapshot: false, + }, + ] as const)("$name", async ({ command, result, scriptParts, needsSnapshot }) => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = formElements(); + if (needsSnapshot) { + requireSnapshotRefs(await browser.snapshot()); + } + + const action = await browser.execute(command); + + expect(action).toEqual({ + requestId: `req-${command.command}`, + ok: true, + result, + }); + expect(containsScript(browser.tab, ...scriptParts)).toBe(true); + }); + + test("navigate loads the requested HTTP URL in the explicit tab", async () => { + const browser = new BrowserAutomationHarness(); + + const result = await browser.execute({ + command: "navigate", + args: { browserId: BROWSER_A, url: "https://example.com/next" }, + }); expect(result).toEqual({ requestId: "req-navigate", ok: true, result: { command: "navigate", browserId: BROWSER_A, url: "https://example.com/next" }, }); - expect(tab.loadedUrls).toEqual(["https://example.com/next"]); + expect(browser.tab.loadedUrls).toEqual(["https://example.com/next"]); + }); + + test("navigate denies non-http URLs before loading the explicit tab", async () => { + const browser = new BrowserAutomationHarness(); + + const result = await browser.execute({ + command: "navigate", + args: { browserId: BROWSER_A, url: "file:///tmp/secret.txt" }, + }); + + expect(result).toEqual({ + requestId: "req-navigate", + ok: false, + error: { + code: "browser_denied", + message: "Browser navigation only supports http and https URLs.", + retryable: false, + }, + }); + expect(browser.tab.loadedUrls).toEqual([]); }); test("navigation actions dispatch to the explicit tab", () => { - const tab = new FakeTab(1, "https://a.test", "A"); - const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tab); + const browser = new BrowserAutomationHarness(); const back = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-back", - workspaceId: "workspace-a", - command: { command: "back", args: { browserId: BROWSER_A } }, - }, - registry, + automationRequest( + { command: "back", args: { browserId: BROWSER_A } }, + { requestId: "req-back" }, + ), + browser.registry, ); const forward = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-forward", - workspaceId: "workspace-a", - command: { command: "forward", args: { browserId: BROWSER_A } }, - }, - registry, + automationRequest( + { command: "forward", args: { browserId: BROWSER_A } }, + { requestId: "req-forward" }, + ), + browser.registry, ); const reload = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "req-reload", - workspaceId: "workspace-a", - command: { command: "reload", args: { browserId: BROWSER_A } }, - }, - registry, + automationRequest( + { command: "reload", args: { browserId: BROWSER_A } }, + { requestId: "req-reload" }, + ), + browser.registry, ); expect(back).toEqual({ @@ -416,23 +863,127 @@ describe("executeAutomationCommand", () => { ok: true, result: { command: "reload", browserId: BROWSER_A }, }); - expect(tab.actions).toEqual(["back", "forward", "reload"]); + expect(browser.tab.actions).toEqual(["back", "forward", "reload"]); }); - test("screenshot captures the explicit tab viewport", async () => { - const tab = new FakeTab(1, "https://a.test", "A"); - const registry = new FakeRegistry(); - registry.register(BROWSER_A, "workspace-a", tab); - - const result = await executeAutomationCommand( + test("logs returns bounded console messages and network entries from the explicit tab", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.consoleMessages = [ + { level: "debug", message: "boot", timestamp: 1 }, + { level: "info", message: "ready", source: "console", line: 3, timestamp: 2 }, + ]; + browser.tab.networkEntries = [ + { url: "https://a.test/ignored", startTime: 1, duration: 2 }, { - type: "browser.automation.execute.request", - requestId: "req-screenshot", - workspaceId: "workspace-a", - command: { command: "screenshot", args: { browserId: BROWSER_A } }, + url: "https://a.test/app.js", + method: "GET", + status: 200, + type: "script", + startTime: 3, + duration: 4, + transferSize: 123, }, - registry, - ); + ]; + + const result = await browser.execute({ + command: "logs", + args: { browserId: BROWSER_A, maxEntries: 1 }, + }); + + expect(result).toEqual({ + requestId: "req-logs", + ok: true, + result: { + command: "logs", + browserId: BROWSER_A, + console: [{ level: "info", message: "ready", source: "console", line: 3, timestamp: 2 }], + network: [ + { + url: "https://a.test/app.js", + method: "GET", + status: 200, + type: "script", + startTime: 3, + duration: 4, + transferSize: 123, + }, + ], + }, + }); + }); + + test("storage reads cookies and web storage from the explicit tab", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.cookies = [ + { name: "theme", value: "dark", domain: "a.test", path: "/", secure: true }, + ]; + browser.tab.storageState = { + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }; + + const result = await browser.execute({ + command: "storage", + args: { browserId: BROWSER_A }, + }); + + expect(result).toEqual({ + requestId: "req-storage", + ok: true, + result: { + command: "storage", + browserId: BROWSER_A, + url: "https://a.test/form", + cookies: [{ name: "theme", value: "dark", domain: "a.test", path: "/", secure: true }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + }); + }); + + test("environment applies viewport and geolocation before reporting the current viewport", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.viewport = { width: 390, height: 844, deviceScaleFactor: 3 }; + + const result = await browser.execute({ + command: "environment", + args: { + browserId: BROWSER_A, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }); + + expect(result).toEqual({ + requestId: "req-environment", + ok: true, + result: { + command: "environment", + browserId: BROWSER_A, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }); + expect(browser.tab.debugCommands).toEqual([ + { + command: "Emulation.setDeviceMetricsOverride", + params: { width: 390, height: 844, deviceScaleFactor: 3, mobile: false }, + }, + { + command: "Emulation.setGeolocationOverride", + params: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + ]); + expect(containsScript(browser.tab, "navigator", "geolocation")).toBe(true); + }); + + test("screenshot serializes the painted viewport and restores throttling", async () => { + const browser = new BrowserAutomationHarness(); + + const result = await browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A }, + }); expect(result).toEqual({ requestId: "req-screenshot", @@ -441,11 +992,245 @@ describe("executeAutomationCommand", () => { command: "screenshot", browserId: BROWSER_A, mimeType: "image/png", - dataBase64: "iVBORw==", - width: 10, - height: 5, + dataBase64: "iVBORwECAw==", + width: 640, + height: 480, }, }); - expect(tab.capturedViewports).toEqual([{ stayHidden: false }]); + expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]); + expect(browser.tab.actions).toEqual([ + "background:false", + "invalidate", + "capture", + "background:true", + ]); + }); + + test("screenshot returns no-frame when the viewport never paints", async () => { + vi.useFakeTimers(); + try { + const browser = new BrowserAutomationHarness(); + browser.tab.captureNeverPaints = true; + + const resultPromise = browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A }, + }); + await vi.advanceTimersByTimeAsync(5_000); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-screenshot", + ok: false, + error: { + code: "screenshot_no_frame", + message: + "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }); + expect(browser.tab.actions).toEqual([ + "background:false", + "invalidate", + "capture", + "background:true", + ]); + } finally { + vi.useRealTimers(); + } + }); + + test("full page screenshot captures the page content area through CDP", async () => { + const browser = new BrowserAutomationHarness(); + + const result = await browser.execute({ + command: "full_page_screenshot", + args: { browserId: BROWSER_A }, + }); + + expect(result).toEqual({ + requestId: "req-full_page_screenshot", + ok: true, + result: { + command: "full_page_screenshot", + browserId: BROWSER_A, + mimeType: "image/png", + dataBase64: "fullPagePng", + width: 390, + height: 1200, + }, + }); + expect(browser.tab.debugCommands).toEqual([ + { command: "Page.getLayoutMetrics" }, + { + command: "Page.captureScreenshot", + params: { + format: "png", + captureBeyondViewport: true, + clip: { x: 0, y: 0, width: 390, height: 1200, scale: 1 }, + }, + }, + ]); + }); + + test("pdf exports the explicit tab with requested print options", async () => { + const browser = new BrowserAutomationHarness(); + + const result = await browser.execute({ + command: "pdf", + args: { browserId: BROWSER_A, landscape: true, printBackground: false }, + }); + + expect(result).toEqual({ + requestId: "req-pdf", + ok: true, + result: { + command: "pdf", + browserId: BROWSER_A, + mimeType: "application/pdf", + dataBase64: "JVBERg==", + }, + }); + expect(browser.tab.pdfOptions).toEqual([{ printBackground: false, landscape: true }]); + }); + + test("download saves the requested HTTP URL through the explicit tab", async () => { + const browser = new BrowserAutomationHarness(); + + const result = await browser.execute({ + command: "download", + args: { + browserId: BROWSER_A, + url: "https://a.test/file.txt", + fileName: "file.txt", + }, + }); + + expect(result).toEqual({ + requestId: "req-download", + ok: true, + result: { + command: "download", + browserId: BROWSER_A, + url: "https://a.test/file.txt", + filePath: "/workspace/downloads/file.txt", + totalBytes: 42, + state: "completed", + }, + }); + expect(browser.tab.downloads).toEqual([ + { url: "https://a.test/file.txt", fileName: "file.txt" }, + ]); + }); + + test("upload resolves workspace files before setting them on the file input", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = [ + { + role: "textbox", + tagName: "input", + text: "", + selector: "#file", + attributes: { id: "file", type: "file" }, + }, + ]; + const workspaceRoot = resolvePath("/workspace/project"); + + await browser.snapshot(); + const result = await browser.execute( + { + command: "upload", + args: { browserId: BROWSER_A, ref: "@e1", filePaths: ["uploads/a.txt"] }, + }, + { cwd: workspaceRoot }, + ); + + expect(result).toEqual({ + requestId: "req-upload", + ok: true, + result: { + command: "upload", + browserId: BROWSER_A, + ref: "@e1", + filePaths: [resolvePath(workspaceRoot, "uploads/a.txt")], + }, + }); + expect(browser.tab.debugCommands).toEqual([ + { command: "DOM.getDocument", params: { depth: -1, pierce: true } }, + { command: "DOM.querySelector", params: { nodeId: 1, selector: "#file" } }, + { + command: "DOM.setFileInputFiles", + params: { nodeId: 2, files: [resolvePath(workspaceRoot, "uploads/a.txt")] }, + }, + ]); + }); + + test("upload denies paths outside the agent workspace", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = [ + { + role: "textbox", + tagName: "input", + text: "", + selector: "#file", + attributes: { id: "file", type: "file" }, + }, + ]; + const workspaceRoot = resolvePath("/workspace/project"); + + await browser.snapshot(); + const result = await browser.execute( + { + command: "upload", + args: { browserId: BROWSER_A, ref: "@e1", filePaths: ["../secret.txt"] }, + }, + { cwd: workspaceRoot }, + ); + + expect(result).toEqual({ + requestId: "req-upload", + ok: false, + error: { + code: "browser_unsupported", + message: "browser_upload only accepts files inside the agent workspace.", + retryable: false, + }, + }); + expect(browser.tab.debugCommands).toEqual([ + { command: "DOM.getDocument", params: { depth: -1, pierce: true } }, + { command: "DOM.querySelector", params: { nodeId: 1, selector: "#file" } }, + ]); + }); + + test("upload reports missing cwd before setting files on the file input", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotElements = [ + { + role: "textbox", + tagName: "input", + text: "", + selector: "#file", + attributes: { id: "file", type: "file" }, + }, + ]; + + await browser.snapshot(); + const result = await browser.execute({ + command: "upload", + args: { browserId: BROWSER_A, ref: "@e1", filePaths: ["uploads/a.txt"] }, + }); + + expect(result).toEqual({ + requestId: "req-upload", + ok: false, + error: { + code: "browser_unsupported", + message: "browser_upload requires request cwd", + retryable: false, + }, + }); + expect(browser.tab.debugCommands).toEqual([ + { command: "DOM.getDocument", params: { depth: -1, pierce: true } }, + { command: "DOM.querySelector", params: { nodeId: 1, selector: "#file" } }, + ]); }); }); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.test.ts b/packages/protocol/src/browser-automation/rpc-schemas.test.ts index 5e240c473..2f5d0c7f1 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.test.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.test.ts @@ -11,6 +11,473 @@ const BROWSER_ID_MESSAGE = "browserId must be a real id returned by browser_new_tab or browser_list_tabs"; const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url"; +const commandParseCases = [ + { + name: "click", + command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } }, + expected: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } }, + }, + { + name: "fill", + command: { + command: "fill", + args: { browserId: BROWSER_ID, ref: "@e1", value: "Ada" }, + }, + expected: { + command: "fill", + args: { browserId: BROWSER_ID, ref: "@e1", value: "Ada" }, + }, + }, + { + name: "type", + command: { + command: "type", + args: { browserId: BROWSER_ID, ref: "@e1", text: "Ada" }, + }, + expected: { + command: "type", + args: { browserId: BROWSER_ID, ref: "@e1", text: "Ada" }, + }, + }, + { + name: "keypress", + command: { command: "keypress", args: { browserId: BROWSER_ID, key: "Enter" } }, + expected: { command: "keypress", args: { browserId: BROWSER_ID, key: "Enter" } }, + }, + { + name: "navigate", + command: { + command: "navigate", + args: { browserId: BROWSER_ID, url: "https://example.com/next" }, + }, + expected: { + command: "navigate", + args: { browserId: BROWSER_ID, url: "https://example.com/next" }, + }, + }, + { + name: "back", + command: { command: "back", args: { browserId: BROWSER_ID } }, + expected: { command: "back", args: { browserId: BROWSER_ID } }, + }, + { + name: "forward", + command: { command: "forward", args: { browserId: BROWSER_ID } }, + expected: { command: "forward", args: { browserId: BROWSER_ID } }, + }, + { + name: "reload", + command: { command: "reload", args: { browserId: BROWSER_ID } }, + expected: { command: "reload", args: { browserId: BROWSER_ID } }, + }, + { + name: "screenshot", + command: { command: "screenshot", args: { browserId: BROWSER_ID } }, + expected: { command: "screenshot", args: { browserId: BROWSER_ID } }, + }, + { + name: "full page screenshot", + command: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } }, + expected: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } }, + }, + { + name: "pdf", + command: { command: "pdf", args: { browserId: BROWSER_ID } }, + expected: { command: "pdf", args: { browserId: BROWSER_ID, printBackground: true } }, + }, + { + name: "download", + command: { + command: "download", + args: { browserId: BROWSER_ID, url: "https://example.com/file.txt" }, + }, + expected: { + command: "download", + args: { browserId: BROWSER_ID, url: "https://example.com/file.txt" }, + }, + }, + { + name: "upload", + command: { + command: "upload", + args: { browserId: BROWSER_ID, ref: "@e1", filePaths: ["/tmp/file.txt"] }, + }, + expected: { + command: "upload", + args: { browserId: BROWSER_ID, ref: "@e1", filePaths: ["/tmp/file.txt"] }, + }, + }, + { + name: "focus", + command: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } }, + expected: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } }, + }, + { + name: "clear", + command: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } }, + expected: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } }, + }, + { + name: "check", + command: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2" } }, + expected: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2", checked: true } }, + }, + { + name: "select", + command: { + command: "select", + args: { browserId: BROWSER_ID, ref: "@e3", value: "us" }, + }, + expected: { + command: "select", + args: { browserId: BROWSER_ID, ref: "@e3", value: "us" }, + }, + }, + { + name: "hover", + command: { command: "hover", args: { browserId: BROWSER_ID, ref: "@e4" } }, + expected: { command: "hover", args: { browserId: BROWSER_ID, ref: "@e4" } }, + }, + { + name: "drag", + command: { + command: "drag", + args: { browserId: BROWSER_ID, sourceRef: "@e4", targetRef: "@e5" }, + }, + expected: { + command: "drag", + args: { browserId: BROWSER_ID, sourceRef: "@e4", targetRef: "@e5" }, + }, + }, + { + name: "logs", + command: { command: "logs", args: { browserId: BROWSER_ID } }, + expected: { command: "logs", args: { browserId: BROWSER_ID, maxEntries: 50 } }, + }, + { + name: "storage", + command: { command: "storage", args: { browserId: BROWSER_ID } }, + expected: { command: "storage", args: { browserId: BROWSER_ID } }, + }, + { + name: "environment", + command: { + command: "environment", + args: { + browserId: BROWSER_ID, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + expected: { + command: "environment", + args: { + browserId: BROWSER_ID, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + }, + { + name: "set background", + command: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } }, + expected: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } }, + }, +] as const; + +const resultParseCases = [ + { + name: "page info", + result: { + command: "page_info", + tab: { + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + }, + }, + expected: { + command: "page_info", + tab: { + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + isActive: false, + isLoading: false, + }, + }, + }, + { + name: "snapshot", + result: { + command: "snapshot", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + ], + }, + expected: { + command: "snapshot", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + ], + }, + }, + { + name: "click", + result: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, + expected: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, + }, + { + name: "fill", + result: { command: "fill", browserId: BROWSER_ID, ref: "@e1" }, + expected: { command: "fill", browserId: BROWSER_ID, ref: "@e1" }, + }, + { + name: "wait", + result: { command: "wait", browserId: BROWSER_ID, matched: "text" }, + expected: { command: "wait", browserId: BROWSER_ID, matched: "text" }, + }, + { + name: "type", + result: { command: "type", browserId: BROWSER_ID, ref: "@e1" }, + expected: { command: "type", browserId: BROWSER_ID, ref: "@e1" }, + }, + { + name: "keypress", + result: { command: "keypress", browserId: BROWSER_ID, key: "Enter" }, + expected: { command: "keypress", browserId: BROWSER_ID, key: "Enter" }, + }, + { + name: "navigate", + result: { command: "navigate", browserId: BROWSER_ID, url: "https://example.com/next" }, + expected: { command: "navigate", browserId: BROWSER_ID, url: "https://example.com/next" }, + }, + { + name: "back", + result: { command: "back", browserId: BROWSER_ID }, + expected: { command: "back", browserId: BROWSER_ID }, + }, + { + name: "forward", + result: { command: "forward", browserId: BROWSER_ID }, + expected: { command: "forward", browserId: BROWSER_ID }, + }, + { + name: "reload", + result: { command: "reload", browserId: BROWSER_ID }, + expected: { command: "reload", browserId: BROWSER_ID }, + }, + { + name: "screenshot", + result: { + command: "screenshot", + browserId: BROWSER_ID, + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 100, + height: 50, + }, + expected: { + command: "screenshot", + browserId: BROWSER_ID, + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 100, + height: 50, + }, + }, + { + name: "full page screenshot", + result: { + command: "full_page_screenshot", + browserId: BROWSER_ID, + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 390, + height: 1200, + }, + expected: { + command: "full_page_screenshot", + browserId: BROWSER_ID, + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 390, + height: 1200, + }, + }, + { + name: "pdf", + result: { + command: "pdf", + browserId: BROWSER_ID, + mimeType: "application/pdf", + dataBase64: "JVBERg==", + }, + expected: { + command: "pdf", + browserId: BROWSER_ID, + mimeType: "application/pdf", + dataBase64: "JVBERg==", + }, + }, + { + name: "download", + result: { + command: "download", + browserId: BROWSER_ID, + url: "https://example.com/file.txt", + filePath: "/tmp/file.txt", + totalBytes: 5, + state: "completed", + }, + expected: { + command: "download", + browserId: BROWSER_ID, + url: "https://example.com/file.txt", + filePath: "/tmp/file.txt", + totalBytes: 5, + state: "completed", + }, + }, + { + name: "upload", + result: { + command: "upload", + browserId: BROWSER_ID, + ref: "@e1", + filePaths: ["/tmp/file.txt"], + }, + expected: { + command: "upload", + browserId: BROWSER_ID, + ref: "@e1", + filePaths: ["/tmp/file.txt"], + }, + }, + { + name: "focus", + result: { command: "focus", browserId: BROWSER_ID, ref: "@e1" }, + expected: { command: "focus", browserId: BROWSER_ID, ref: "@e1" }, + }, + { + name: "clear", + result: { command: "clear", browserId: BROWSER_ID, ref: "@e1" }, + expected: { command: "clear", browserId: BROWSER_ID, ref: "@e1" }, + }, + { + name: "check", + result: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false }, + expected: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false }, + }, + { + name: "select", + result: { command: "select", browserId: BROWSER_ID, ref: "@e3", value: "us" }, + expected: { command: "select", browserId: BROWSER_ID, ref: "@e3", value: "us" }, + }, + { + name: "hover", + result: { command: "hover", browserId: BROWSER_ID, ref: "@e4" }, + expected: { command: "hover", browserId: BROWSER_ID, ref: "@e4" }, + }, + { + name: "drag", + result: { command: "drag", browserId: BROWSER_ID, sourceRef: "@e4", targetRef: "@e5" }, + expected: { command: "drag", browserId: BROWSER_ID, sourceRef: "@e4", targetRef: "@e5" }, + }, + { + name: "logs", + result: { + command: "logs", + browserId: BROWSER_ID, + console: [{ level: "info", message: "ready", timestamp: 10 }], + network: [ + { + url: "https://example.com/app.js", + type: "script", + startTime: 1, + duration: 2, + }, + ], + }, + expected: { + command: "logs", + browserId: BROWSER_ID, + console: [{ level: "info", message: "ready", timestamp: 10 }], + network: [ + { + url: "https://example.com/app.js", + type: "script", + startTime: 1, + duration: 2, + }, + ], + }, + }, + { + name: "storage", + result: { + command: "storage", + browserId: BROWSER_ID, + url: "https://example.com", + cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + expected: { + command: "storage", + browserId: BROWSER_ID, + url: "https://example.com", + cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + }, + { + name: "environment", + result: { + command: "environment", + browserId: BROWSER_ID, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + expected: { + command: "environment", + browserId: BROWSER_ID, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + { + name: "set background", + result: { command: "set_background", browserId: BROWSER_ID, color: "red" }, + expected: { command: "set_background", browserId: BROWSER_ID, color: "red" }, + }, +] as const; + describe("browser automation execute RPC schemas", () => { test("list tabs reads workspace from the request envelope", () => { const parsed = BrowserAutomationExecuteRequestSchema.parse({ @@ -161,6 +628,20 @@ describe("browser automation execute RPC schemas", () => { }); }); + test.each(commandParseCases)( + "$name requests parse explicit browser ids", + ({ command, expected }) => { + const parsed = BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-command", + workspaceId: "workspace-1", + command, + }); + + expect(parsed.command).toEqual(expected); + }, + ); + test("navigate rejects non-http URLs at the protocol boundary", () => { const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ type: "browser.automation.execute.request", @@ -177,6 +658,22 @@ describe("browser automation execute RPC schemas", () => { }); }); + test("download rejects non-http URLs at the protocol boundary", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-download", + command: { + command: "download", + args: { browserId: BROWSER_ID, url: "file:///tmp/secret.txt" }, + }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: "URL must use http or https" })] }, + }); + }); + test("new tab responses declare the generated browser id shape", () => { const parsed = BrowserAutomationExecuteResponseSchema.parse({ type: "browser.automation.execute.response", @@ -227,4 +724,48 @@ describe("browser automation execute RPC schemas", () => { error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] }, }); }); + + test.each(resultParseCases)( + "$name results parse under the response payload", + ({ result, expected }) => { + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-result", + ok: true, + result, + }, + }); + + expect(parsed.payload).toEqual({ + requestId: "req-result", + ok: true, + result: expected, + }); + }, + ); + + test("error responses keep stable codes, messages, and retry defaults", () => { + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-error", + ok: false, + error: { + code: "browser_no_desktop", + message: "No desktop browser automation client is connected.", + }, + }, + }); + + expect(parsed.payload).toEqual({ + requestId: "req-error", + ok: false, + error: { + code: "browser_no_desktop", + message: "No desktop browser automation client is connected.", + retryable: false, + }, + }); + }); }); diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index 37ddb8115..94a4ee154 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -153,6 +153,418 @@ function screenshotPayload(): Extract }; } +const routedToolCases = [ + { + name: "set background", + toolName: "browser_set_background", + input: { browserId: BROWSER_ID, color: "red" }, + command: { command: "set_background", args: { browserId: BROWSER_ID, color: "red" } }, + payload: { + requestId: "req-set-background", + ok: true, + result: { command: "set_background", browserId: BROWSER_ID, color: "red" }, + }, + content: [{ type: "text", text: "Set browser page background to red." }], + }, + { + name: "click", + toolName: "browser_click", + input: { browserId: BROWSER_ID, ref: "@e2" }, + command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e2" } }, + payload: { + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_ID, ref: "@e2" }, + }, + content: [{ type: "text", text: "Clicked browser element @e2." }], + }, + { + name: "fill", + toolName: "browser_fill", + input: { browserId: BROWSER_ID, ref: "@e1", value: "Ada" }, + command: { command: "fill", args: { browserId: BROWSER_ID, ref: "@e1", value: "Ada" } }, + payload: { + requestId: "req-fill", + ok: true, + result: { command: "fill", browserId: BROWSER_ID, ref: "@e1" }, + }, + content: [{ type: "text", text: "Filled browser element @e1." }], + }, + { + name: "type", + toolName: "browser_type", + input: { browserId: BROWSER_ID, ref: "@e1", text: "Ada" }, + command: { command: "type", args: { browserId: BROWSER_ID, ref: "@e1", text: "Ada" } }, + payload: { + requestId: "req-type", + ok: true, + result: { command: "type", browserId: BROWSER_ID, ref: "@e1" }, + }, + content: [{ type: "text", text: "Typed into browser element @e1." }], + }, + { + name: "keypress", + toolName: "browser_keypress", + input: { browserId: BROWSER_ID, key: "Enter" }, + command: { command: "keypress", args: { browserId: BROWSER_ID, key: "Enter" } }, + payload: { + requestId: "req-keypress", + ok: true, + result: { command: "keypress", browserId: BROWSER_ID, key: "Enter" }, + }, + content: [{ type: "text", text: "Pressed Enter in the browser." }], + }, + { + name: "back", + toolName: "browser_back", + input: { browserId: BROWSER_ID }, + command: { command: "back", args: { browserId: BROWSER_ID } }, + payload: { + requestId: "req-back", + ok: true, + result: { command: "back", browserId: BROWSER_ID }, + }, + content: [{ type: "text", text: "Browser back complete." }], + }, + { + name: "screenshot", + toolName: "browser_screenshot", + input: { browserId: BROWSER_ID }, + command: { command: "screenshot", args: { browserId: BROWSER_ID } }, + payload: screenshotPayload(), + content: [ + { type: "text", text: "Captured browser screenshot (800x600)." }, + { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }, + ], + }, + { + name: "logs", + toolName: "browser_logs", + input: { browserId: BROWSER_ID }, + command: { command: "logs", args: { browserId: BROWSER_ID, maxEntries: 50 } }, + payload: { + requestId: "req-logs", + ok: true, + result: { + command: "logs", + browserId: BROWSER_ID, + console: [{ level: "info", message: "ready", timestamp: 10 }], + network: [ + { + url: "https://example.com/app.js", + type: "script", + startTime: 1, + duration: 2, + }, + ], + }, + }, + content: [{ type: "text", text: "Read 1 console log and 1 network entry." }], + }, + { + name: "storage", + toolName: "browser_storage", + input: { browserId: BROWSER_ID }, + command: { command: "storage", args: { browserId: BROWSER_ID } }, + payload: { + requestId: "req-storage", + ok: true, + result: { + command: "storage", + browserId: BROWSER_ID, + url: "https://example.com", + cookies: [{ name: "theme", value: "dark" }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + }, + content: [ + { type: "text", text: "Read 1 cookie, 1 localStorage entry, and 1 sessionStorage entry." }, + ], + }, + { + name: "environment", + toolName: "browser_environment", + input: { + browserId: BROWSER_ID, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + command: { + command: "environment", + args: { + browserId: BROWSER_ID, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + payload: { + requestId: "req-environment", + ok: true, + result: { + command: "environment", + browserId: BROWSER_ID, + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + content: [{ type: "text", text: "Browser environment viewport is 390x844." }], + }, + { + name: "full page screenshot", + toolName: "browser_full_page_screenshot", + input: { browserId: BROWSER_ID }, + command: { command: "full_page_screenshot", args: { browserId: BROWSER_ID } }, + payload: { + requestId: "req-full-page", + ok: true, + result: { + command: "full_page_screenshot", + browserId: BROWSER_ID, + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 390, + height: 1200, + }, + }, + content: [ + { type: "text", text: "Captured full-page browser screenshot (390x1200)." }, + { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }, + ], + }, + { + name: "pdf", + toolName: "browser_pdf", + input: { browserId: BROWSER_ID, landscape: true }, + command: { + command: "pdf", + args: { browserId: BROWSER_ID, landscape: true, printBackground: true }, + }, + payload: { + requestId: "req-pdf", + ok: true, + result: { + command: "pdf", + browserId: BROWSER_ID, + mimeType: "application/pdf", + dataBase64: "JVBERg==", + }, + }, + content: [{ type: "text", text: "Exported browser page PDF." }], + }, + { + name: "download", + toolName: "browser_download", + input: { browserId: BROWSER_ID, url: "https://example.com/file.txt", fileName: "file.txt" }, + command: { + command: "download", + args: { + browserId: BROWSER_ID, + url: "https://example.com/file.txt", + fileName: "file.txt", + }, + }, + payload: { + requestId: "req-download", + ok: true, + result: { + command: "download", + browserId: BROWSER_ID, + url: "https://example.com/file.txt", + filePath: "/tmp/file.txt", + totalBytes: 5, + state: "completed", + }, + }, + content: [{ type: "text", text: "Downloaded browser file to /tmp/file.txt." }], + }, + { + name: "upload", + toolName: "browser_upload", + input: { browserId: BROWSER_ID, ref: "@e1", filePaths: ["/tmp/file.txt"] }, + command: { + command: "upload", + args: { browserId: BROWSER_ID, ref: "@e1", filePaths: ["/tmp/file.txt"] }, + }, + payload: { + requestId: "req-upload", + ok: true, + result: { + command: "upload", + browserId: BROWSER_ID, + ref: "@e1", + filePaths: ["/tmp/file.txt"], + }, + }, + content: [{ type: "text", text: "Uploaded 1 file to browser element @e1." }], + }, + { + name: "focus", + toolName: "browser_focus", + input: { browserId: BROWSER_ID, ref: "@e1" }, + command: { command: "focus", args: { browserId: BROWSER_ID, ref: "@e1" } }, + payload: { + requestId: "req-focus", + ok: true, + result: { command: "focus", browserId: BROWSER_ID, ref: "@e1" }, + }, + content: [{ type: "text", text: "Focused browser element @e1." }], + }, + { + name: "clear", + toolName: "browser_clear", + input: { browserId: BROWSER_ID, ref: "@e1" }, + command: { command: "clear", args: { browserId: BROWSER_ID, ref: "@e1" } }, + payload: { + requestId: "req-clear", + ok: true, + result: { command: "clear", browserId: BROWSER_ID, ref: "@e1" }, + }, + content: [{ type: "text", text: "Cleared browser element @e1." }], + }, + { + name: "check", + toolName: "browser_check", + input: { browserId: BROWSER_ID, ref: "@e2", checked: false }, + command: { command: "check", args: { browserId: BROWSER_ID, ref: "@e2", checked: false } }, + payload: { + requestId: "req-check", + ok: true, + result: { command: "check", browserId: BROWSER_ID, ref: "@e2", checked: false }, + }, + content: [{ type: "text", text: "Unchecked browser element @e2." }], + }, + { + name: "select", + toolName: "browser_select", + input: { browserId: BROWSER_ID, ref: "@e3", value: "us" }, + command: { command: "select", args: { browserId: BROWSER_ID, ref: "@e3", value: "us" } }, + payload: { + requestId: "req-select", + ok: true, + result: { command: "select", browserId: BROWSER_ID, ref: "@e3", value: "us" }, + }, + content: [{ type: "text", text: "Selected us in browser element @e3." }], + }, + { + name: "hover", + toolName: "browser_hover", + input: { browserId: BROWSER_ID, ref: "@e4" }, + command: { command: "hover", args: { browserId: BROWSER_ID, ref: "@e4" } }, + payload: { + requestId: "req-hover", + ok: true, + result: { command: "hover", browserId: BROWSER_ID, ref: "@e4" }, + }, + content: [{ type: "text", text: "Hovered browser element @e4." }], + }, + { + name: "drag", + toolName: "browser_drag", + input: { browserId: BROWSER_ID, sourceRef: "@e4", targetRef: "@e5" }, + command: { + command: "drag", + args: { browserId: BROWSER_ID, sourceRef: "@e4", targetRef: "@e5" }, + }, + payload: { + requestId: "req-drag", + ok: true, + result: { command: "drag", browserId: BROWSER_ID, sourceRef: "@e4", targetRef: "@e5" }, + }, + content: [{ type: "text", text: "Dragged browser element @e4 to @e5." }], + }, +] satisfies Array<{ + name: string; + toolName: string; + input: Record; + command: BrowserToolsExecuteInput["command"]; + payload: Extract; + content: PaseoToolResult["content"]; +}>; + +const brokerErrorCases = [ + { + name: "disabled browser tools", + toolName: "browser_list_tabs", + input: {}, + payload: { + requestId: "req-disabled", + ok: false, + error: { + code: "browser_disabled", + message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.", + retryable: false, + }, + }, + content: [ + { + type: "text", + text: "Browser tools are disabled. Enable desktop browser tools on the host, then try again.", + }, + ], + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, + }, + { + name: "typed timeout errors", + toolName: "browser_page_info", + input: { browserId: BROWSER_ID }, + payload: { + requestId: "req-timeout", + ok: false, + error: { + code: "browser_timeout", + message: "Browser automation timed out after 15000ms.", + retryable: true, + }, + }, + content: [ + { + type: "text", + text: "The browser did not respond before the timeout. Try again or check the desktop app.", + }, + ], + context: { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + browserId: BROWSER_ID, + }, + }, + { + name: "screenshot no-frame errors", + toolName: "browser_screenshot", + input: { browserId: BROWSER_ID }, + payload: { + requestId: "req-no-frame", + ok: false, + error: { + code: "screenshot_no_frame", + message: "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }, + content: [ + { + type: "text", + text: "The browser tab has no painted frame. Focus the tab in the app, then try again.", + }, + ], + context: { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + browserId: BROWSER_ID, + }, + }, +] satisfies Array<{ + name: string; + toolName: string; + input: Record; + payload: Extract; + content: PaseoToolResult["content"]; + context: Record; +}>; + describe("registerBrowserTools", () => { test("list tabs sends workspace in the request envelope", async () => { const harness = new BrowserToolHarness(); @@ -429,6 +841,53 @@ describe("registerBrowserTools", () => { }); }); + test.each(routedToolCases)( + "$name routes browser id in command args and workspace id in the envelope", + async ({ toolName, input, command, payload, content }) => { + const harness = new BrowserToolHarness(); + harness.broker.setResponse(payload); + + const response = await harness.execute(toolName, input); + + expect(harness.broker.calls).toEqual([ + { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + command, + }, + ]); + expect(response.content).toEqual(content); + expect(response.structuredContent).toEqual({ + ok: payload.ok, + result: payload.result, + context: { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + browserId: BROWSER_ID, + }, + }); + }, + ); + + test.each(brokerErrorCases)( + "$name keep broker error summaries model-actionable", + async ({ toolName, input, payload, content, context }) => { + const harness = new BrowserToolHarness(); + harness.broker.setResponse(payload); + + const response = await harness.execute(toolName, input); + + expect(response.content).toEqual(content); + expect(response.structuredContent).toEqual({ + ok: false, + error: payload.error, + context, + }); + }, + ); + test("wait rejects calls without a condition", () => { const harness = new BrowserToolHarness(); diff --git a/packages/server/src/server/websocket-server.browser-tools.test.ts b/packages/server/src/server/websocket-server.browser-tools.test.ts index f3c8d0c97..1ef750c35 100644 --- a/packages/server/src/server/websocket-server.browser-tools.test.ts +++ b/packages/server/src/server/websocket-server.browser-tools.test.ts @@ -7,7 +7,7 @@ import type { } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; import type pino from "pino"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; @@ -228,17 +228,17 @@ function createVoiceAssistantWebSocketServer(params: { }): VoiceAssistantWebSocketServer { const { httpServer, broker } = params; const agentManager = { - setAgentAttentionCallback: vi.fn(), - subscribe: vi.fn(() => () => {}), - getMetricsSnapshot: vi.fn(() => ({ + setAgentAttentionCallback() {}, + subscribe: () => () => {}, + getMetricsSnapshot: () => ({ total: 0, byLifecycle: {}, withActiveForegroundTurn: 0, timelineStats: { totalItems: 0, maxItemsPerAgent: 0 }, - })), + }), }; const daemonConfigStore = { - onChange: vi.fn(() => () => {}), + onChange: () => () => {}, }; return new VoiceAssistantWebSocketServer( @@ -264,15 +264,15 @@ function createVoiceAssistantWebSocketServer(params: { createStub({}), createStub({}), createStub({ - subscribe: vi.fn(), - scheduleRefreshForCwd: vi.fn(), - getMetrics: vi.fn(() => ({ + subscribe: () => {}, + scheduleRefreshForCwd: () => {}, + getMetrics: () => ({ checkoutDiffTargetCount: 0, checkoutDiffSubscriptionCount: 0, checkoutDiffWatcherCount: 0, checkoutDiffFallbackRefreshTargetCount: 0, - })), - dispose: vi.fn(), + }), + dispose: () => {}, }), undefined, undefined, @@ -348,12 +348,12 @@ function createBrowserRequestQueue(): QueuedBrowserRequests { function createLogger() { const logger = { - child: vi.fn(() => logger), - trace: vi.fn(), - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), + child: () => logger, + trace() {}, + debug() {}, + info() {}, + warn() {}, + error() {}, }; return logger; }