Add opt-in browser tools for desktop tabs
* Add opt-in browser tools for desktop tabs Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs. * Fix browser tools CI expectations * Address browser tools review findings * Restrict browser file automation paths * Fix browser upload test on Windows * Harden browser navigation inputs * Make browser tools create usable tabs * Update browser MCP empty-state test * Fail browser tab creation when registration times out * Fix browser screenshots for agents * Hide disabled browser tools from agents * Address browser tools architecture review * Replace browser tools review tests * Wrap browser tab registration errors * Mock Expo Router in app unit tests * Handle invalid browser automation requests * Return browser failure on desktop disconnect * Update browser disconnect websocket test * Relax browser timeout polling test * Handle invalid browser responses * Return browser failure when send fails * Remove local diagnostics and fixture paths * Fix dev service home fallback * Use worktree home for dev services * Use managed daemon in desktop dev * fix(browser): keep agent tabs addressable Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution. * refactor(browser): register tools through catalog Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter. * fix(settings): translate browser tools host error
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import { afterEach, expect, expectTypeOf, test, vi } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { DaemonClient, type DaemonTransport, type Logger } from "./daemon-client";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import {
|
||||
decodeFileTransferFrame,
|
||||
encodeFileTransferFrame,
|
||||
@@ -136,6 +137,58 @@ afterEach(async () => {
|
||||
await Promise.all(clients.map((client) => client.close()));
|
||||
clients.length = 0;
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
test("does not infer browser automation capabilities from Electron runtime", async () => {
|
||||
vi.stubGlobal("navigator", {
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Paseo/0.1.89 Chrome/146 Electron/41.2.0 Safari/537.36",
|
||||
});
|
||||
const mock = createMockTransport();
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "electron_unit_test",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
const connectPromise = client.connect();
|
||||
mock.triggerOpen({ preserveSent: true });
|
||||
await connectPromise;
|
||||
|
||||
const hello = z
|
||||
.object({
|
||||
type: z.literal("hello"),
|
||||
capabilities: z.record(z.unknown()),
|
||||
})
|
||||
.parse(JSON.parse(assertStr(mock.sent[0])));
|
||||
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBeUndefined();
|
||||
});
|
||||
|
||||
test("advertises consumer-provided browser automation capabilities", async () => {
|
||||
const mock = createMockTransport();
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "browser_capability_unit_test",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
const connectPromise = client.connect();
|
||||
mock.triggerOpen({ preserveSent: true });
|
||||
await connectPromise;
|
||||
|
||||
const hello = z
|
||||
.object({
|
||||
type: z.literal("hello"),
|
||||
capabilities: z.record(z.unknown()),
|
||||
})
|
||||
.parse(JSON.parse(assertStr(mock.sent[0])));
|
||||
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBe(true);
|
||||
});
|
||||
|
||||
const noopLogger: Logger = {
|
||||
@@ -453,6 +506,7 @@ test("advertises client capabilities in hello", async () => {
|
||||
logger,
|
||||
reconnect: { enabled: false },
|
||||
transportFactory: () => mock.transport,
|
||||
capabilities: { desktop_browser_automation: true },
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
@@ -470,6 +524,43 @@ test("advertises client capabilities in hello", async () => {
|
||||
custom_mode_icons: true,
|
||||
reasoning_merge_enum: true,
|
||||
terminal_reflowable_snapshot: true,
|
||||
desktop_browser_automation: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("sends typed browser automation execute responses", async () => {
|
||||
const logger = createMockLogger();
|
||||
const mock = createMockTransport();
|
||||
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "clsk_unit_test",
|
||||
logger,
|
||||
reconnect: { enabled: false },
|
||||
transportFactory: () => mock.transport,
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
const connectPromise = client.connect();
|
||||
mock.triggerOpen();
|
||||
await connectPromise;
|
||||
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
},
|
||||
});
|
||||
|
||||
expect(parseSentFrame(mock.sent[0])).toEqual({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { z } from "zod";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import { CLIENT_CAPS, type ClientCapability } from "@getpaseo/protocol/client-capabilities";
|
||||
import {
|
||||
AgentCreateFailedStatusPayloadSchema,
|
||||
AgentCreatedStatusPayloadSchema,
|
||||
@@ -119,6 +119,10 @@ import {
|
||||
} from "./daemon-client-transport.js";
|
||||
import { DaemonClientRuntimeMetrics } from "./daemon-client-runtime-metrics.js";
|
||||
import { TerminalStreamRouter, type TerminalStreamEvent } from "./terminal-stream-router.js";
|
||||
import type {
|
||||
BrowserAutomationExecuteRequest,
|
||||
BrowserAutomationExecuteResponse,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
|
||||
export interface Logger {
|
||||
debug(obj: object, msg?: string): void;
|
||||
@@ -221,6 +225,8 @@ export type DaemonEvent =
|
||||
| { type: "error"; message: string };
|
||||
|
||||
export type DaemonEventHandler = (event: DaemonEvent) => void;
|
||||
export type BrowserAutomationExecuteRequestMessage = BrowserAutomationExecuteRequest;
|
||||
export type BrowserAutomationExecuteResponseMessage = BrowserAutomationExecuteResponse;
|
||||
|
||||
export interface DaemonClientConfig {
|
||||
url: string;
|
||||
@@ -246,6 +252,7 @@ export interface DaemonClientConfig {
|
||||
};
|
||||
runtimeMetricsIntervalMs?: number;
|
||||
runtimeMetricsWindowMs?: number;
|
||||
capabilities?: Partial<Record<ClientCapability, boolean>>;
|
||||
}
|
||||
|
||||
export interface SendMessageOptions {
|
||||
@@ -3850,6 +3857,10 @@ export class DaemonClient {
|
||||
});
|
||||
}
|
||||
|
||||
sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void {
|
||||
this.sendSessionMessageStrict(response);
|
||||
}
|
||||
|
||||
async readProjectConfig(repoRoot: string, requestId?: string): Promise<ReadProjectConfigPayload> {
|
||||
return this.sendCorrelatedSessionRequest({
|
||||
requestId,
|
||||
@@ -4648,6 +4659,7 @@ export class DaemonClient {
|
||||
[CLIENT_CAPS.customModeIcons]: true,
|
||||
[CLIENT_CAPS.reasoningMergeEnum]: true,
|
||||
[CLIENT_CAPS.terminalReflowableSnapshot]: true,
|
||||
...this.config.capabilities,
|
||||
},
|
||||
...(this.config.appVersion ? { appVersion: this.config.appVersion } : {}),
|
||||
}),
|
||||
|
||||
@@ -674,6 +674,7 @@ test("config actions delegate to existing daemon config RPCs", async () => {
|
||||
config: {
|
||||
mcp: { injectIntoAgents: true },
|
||||
providers: {},
|
||||
browserTools: { enabled: false },
|
||||
metadataGeneration: { providers: [] },
|
||||
autoArchiveAfterMerge: false,
|
||||
enableTerminalAgentHooks: false,
|
||||
@@ -728,6 +729,7 @@ test("config actions delegate to existing daemon config RPCs", async () => {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
browserTools: { enabled: false },
|
||||
metadataGeneration: { providers: [] },
|
||||
autoArchiveAfterMerge: false,
|
||||
enableTerminalAgentHooks: false,
|
||||
|
||||
@@ -30,6 +30,8 @@ export { DaemonClient };
|
||||
export type {
|
||||
DaemonClientConfig,
|
||||
DaemonEvent,
|
||||
BrowserAutomationExecuteRequestMessage,
|
||||
BrowserAutomationExecuteResponseMessage,
|
||||
WebSocketFactory,
|
||||
WebSocketLike,
|
||||
} from "./daemon-client.js";
|
||||
|
||||
Reference in New Issue
Block a user