Add opt-in browser tools for desktop tabs

* Add opt-in browser tools for desktop tabs

Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs.

* Fix browser tools CI expectations

* Address browser tools review findings

* Restrict browser file automation paths

* Fix browser upload test on Windows

* Harden browser navigation inputs

* Make browser tools create usable tabs

* Update browser MCP empty-state test

* Fail browser tab creation when registration times out

* Fix browser screenshots for agents

* Hide disabled browser tools from agents

* Address browser tools architecture review

* Replace browser tools review tests

* Wrap browser tab registration errors

* Mock Expo Router in app unit tests

* Handle invalid browser automation requests

* Return browser failure on desktop disconnect

* Update browser disconnect websocket test

* Relax browser timeout polling test

* Handle invalid browser responses

* Return browser failure when send fails

* Remove local diagnostics and fixture paths

* Fix dev service home fallback

* Use worktree home for dev services

* Use managed daemon in desktop dev

* fix(browser): keep agent tabs addressable

Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution.

* refactor(browser): register tools through catalog

Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter.

* fix(settings): translate browser tools host error
This commit is contained in:
Mohamed Boudra
2026-06-30 22:33:18 +02:00
committed by GitHub
parent db03b1f3fd
commit 20385bdb50
71 changed files with 10908 additions and 115 deletions
@@ -25,6 +25,11 @@ import {
} from "@/desktop/host";
import { isDev } from "@/constants/platform";
import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/stores/browser-store";
import {
prepareBrowserWebview,
releaseResidentBrowserWebview,
takeResidentBrowserWebview,
} from "./browser-webview-resident";
type ElectronWebview = HTMLElement & {
canGoBack?: () => boolean;
@@ -404,16 +409,16 @@ export function BrowserPane({
initialUrlRef.current,
browserErrorLabelsRef.current,
);
const webview = document.createElement("webview") as ElectronWebview;
const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null;
const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview);
webviewRef.current = webview;
webview.setAttribute("partition", `persist:paseo-browser-${browserId}`);
webview.setAttribute("allowpopups", "true");
webview.setAttribute("spellcheck", "false");
webview.setAttribute("autosize", "on");
webview.setAttribute(
"src",
initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current,
);
void getDesktopHost()?.browser?.registerWorkspaceBrowser?.({ browserId, workspaceId });
if (!residentWebview) {
prepareBrowserWebview(webview, {
browserId,
initialUrl: initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current,
});
}
webview.style.display = "flex";
webview.style.flex = "1";
webview.style.width = "100%";
@@ -528,7 +533,14 @@ export function BrowserPane({
webview.removeEventListener("focus", handleWebviewFocus);
webview.removeEventListener("mousedown", handleWebviewFocus);
if (host.contains(webview)) {
host.removeChild(webview);
const browserStillExists = Boolean(
useBrowserStore.getState().browsersById[browserIdRef.current],
);
if (browserStillExists) {
releaseResidentBrowserWebview(browserIdRef.current, webview);
} else {
host.removeChild(webview);
}
}
if (webviewRef.current === webview) {
webviewRef.current = null;
@@ -0,0 +1,60 @@
import { afterEach, describe, expect, it } from "vitest";
import {
clearResidentBrowserWebviewsForTests,
ensureResidentBrowserWebview,
releaseResidentBrowserWebview,
removeResidentBrowserWebview,
takeResidentBrowserWebview,
} from "./browser-webview-resident";
describe("resident browser webviews", () => {
afterEach(() => {
clearResidentBrowserWebviewsForTests();
});
it("keeps a browser webview mounted offscreen and reuses the same node", () => {
const host = document.createElement("div");
const webview = document.createElement("webview");
host.appendChild(webview);
document.body.appendChild(host);
releaseResidentBrowserWebview("browser-a", webview);
expect(host.children).toHaveLength(0);
expect(webview.isConnected).toBe(true);
expect(webview.style.width).toBe("1280px");
expect(webview.style.height).toBe("800px");
const reused = takeResidentBrowserWebview("browser-a");
expect(reused).toBe(webview);
expect(takeResidentBrowserWebview("browser-a")).toBeNull();
});
it("creates a resident webview for an agent-created unfocused tab", () => {
const webview = ensureResidentBrowserWebview({
browserId: "browser-agent",
url: "https://example.com",
});
expect(webview).not.toBeNull();
expect(webview?.isConnected).toBe(true);
expect(webview?.getAttribute("data-paseo-browser-id")).toBe("browser-agent");
expect(webview?.getAttribute("partition")).toBe("persist:paseo-browser-browser-agent");
expect((webview as HTMLUnknownElement & { src?: string })?.src).toContain(
"https://example.com",
);
});
it("removes a resident webview when its browser tab closes", () => {
const webview = ensureResidentBrowserWebview({
browserId: "browser-closed",
url: "https://example.com",
});
removeResidentBrowserWebview("browser-closed");
expect(webview?.isConnected).toBe(false);
expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
});
});
@@ -0,0 +1,153 @@
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
const RESIDENT_VIEWPORT_WIDTH = 1280;
const RESIDENT_VIEWPORT_HEIGHT = 800;
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
interface BrowserWebviewElement extends HTMLElement {
src: string;
}
function trimNonEmpty(value: string | null | undefined): string | null {
if (typeof value !== "string") {
return null;
}
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null;
}
function readDocument(): Document | null {
return typeof document === "undefined" ? null : document;
}
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
if (existing) {
return existing;
}
const host = ownerDocument.createElement("div");
host.id = RESIDENT_BROWSER_HOST_ID;
host.setAttribute("aria-hidden", "true");
host.style.position = "fixed";
host.style.left = "-20000px";
host.style.top = "0";
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
host.style.overflow = "hidden";
host.style.opacity = "0";
host.style.pointerEvents = "none";
ownerDocument.body.appendChild(host);
return host;
}
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
return element as HTMLElement;
}
}
return null;
}
function applyResidentWebviewStyle(webview: HTMLElement): void {
webview.style.display = "block";
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
webview.style.border = "0";
webview.style.background = "transparent";
}
export function prepareBrowserWebview(
webview: HTMLElement,
input: { browserId: string; initialUrl?: string | null },
): void {
webview.setAttribute(BROWSER_ID_ATTRIBUTE, input.browserId);
webview.setAttribute("partition", `persist:paseo-browser-${input.browserId}`);
webview.setAttribute("allowpopups", "true");
webview.setAttribute("spellcheck", "false");
webview.setAttribute("autosize", "on");
if (input.initialUrl) {
(webview as BrowserWebviewElement).src = input.initialUrl;
}
}
export function ensureResidentBrowserWebview(input: {
browserId: string;
url: string;
}): HTMLElement | null {
const browserId = trimNonEmpty(input.browserId);
if (!browserId) {
return null;
}
const ownerDocument = readDocument();
if (!ownerDocument) {
return null;
}
const resident = residentWebviewsByBrowserId.get(browserId) ?? null;
if (resident?.isConnected) {
return resident;
}
const existing = findBrowserWebview(browserId, ownerDocument);
if (existing) {
return existing;
}
const webview = ownerDocument.createElement("webview") as BrowserWebviewElement;
prepareBrowserWebview(webview, { browserId, initialUrl: input.url });
releaseResidentBrowserWebview(browserId, webview);
return webview;
}
export function takeResidentBrowserWebview(browserId: string): HTMLElement | null {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
return null;
}
const webview = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
if (!webview) {
return null;
}
residentWebviewsByBrowserId.delete(normalizedBrowserId);
return webview;
}
export function releaseResidentBrowserWebview(browserId: string, webview: HTMLElement): void {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
webview.remove();
return;
}
const ownerDocument = readDocument();
if (!ownerDocument) {
return;
}
residentWebviewsByBrowserId.set(normalizedBrowserId, webview);
applyResidentWebviewStyle(webview);
getResidentBrowserHost(ownerDocument).appendChild(webview);
}
export function removeResidentBrowserWebview(browserId: string): void {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
return;
}
const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
residentWebviewsByBrowserId.delete(normalizedBrowserId);
resident?.remove();
}
export function clearResidentBrowserWebviewsForTests(): void {
for (const webview of residentWebviewsByBrowserId.values()) {
webview.remove();
}
residentWebviewsByBrowserId.clear();
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
}