Add opt-in browser tools for desktop tabs
* Add opt-in browser tools for desktop tabs Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs. * Fix browser tools CI expectations * Address browser tools review findings * Restrict browser file automation paths * Fix browser upload test on Windows * Harden browser navigation inputs * Make browser tools create usable tabs * Update browser MCP empty-state test * Fail browser tab creation when registration times out * Fix browser screenshots for agents * Hide disabled browser tools from agents * Address browser tools architecture review * Replace browser tools review tests * Wrap browser tab registration errors * Mock Expo Router in app unit tests * Handle invalid browser automation requests * Return browser failure on desktop disconnect * Update browser disconnect websocket test * Relax browser timeout polling test * Handle invalid browser responses * Return browser failure when send fails * Remove local diagnostics and fixture paths * Fix dev service home fallback * Use worktree home for dev services * Use managed daemon in desktop dev * fix(browser): keep agent tabs addressable Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution. * refactor(browser): register tools through catalog Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter. * fix(settings): translate browser tools host error
This commit is contained in:
@@ -25,6 +25,11 @@ import {
|
||||
} from "@/desktop/host";
|
||||
import { isDev } from "@/constants/platform";
|
||||
import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/stores/browser-store";
|
||||
import {
|
||||
prepareBrowserWebview,
|
||||
releaseResidentBrowserWebview,
|
||||
takeResidentBrowserWebview,
|
||||
} from "./browser-webview-resident";
|
||||
|
||||
type ElectronWebview = HTMLElement & {
|
||||
canGoBack?: () => boolean;
|
||||
@@ -404,16 +409,16 @@ export function BrowserPane({
|
||||
initialUrlRef.current,
|
||||
browserErrorLabelsRef.current,
|
||||
);
|
||||
const webview = document.createElement("webview") as ElectronWebview;
|
||||
const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null;
|
||||
const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview);
|
||||
webviewRef.current = webview;
|
||||
webview.setAttribute("partition", `persist:paseo-browser-${browserId}`);
|
||||
webview.setAttribute("allowpopups", "true");
|
||||
webview.setAttribute("spellcheck", "false");
|
||||
webview.setAttribute("autosize", "on");
|
||||
webview.setAttribute(
|
||||
"src",
|
||||
initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current,
|
||||
);
|
||||
void getDesktopHost()?.browser?.registerWorkspaceBrowser?.({ browserId, workspaceId });
|
||||
if (!residentWebview) {
|
||||
prepareBrowserWebview(webview, {
|
||||
browserId,
|
||||
initialUrl: initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current,
|
||||
});
|
||||
}
|
||||
webview.style.display = "flex";
|
||||
webview.style.flex = "1";
|
||||
webview.style.width = "100%";
|
||||
@@ -528,7 +533,14 @@ export function BrowserPane({
|
||||
webview.removeEventListener("focus", handleWebviewFocus);
|
||||
webview.removeEventListener("mousedown", handleWebviewFocus);
|
||||
if (host.contains(webview)) {
|
||||
host.removeChild(webview);
|
||||
const browserStillExists = Boolean(
|
||||
useBrowserStore.getState().browsersById[browserIdRef.current],
|
||||
);
|
||||
if (browserStillExists) {
|
||||
releaseResidentBrowserWebview(browserIdRef.current, webview);
|
||||
} else {
|
||||
host.removeChild(webview);
|
||||
}
|
||||
}
|
||||
if (webviewRef.current === webview) {
|
||||
webviewRef.current = null;
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
clearResidentBrowserWebviewsForTests,
|
||||
ensureResidentBrowserWebview,
|
||||
releaseResidentBrowserWebview,
|
||||
removeResidentBrowserWebview,
|
||||
takeResidentBrowserWebview,
|
||||
} from "./browser-webview-resident";
|
||||
|
||||
describe("resident browser webviews", () => {
|
||||
afterEach(() => {
|
||||
clearResidentBrowserWebviewsForTests();
|
||||
});
|
||||
|
||||
it("keeps a browser webview mounted offscreen and reuses the same node", () => {
|
||||
const host = document.createElement("div");
|
||||
const webview = document.createElement("webview");
|
||||
host.appendChild(webview);
|
||||
document.body.appendChild(host);
|
||||
|
||||
releaseResidentBrowserWebview("browser-a", webview);
|
||||
|
||||
expect(host.children).toHaveLength(0);
|
||||
expect(webview.isConnected).toBe(true);
|
||||
expect(webview.style.width).toBe("1280px");
|
||||
expect(webview.style.height).toBe("800px");
|
||||
|
||||
const reused = takeResidentBrowserWebview("browser-a");
|
||||
|
||||
expect(reused).toBe(webview);
|
||||
expect(takeResidentBrowserWebview("browser-a")).toBeNull();
|
||||
});
|
||||
|
||||
it("creates a resident webview for an agent-created unfocused tab", () => {
|
||||
const webview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-agent",
|
||||
url: "https://example.com",
|
||||
});
|
||||
|
||||
expect(webview).not.toBeNull();
|
||||
expect(webview?.isConnected).toBe(true);
|
||||
expect(webview?.getAttribute("data-paseo-browser-id")).toBe("browser-agent");
|
||||
expect(webview?.getAttribute("partition")).toBe("persist:paseo-browser-browser-agent");
|
||||
expect((webview as HTMLUnknownElement & { src?: string })?.src).toContain(
|
||||
"https://example.com",
|
||||
);
|
||||
});
|
||||
|
||||
it("removes a resident webview when its browser tab closes", () => {
|
||||
const webview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-closed",
|
||||
url: "https://example.com",
|
||||
});
|
||||
|
||||
removeResidentBrowserWebview("browser-closed");
|
||||
|
||||
expect(webview?.isConnected).toBe(false);
|
||||
expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,153 @@
|
||||
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
|
||||
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
|
||||
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||
|
||||
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
|
||||
|
||||
interface BrowserWebviewElement extends HTMLElement {
|
||||
src: string;
|
||||
}
|
||||
|
||||
function trimNonEmpty(value: string | null | undefined): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function readDocument(): Document | null {
|
||||
return typeof document === "undefined" ? null : document;
|
||||
}
|
||||
|
||||
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
||||
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const host = ownerDocument.createElement("div");
|
||||
host.id = RESIDENT_BROWSER_HOST_ID;
|
||||
host.setAttribute("aria-hidden", "true");
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "-20000px";
|
||||
host.style.top = "0";
|
||||
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "0";
|
||||
host.style.pointerEvents = "none";
|
||||
ownerDocument.body.appendChild(host);
|
||||
return host;
|
||||
}
|
||||
|
||||
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
|
||||
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
|
||||
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
|
||||
return element as HTMLElement;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function applyResidentWebviewStyle(webview: HTMLElement): void {
|
||||
webview.style.display = "block";
|
||||
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
webview.style.border = "0";
|
||||
webview.style.background = "transparent";
|
||||
}
|
||||
|
||||
export function prepareBrowserWebview(
|
||||
webview: HTMLElement,
|
||||
input: { browserId: string; initialUrl?: string | null },
|
||||
): void {
|
||||
webview.setAttribute(BROWSER_ID_ATTRIBUTE, input.browserId);
|
||||
webview.setAttribute("partition", `persist:paseo-browser-${input.browserId}`);
|
||||
webview.setAttribute("allowpopups", "true");
|
||||
webview.setAttribute("spellcheck", "false");
|
||||
webview.setAttribute("autosize", "on");
|
||||
if (input.initialUrl) {
|
||||
(webview as BrowserWebviewElement).src = input.initialUrl;
|
||||
}
|
||||
}
|
||||
|
||||
export function ensureResidentBrowserWebview(input: {
|
||||
browserId: string;
|
||||
url: string;
|
||||
}): HTMLElement | null {
|
||||
const browserId = trimNonEmpty(input.browserId);
|
||||
if (!browserId) {
|
||||
return null;
|
||||
}
|
||||
const ownerDocument = readDocument();
|
||||
if (!ownerDocument) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const resident = residentWebviewsByBrowserId.get(browserId) ?? null;
|
||||
if (resident?.isConnected) {
|
||||
return resident;
|
||||
}
|
||||
|
||||
const existing = findBrowserWebview(browserId, ownerDocument);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const webview = ownerDocument.createElement("webview") as BrowserWebviewElement;
|
||||
prepareBrowserWebview(webview, { browserId, initialUrl: input.url });
|
||||
releaseResidentBrowserWebview(browserId, webview);
|
||||
return webview;
|
||||
}
|
||||
|
||||
export function takeResidentBrowserWebview(browserId: string): HTMLElement | null {
|
||||
const normalizedBrowserId = trimNonEmpty(browserId);
|
||||
if (!normalizedBrowserId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const webview = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
|
||||
if (!webview) {
|
||||
return null;
|
||||
}
|
||||
|
||||
residentWebviewsByBrowserId.delete(normalizedBrowserId);
|
||||
return webview;
|
||||
}
|
||||
|
||||
export function releaseResidentBrowserWebview(browserId: string, webview: HTMLElement): void {
|
||||
const normalizedBrowserId = trimNonEmpty(browserId);
|
||||
if (!normalizedBrowserId) {
|
||||
webview.remove();
|
||||
return;
|
||||
}
|
||||
const ownerDocument = readDocument();
|
||||
if (!ownerDocument) {
|
||||
return;
|
||||
}
|
||||
|
||||
residentWebviewsByBrowserId.set(normalizedBrowserId, webview);
|
||||
applyResidentWebviewStyle(webview);
|
||||
getResidentBrowserHost(ownerDocument).appendChild(webview);
|
||||
}
|
||||
|
||||
export function removeResidentBrowserWebview(browserId: string): void {
|
||||
const normalizedBrowserId = trimNonEmpty(browserId);
|
||||
if (!normalizedBrowserId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
|
||||
residentWebviewsByBrowserId.delete(normalizedBrowserId);
|
||||
resident?.remove();
|
||||
}
|
||||
|
||||
export function clearResidentBrowserWebviewsForTests(): void {
|
||||
for (const webview of residentWebviewsByBrowserId.values()) {
|
||||
webview.remove();
|
||||
}
|
||||
residentWebviewsByBrowserId.clear();
|
||||
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
|
||||
}
|
||||
Reference in New Issue
Block a user