Add opt-in browser tools for desktop tabs

* Add opt-in browser tools for desktop tabs

Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs.

* Fix browser tools CI expectations

* Address browser tools review findings

* Restrict browser file automation paths

* Fix browser upload test on Windows

* Harden browser navigation inputs

* Make browser tools create usable tabs

* Update browser MCP empty-state test

* Fail browser tab creation when registration times out

* Fix browser screenshots for agents

* Hide disabled browser tools from agents

* Address browser tools architecture review

* Replace browser tools review tests

* Wrap browser tab registration errors

* Mock Expo Router in app unit tests

* Handle invalid browser automation requests

* Return browser failure on desktop disconnect

* Update browser disconnect websocket test

* Relax browser timeout polling test

* Handle invalid browser responses

* Return browser failure when send fails

* Remove local diagnostics and fixture paths

* Fix dev service home fallback

* Use worktree home for dev services

* Use managed daemon in desktop dev

* fix(browser): keep agent tabs addressable

Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution.

* refactor(browser): register tools through catalog

Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter.

* fix(settings): translate browser tools host error
This commit is contained in:
Mohamed Boudra
2026-06-30 22:33:18 +02:00
committed by GitHub
parent db03b1f3fd
commit 20385bdb50
71 changed files with 10908 additions and 115 deletions
@@ -0,0 +1,70 @@
import React, { useCallback } from "react";
import { Text, View } from "react-native";
import { useMutation } from "@tanstack/react-query";
import { useTranslation } from "react-i18next";
import { Switch } from "@/components/ui/switch";
import { useDaemonConfig } from "@/hooks/use-daemon-config";
import { useHostRuntimeIsConnected } from "@/runtime/host-runtime";
import { settingsStyles } from "@/styles/settings";
import {
createBrowserToolsPatch,
getBrowserToolsCardState,
getBrowserToolsMutationViewState,
} from "./browser-tools-config";
export function BrowserToolsOptInCard({ serverId }: { serverId: string }) {
const { t } = useTranslation();
const isConnected = useHostRuntimeIsConnected(serverId);
const { config, patchConfig } = useDaemonConfig(serverId);
const state = getBrowserToolsCardState({ isConnected, config });
const mutation = useMutation({
mutationFn: async (next: boolean) => {
const result = await patchConfig(createBrowserToolsPatch(next));
if (!result) {
throw new Error(t("workspace.terminal.hostDisconnected"));
}
return result;
},
});
const mutationView = getBrowserToolsMutationViewState({
isPending: mutation.isPending,
error: mutation.error,
});
const handleValueChange = useCallback(
(next: boolean) => {
mutation.mutate(next);
},
[mutation],
);
if (!state.isVisible) return null;
return (
<View style={settingsStyles.card} testID="host-page-browser-tools-card">
<View style={settingsStyles.row}>
<View style={settingsStyles.rowContent}>
<Text style={settingsStyles.rowTitle}>{state.title}</Text>
<Text style={settingsStyles.rowHint}>{state.warning}</Text>
{mutationView.loadingText ? (
<Text style={settingsStyles.rowHint} testID="host-page-browser-tools-loading">
{mutationView.loadingText}
</Text>
) : null}
{mutationView.errorText ? (
<Text style={settingsStyles.rowError} testID="host-page-browser-tools-error">
{mutationView.errorText}
</Text>
) : null}
</View>
<Switch
value={state.isEnabled}
onValueChange={handleValueChange}
disabled={mutationView.isSwitchDisabled}
accessibilityLabel="Enable browser tools"
testID="host-page-browser-tools-switch"
/>
</View>
</View>
);
}
@@ -0,0 +1,70 @@
import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
import { describe, expect, it } from "vitest";
import {
BROWSER_TOOLS_WARNING,
createBrowserToolsPatch,
getBrowserToolsCardState,
getBrowserToolsMutationViewState,
} from "./browser-tools-config";
function makeConfig(browserToolsEnabled = false): MutableDaemonConfig {
return {
mcp: { injectIntoAgents: false },
browserTools: { enabled: browserToolsEnabled },
providers: {},
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
enableTerminalAgentHooks: false,
appendSystemPrompt: "",
};
}
describe("browser tools opt-in config", () => {
it("shows the card with the logged-in browser state warning when connected", () => {
expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(false) })).toEqual({
isVisible: true,
isEnabled: false,
title: "Browser tools",
warning: BROWSER_TOOLS_WARNING,
});
});
it("reads enabled state from daemon config", () => {
expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(true) })).toMatchObject(
{
isEnabled: true,
},
);
});
it("hides the card when the host is disconnected", () => {
expect(
getBrowserToolsCardState({ isConnected: false, config: makeConfig(true) }),
).toMatchObject({
isVisible: false,
});
});
it("writes daemon.browserTools.enabled when toggled", () => {
expect(createBrowserToolsPatch(true)).toEqual({ browserTools: { enabled: true } });
expect(createBrowserToolsPatch(false)).toEqual({ browserTools: { enabled: false } });
});
it("shows loading and disables the toggle while browser tool settings save", () => {
expect(getBrowserToolsMutationViewState({ isPending: true, error: null })).toEqual({
isSwitchDisabled: true,
loadingText: "Updating browser tools…",
errorText: null,
});
});
it("shows the save error when browser tool settings fail", () => {
expect(
getBrowserToolsMutationViewState({ isPending: false, error: new Error("Disk full") }),
).toEqual({
isSwitchDisabled: false,
loadingText: null,
errorText: "Disk full",
});
});
});
@@ -0,0 +1,49 @@
import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
export const BROWSER_TOOLS_TITLE = "Browser tools";
export const BROWSER_TOOLS_WARNING =
"Allow agents to access and control Paseo desktop browser tabs, including logged-in browser state. Only enable this for agents you trust.";
export interface BrowserToolsCardState {
isVisible: boolean;
isEnabled: boolean;
title: string;
warning: string;
}
export interface BrowserToolsMutationViewState {
isSwitchDisabled: boolean;
loadingText: string | null;
errorText: string | null;
}
export function getBrowserToolsCardState(input: {
isConnected: boolean;
config: MutableDaemonConfig | null;
}): BrowserToolsCardState {
return {
isVisible: input.isConnected,
isEnabled: input.config?.browserTools.enabled === true,
title: BROWSER_TOOLS_TITLE,
warning: BROWSER_TOOLS_WARNING,
};
}
export function createBrowserToolsPatch(enabled: boolean): Partial<MutableDaemonConfig> {
return { browserTools: { enabled } };
}
export function getBrowserToolsMutationViewState(input: {
isPending: boolean;
error: unknown;
}): BrowserToolsMutationViewState {
return {
isSwitchDisabled: input.isPending,
loadingText: input.isPending ? "Updating browser tools…" : null,
errorText: input.error ? toErrorMessage(input.error) : null,
};
}
function toErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
@@ -61,6 +61,7 @@ import { formatLatency } from "@/utils/latency";
import { ICON_SIZE } from "@/styles/theme";
import type { Theme } from "@/styles/theme";
import { getProviderIcon } from "@/components/provider-icons";
import { BrowserToolsOptInCard } from "./browser-tools-card";
const ThemedArrowUp = withUnistyles(ArrowUp);
const ThemedArrowDown = withUnistyles(ArrowDown);
@@ -262,6 +263,7 @@ export function HostAgentsPage({ serverId }: { serverId: string }) {
{isConnected ? (
<SettingsSection title={t("settings.hostSections.agents")}>
<InjectPaseoToolsCard serverId={serverId} />
<BrowserToolsOptInCard serverId={serverId} />
<AppendSystemPromptCard serverId={serverId} />
</SettingsSection>
) : (
@@ -220,6 +220,7 @@ const disabledCodexEntry: ProviderSnapshotEntry = {
function makeConfig(providers: MutableDaemonConfig["providers"] = {}): MutableDaemonConfig {
return {
mcp: { injectIntoAgents: false },
browserTools: { enabled: false },
providers,
metadataGeneration: { providers: [] },
autoArchiveAfterMerge: false,
@@ -105,6 +105,7 @@ import type { CheckoutStatusPayload } from "@/git/use-status-query";
import { confirmDialog } from "@/utils/confirm-dialog";
import { useArchiveAgent } from "@/hooks/use-archive-agent";
import { useStableEvent } from "@/hooks/use-stable-event";
import { removeResidentBrowserWebview } from "@/components/browser-webview-resident";
import { createWorkspaceBrowser, useBrowserStore } from "@/stores/browser-store";
import { getDesktopHost } from "@/desktop/host";
import { buildProviderCommand } from "@/utils/provider-command-templates";
@@ -304,19 +305,22 @@ function decodeSegment(value: string): string {
function useSyncWorkspaceActiveBrowser(input: {
workspaceLayout: WorkspaceLayout | null;
isRouteFocused: boolean;
workspaceId: string;
}) {
const focusedBrowserId = useMemo(
() => getFocusedBrowserId(input.workspaceLayout),
[input.workspaceLayout],
);
const desktopActiveBrowserId = input.isRouteFocused ? focusedBrowserId : null;
useEffect(() => {
if (!getIsElectron()) {
return;
}
void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.(desktopActiveBrowserId);
}, [desktopActiveBrowserId]);
void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.({
workspaceId: input.workspaceId,
browserId: focusedBrowserId,
});
}, [focusedBrowserId, input.workspaceId]);
}
function getFallbackTabOptionLabel(
@@ -1876,7 +1880,11 @@ function WorkspaceScreenContent({
() => (workspaceLayout ? collectAllTabs(workspaceLayout.root) : EMPTY_UI_TABS),
[workspaceLayout],
);
useSyncWorkspaceActiveBrowser({ workspaceLayout, isRouteFocused });
useSyncWorkspaceActiveBrowser({
workspaceLayout,
isRouteFocused,
workspaceId: normalizedWorkspaceId,
});
const openWorkspaceTabInBackground = useWorkspaceLayoutStore(
(state) => state.openTabInBackground,
);
@@ -1921,6 +1929,7 @@ function WorkspaceScreenContent({
if (input.target?.kind === "browser") {
const { browserId } = input.target;
useBrowserStore.getState().removeBrowser(browserId);
removeResidentBrowserWebview(browserId);
void getDesktopHost()?.browser?.clearPartition?.(browserId);
}
closeWorkspaceTab(persistenceKey, normalizedTabId);