Add opt-in browser tools for desktop tabs
* Add opt-in browser tools for desktop tabs Adds the daemon opt-in, desktop tab routing, MCP tools, and real browser automation surfaces for Paseo desktop browser tabs. * Fix browser tools CI expectations * Address browser tools review findings * Restrict browser file automation paths * Fix browser upload test on Windows * Harden browser navigation inputs * Make browser tools create usable tabs * Update browser MCP empty-state test * Fail browser tab creation when registration times out * Fix browser screenshots for agents * Hide disabled browser tools from agents * Address browser tools architecture review * Replace browser tools review tests * Wrap browser tab registration errors * Mock Expo Router in app unit tests * Handle invalid browser automation requests * Return browser failure on desktop disconnect * Update browser disconnect websocket test * Relax browser timeout polling test * Handle invalid browser responses * Return browser failure when send fails * Remove local diagnostics and fixture paths * Fix dev service home fallback * Use worktree home for dev services * Use managed daemon in desktop dev * fix(browser): keep agent tabs addressable Track agent-active browser targets separately from human-focused tabs and keep resident webviews alive for automation. Browser tool visibility now comes from registration while the broker reports disabled execution. * refactor(browser): register tools through catalog Move browser tool registration onto the shared Paseo tool catalog so the MCP server remains only the transport adapter. * fix(settings): translate browser tools host error
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
import React, { useCallback } from "react";
|
||||
import { Text, View } from "react-native";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { useDaemonConfig } from "@/hooks/use-daemon-config";
|
||||
import { useHostRuntimeIsConnected } from "@/runtime/host-runtime";
|
||||
import { settingsStyles } from "@/styles/settings";
|
||||
import {
|
||||
createBrowserToolsPatch,
|
||||
getBrowserToolsCardState,
|
||||
getBrowserToolsMutationViewState,
|
||||
} from "./browser-tools-config";
|
||||
|
||||
export function BrowserToolsOptInCard({ serverId }: { serverId: string }) {
|
||||
const { t } = useTranslation();
|
||||
const isConnected = useHostRuntimeIsConnected(serverId);
|
||||
const { config, patchConfig } = useDaemonConfig(serverId);
|
||||
const state = getBrowserToolsCardState({ isConnected, config });
|
||||
const mutation = useMutation({
|
||||
mutationFn: async (next: boolean) => {
|
||||
const result = await patchConfig(createBrowserToolsPatch(next));
|
||||
if (!result) {
|
||||
throw new Error(t("workspace.terminal.hostDisconnected"));
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
const mutationView = getBrowserToolsMutationViewState({
|
||||
isPending: mutation.isPending,
|
||||
error: mutation.error,
|
||||
});
|
||||
|
||||
const handleValueChange = useCallback(
|
||||
(next: boolean) => {
|
||||
mutation.mutate(next);
|
||||
},
|
||||
[mutation],
|
||||
);
|
||||
|
||||
if (!state.isVisible) return null;
|
||||
|
||||
return (
|
||||
<View style={settingsStyles.card} testID="host-page-browser-tools-card">
|
||||
<View style={settingsStyles.row}>
|
||||
<View style={settingsStyles.rowContent}>
|
||||
<Text style={settingsStyles.rowTitle}>{state.title}</Text>
|
||||
<Text style={settingsStyles.rowHint}>{state.warning}</Text>
|
||||
{mutationView.loadingText ? (
|
||||
<Text style={settingsStyles.rowHint} testID="host-page-browser-tools-loading">
|
||||
{mutationView.loadingText}
|
||||
</Text>
|
||||
) : null}
|
||||
{mutationView.errorText ? (
|
||||
<Text style={settingsStyles.rowError} testID="host-page-browser-tools-error">
|
||||
{mutationView.errorText}
|
||||
</Text>
|
||||
) : null}
|
||||
</View>
|
||||
<Switch
|
||||
value={state.isEnabled}
|
||||
onValueChange={handleValueChange}
|
||||
disabled={mutationView.isSwitchDisabled}
|
||||
accessibilityLabel="Enable browser tools"
|
||||
testID="host-page-browser-tools-switch"
|
||||
/>
|
||||
</View>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
BROWSER_TOOLS_WARNING,
|
||||
createBrowserToolsPatch,
|
||||
getBrowserToolsCardState,
|
||||
getBrowserToolsMutationViewState,
|
||||
} from "./browser-tools-config";
|
||||
|
||||
function makeConfig(browserToolsEnabled = false): MutableDaemonConfig {
|
||||
return {
|
||||
mcp: { injectIntoAgents: false },
|
||||
browserTools: { enabled: browserToolsEnabled },
|
||||
providers: {},
|
||||
metadataGeneration: { providers: [] },
|
||||
autoArchiveAfterMerge: false,
|
||||
enableTerminalAgentHooks: false,
|
||||
appendSystemPrompt: "",
|
||||
};
|
||||
}
|
||||
|
||||
describe("browser tools opt-in config", () => {
|
||||
it("shows the card with the logged-in browser state warning when connected", () => {
|
||||
expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(false) })).toEqual({
|
||||
isVisible: true,
|
||||
isEnabled: false,
|
||||
title: "Browser tools",
|
||||
warning: BROWSER_TOOLS_WARNING,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads enabled state from daemon config", () => {
|
||||
expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(true) })).toMatchObject(
|
||||
{
|
||||
isEnabled: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("hides the card when the host is disconnected", () => {
|
||||
expect(
|
||||
getBrowserToolsCardState({ isConnected: false, config: makeConfig(true) }),
|
||||
).toMatchObject({
|
||||
isVisible: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("writes daemon.browserTools.enabled when toggled", () => {
|
||||
expect(createBrowserToolsPatch(true)).toEqual({ browserTools: { enabled: true } });
|
||||
expect(createBrowserToolsPatch(false)).toEqual({ browserTools: { enabled: false } });
|
||||
});
|
||||
|
||||
it("shows loading and disables the toggle while browser tool settings save", () => {
|
||||
expect(getBrowserToolsMutationViewState({ isPending: true, error: null })).toEqual({
|
||||
isSwitchDisabled: true,
|
||||
loadingText: "Updating browser tools…",
|
||||
errorText: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("shows the save error when browser tool settings fail", () => {
|
||||
expect(
|
||||
getBrowserToolsMutationViewState({ isPending: false, error: new Error("Disk full") }),
|
||||
).toEqual({
|
||||
isSwitchDisabled: false,
|
||||
loadingText: null,
|
||||
errorText: "Disk full",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
|
||||
|
||||
export const BROWSER_TOOLS_TITLE = "Browser tools";
|
||||
export const BROWSER_TOOLS_WARNING =
|
||||
"Allow agents to access and control Paseo desktop browser tabs, including logged-in browser state. Only enable this for agents you trust.";
|
||||
|
||||
export interface BrowserToolsCardState {
|
||||
isVisible: boolean;
|
||||
isEnabled: boolean;
|
||||
title: string;
|
||||
warning: string;
|
||||
}
|
||||
|
||||
export interface BrowserToolsMutationViewState {
|
||||
isSwitchDisabled: boolean;
|
||||
loadingText: string | null;
|
||||
errorText: string | null;
|
||||
}
|
||||
|
||||
export function getBrowserToolsCardState(input: {
|
||||
isConnected: boolean;
|
||||
config: MutableDaemonConfig | null;
|
||||
}): BrowserToolsCardState {
|
||||
return {
|
||||
isVisible: input.isConnected,
|
||||
isEnabled: input.config?.browserTools.enabled === true,
|
||||
title: BROWSER_TOOLS_TITLE,
|
||||
warning: BROWSER_TOOLS_WARNING,
|
||||
};
|
||||
}
|
||||
|
||||
export function createBrowserToolsPatch(enabled: boolean): Partial<MutableDaemonConfig> {
|
||||
return { browserTools: { enabled } };
|
||||
}
|
||||
|
||||
export function getBrowserToolsMutationViewState(input: {
|
||||
isPending: boolean;
|
||||
error: unknown;
|
||||
}): BrowserToolsMutationViewState {
|
||||
return {
|
||||
isSwitchDisabled: input.isPending,
|
||||
loadingText: input.isPending ? "Updating browser tools…" : null,
|
||||
errorText: input.error ? toErrorMessage(input.error) : null,
|
||||
};
|
||||
}
|
||||
|
||||
function toErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
@@ -61,6 +61,7 @@ import { formatLatency } from "@/utils/latency";
|
||||
import { ICON_SIZE } from "@/styles/theme";
|
||||
import type { Theme } from "@/styles/theme";
|
||||
import { getProviderIcon } from "@/components/provider-icons";
|
||||
import { BrowserToolsOptInCard } from "./browser-tools-card";
|
||||
|
||||
const ThemedArrowUp = withUnistyles(ArrowUp);
|
||||
const ThemedArrowDown = withUnistyles(ArrowDown);
|
||||
@@ -262,6 +263,7 @@ export function HostAgentsPage({ serverId }: { serverId: string }) {
|
||||
{isConnected ? (
|
||||
<SettingsSection title={t("settings.hostSections.agents")}>
|
||||
<InjectPaseoToolsCard serverId={serverId} />
|
||||
<BrowserToolsOptInCard serverId={serverId} />
|
||||
<AppendSystemPromptCard serverId={serverId} />
|
||||
</SettingsSection>
|
||||
) : (
|
||||
|
||||
@@ -220,6 +220,7 @@ const disabledCodexEntry: ProviderSnapshotEntry = {
|
||||
function makeConfig(providers: MutableDaemonConfig["providers"] = {}): MutableDaemonConfig {
|
||||
return {
|
||||
mcp: { injectIntoAgents: false },
|
||||
browserTools: { enabled: false },
|
||||
providers,
|
||||
metadataGeneration: { providers: [] },
|
||||
autoArchiveAfterMerge: false,
|
||||
|
||||
@@ -105,6 +105,7 @@ import type { CheckoutStatusPayload } from "@/git/use-status-query";
|
||||
import { confirmDialog } from "@/utils/confirm-dialog";
|
||||
import { useArchiveAgent } from "@/hooks/use-archive-agent";
|
||||
import { useStableEvent } from "@/hooks/use-stable-event";
|
||||
import { removeResidentBrowserWebview } from "@/components/browser-webview-resident";
|
||||
import { createWorkspaceBrowser, useBrowserStore } from "@/stores/browser-store";
|
||||
import { getDesktopHost } from "@/desktop/host";
|
||||
import { buildProviderCommand } from "@/utils/provider-command-templates";
|
||||
@@ -304,19 +305,22 @@ function decodeSegment(value: string): string {
|
||||
function useSyncWorkspaceActiveBrowser(input: {
|
||||
workspaceLayout: WorkspaceLayout | null;
|
||||
isRouteFocused: boolean;
|
||||
workspaceId: string;
|
||||
}) {
|
||||
const focusedBrowserId = useMemo(
|
||||
() => getFocusedBrowserId(input.workspaceLayout),
|
||||
[input.workspaceLayout],
|
||||
);
|
||||
const desktopActiveBrowserId = input.isRouteFocused ? focusedBrowserId : null;
|
||||
|
||||
useEffect(() => {
|
||||
if (!getIsElectron()) {
|
||||
return;
|
||||
}
|
||||
void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.(desktopActiveBrowserId);
|
||||
}, [desktopActiveBrowserId]);
|
||||
void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.({
|
||||
workspaceId: input.workspaceId,
|
||||
browserId: focusedBrowserId,
|
||||
});
|
||||
}, [focusedBrowserId, input.workspaceId]);
|
||||
}
|
||||
|
||||
function getFallbackTabOptionLabel(
|
||||
@@ -1876,7 +1880,11 @@ function WorkspaceScreenContent({
|
||||
() => (workspaceLayout ? collectAllTabs(workspaceLayout.root) : EMPTY_UI_TABS),
|
||||
[workspaceLayout],
|
||||
);
|
||||
useSyncWorkspaceActiveBrowser({ workspaceLayout, isRouteFocused });
|
||||
useSyncWorkspaceActiveBrowser({
|
||||
workspaceLayout,
|
||||
isRouteFocused,
|
||||
workspaceId: normalizedWorkspaceId,
|
||||
});
|
||||
const openWorkspaceTabInBackground = useWorkspaceLayoutStore(
|
||||
(state) => state.openTabInBackground,
|
||||
);
|
||||
@@ -1921,6 +1929,7 @@ function WorkspaceScreenContent({
|
||||
if (input.target?.kind === "browser") {
|
||||
const { browserId } = input.target;
|
||||
useBrowserStore.getState().removeBrowser(browserId);
|
||||
removeResidentBrowserWebview(browserId);
|
||||
void getDesktopHost()?.browser?.clearPartition?.(browserId);
|
||||
}
|
||||
closeWorkspaceTab(persistenceKey, normalizedTabId);
|
||||
|
||||
Reference in New Issue
Block a user