feat(server): add Playwright MCP with headless mode to spawned Claude agents

Spawned agents now get Playwright browser automation capabilities by default.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
2025-11-29 05:08:56 +00:00
co-authored by Claude
parent 53d91a319e
commit 7c1517d830
5 changed files with 158 additions and 60 deletions
@@ -131,10 +131,13 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void) | null ((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void) | null
>(null); >(null);
// Stabilize audio level callback to prevent recorder recreation on every render
const handleRecordingAudioLevel = useCallback((level: number) => {
setRecordingVolume(level);
}, []);
const audioRecorder = useAudioRecorder({ const audioRecorder = useAudioRecorder({
onAudioLevel: (level) => { onAudioLevel: handleRecordingAudioLevel,
setRecordingVolume(level);
},
}); });
const debugInputHeight = (label: string, payload: Record<string, unknown>) => { const debugInputHeight = (label: string, payload: Record<string, unknown>) => {
@@ -217,7 +220,10 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
setRecordingDuration((prev) => prev + 1); setRecordingDuration((prev) => prev + 1);
}, 1000); }, 1000);
} catch (error) { } catch (error) {
console.error("[AgentInput] Failed to start recording:", error); const isCancelled = error instanceof Error && error.message.includes("Recording cancelled");
if (!isCancelled) {
console.error("[AgentInput] Failed to start recording:", error);
}
} }
} }
@@ -406,10 +406,14 @@ function AgentFlowModal({
}); });
const prevVisibilityRef = useRef(isVisible); const prevVisibilityRef = useRef(isVisible);
const dictationRequestIdRef = useRef<string | null>(null); const dictationRequestIdRef = useRef<string | null>(null);
// Stabilize audio level callback to prevent recorder recreation on every render
const handleDictationAudioLevel = useCallback((level: number) => {
setDictationVolume(level);
}, []);
const dictationRecorder = useAudioRecorder({ const dictationRecorder = useAudioRecorder({
onAudioLevel: (level) => { onAudioLevel: handleDictationAudioLevel,
setDictationVolume(level);
},
}); });
const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED; const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED;
const hasPendingCreateOrResume = pendingRequestIdRef.current !== null; const hasPendingCreateOrResume = pendingRequestIdRef.current !== null;
@@ -916,11 +920,11 @@ function AgentFlowModal({
setIsDictationProcessing(false); setIsDictationProcessing(false);
setDictationVolume(0); setDictationVolume(0);
setDictationDebugInfo(null); setDictationDebugInfo(null);
if (dictationRecorder.isRecording?.()) { if (isDictating) {
void dictationRecorder.stop().catch(() => {}); void dictationRecorder.stop().catch(() => {});
} }
cancelRepoInfo(); cancelRepoInfo();
}, [cancelRepoInfo, resetRepoInfo, setPromptHeight]); }, [cancelRepoInfo, dictationRecorder, isDictating, resetRepoInfo, setPromptHeight]);
const handleDictationStart = useCallback(async () => { const handleDictationStart = useCallback(async () => {
if (!isCreateFlow || isLoading || isDictating || isDictationProcessing) { if (!isCreateFlow || isLoading || isDictating || isDictationProcessing) {
@@ -937,12 +941,17 @@ function AgentFlowModal({
if (shouldShowAudioDebug) { if (shouldShowAudioDebug) {
setDictationDebugInfo(null); setDictationDebugInfo(null);
} }
await dictationRecorder.start();
setIsDictating(true); setIsDictating(true);
setDictationVolume(0); setDictationVolume(0);
setErrorMessage(""); setErrorMessage("");
await dictationRecorder.start();
} catch (error) { } catch (error) {
console.error("[CreateAgentModal] Failed to start dictation:", error); const isCancelled = error instanceof Error && error.message.includes("Recording cancelled");
if (!isCancelled) {
console.error("[CreateAgentModal] Failed to start dictation:", error);
}
setIsDictating(false);
setDictationVolume(0);
} }
}, [ }, [
daemonAvailabilityError, daemonAvailabilityError,
@@ -989,9 +998,9 @@ function AgentFlowModal({
try { try {
const audioData = await dictationRecorder.stop(); const audioData = await dictationRecorder.stop();
setIsDictating(false);
setDictationVolume(0); setDictationVolume(0);
if (!audioData) { if (!audioData) {
setIsDictating(false);
setIsDictationProcessing(false); setIsDictationProcessing(false);
return; return;
} }
@@ -1003,10 +1012,15 @@ function AgentFlowModal({
requestId, requestId,
{ mode: "transcribe_only" } { mode: "transcribe_only" }
); );
// Only clear isDictating after successful transcription send
setIsDictating(false);
} catch (error) { } catch (error) {
console.error("[CreateAgentModal] Failed to complete dictation:", error); console.error("[CreateAgentModal] Failed to complete dictation:", error);
dictationRequestIdRef.current = null; dictationRequestIdRef.current = null;
setIsDictationProcessing(false); setIsDictationProcessing(false);
// CRITICAL FIX: Always reset dictation state on error
setIsDictating(false);
setDictationVolume(0);
} }
}, [ }, [
daemonAvailabilityError, daemonAvailabilityError,
@@ -1152,21 +1166,38 @@ function AgentFlowModal({
]); ]);
useEffect(() => { useEffect(() => {
if (isVisible || !dictationRecorder.isRecording?.()) { if (isVisible || !isDictating) {
return; return;
} }
void dictationRecorder.stop().catch(() => {}); let cancelled = false;
setIsDictating(false);
setDictationVolume(0); const stopRecording = async () => {
}, [dictationRecorder, isVisible]); try {
await dictationRecorder.stop();
} catch (error) {
console.error("[CreateAgentModal] Failed to stop dictation while hidden:", error);
} finally {
if (!cancelled) {
setIsDictating(false);
setDictationVolume(0);
}
}
};
void stopRecording();
return () => {
cancelled = true;
};
}, [dictationRecorder, isDictating, isVisible]);
useEffect(() => { useEffect(() => {
return () => { return () => {
if (dictationRecorder.isRecording?.()) { if (isDictating) {
void dictationRecorder.stop().catch(() => {}); void dictationRecorder.stop().catch(() => {});
} }
}; };
}, [dictationRecorder]); }, [dictationRecorder, isDictating]);
useEffect(() => { useEffect(() => {
return () => { return () => {
@@ -78,6 +78,9 @@ function resolveNextConnectionState(
} }
function logConnectionLifecycle(daemon: DaemonProfile, previous: ConnectionState, next: ConnectionState) { function logConnectionLifecycle(daemon: DaemonProfile, previous: ConnectionState, next: ConnectionState) {
const severity: "info" | "warn" = next.status === "error" ? "warn" : "info";
const logger = severity === "warn" ? console.warn : console.info;
const logPayload = { const logPayload = {
event: "daemon_connection_state", event: "daemon_connection_state",
daemonId: daemon.id, daemonId: daemon.id,
@@ -87,15 +90,9 @@ function logConnectionLifecycle(daemon: DaemonProfile, previous: ConnectionState
lastError: next.lastError ?? null, lastError: next.lastError ?? null,
lastOnlineAt: next.lastOnlineAt ?? null, lastOnlineAt: next.lastOnlineAt ?? null,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
severity,
}; };
const logger =
next.status === "error"
? console.error
: next.status === "offline"
? console.warn
: console.info;
logger("[DaemonConnection]", logPayload); logger("[DaemonConnection]", logPayload);
} }
@@ -1,6 +1,6 @@
import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio'; import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
import { Paths, File, Directory, FileInfo } from 'expo-file-system'; import { Paths, File, Directory, FileInfo } from 'expo-file-system';
import { useState, useEffect } from 'react'; import { useState, useEffect, useRef, useMemo, useCallback } from 'react';
export interface AudioCaptureConfig { export interface AudioCaptureConfig {
sampleRate?: number; sampleRate?: number;
@@ -126,11 +126,18 @@ async function uriToBlob(uri: string): Promise<Blob> {
* - Echo cancellation, noise suppression, auto gain control (voice_communication on Android) * - Echo cancellation, noise suppression, auto gain control (voice_communication on Android)
*/ */
export function useAudioRecorder(config?: AudioCaptureConfig) { export function useAudioRecorder(config?: AudioCaptureConfig) {
const [isRecording, setIsRecording] = useState(false);
const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null); const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
const recordingIntentRef = useRef(false);
const stopRequestedRef = useRef(false);
// Create recording options matching web app constraints // Store config callbacks in refs so they can update without recreating the recorder
const recordingOptions: RecordingOptions = { const configRef = useRef(config);
useEffect(() => {
configRef.current = config;
}, [config]);
// Create stable recording options - only recreate if actual config values change
const recordingOptions: RecordingOptions = useMemo(() => ({
...RecordingPresets.HIGH_QUALITY, ...RecordingPresets.HIGH_QUALITY,
sampleRate: config?.sampleRate || 16000, sampleRate: config?.sampleRate || 16000,
numberOfChannels: config?.numberOfChannels || 1, numberOfChannels: config?.numberOfChannels || 1,
@@ -156,14 +163,22 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
mimeType: 'audio/webm;codecs=opus', mimeType: 'audio/webm;codecs=opus',
bitsPerSecond: config?.bitRate || 128000, bitsPerSecond: config?.bitRate || 128000,
}, },
}; }), [config?.sampleRate, config?.numberOfChannels, config?.bitRate, config?.onAudioLevel]);
const audioRecorder = useExpoAudioRecorder(recordingOptions); const audioRecorder = useExpoAudioRecorder(recordingOptions);
const recorderState = useAudioRecorderState(audioRecorder, 100); const recorderState = useAudioRecorderState(audioRecorder, 100);
// Monitor audio levels if metering is enabled // Store recorder in ref for stable access across re-renders
const recorderRef = useRef(audioRecorder);
useEffect(() => { useEffect(() => {
if (!config?.onAudioLevel || !isRecording) return; recorderRef.current = audioRecorder;
}, [audioRecorder]);
// Monitor audio levels if metering is enabled
// Use configRef to access the latest callback without recreating the effect
useEffect(() => {
const currentConfig = configRef.current;
if (!currentConfig?.onAudioLevel || !recorderRef.current.isRecording) return;
const interval = setInterval(() => { const interval = setInterval(() => {
const metering = recorderState.metering; const metering = recorderState.metering;
@@ -172,22 +187,35 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
// Convert to 0-1 range where 0 is silence and 1 is loud // Convert to 0-1 range where 0 is silence and 1 is loud
// We'll use -40 dB as the threshold for "loud" // We'll use -40 dB as the threshold for "loud"
const normalized = Math.max(0, Math.min(1, (metering + 40) / 40)); const normalized = Math.max(0, Math.min(1, (metering + 40) / 40));
config.onAudioLevel?.(normalized); configRef.current?.onAudioLevel?.(normalized);
} }
}, 100); // Check every 100ms }, 100); // Check every 100ms
return () => clearInterval(interval); return () => clearInterval(interval);
}, [isRecording, config, recorderState.metering]); }, [recorderState.metering, recorderState.isRecording]);
async function start(): Promise<void> { const start = useCallback(async (): Promise<void> => {
if (isRecording) { const recorder = recorderRef.current;
// Use expo's isRecording as single source of truth
if (recorder.isRecording) {
throw new Error('Already recording'); throw new Error('Already recording');
} }
try { try {
recordingIntentRef.current = true;
const ensureNotCancelled = () => {
if (stopRequestedRef.current) {
throw new Error('Recording cancelled');
}
};
ensureNotCancelled();
// Request microphone permissions // Request microphone permissions
console.log('[AudioRecorder] Requesting recording permissions...'); console.log('[AudioRecorder] Requesting recording permissions...');
const permissionResponse = await requestRecordingPermissionsAsync(); const permissionResponse = await requestRecordingPermissionsAsync();
ensureNotCancelled();
if (!permissionResponse.granted) { if (!permissionResponse.granted) {
throw new Error('Microphone permission denied. Please enable microphone access in your device settings.'); throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
@@ -199,6 +227,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
playsInSilentMode: true, playsInSilentMode: true,
allowsRecording: true, allowsRecording: true,
}); });
ensureNotCancelled();
console.log('[AudioRecorder] Starting recording with options:', { console.log('[AudioRecorder] Starting recording with options:', {
sampleRate: recordingOptions.sampleRate, sampleRate: recordingOptions.sampleRate,
@@ -208,35 +237,56 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
const startTime = new Date(); const startTime = new Date();
setRecordingStartTime(startTime); setRecordingStartTime(startTime);
ensureNotCancelled();
// Prepare the recorder before recording (required step) // Prepare the recorder before recording (required step)
console.log('[AudioRecorder] Preparing recorder...'); console.log('[AudioRecorder] Preparing recorder...');
await audioRecorder.prepareToRecordAsync(); await recorder.prepareToRecordAsync();
ensureNotCancelled();
console.log('[AudioRecorder] Starting recording...'); console.log('[AudioRecorder] Starting recording...');
await audioRecorder.record(); await recorder.record();
setIsRecording(true); ensureNotCancelled();
console.log('[AudioRecorder] Recording started at:', startTime.toISOString()); console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
console.log('[AudioRecorder] Recorder isRecording:', audioRecorder.isRecording); console.log('[AudioRecorder] Recorder isRecording:', recorder.isRecording);
} catch (error: any) { } catch (error: any) {
console.error('[AudioRecorder] Failed to start recording:', error); recordingIntentRef.current = false;
setRecordingStartTime(null);
if (error?.message === 'Recording cancelled') {
console.log('[AudioRecorder] Recording start cancelled.');
} else {
console.error('[AudioRecorder] Failed to start recording:', error);
}
throw new Error(`Failed to start audio recording: ${error.message}`); throw new Error(`Failed to start audio recording: ${error.message}`);
} }
} }, [recordingOptions.sampleRate, recordingOptions.numberOfChannels, recordingOptions.bitRate]);
async function stop(): Promise<Blob> { const stop = useCallback(async (): Promise<Blob> => {
if (!isRecording) { const recorder = recorderRef.current;
// Guard against duplicate stop calls
if (!recorder.isRecording && !recordingIntentRef.current) {
console.error('[AudioRecorder] Attempted to stop when not recording. Recorder state:', {
isRecording: recorder.isRecording,
hasRecordingIntent: recordingIntentRef.current,
hasRecordingStartTime: !!recordingStartTime,
});
throw new Error('Not recording'); throw new Error('Not recording');
} }
stopRequestedRef.current = true;
try { try {
// Stop recording // Stop recording
await audioRecorder.stop(); if (recorder.isRecording) {
setIsRecording(false); await recorder.stop();
} else {
console.warn('[AudioRecorder] Recorder already stopped before stop() call, continuing cleanup.');
}
// Get URI from recorder // Get URI from recorder
let uri = audioRecorder.uri; let uri = recorder.uri;
console.log('[AudioRecorder] Initial URI from recorder:', uri); console.log('[AudioRecorder] Initial URI from recorder:', uri);
// Workaround for Expo SDK 54 Android bug - find actual recording file // Workaround for Expo SDK 54 Android bug - find actual recording file
@@ -273,28 +323,37 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
// Clean up the temporary file // Clean up the temporary file
file.delete(); file.delete();
// Reset start time // Reset start time and clear recording intent
setRecordingStartTime(null); setRecordingStartTime(null);
recordingIntentRef.current = false;
return audioBlob; return audioBlob;
} catch (error: any) { } catch (error: any) {
setIsRecording(false); recordingIntentRef.current = false;
setRecordingStartTime(null); setRecordingStartTime(null);
console.error('[AudioRecorder] Failed to stop recording:', error); console.error('[AudioRecorder] Failed to stop recording:', error);
throw new Error(`Failed to stop audio recording: ${error.message}`); throw new Error(`Failed to stop audio recording: ${error.message}`);
} finally {
stopRequestedRef.current = false;
} }
} }, [recordingStartTime]);
function getSupportedMimeType(): string | null { const getSupportedMimeType = useCallback((): string | null => {
// On native platforms, expo-audio uses m4a/AAC // On native platforms, expo-audio uses m4a/AAC
// On web, it can use webm/opus // On web, it can use webm/opus
return 'audio/m4a'; return 'audio/m4a';
} }, []);
return { const isRecording = useCallback(() => {
// Treat local intent as backup so callers can detect pending sessions
return recorderRef.current.isRecording || recordingIntentRef.current;
}, []);
// Return stable object using useMemo
return useMemo(() => ({
start, start,
stop, stop,
isRecording: () => isRecording, isRecording,
getSupportedMimeType, getSupportedMimeType,
}; }), [start, stop, isRecording, getSupportedMimeType]);
} }
@@ -496,20 +496,25 @@ class ClaudeAgentSession implements AgentSession {
// Always include the agent-control MCP server so agents can launch other agents // Always include the agent-control MCP server so agents can launch other agents
const agentControlConfig = this.agentControlMcp ?? DEFAULT_AGENT_CONTROL_MCP; const agentControlConfig = this.agentControlMcp ?? DEFAULT_AGENT_CONTROL_MCP;
const agentControlMcp: Record<string, ClaudeMcpServerConfig> = { const defaultMcpServers: Record<string, ClaudeMcpServerConfig> = {
"agent-control": { "agent-control": {
type: "http", type: "http",
url: agentControlConfig.url, url: agentControlConfig.url,
...(agentControlConfig.headers ? { headers: agentControlConfig.headers } : {}), ...(agentControlConfig.headers ? { headers: agentControlConfig.headers } : {}),
}, },
playwright: {
type: "stdio",
command: "npx",
args: ["@playwright/mcp", "--headless"],
},
}; };
if (this.config.mcpServers) { if (this.config.mcpServers) {
const normalizedUserServers = this.normalizeMcpServers(this.config.mcpServers); const normalizedUserServers = this.normalizeMcpServers(this.config.mcpServers);
// Merge user-provided MCP servers with agent-control, user servers take precedence // Merge user-provided MCP servers with defaults, user servers take precedence
base.mcpServers = { ...agentControlMcp, ...normalizedUserServers }; base.mcpServers = { ...defaultMcpServers, ...normalizedUserServers };
} else { } else {
base.mcpServers = agentControlMcp; base.mcpServers = defaultMcpServers;
} }
if (this.config.model) { if (this.config.model) {