diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index 9acca11c1..155944e7f 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -21,7 +21,7 @@ import Animated, { FadeOut, } from "react-native-reanimated"; import { useRealtime } from "@/contexts/realtime-context"; -import { useAudioRecorder } from "@/hooks/use-audio-recorder"; +import { useDictation } from "@/hooks/use-dictation"; import { FOOTER_HEIGHT } from "@/contexts/footer-controls-context"; import { VoiceNoteRecordingOverlay } from "./voice-note-recording-overlay"; import { generateMessageId } from "@/types/stream"; @@ -107,39 +107,111 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { const [userInput, setUserInput] = useState(""); const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT); - const [isRecording, setIsRecording] = useState(false); const [isProcessing, setIsProcessing] = useState(false); const [selectedImages, setSelectedImages] = useState>([]); - const [recordingVolume, setRecordingVolume] = useState(0); - const [recordingDuration, setRecordingDuration] = useState(0); - const [transcribingRequestId, setTranscribingRequestId] = useState(null); const [isCancellingAgent, setIsCancellingAgent] = useState(false); const [audioDebugInfo, setAudioDebugInfo] = useState(null); - const recordingIntervalRef = useRef | null>(null); const textInputRef = useRef unknown }) | null>(null); const inputHeightRef = useRef(MIN_INPUT_HEIGHT); const baselineInputHeightRef = useRef(null); const overlayTransition = useSharedValue(0); const { pickImages } = useImageAttachmentPicker(); const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED; - const pendingTranscriptionRef = useRef<{ requestId: string } | null>(null); const agentIdRef = useRef(agentId); const sendAgentMessageRef = useRef(sendAgentMessage); const agentStatusRef = useRef(undefined); const updateQueueRef = useRef< ((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void) | null >(null); + const handleDictationTranscript = useCallback( + (text: string) => { + if (!text) { + return; + } + const shouldQueue = agentStatusRef.current === "running"; + if (shouldQueue) { + updateQueueRef.current?.((current) => [ + ...current, + { + id: generateMessageId(), + text, + }, + ]); + return; + } + void (async () => { + try { + await sendAgentMessageRef.current?.(agentIdRef.current, text); + } catch (error) { + console.error("[AgentInput] Failed to send transcribed message:", error); + updateQueueRef.current?.((current) => [ + ...current, + { + id: generateMessageId(), + text, + }, + ]); + } + })(); + }, + [] + ); - // Stabilize audio level callback to prevent recorder recreation on every render - const handleRecordingAudioLevel = useCallback((level: number) => { - setRecordingVolume(level); + const handleDictationError = useCallback((error: Error) => { + console.error("[AgentInput] Dictation error:", error); }, []); - const audioRecorder = useAudioRecorder({ - onAudioLevel: handleRecordingAudioLevel, + const canStartDictation = useCallback(() => { + const allowed = !isRealtimeMode && ws.isConnected; + console.log("[AgentInput] canStartDictation", { + allowed, + isRealtimeMode, + wsConnected: ws.isConnected, + }); + return allowed; + }, [isRealtimeMode, ws.isConnected]); + + const canConfirmDictation = useCallback(() => { + const allowed = ws.isConnected; + console.log("[AgentInput] canConfirmDictation", { + allowed, + wsConnected: ws.isConnected, + }); + return allowed; + }, [ws.isConnected]); + + const { + isRecording: isDictating, + isProcessing: isDictationProcessing, + volume: dictationVolume, + duration: dictationDuration, + pendingRequestId: dictationPendingRequestId, + startDictation, + cancelDictation, + confirmDictation, + } = useDictation({ + agentId, + sendAgentAudio, + ws, + mode: "transcribe_only", + onTranscript: handleDictationTranscript, + onError: handleDictationError, + canStart: canStartDictation, + canConfirm: canConfirmDictation, + enableDuration: true, }); + const dictationRequestIdRef = useRef(null); + useEffect(() => { + dictationRequestIdRef.current = dictationPendingRequestId; + }, [dictationPendingRequestId]); + + useEffect(() => { + const shouldShowOverlay = isDictating || isDictationProcessing; + overlayTransition.value = withTiming(shouldShowOverlay ? 1 : 0, { duration: 250 }); + }, [isDictating, isDictationProcessing, overlayTransition]); + const debugInputHeight = (label: string, payload: Record) => { if (!SHOULD_DEBUG_INPUT_HEIGHT) { return; @@ -196,11 +268,12 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { } async function handleVoicePress() { - if (isRecording) { - // This shouldn't happen as button is hidden when recording - return; - } - if (isRealtimeMode) { + console.log("[AgentInput] handleVoicePress", { + isDictating, + isRealtimeMode, + wsConnected: ws.isConnected, + }); + if (isDictating || isRealtimeMode) { return; } @@ -208,19 +281,12 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { setAudioDebugInfo(null); } - // Start recording try { - await audioRecorder.start(); - setIsRecording(true); - setRecordingDuration(0); - overlayTransition.value = withTiming(1, { duration: 250 }); - - // Start duration timer - recordingIntervalRef.current = setInterval(() => { - setRecordingDuration((prev) => prev + 1); - }, 1000); + await startDictation(); + console.log("[AgentInput] startDictation invoked"); } catch (error) { - const isCancelled = error instanceof Error && error.message.includes("Recording cancelled"); + const isCancelled = + error instanceof Error && error.message.includes("Recording cancelled"); if (!isCancelled) { console.error("[AgentInput] Failed to start recording:", error); } @@ -228,152 +294,66 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { } async function handleCancelRecording() { + console.log("[AgentInput] handleCancelRecording", { + isDictating, + isDictationProcessing, + }); + if (!isDictating && !isDictationProcessing) { + return; + } try { - // Stop recording without sending - await audioRecorder.stop(); - setIsRecording(false); - overlayTransition.value = withTiming(0, { duration: 250 }); - - // Clear timer - if (recordingIntervalRef.current) { - clearInterval(recordingIntervalRef.current); - recordingIntervalRef.current = null; - } - setRecordingDuration(0); - setRecordingVolume(0); + await cancelDictation(); } catch (error) { console.error("[AgentInput] Failed to cancel recording:", error); - setIsRecording(false); - overlayTransition.value = withTiming(0, { duration: 250 }); } } async function handleSendRecording() { + console.log("[AgentInput] handleSendRecording", { + isDictating, + isDictationProcessing, + }); + if (!isDictating) { + return; + } try { - const audioData = await audioRecorder.stop(); - setIsRecording(false); - - // Clear timer - if (recordingIntervalRef.current) { - clearInterval(recordingIntervalRef.current); - recordingIntervalRef.current = null; - } - setRecordingDuration(0); - setRecordingVolume(0); - - if (audioData) { - // Generate request ID for tracking transcription - const requestId = generateMessageId(); - setTranscribingRequestId(requestId); - pendingTranscriptionRef.current = { - requestId, - }; - console.log("[AgentInput] Audio recorded:", audioData.size, "bytes", "requestId:", requestId); - - try { - // Send audio to agent for transcription and processing - await sendAgentAudio(agentId, audioData, requestId, { mode: "transcribe_only" }); - console.log("[AgentInput] Audio sent to agent"); - } catch (error) { - console.error("[AgentInput] Failed to send audio:", error); - // Clear transcribing state on error - setTranscribingRequestId(null); - pendingTranscriptionRef.current = null; - overlayTransition.value = withTiming(0, { duration: 250 }); - } - } else { - // No audio data, dismiss overlay immediately - overlayTransition.value = withTiming(0, { duration: 250 }); - } + await confirmDictation(); } catch (error) { - console.error("[AgentInput] Failed to stop recording:", error); - setIsRecording(false); - setTranscribingRequestId(null); - pendingTranscriptionRef.current = null; - overlayTransition.value = withTiming(0, { duration: 250 }); + console.error("[AgentInput] Failed to send recording:", error); } } useEffect(() => { + if (!shouldShowAudioDebug) { + return; + } const unsubscribe = ws.on("transcription_result", (message: SessionOutboundMessage) => { if (message.type !== "transcription_result") { return; } - const pending = pendingTranscriptionRef.current; - if (!pending || !message.payload.requestId) { + const pendingRequestId = dictationRequestIdRef.current; + if (!pendingRequestId || message.payload.requestId !== pendingRequestId) { return; } - if (message.payload.requestId !== pending.requestId) { - return; - } - - console.log("[AgentInput] Transcription completed for requestId:", pending.requestId); - pendingTranscriptionRef.current = null; - setTranscribingRequestId(null); - overlayTransition.value = withTiming(0, { duration: 250 }); - - if (shouldShowAudioDebug) { - setAudioDebugInfo({ - requestId: pending.requestId, - transcript: message.payload.text?.trim(), - debugRecordingPath: message.payload.debugRecordingPath ?? undefined, - format: message.payload.format, - byteLength: message.payload.byteLength, - duration: message.payload.duration, - avgLogprob: message.payload.avgLogprob, - isLowConfidence: message.payload.isLowConfidence, - }); - } - - const transcriptText = message.payload.text?.trim(); - if (!transcriptText) { - return; - } - - const shouldQueue = agentStatusRef.current === "running"; - if (shouldQueue) { - updateQueueRef.current?.((current) => [ - ...current, - { - id: generateMessageId(), - text: transcriptText, - }, - ]); - return; - } - - void (async () => { - try { - await sendAgentMessageRef.current?.(agentIdRef.current, transcriptText); - } catch (error) { - console.error("[AgentInput] Failed to send transcribed message:", error); - updateQueueRef.current?.((current) => [ - ...current, - { - id: generateMessageId(), - text: transcriptText, - }, - ]); - } - })(); + dictationRequestIdRef.current = null; + setAudioDebugInfo({ + requestId: pendingRequestId, + transcript: message.payload.text?.trim(), + debugRecordingPath: message.payload.debugRecordingPath ?? undefined, + format: message.payload.format, + byteLength: message.payload.byteLength, + duration: message.payload.duration, + avgLogprob: message.payload.avgLogprob, + isLowConfidence: message.payload.isLowConfidence, + }); }); return () => { unsubscribe(); }; - }, [ws, overlayTransition, shouldShowAudioDebug]); - - // Cleanup timer on unmount - useEffect(() => { - return () => { - if (recordingIntervalRef.current) { - clearInterval(recordingIntervalRef.current); - } - }; - }, []); - + }, [shouldShowAudioDebug, ws]); function isTextAreaLike(value: unknown): value is TextAreaHandle { if (!value || typeof value !== "object") { @@ -600,7 +580,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { event.preventDefault(); event.stopPropagation(); - if (isRecording) { + if (isDictating) { void handleSendRecording(); return; } @@ -611,7 +591,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { return; } - if (key === "escape" && isRecording) { + if (key === "escape" && isDictating) { event.preventDefault(); event.stopPropagation(); void handleCancelRecording(); @@ -623,7 +603,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { window.removeEventListener("keydown", listener, true); }; }, [ - isRecording, + isDictating, isRealtimeMode, shouldShowVoiceControls, ws, @@ -768,10 +748,10 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { style={[ styles.voiceButton as any, (!ws.isConnected || isRealtimeMode ? styles.buttonDisabled : undefined) as any, - (isRecording ? styles.voiceButtonRecording : undefined) as any, + (isDictating ? styles.voiceButtonRecording : undefined) as any, ]} > - {isRecording ? ( + {isDictating ? ( ) : ( @@ -858,7 +838,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { multiline scrollEnabled={inputHeight >= MAX_INPUT_HEIGHT} onContentSizeChange={handleContentSizeChange} - editable={!isRecording && ws.isConnected} + editable={!isDictating && ws.isConnected} onKeyPress={shouldHandleDesktopSubmit ? handleDesktopSubmitKeyPress : undefined} /> @@ -942,11 +922,11 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) { diff --git a/packages/app/src/components/create-agent-modal.tsx b/packages/app/src/components/create-agent-modal.tsx index 914e0d9e1..dfd4b47b9 100644 --- a/packages/app/src/components/create-agent-modal.tsx +++ b/packages/app/src/components/create-agent-modal.tsx @@ -41,7 +41,7 @@ import { theme as defaultTheme } from "@/styles/theme"; import { useRecentPaths } from "@/hooks/use-recent-paths"; import { useRouter } from "expo-router"; import { generateMessageId } from "@/types/stream"; -import { useAudioRecorder } from "@/hooks/use-audio-recorder"; +import { useDictation } from "@/hooks/use-dictation"; import { VolumeMeter } from "@/components/volume-meter"; import { AUDIO_DEBUG_ENABLED } from "@/config/audio-debug"; import { AudioDebugNotice, type AudioDebugInfo } from "./audio-debug-notice"; @@ -280,7 +280,10 @@ function AgentFlowModal({ const effectiveWs = ws ?? inertWebSocket; const createAgent = session?.createAgent; const resumeAgent = session?.resumeAgent; - const sendAgentAudio = session?.sendAgentAudio; + const sessionSendAgentAudio = session?.sendAgentAudio; + const noopSendAgentAudio = useCallback(async () => {}, []); + const sendAgentAudio = sessionSendAgentAudio ?? noopSendAgentAudio; + const hasSendAgentAudio = Boolean(sessionSendAgentAudio); const agents = session?.agents; const providerModels = session?.providerModels; const requestProviderModels = session?.requestProviderModels; @@ -385,9 +388,6 @@ function AgentFlowModal({ ); const [isImportLoading, setIsImportLoading] = useState(false); const [importError, setImportError] = useState(null); - const [isDictating, setIsDictating] = useState(false); - const [isDictationProcessing, setIsDictationProcessing] = useState(false); - const [dictationVolume, setDictationVolume] = useState(0); const [dictationDebugInfo, setDictationDebugInfo] = useState(null); const [openDropdown, setOpenDropdown] = useState(null); const pendingRequestIdRef = useRef(null); @@ -395,6 +395,7 @@ function AgentFlowModal({ const promptInputRef = useRef< TextInput | (TextInput & { getNativeRef?: () => unknown }) | null >(null); + const focusPromptInputRef = useRef<() => void>(() => {}); const promptInputHeightRef = useRef(PROMPT_MIN_HEIGHT); const promptBaselineHeightRef = useRef(null); const formPreferencesHydratedRef = useRef(false); @@ -407,17 +408,80 @@ function AgentFlowModal({ const prevVisibilityRef = useRef(isVisible); const dictationRequestIdRef = useRef(null); - // Stabilize audio level callback to prevent recorder recreation on every render - const handleDictationAudioLevel = useCallback((level: number) => { - setDictationVolume(level); - }, []); + const handleDictationTranscript = useCallback( + (text: string, _meta: { requestId: string }) => { + if (!isCreateFlow || !text) { + return; + } + setInitialPrompt((prev) => { + if (!prev) { + return text; + } + const needsSpace = /\s$/.test(prev); + return `${prev}${needsSpace ? "" : " "}${text}`; + }); + focusPromptInputRef.current?.(); + }, + [isCreateFlow, setInitialPrompt] + ); - const dictationRecorder = useAudioRecorder({ - onAudioLevel: handleDictationAudioLevel, + const handleDictationError = useCallback( + (dictationError: Error) => { + setErrorMessage(dictationError.message); + }, + [] + ); + + const canStartDictation = useCallback(() => { + const allowed = isCreateFlow && !isLoading && isTargetDaemonReady && isWsConnected; + console.log("[CreateAgentModal] canStartDictation", { + allowed, + isCreateFlow, + isLoading, + isTargetDaemonReady, + isWsConnected, + }); + return allowed; + }, [isCreateFlow, isLoading, isTargetDaemonReady, isWsConnected]); + + const canConfirmDictation = useCallback(() => { + const allowed = isTargetDaemonReady && hasSendAgentAudio; + console.log("[CreateAgentModal] canConfirmDictation", { + allowed, + isTargetDaemonReady, + hasSendAgentAudio, + }); + return allowed; + }, [hasSendAgentAudio, isTargetDaemonReady]); + + const { + isRecording: isDictating, + isProcessing: isDictationProcessing, + volume: dictationVolume, + pendingRequestId: dictationPendingRequestId, + startDictation, + cancelDictation, + confirmDictation, + reset: resetDictation, + } = useDictation({ + agentId: DICTATION_AGENT_ID, + sendAgentAudio, + ws: effectiveWs, + mode: "transcribe_only", + onTranscript: handleDictationTranscript, + onError: handleDictationError, + canStart: canStartDictation, + canConfirm: canConfirmDictation, + autoStopWhenHidden: { isVisible }, }); const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED; + + useEffect(() => { + dictationRequestIdRef.current = dictationPendingRequestId; + }, [dictationPendingRequestId]); + const hasPendingCreateOrResume = pendingRequestIdRef.current !== null; - const hasPendingDictation = dictationRequestIdRef.current !== null; + const hasPendingDictation = dictationPendingRequestId !== null; const shouldListenForStatus = isVisible || hasPendingCreateOrResume; const shouldListenForDictation = isCreateFlow && (isVisible || hasPendingDictation); @@ -800,6 +864,9 @@ function AgentFlowModal({ } } }, [shouldAutoFocusPrompt]); + useEffect(() => { + focusPromptInputRef.current = focusPromptInput; + }, [focusPromptInput]); const activeSessionIds = useMemo(() => { const ids = new Set(); if (!agents) { @@ -916,17 +983,21 @@ function AgentFlowModal({ pendingNavigationServerIdRef.current = null; shouldSyncBaseBranchRef.current = true; dictationRequestIdRef.current = null; - setIsDictating(false); - setIsDictationProcessing(false); - setDictationVolume(0); setDictationDebugInfo(null); - if (isDictating) { - void dictationRecorder.stop().catch(() => {}); - } + void cancelDictation(); + resetDictation(); cancelRepoInfo(); - }, [cancelRepoInfo, dictationRecorder, isDictating, resetRepoInfo, setPromptHeight]); + }, [cancelRepoInfo, cancelDictation, resetDictation, resetRepoInfo, setPromptHeight]); const handleDictationStart = useCallback(async () => { + console.log("[CreateAgentModal] handleDictationStart", { + isCreateFlow, + isLoading, + isDictating, + isDictationProcessing, + isTargetDaemonReady, + isWsConnected, + }); if (!isCreateFlow || isLoading || isDictating || isDictationProcessing) { return; } @@ -941,21 +1012,17 @@ function AgentFlowModal({ if (shouldShowAudioDebug) { setDictationDebugInfo(null); } - setIsDictating(true); - setDictationVolume(0); setErrorMessage(""); - await dictationRecorder.start(); + await startDictation(); + console.log("[CreateAgentModal] startDictation invoked"); } catch (error) { const isCancelled = error instanceof Error && error.message.includes("Recording cancelled"); if (!isCancelled) { console.error("[CreateAgentModal] Failed to start dictation:", error); } - setIsDictating(false); - setDictationVolume(0); } }, [ daemonAvailabilityError, - dictationRecorder, isCreateFlow, isDictating, isDictationProcessing, @@ -964,27 +1031,34 @@ function AgentFlowModal({ isWsConnected, logOfflineDaemonAction, shouldShowAudioDebug, + startDictation, ]); const handleDictationCancel = useCallback(async () => { + console.log("[CreateAgentModal] handleDictationCancel", { + isDictating, + }); if (!isDictating) { return; } try { - await dictationRecorder.stop(); + await cancelDictation(); } catch (error) { console.error("[CreateAgentModal] Failed to cancel dictation:", error); - } finally { - setIsDictating(false); - setDictationVolume(0); } - }, [dictationRecorder, isDictating]); + }, [cancelDictation, isDictating]); const handleDictationConfirm = useCallback(async () => { + console.log("[CreateAgentModal] handleDictationConfirm", { + isDictating, + isDictationProcessing, + isTargetDaemonReady, + hasSendAgentAudio, + }); if (!isDictating || isDictationProcessing) { return; } - if (!sendAgentAudio || !isTargetDaemonReady) { + if (!isTargetDaemonReady || !hasSendAgentAudio) { logOfflineDaemonAction("dictation"); setErrorMessage( daemonAvailabilityError ?? @@ -992,44 +1066,19 @@ function AgentFlowModal({ ); return; } - - // Set processing state BEFORE stopping to ensure UI updates properly - setIsDictationProcessing(true); - try { - const audioData = await dictationRecorder.stop(); - setDictationVolume(0); - if (!audioData) { - setIsDictating(false); - setIsDictationProcessing(false); - return; - } - const requestId = generateMessageId(); - dictationRequestIdRef.current = requestId; - await sendAgentAudio( - DICTATION_AGENT_ID, - audioData, - requestId, - { mode: "transcribe_only" } - ); - // Only clear isDictating after successful transcription send - setIsDictating(false); + await confirmDictation(); } catch (error) { console.error("[CreateAgentModal] Failed to complete dictation:", error); - dictationRequestIdRef.current = null; - setIsDictationProcessing(false); - // CRITICAL FIX: Always reset dictation state on error - setIsDictating(false); - setDictationVolume(0); } }, [ daemonAvailabilityError, - dictationRecorder, + confirmDictation, + hasSendAgentAudio, isDictating, isDictationProcessing, isTargetDaemonReady, logOfflineDaemonAction, - sendAgentAudio, ]); const navigateToAgentIfNeeded = useCallback(() => { @@ -1165,39 +1214,6 @@ function AgentFlowModal({ handleCloseAnimationComplete, ]); - useEffect(() => { - if (isVisible || !isDictating) { - return; - } - let cancelled = false; - - const stopRecording = async () => { - try { - await dictationRecorder.stop(); - } catch (error) { - console.error("[CreateAgentModal] Failed to stop dictation while hidden:", error); - } finally { - if (!cancelled) { - setIsDictating(false); - setDictationVolume(0); - } - } - }; - - void stopRecording(); - - return () => { - cancelled = true; - }; - }, [dictationRecorder, isDictating, isVisible]); - - useEffect(() => { - return () => { - if (isDictating) { - void dictationRecorder.stop().catch(() => {}); - } - }; - }, [dictationRecorder, isDictating]); useEffect(() => { return () => { @@ -1656,7 +1672,7 @@ function AgentFlowModal({ }, [handleClose, shouldListenForStatus, ws]); useEffect(() => { - if (!shouldListenForDictation || !ws) { + if (!shouldListenForDictation || !ws || !shouldShowAudioDebug) { return; } const unsubscribe = ws.on("transcription_result", (message) => { @@ -1668,37 +1684,22 @@ function AgentFlowModal({ return; } dictationRequestIdRef.current = null; - setIsDictationProcessing(false); - if (shouldShowAudioDebug) { - setDictationDebugInfo({ - requestId: pendingId, - transcript: message.payload.text?.trim(), - debugRecordingPath: message.payload.debugRecordingPath ?? undefined, - format: message.payload.format, - byteLength: message.payload.byteLength, - duration: message.payload.duration, - avgLogprob: message.payload.avgLogprob, - isLowConfidence: message.payload.isLowConfidence, - }); - } - const transcriptText = message.payload.text?.trim(); - if (!transcriptText) { - return; - } - setInitialPrompt((prev) => { - if (!prev) { - return transcriptText; - } - const needsSpace = /\s$/.test(prev); - return `${prev}${needsSpace ? "" : " "}${transcriptText}`; + setDictationDebugInfo({ + requestId: pendingId, + transcript: message.payload.text?.trim(), + debugRecordingPath: message.payload.debugRecordingPath ?? undefined, + format: message.payload.format, + byteLength: message.payload.byteLength, + duration: message.payload.duration, + avgLogprob: message.payload.avgLogprob, + isLowConfidence: message.payload.isLowConfidence, }); - focusPromptInput(); }); return () => { unsubscribe(); }; - }, [focusPromptInput, shouldListenForDictation, ws, shouldShowAudioDebug]); + }, [shouldListenForDictation, shouldShowAudioDebug, ws]); useEffect(() => { if (!isVisible || !isImportFlow) { diff --git a/packages/app/src/hooks/use-audio-recorder.web.ts b/packages/app/src/hooks/use-audio-recorder.web.ts index 159d7a3c8..63e0a09dc 100644 --- a/packages/app/src/hooks/use-audio-recorder.web.ts +++ b/packages/app/src/hooks/use-audio-recorder.web.ts @@ -133,14 +133,28 @@ export function useAudioRecorder(config?: AudioCaptureConfig) { throw new Error("Already recording"); } - if ( + const missingNavigator = typeof navigator === "undefined" || !navigator.mediaDevices || - typeof navigator.mediaDevices.getUserMedia !== "function" - ) { + typeof navigator.mediaDevices.getUserMedia !== "function"; + + const secureContext = + typeof window !== "undefined" && typeof window.isSecureContext === "boolean" + ? window.isSecureContext + : true; + const currentOrigin = + typeof window !== "undefined" && window.location ? window.location.origin : "unknown"; + + if (missingNavigator) { throw new Error("Microphone capture is not supported in this environment"); } + if (!secureContext) { + throw new Error( + `Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}` + ); + } + const options = configRef.current; const constraints: MediaStreamConstraints = { audio: { diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts new file mode 100644 index 000000000..353df932e --- /dev/null +++ b/packages/app/src/hooks/use-dictation.ts @@ -0,0 +1,388 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import type { SessionContextValue } from "@/contexts/session-context"; +import { useAudioRecorder } from "@/hooks/use-audio-recorder"; +import type { UseWebSocketReturn } from "@/hooks/use-websocket"; +import { generateMessageId } from "@/types/stream"; +import type { SessionOutboundMessage } from "@server/server/messages"; + +export type UseDictationOptions = { + agentId: string; + sendAgentAudio: SessionContextValue["sendAgentAudio"]; + ws: UseWebSocketReturn; + mode?: "transcribe_only" | "auto_run"; + onTranscript: (text: string, meta: { requestId: string }) => void; + onError?: (error: Error) => void; + canStart?: () => boolean; + canConfirm?: () => boolean; + autoStopWhenHidden?: { isVisible: boolean }; + enableDuration?: boolean; +}; + +export type UseDictationResult = { + isRecording: boolean; + isProcessing: boolean; + volume: number; + duration: number; + pendingRequestId: string | null; + error: string | null; + startDictation: () => Promise; + cancelDictation: () => Promise; + confirmDictation: () => Promise; + reset: () => void; +}; + +const DURATION_TICK_MS = 1000; + +const toError = (error: unknown): Error => { + if (error instanceof Error) { + return error; + } + if (typeof error === "string" && error.trim().length > 0) { + return new Error(error); + } + return new Error("An unexpected error occurred while handling dictation."); +}; + +export function useDictation(options: UseDictationOptions): UseDictationResult { + const { + agentId, + sendAgentAudio, + ws, + mode = "transcribe_only", + onTranscript, + onError, + canStart, + canConfirm, + autoStopWhenHidden, + enableDuration = false, + } = options; + + const [isRecording, setIsRecording] = useState(false); + const [isProcessing, setIsProcessing] = useState(false); + const [volume, setVolume] = useState(0); + const [duration, setDuration] = useState(0); + const [pendingRequestId, setPendingRequestId] = useState(null); + const [error, setError] = useState(null); + + const handleAudioLevel = useCallback((level: number) => { + setVolume(level); + }, []); + + const recorder = useAudioRecorder({ onAudioLevel: handleAudioLevel }); + const recorderRef = useRef(recorder); + useEffect(() => { + recorderRef.current = recorder; + }, [recorder]); + + const onTranscriptRef = useRef(onTranscript); + useEffect(() => { + onTranscriptRef.current = onTranscript; + }, [onTranscript]); + + const onErrorRef = useRef(onError); + useEffect(() => { + onErrorRef.current = onError; + }, [onError]); + + const isRecordingRef = useRef(isRecording); + useEffect(() => { + isRecordingRef.current = isRecording; + }, [isRecording]); + + const pendingRequestIdRef = useRef(null); + const activeStopPromiseRef = useRef | null>(null); + const durationIntervalRef = useRef | null>(null); + + const stopDurationTracking = useCallback(() => { + if (durationIntervalRef.current) { + clearInterval(durationIntervalRef.current); + durationIntervalRef.current = null; + } + }, []); + + const startDurationTracking = useCallback(() => { + if (!enableDuration) { + console.log("[useDictation] startDictation blocked by canStart()"); + return; + } + if (durationIntervalRef.current) { + return; + } + durationIntervalRef.current = setInterval(() => { + setDuration((prev) => prev + 1); + }, DURATION_TICK_MS); + }, [enableDuration]); + + useEffect(() => { + if (!enableDuration) { + stopDurationTracking(); + setDuration(0); + } + }, [enableDuration, stopDurationTracking]); + + const reportError = useCallback( + (err: unknown, context?: string) => { + const normalized = toError(err); + if (context) { + console.error(`[useDictation] ${context}`, normalized); + } else { + console.error("[useDictation]", normalized); + } + setError(normalized.message); + onErrorRef.current?.(normalized); + }, + [setError] + ); + + const stopRecorder = useCallback(async (): Promise => { + if (activeStopPromiseRef.current) { + return activeStopPromiseRef.current; + } + const recorderInstance = recorderRef.current; + if (!recorderInstance) { + throw new Error("Recorder unavailable"); + } + const stopPromise = (async () => { + try { + return await recorderInstance.stop(); + } finally { + activeStopPromiseRef.current = null; + } + })(); + activeStopPromiseRef.current = stopPromise; + return stopPromise; + }, []); + + const startDictation = useCallback(async () => { + console.log("[useDictation] startDictation requested", { + isRecording: isRecordingRef.current, + isProcessing, + }); + if (isRecordingRef.current || isProcessing) { + console.log("[useDictation] startDictation aborted: already recording/processing", { + isRecording: isRecordingRef.current, + isProcessing, + }); + return; + } + const startAllowed = canStart ? canStart() : true; + if (!startAllowed) { + console.log("[useDictation] startDictation blocked by canStart()"); + return; + } + + setError(null); + setVolume(0); + setDuration(0); + setIsProcessing(false); + pendingRequestIdRef.current = null; + setPendingRequestId(null); + + try { + const recorderInstance = recorderRef.current; + if (!recorderInstance) { + throw new Error("Recorder unavailable"); + } + await recorderInstance.start(); + console.log("[useDictation] recorder.start succeeded"); + isRecordingRef.current = true; + setIsRecording(true); + if (enableDuration) { + startDurationTracking(); + } + } catch (err) { + stopDurationTracking(); + isRecordingRef.current = false; + setIsRecording(false); + reportError(err, "Failed to start dictation"); + } + }, [ + canStart, + enableDuration, + isProcessing, + reportError, + startDurationTracking, + stopDurationTracking, + ]); + + const cancelDictation = useCallback(async () => { + console.log("[useDictation] cancelDictation requested", { + isRecording: isRecordingRef.current, + hasActiveStop: Boolean(activeStopPromiseRef.current), + }); + if (!isRecordingRef.current && !activeStopPromiseRef.current) { + console.log("[useDictation] cancelDictation ignored: nothing to cancel"); + return; + } + stopDurationTracking(); + setDuration(0); + setError(null); + + try { + await stopRecorder(); + } catch (err) { + reportError(err, "Failed to cancel dictation"); + } finally { + isRecordingRef.current = false; + setIsRecording(false); + setIsProcessing(false); + setVolume(0); + } + }, [reportError, stopDurationTracking, stopRecorder]); + + const confirmDictation = useCallback(async () => { + console.log("[useDictation] confirmDictation requested", { + isRecording: isRecordingRef.current, + isProcessing, + }); + if (!isRecordingRef.current || isProcessing) { + console.log("[useDictation] confirmDictation ignored: recording flag", { + isRecording: isRecordingRef.current, + isProcessing, + }); + return; + } + const confirmAllowed = canConfirm ? canConfirm() : true; + if (!confirmAllowed) { + console.log("[useDictation] confirmDictation blocked by canConfirm()"); + return; + } + + setError(null); + stopDurationTracking(); + setDuration(0); + setIsProcessing(true); + + try { + const audioData = await stopRecorder(); + isRecordingRef.current = false; + setIsRecording(false); + setVolume(0); + + const requestId = generateMessageId(); + pendingRequestIdRef.current = requestId; + setPendingRequestId(requestId); + + await sendAgentAudio(agentId, audioData, requestId, { mode }); + console.log("[useDictation] sendAgentAudio resolved", { requestId }); + } catch (err) { + pendingRequestIdRef.current = null; + setPendingRequestId(null); + setIsProcessing(false); + isRecordingRef.current = false; + setIsRecording(false); + setVolume(0); + reportError(err, "Failed to complete dictation"); + } + }, [ + agentId, + canConfirm, + isProcessing, + mode, + reportError, + sendAgentAudio, + stopDurationTracking, + stopRecorder, + ]); + + const reset = useCallback(() => { + pendingRequestIdRef.current = null; + setPendingRequestId(null); + setIsRecording(false); + isRecordingRef.current = false; + setIsProcessing(false); + stopDurationTracking(); + setDuration(0); + setVolume(0); + setError(null); + }, [stopDurationTracking]); + + const cancelRef = useRef<(() => void) | null>(null); + useEffect(() => { + cancelRef.current = () => { + void cancelDictation(); + }; + }, [cancelDictation]); + + const visibilityRef = useRef( + typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null + ); + useEffect(() => { + const nextVisible = + typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null; + const prevVisible = visibilityRef.current; + visibilityRef.current = nextVisible; + + if (prevVisible === true && nextVisible === false && isRecordingRef.current) { + cancelRef.current?.(); + } + }, [autoStopWhenHidden?.isVisible]); + + useEffect(() => { + pendingRequestIdRef.current = null; + setPendingRequestId(null); + setIsProcessing(false); + }, [agentId]); + + useEffect(() => { + const unsubscribe = ws.on("transcription_result", (message: SessionOutboundMessage) => { + if (message.type !== "transcription_result") { + return; + } + const expectedId = pendingRequestIdRef.current; + if (!expectedId || message.payload.requestId !== expectedId) { + return; + } + + pendingRequestIdRef.current = null; + setPendingRequestId(null); + setIsProcessing(false); + + const transcriptText = message.payload.text?.trim(); + if (!transcriptText) { + return; + } + + console.log("[useDictation] transcription_result received", { + requestId: expectedId, + textLength: transcriptText.length, + }); + onTranscriptRef.current?.(transcriptText, { + requestId: message.payload.requestId ?? expectedId, + }); + }); + + return () => { + unsubscribe(); + }; + }, [ws]); + + useEffect(() => { + return () => { + stopDurationTracking(); + const activeStop = activeStopPromiseRef.current; + if (activeStop) { + void activeStop.catch(() => undefined); + return; + } + const recorderInstance = recorderRef.current; + if (recorderInstance?.isRecording?.()) { + void recorderInstance.stop().catch(() => undefined); + } + }; + }, [stopDurationTracking]); + + return { + isRecording, + isProcessing, + volume, + duration, + pendingRequestId, + error, + startDictation, + cancelDictation, + confirmDictation, + reset, + }; +}