import { useCallback, useEffect, useRef, useState } from "react"; import type { SessionContextValue } from "@/contexts/session-context"; import { useAudioRecorder } from "@/hooks/use-audio-recorder"; import { useSessionRpc } from "@/hooks/use-session-rpc"; import type { RpcFailureReason, RpcRetryAttemptEvent } from "@/hooks/use-session-rpc"; import type { UseWebSocketReturn } from "@/hooks/use-websocket"; import { generateMessageId } from "@/types/stream"; export type DictationStatus = "idle" | "recording" | "uploading" | "retrying" | "failed"; export type DictationRetryInfo = { attempt: number; maxAttempts: number; reason: RpcFailureReason; errorMessage: string; nextRetryMs: number; }; export type FailedDictationRecording = { requestId: string; durationSeconds: number; sizeBytes: number; format: string; recordedAt: number; errorMessage: string; }; export type DictationOutcome = | { type: "success"; requestId: string; timestamp: number } | { type: "failure"; requestId: string; errorMessage: string; timestamp: number }; export type UseDictationOptions = { agentId: string; sendAgentAudio: SessionContextValue["sendAgentAudio"]; ws: UseWebSocketReturn; mode?: "transcribe_only" | "auto_run"; onTranscript: (text: string, meta: { requestId: string }) => void; onError?: (error: Error) => void; onRetryAttempt?: (info: DictationRetryInfo) => void; onPermanentFailure?: (error: Error, context: { requestId: string }) => void; canStart?: () => boolean; canConfirm?: () => boolean; autoStopWhenHidden?: { isVisible: boolean }; enableDuration?: boolean; }; export type UseDictationResult = { isRecording: boolean; isProcessing: boolean; volume: number; duration: number; pendingRequestId: string | null; error: string | null; status: DictationStatus; retryAttempt: number; maxRetryAttempts: number; retryInfo: DictationRetryInfo | null; failedRecording: FailedDictationRecording | null; lastOutcome: DictationOutcome | null; startDictation: () => Promise; cancelDictation: () => Promise; confirmDictation: () => Promise; retryFailedDictation: () => Promise; discardFailedDictation: () => void; reset: () => void; }; const DURATION_TICK_MS = 1000; const MAX_AUTO_RETRY_ATTEMPTS = 5; const RETRY_BASE_DELAY_MS = 2000; const RETRY_MAX_DELAY_MS = 12000; const RETRY_BACKOFF_FACTOR = 1.8; const RETRY_JITTER_MS = 400; const TRANSCRIPTION_TIMEOUT_MS = 120000; const toError = (error: unknown): Error => { if (error instanceof Error) { return error; } if (typeof error === "string" && error.trim().length > 0) { return new Error(error); } return new Error("An unexpected error occurred while handling dictation."); }; type CapturedAudioPayload = { blob: Blob; format: string; sizeBytes: number; durationSeconds: number; recordedAt: number; }; const deriveFormatFromMime = (mimeType?: string): string => { if (!mimeType || mimeType.length === 0) { return "webm"; } const slashIndex = mimeType.indexOf("/"); let formatPart = slashIndex >= 0 ? mimeType.slice(slashIndex + 1) : mimeType; const semicolonIndex = formatPart.indexOf(";"); if (semicolonIndex >= 0) { formatPart = formatPart.slice(0, semicolonIndex); } return formatPart.trim().length > 0 ? formatPart.trim() : "webm"; }; const buildCapturedAudioPayload = (blob: Blob, durationSeconds: number): CapturedAudioPayload => ({ blob, format: deriveFormatFromMime(blob.type), sizeBytes: typeof blob.size === "number" ? blob.size : 0, durationSeconds, recordedAt: Date.now(), }); export function useDictation(options: UseDictationOptions): UseDictationResult { const { agentId, sendAgentAudio, ws, mode = "transcribe_only", onTranscript, onError, onRetryAttempt, onPermanentFailure, canStart, canConfirm, autoStopWhenHidden, enableDuration = false, } = options; const [isRecording, setIsRecording] = useState(false); const [isProcessing, setIsProcessing] = useState(false); const [volume, setVolume] = useState(0); const [duration, setDuration] = useState(0); const [pendingRequestId, setPendingRequestId] = useState(null); const [error, setError] = useState(null); const [status, setStatus] = useState("idle"); const [retryAttempt, setRetryAttempt] = useState(0); const [retryInfo, setRetryInfo] = useState(null); const [failedRecording, setFailedRecording] = useState(null); const [lastOutcome, setLastOutcome] = useState(null); const maxRetryAttempts = MAX_AUTO_RETRY_ATTEMPTS; const { waitForResponse: waitForTranscriptionResponse, reset: resetTranscriptionRpc } = useSessionRpc({ ws, requestType: "send_agent_audio", responseType: "transcription_result", }); const pendingAudioRef = useRef(null); const handleAudioLevel = useCallback((level: number) => { setVolume(level); }, []); const durationRef = useRef(0); useEffect(() => { durationRef.current = duration; }, [duration]); const recorder = useAudioRecorder({ onAudioLevel: handleAudioLevel }); const recorderRef = useRef(recorder); useEffect(() => { recorderRef.current = recorder; }, [recorder]); const onTranscriptRef = useRef(onTranscript); useEffect(() => { onTranscriptRef.current = onTranscript; }, [onTranscript]); const onErrorRef = useRef(onError); useEffect(() => { onErrorRef.current = onError; }, [onError]); const onRetryAttemptRef = useRef(onRetryAttempt); useEffect(() => { onRetryAttemptRef.current = onRetryAttempt; }, [onRetryAttempt]); const onPermanentFailureRef = useRef(onPermanentFailure); useEffect(() => { onPermanentFailureRef.current = onPermanentFailure; }, [onPermanentFailure]); const isRecordingRef = useRef(isRecording); useEffect(() => { isRecordingRef.current = isRecording; }, [isRecording]); const pendingRequestIdRef = useRef(null); const activeStopPromiseRef = useRef | null>(null); const durationIntervalRef = useRef | null>(null); const stopDurationTracking = useCallback(() => { if (durationIntervalRef.current) { clearInterval(durationIntervalRef.current); durationIntervalRef.current = null; } }, []); const startDurationTracking = useCallback(() => { if (!enableDuration) { console.log("[useDictation] startDictation blocked by canStart()"); return; } if (durationIntervalRef.current) { return; } durationIntervalRef.current = setInterval(() => { setDuration((prev) => prev + 1); }, DURATION_TICK_MS); }, [enableDuration]); useEffect(() => { if (!enableDuration) { stopDurationTracking(); setDuration(0); } }, [enableDuration, stopDurationTracking]); const reportError = useCallback( (err: unknown, context?: string) => { const normalized = toError(err); if (context) { console.error(`[useDictation] ${context}`, normalized); } else { console.error("[useDictation]", normalized); } setError(normalized.message); onErrorRef.current?.(normalized); }, [setError] ); const stopRecorder = useCallback(async (): Promise => { if (activeStopPromiseRef.current) { return activeStopPromiseRef.current; } const recorderInstance = recorderRef.current; if (!recorderInstance) { throw new Error("Recorder unavailable"); } const stopPromise = (async () => { try { return await recorderInstance.stop(); } finally { activeStopPromiseRef.current = null; } })(); activeStopPromiseRef.current = stopPromise; return stopPromise; }, []); const transmitDictation = useCallback( async (requestId: string) => { const capturedAudio = pendingAudioRef.current; if (!capturedAudio) { throw new Error("No recorded audio available for transcription"); } setRetryAttempt(1); setRetryInfo(null); console.info("[useDictation] sending transcription request", { requestId, attempt: 1, agentId, size: capturedAudio.sizeBytes, durationSeconds: capturedAudio.durationSeconds, }); const transcription = await waitForTranscriptionResponse({ requestId, dispatch: async (_id, attempt) => { setStatus("uploading"); setRetryAttempt(attempt); await sendAgentAudio(agentId, capturedAudio.blob, requestId, { mode }); }, retry: { maxAttempts: MAX_AUTO_RETRY_ATTEMPTS, baseDelayMs: RETRY_BASE_DELAY_MS, maxDelayMs: RETRY_MAX_DELAY_MS, backoffFactor: RETRY_BACKOFF_FACTOR, jitterMs: RETRY_JITTER_MS, shouldRetry: ({ attempt, maxAttempts }) => attempt < maxAttempts, onRetryAttempt: (event: RpcRetryAttemptEvent) => { setStatus("retrying"); setRetryAttempt(event.attempt); const info: DictationRetryInfo = { attempt: event.attempt, maxAttempts: event.maxAttempts, reason: event.reason, errorMessage: event.error.message, nextRetryMs: event.nextDelayMs, }; setRetryInfo(info); onRetryAttemptRef.current?.(info); console.warn("[useDictation] retry scheduled", { requestId, attempt: event.attempt, maxAttempts: event.maxAttempts, reason: event.reason, error: event.error.message, nextRetryMs: event.nextDelayMs, }); }, }, timeoutMs: TRANSCRIPTION_TIMEOUT_MS, }); return transcription; }, [agentId, mode, onRetryAttemptRef, sendAgentAudio, waitForTranscriptionResponse] ); const handleTranscriptionSuccess = useCallback( (transcription: Awaited>, requestId: string) => { pendingRequestIdRef.current = null; setPendingRequestId(null); setIsProcessing(false); setDuration(0); setStatus("idle"); setRetryAttempt(0); setRetryInfo(null); setFailedRecording(null); pendingAudioRef.current = null; setLastOutcome({ type: "success", requestId, timestamp: Date.now() }); const transcriptText = transcription.text?.trim(); if (!transcriptText) { return; } console.log("[useDictation] transcription_result received", { requestId, textLength: transcriptText.length, }); onTranscriptRef.current?.(transcriptText, { requestId: transcription.requestId ?? requestId, }); }, [onTranscriptRef] ); const handleDictationFailure = useCallback( (failure: unknown, requestId: string | null) => { const normalized = toError(failure); pendingRequestIdRef.current = null; setPendingRequestId(null); setIsProcessing(false); isRecordingRef.current = false; setIsRecording(false); setVolume(0); setRetryInfo(null); const capturedAudio = pendingAudioRef.current; if (capturedAudio) { setStatus("failed"); setFailedRecording({ requestId: requestId ?? generateMessageId(), durationSeconds: capturedAudio.durationSeconds, sizeBytes: capturedAudio.sizeBytes, format: capturedAudio.format, recordedAt: capturedAudio.recordedAt, errorMessage: normalized.message, }); if (requestId) { onPermanentFailureRef.current?.(normalized, { requestId }); } } else { setStatus("idle"); } setRetryAttempt(0); setLastOutcome({ type: "failure", requestId: requestId ?? generateMessageId(), errorMessage: normalized.message, timestamp: Date.now(), }); reportError(normalized, "Failed to complete dictation"); }, [onPermanentFailureRef, reportError] ); const startDictation = useCallback(async () => { console.log("[useDictation] startDictation requested", { isRecording: isRecordingRef.current, isProcessing, }); if (isRecordingRef.current || isProcessing) { console.log("[useDictation] startDictation aborted: already recording/processing", { isRecording: isRecordingRef.current, isProcessing, }); return; } const startAllowed = canStart ? canStart() : true; if (!startAllowed) { console.log("[useDictation] startDictation blocked by canStart()"); return; } setError(null); setVolume(0); setDuration(0); setIsProcessing(false); setStatus("recording"); setRetryAttempt(0); setRetryInfo(null); setFailedRecording(null); pendingAudioRef.current = null; setLastOutcome(null); pendingRequestIdRef.current = null; setPendingRequestId(null); try { const recorderInstance = recorderRef.current; if (!recorderInstance) { throw new Error("Recorder unavailable"); } await recorderInstance.start(); console.log("[useDictation] recorder.start succeeded"); isRecordingRef.current = true; setIsRecording(true); if (enableDuration) { startDurationTracking(); } } catch (err) { stopDurationTracking(); isRecordingRef.current = false; setIsRecording(false); reportError(err, "Failed to start dictation"); } }, [ canStart, enableDuration, isProcessing, reportError, startDurationTracking, stopDurationTracking, ]); const cancelDictation = useCallback(async () => { console.log("[useDictation] cancelDictation requested", { isRecording: isRecordingRef.current, hasActiveStop: Boolean(activeStopPromiseRef.current), }); if (!isRecordingRef.current && !activeStopPromiseRef.current) { console.log("[useDictation] cancelDictation ignored: nothing to cancel"); return; } stopDurationTracking(); setDuration(0); setError(null); try { await stopRecorder(); } catch (err) { reportError(err, "Failed to cancel dictation"); } finally { isRecordingRef.current = false; setIsRecording(false); setIsProcessing(false); setVolume(0); setStatus("idle"); setRetryAttempt(0); setRetryInfo(null); pendingAudioRef.current = null; setFailedRecording(null); setLastOutcome(null); } }, [reportError, stopDurationTracking, stopRecorder]); const confirmDictation = useCallback(async () => { console.log("[useDictation] confirmDictation requested", { isRecording: isRecordingRef.current, isProcessing, }); if (!isRecordingRef.current || isProcessing) { console.log("[useDictation] confirmDictation ignored: recording flag", { isRecording: isRecordingRef.current, isProcessing, }); return; } const confirmAllowed = canConfirm ? canConfirm() : true; if (!confirmAllowed) { console.log("[useDictation] confirmDictation blocked by canConfirm()"); return; } setError(null); stopDurationTracking(); setIsProcessing(true); setRetryInfo(null); setRetryAttempt(0); setLastOutcome(null); let requestId: string | null = null; try { const audioData = await stopRecorder(); const recordedDurationSeconds = durationRef.current; pendingAudioRef.current = buildCapturedAudioPayload(audioData, recordedDurationSeconds); setStatus("uploading"); isRecordingRef.current = false; setIsRecording(false); setVolume(0); requestId = generateMessageId(); pendingRequestIdRef.current = requestId; setPendingRequestId(requestId); const transcription = await transmitDictation(requestId); handleTranscriptionSuccess(transcription, requestId); } catch (err) { resetTranscriptionRpc(); handleDictationFailure(err, requestId); } }, [ agentId, canConfirm, isProcessing, mode, handleDictationFailure, handleTranscriptionSuccess, resetTranscriptionRpc, stopDurationTracking, stopRecorder, transmitDictation, ]); const retryFailedDictation = useCallback(async () => { if (!pendingAudioRef.current) { return; } setError(null); setRetryInfo(null); setRetryAttempt(0); setStatus("uploading"); setIsProcessing(true); setLastOutcome(null); const requestId = generateMessageId(); pendingRequestIdRef.current = requestId; setPendingRequestId(requestId); try { const transcription = await transmitDictation(requestId); handleTranscriptionSuccess(transcription, requestId); } catch (err) { resetTranscriptionRpc(); handleDictationFailure(err, requestId); } }, [ handleDictationFailure, handleTranscriptionSuccess, resetTranscriptionRpc, transmitDictation, ]); const discardFailedDictation = useCallback(() => { pendingAudioRef.current = null; pendingRequestIdRef.current = null; setPendingRequestId(null); setIsProcessing(false); setDuration(0); setFailedRecording(null); setStatus("idle"); setRetryAttempt(0); setRetryInfo(null); setError(null); setLastOutcome(null); }, []); const reset = useCallback(() => { pendingRequestIdRef.current = null; setPendingRequestId(null); setIsRecording(false); isRecordingRef.current = false; setIsProcessing(false); stopDurationTracking(); setDuration(0); setVolume(0); setError(null); setStatus("idle"); setRetryAttempt(0); setRetryInfo(null); setFailedRecording(null); pendingAudioRef.current = null; setLastOutcome(null); resetTranscriptionRpc(); }, [resetTranscriptionRpc, stopDurationTracking]); const cancelRef = useRef<(() => void) | null>(null); useEffect(() => { cancelRef.current = () => { void cancelDictation(); }; }, [cancelDictation]); const visibilityRef = useRef( typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null ); useEffect(() => { const nextVisible = typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null; const prevVisible = visibilityRef.current; visibilityRef.current = nextVisible; if (prevVisible === true && nextVisible === false && isRecordingRef.current) { cancelRef.current?.(); } }, [autoStopWhenHidden?.isVisible]); useEffect(() => { pendingRequestIdRef.current = null; setPendingRequestId(null); setIsProcessing(false); pendingAudioRef.current = null; setStatus("idle"); setRetryAttempt(0); setRetryInfo(null); setFailedRecording(null); resetTranscriptionRpc(); }, [agentId, resetTranscriptionRpc]); useEffect(() => { return () => { stopDurationTracking(); const activeStop = activeStopPromiseRef.current; if (activeStop) { void activeStop.catch(() => undefined); return; } const recorderInstance = recorderRef.current; if (recorderInstance?.isRecording?.()) { void recorderInstance.stop().catch(() => undefined); } }; }, [stopDurationTracking]); return { isRecording, isProcessing, volume, duration, pendingRequestId, error, status, retryAttempt, maxRetryAttempts, retryInfo, failedRecording, lastOutcome, startDictation, cancelDictation, confirmDictation, retryFailedDictation, discardFailedDictation, reset, }; }