From 6aa9b1c72e79a3bccb05049423e3c9865ef776af Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sun, 23 Nov 2025 14:32:08 +0100 Subject: [PATCH] Add web dictation recorder and shortcuts --- .../app/src/components/agent-input-area.tsx | 39 +++ .../app/src/hooks/use-audio-recorder.d.ts | 1 + ...corder.ts => use-audio-recorder.native.ts} | 0 .../app/src/hooks/use-audio-recorder.web.ts | 286 ++++++++++++++++++ 4 files changed, 326 insertions(+) create mode 100644 packages/app/src/hooks/use-audio-recorder.d.ts rename packages/app/src/hooks/{use-audio-recorder.ts => use-audio-recorder.native.ts} (100%) create mode 100644 packages/app/src/hooks/use-audio-recorder.web.ts diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index cba4b1e71..282446954 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -261,6 +261,7 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { }; }, []); + function isTextAreaLike(value: unknown): value is TextAreaHandle { if (!value || typeof value !== "object") { return false; @@ -457,6 +458,44 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { } }, [isAgentRunning, ws.isConnected]); + useEffect(() => { + if (!IS_WEB) { + return; + } + + const handleDictationShortcut = (event: KeyboardEvent) => { + const key = event.key.toLowerCase(); + + if (event.metaKey && !event.altKey && !event.ctrlKey && key === "d") { + event.preventDefault(); + if (isRecording) { + void handleSendRecording(); + } else if (!isRealtimeMode && shouldShowVoiceControls && ws.isConnected) { + void handleVoicePress(); + } + return; + } + + if (key === "escape" && isRecording) { + event.preventDefault(); + void handleCancelRecording(); + } + }; + + window.addEventListener("keydown", handleDictationShortcut); + return () => { + window.removeEventListener("keydown", handleDictationShortcut); + }; + }, [ + isRecording, + isRealtimeMode, + shouldShowVoiceControls, + ws, + handleVoicePress, + handleSendRecording, + handleCancelRecording, + ]); + useEffect(() => { focusWebTextInput(); }, [agentId]); diff --git a/packages/app/src/hooks/use-audio-recorder.d.ts b/packages/app/src/hooks/use-audio-recorder.d.ts new file mode 100644 index 000000000..305f6728f --- /dev/null +++ b/packages/app/src/hooks/use-audio-recorder.d.ts @@ -0,0 +1 @@ +export * from "./use-audio-recorder.native"; diff --git a/packages/app/src/hooks/use-audio-recorder.ts b/packages/app/src/hooks/use-audio-recorder.native.ts similarity index 100% rename from packages/app/src/hooks/use-audio-recorder.ts rename to packages/app/src/hooks/use-audio-recorder.native.ts diff --git a/packages/app/src/hooks/use-audio-recorder.web.ts b/packages/app/src/hooks/use-audio-recorder.web.ts new file mode 100644 index 000000000..159d7a3c8 --- /dev/null +++ b/packages/app/src/hooks/use-audio-recorder.web.ts @@ -0,0 +1,286 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +export interface AudioCaptureConfig { + sampleRate?: number; + numberOfChannels?: number; + bitRate?: number; + onAudioLevel?: (level: number) => void; + onSpeechSegment?: (audioBlob: Blob) => void; + enableContinuousRecording?: boolean; +} + +const MIME_TYPE_CANDIDATES = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/ogg;codecs=opus", + "audio/ogg", +]; + +export function useAudioRecorder(config?: AudioCaptureConfig) { + const [isRecording, setIsRecording] = useState(false); + + const mediaRecorderRef = useRef(null); + const mediaStreamRef = useRef(null); + const chunksRef = useRef([]); + const supportedMimeTypeRef = useRef(undefined); + const analyserRef = useRef(null); + const audioContextRef = useRef(null); + const rafIdRef = useRef(null); + const configRef = useRef(config); + + useEffect(() => { + configRef.current = config; + }, [config]); + + const detectSupportedMimeType = useCallback((): string | null => { + if (supportedMimeTypeRef.current !== undefined) { + return supportedMimeTypeRef.current; + } + + if (typeof window === "undefined" || typeof window.MediaRecorder === "undefined") { + supportedMimeTypeRef.current = null; + return null; + } + + const detected = MIME_TYPE_CANDIDATES.find((candidate) => + window.MediaRecorder.isTypeSupported(candidate) + ); + + supportedMimeTypeRef.current = detected ?? null; + return supportedMimeTypeRef.current; + }, []); + + const stopMetering = useCallback(() => { + if (rafIdRef.current !== null) { + cancelAnimationFrame(rafIdRef.current); + rafIdRef.current = null; + } + + analyserRef.current?.disconnect(); + analyserRef.current = null; + + if (audioContextRef.current) { + // Closing can reject if context already closed; ignore. + void audioContextRef.current.close().catch(() => undefined); + audioContextRef.current = null; + } + }, []); + + const cleanupStream = useCallback(() => { + if (mediaStreamRef.current) { + mediaStreamRef.current.getTracks().forEach((track) => { + try { + track.stop(); + } catch { + // Ignore track stop errors. + } + }); + mediaStreamRef.current = null; + } + }, []); + + const startMetering = useCallback((stream: MediaStream) => { + const onAudioLevel = configRef.current?.onAudioLevel; + if (!onAudioLevel) { + return; + } + + const AudioContextCtor = + (typeof window !== "undefined" && + ((window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext || + (window as typeof window & { webkitAudioContext?: typeof AudioContext }) + .webkitAudioContext)) || + null; + + if (!AudioContextCtor) { + console.warn("[AudioRecorder][Web] AudioContext unavailable for metering"); + return; + } + + try { + const audioContext = new AudioContextCtor(); + const source = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 2048; + source.connect(analyser); + + analyserRef.current = analyser; + audioContextRef.current = audioContext; + + const dataArray = new Uint8Array(analyser.fftSize); + + const tick = () => { + analyser.getByteTimeDomainData(dataArray); + let sumSquares = 0; + for (let i = 0; i < dataArray.length; i++) { + const value = (dataArray[i] - 128) / 128; + sumSquares += value * value; + } + const rms = Math.sqrt(sumSquares / dataArray.length); + const normalized = Math.min(1, Math.max(0, rms * 2)); + onAudioLevel(normalized); + rafIdRef.current = requestAnimationFrame(tick); + }; + + tick(); + } catch (error) { + console.warn("[AudioRecorder][Web] Failed to start metering", error); + } + }, []); + + const start = useCallback(async () => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") { + throw new Error("Already recording"); + } + + if ( + typeof navigator === "undefined" || + !navigator.mediaDevices || + typeof navigator.mediaDevices.getUserMedia !== "function" + ) { + throw new Error("Microphone capture is not supported in this environment"); + } + + const options = configRef.current; + const constraints: MediaStreamConstraints = { + audio: { + channelCount: options?.numberOfChannels ?? 1, + sampleRate: options?.sampleRate, + noiseSuppression: true, + echoCancellation: true, + autoGainControl: true, + }, + }; + + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia(constraints); + } catch (error: any) { + throw new Error(`Failed to access microphone: ${error?.message ?? error}`); + } + + mediaStreamRef.current = stream; + + const recorderOptions: MediaRecorderOptions = {}; + const mimeType = detectSupportedMimeType(); + if (mimeType) { + recorderOptions.mimeType = mimeType; + } + if (options?.bitRate) { + recorderOptions.audioBitsPerSecond = options.bitRate; + } + + let recorder: MediaRecorder; + try { + recorder = new MediaRecorder(stream, recorderOptions); + } catch (error: any) { + cleanupStream(); + throw new Error(`Failed to initialize recorder: ${error?.message ?? error}`); + } + + mediaRecorderRef.current = recorder; + chunksRef.current = []; + + recorder.ondataavailable = (event) => { + if (event.data && event.data.size > 0) { + chunksRef.current.push(event.data); + if (options?.enableContinuousRecording && options?.onSpeechSegment) { + options.onSpeechSegment(event.data); + } + } + }; + + recorder.onerror = (event) => { + const error = (event as { error?: Error }).error; + console.error("[AudioRecorder][Web] Recorder error", error ?? event); + }; + + const timeslice = options?.enableContinuousRecording ? 1000 : undefined; + + startMetering(stream); + setIsRecording(true); + + if (timeslice !== undefined) { + recorder.start(timeslice); + } else { + recorder.start(); + } + }, [cleanupStream, detectSupportedMimeType, startMetering]); + + const stop = useCallback(async (): Promise => { + const recorder = mediaRecorderRef.current; + if (!recorder || recorder.state === "inactive") { + throw new Error("Not recording"); + } + + return await new Promise((resolve, reject) => { + const finalize = () => { + recorder.removeEventListener("stop", finalize); + recorder.removeEventListener("error", handleError as EventListener); + + stopMetering(); + cleanupStream(); + mediaRecorderRef.current = null; + setIsRecording(false); + + const mimeType = recorder.mimeType || detectSupportedMimeType() || "audio/webm"; + const blob = new Blob(chunksRef.current, { type: mimeType }); + chunksRef.current = []; + resolve(blob); + }; + + const handleError = (event: unknown) => { + recorder.removeEventListener("stop", finalize); + recorder.removeEventListener("error", handleError as EventListener); + + stopMetering(); + cleanupStream(); + mediaRecorderRef.current = null; + chunksRef.current = []; + setIsRecording(false); + + const error = + typeof event === "object" && event && "error" in event + ? (event as { error?: Error }).error + : null; + reject(error ?? new Error("Failed to stop recording")); + }; + + recorder.addEventListener("stop", finalize); + recorder.addEventListener("error", handleError as EventListener); + + try { + recorder.stop(); + } catch (error) { + recorder.removeEventListener("stop", finalize); + recorder.removeEventListener("error", handleError as EventListener); + handleError(error); + } + }); + }, [cleanupStream, detectSupportedMimeType, stopMetering]); + + useEffect(() => { + return () => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") { + try { + mediaRecorderRef.current.stop(); + } catch { + // Ignore stop during unmount. + } + } + stopMetering(); + cleanupStream(); + }; + }, [cleanupStream, stopMetering]); + + const getSupportedMimeType = useCallback(() => detectSupportedMimeType(), [ + detectSupportedMimeType, + ]); + + return { + start, + stop, + isRecording: () => isRecording, + getSupportedMimeType, + }; +}