fix: share dictation hook across surfaces
- add shared useDictation hook to prevent double-stop races - refactor modal + agent chat to consume shared API - improve web recorder errors with secure-context messaging 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -21,7 +21,7 @@ import Animated, {
|
|||||||
FadeOut,
|
FadeOut,
|
||||||
} from "react-native-reanimated";
|
} from "react-native-reanimated";
|
||||||
import { useRealtime } from "@/contexts/realtime-context";
|
import { useRealtime } from "@/contexts/realtime-context";
|
||||||
import { useAudioRecorder } from "@/hooks/use-audio-recorder";
|
import { useDictation } from "@/hooks/use-dictation";
|
||||||
import { FOOTER_HEIGHT } from "@/contexts/footer-controls-context";
|
import { FOOTER_HEIGHT } from "@/contexts/footer-controls-context";
|
||||||
import { VoiceNoteRecordingOverlay } from "./voice-note-recording-overlay";
|
import { VoiceNoteRecordingOverlay } from "./voice-note-recording-overlay";
|
||||||
import { generateMessageId } from "@/types/stream";
|
import { generateMessageId } from "@/types/stream";
|
||||||
@@ -107,39 +107,111 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
|
|
||||||
const [userInput, setUserInput] = useState("");
|
const [userInput, setUserInput] = useState("");
|
||||||
const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT);
|
const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT);
|
||||||
const [isRecording, setIsRecording] = useState(false);
|
|
||||||
const [isProcessing, setIsProcessing] = useState(false);
|
const [isProcessing, setIsProcessing] = useState(false);
|
||||||
const [selectedImages, setSelectedImages] = useState<Array<{ uri: string; mimeType: string }>>([]);
|
const [selectedImages, setSelectedImages] = useState<Array<{ uri: string; mimeType: string }>>([]);
|
||||||
const [recordingVolume, setRecordingVolume] = useState(0);
|
|
||||||
const [recordingDuration, setRecordingDuration] = useState(0);
|
|
||||||
const [transcribingRequestId, setTranscribingRequestId] = useState<string | null>(null);
|
|
||||||
const [isCancellingAgent, setIsCancellingAgent] = useState(false);
|
const [isCancellingAgent, setIsCancellingAgent] = useState(false);
|
||||||
const [audioDebugInfo, setAudioDebugInfo] = useState<AudioDebugInfo | null>(null);
|
const [audioDebugInfo, setAudioDebugInfo] = useState<AudioDebugInfo | null>(null);
|
||||||
|
|
||||||
const recordingIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
|
||||||
const textInputRef = useRef<TextInput | (TextInput & { getNativeRef?: () => unknown }) | null>(null);
|
const textInputRef = useRef<TextInput | (TextInput & { getNativeRef?: () => unknown }) | null>(null);
|
||||||
const inputHeightRef = useRef(MIN_INPUT_HEIGHT);
|
const inputHeightRef = useRef(MIN_INPUT_HEIGHT);
|
||||||
const baselineInputHeightRef = useRef<number | null>(null);
|
const baselineInputHeightRef = useRef<number | null>(null);
|
||||||
const overlayTransition = useSharedValue(0);
|
const overlayTransition = useSharedValue(0);
|
||||||
const { pickImages } = useImageAttachmentPicker();
|
const { pickImages } = useImageAttachmentPicker();
|
||||||
const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED;
|
const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED;
|
||||||
const pendingTranscriptionRef = useRef<{ requestId: string } | null>(null);
|
|
||||||
const agentIdRef = useRef(agentId);
|
const agentIdRef = useRef(agentId);
|
||||||
const sendAgentMessageRef = useRef(sendAgentMessage);
|
const sendAgentMessageRef = useRef(sendAgentMessage);
|
||||||
const agentStatusRef = useRef<string | undefined>(undefined);
|
const agentStatusRef = useRef<string | undefined>(undefined);
|
||||||
const updateQueueRef = useRef<
|
const updateQueueRef = useRef<
|
||||||
((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void) | null
|
((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void) | null
|
||||||
>(null);
|
>(null);
|
||||||
|
const handleDictationTranscript = useCallback(
|
||||||
|
(text: string) => {
|
||||||
|
if (!text) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const shouldQueue = agentStatusRef.current === "running";
|
||||||
|
if (shouldQueue) {
|
||||||
|
updateQueueRef.current?.((current) => [
|
||||||
|
...current,
|
||||||
|
{
|
||||||
|
id: generateMessageId(),
|
||||||
|
text,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
await sendAgentMessageRef.current?.(agentIdRef.current, text);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[AgentInput] Failed to send transcribed message:", error);
|
||||||
|
updateQueueRef.current?.((current) => [
|
||||||
|
...current,
|
||||||
|
{
|
||||||
|
id: generateMessageId(),
|
||||||
|
text,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
},
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
|
||||||
// Stabilize audio level callback to prevent recorder recreation on every render
|
const handleDictationError = useCallback((error: Error) => {
|
||||||
const handleRecordingAudioLevel = useCallback((level: number) => {
|
console.error("[AgentInput] Dictation error:", error);
|
||||||
setRecordingVolume(level);
|
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const audioRecorder = useAudioRecorder({
|
const canStartDictation = useCallback(() => {
|
||||||
onAudioLevel: handleRecordingAudioLevel,
|
const allowed = !isRealtimeMode && ws.isConnected;
|
||||||
|
console.log("[AgentInput] canStartDictation", {
|
||||||
|
allowed,
|
||||||
|
isRealtimeMode,
|
||||||
|
wsConnected: ws.isConnected,
|
||||||
|
});
|
||||||
|
return allowed;
|
||||||
|
}, [isRealtimeMode, ws.isConnected]);
|
||||||
|
|
||||||
|
const canConfirmDictation = useCallback(() => {
|
||||||
|
const allowed = ws.isConnected;
|
||||||
|
console.log("[AgentInput] canConfirmDictation", {
|
||||||
|
allowed,
|
||||||
|
wsConnected: ws.isConnected,
|
||||||
|
});
|
||||||
|
return allowed;
|
||||||
|
}, [ws.isConnected]);
|
||||||
|
|
||||||
|
const {
|
||||||
|
isRecording: isDictating,
|
||||||
|
isProcessing: isDictationProcessing,
|
||||||
|
volume: dictationVolume,
|
||||||
|
duration: dictationDuration,
|
||||||
|
pendingRequestId: dictationPendingRequestId,
|
||||||
|
startDictation,
|
||||||
|
cancelDictation,
|
||||||
|
confirmDictation,
|
||||||
|
} = useDictation({
|
||||||
|
agentId,
|
||||||
|
sendAgentAudio,
|
||||||
|
ws,
|
||||||
|
mode: "transcribe_only",
|
||||||
|
onTranscript: handleDictationTranscript,
|
||||||
|
onError: handleDictationError,
|
||||||
|
canStart: canStartDictation,
|
||||||
|
canConfirm: canConfirmDictation,
|
||||||
|
enableDuration: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const dictationRequestIdRef = useRef<string | null>(null);
|
||||||
|
useEffect(() => {
|
||||||
|
dictationRequestIdRef.current = dictationPendingRequestId;
|
||||||
|
}, [dictationPendingRequestId]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const shouldShowOverlay = isDictating || isDictationProcessing;
|
||||||
|
overlayTransition.value = withTiming(shouldShowOverlay ? 1 : 0, { duration: 250 });
|
||||||
|
}, [isDictating, isDictationProcessing, overlayTransition]);
|
||||||
|
|
||||||
const debugInputHeight = (label: string, payload: Record<string, unknown>) => {
|
const debugInputHeight = (label: string, payload: Record<string, unknown>) => {
|
||||||
if (!SHOULD_DEBUG_INPUT_HEIGHT) {
|
if (!SHOULD_DEBUG_INPUT_HEIGHT) {
|
||||||
return;
|
return;
|
||||||
@@ -196,11 +268,12 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function handleVoicePress() {
|
async function handleVoicePress() {
|
||||||
if (isRecording) {
|
console.log("[AgentInput] handleVoicePress", {
|
||||||
// This shouldn't happen as button is hidden when recording
|
isDictating,
|
||||||
return;
|
isRealtimeMode,
|
||||||
}
|
wsConnected: ws.isConnected,
|
||||||
if (isRealtimeMode) {
|
});
|
||||||
|
if (isDictating || isRealtimeMode) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,19 +281,12 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
setAudioDebugInfo(null);
|
setAudioDebugInfo(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start recording
|
|
||||||
try {
|
try {
|
||||||
await audioRecorder.start();
|
await startDictation();
|
||||||
setIsRecording(true);
|
console.log("[AgentInput] startDictation invoked");
|
||||||
setRecordingDuration(0);
|
|
||||||
overlayTransition.value = withTiming(1, { duration: 250 });
|
|
||||||
|
|
||||||
// Start duration timer
|
|
||||||
recordingIntervalRef.current = setInterval(() => {
|
|
||||||
setRecordingDuration((prev) => prev + 1);
|
|
||||||
}, 1000);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const isCancelled = error instanceof Error && error.message.includes("Recording cancelled");
|
const isCancelled =
|
||||||
|
error instanceof Error && error.message.includes("Recording cancelled");
|
||||||
if (!isCancelled) {
|
if (!isCancelled) {
|
||||||
console.error("[AgentInput] Failed to start recording:", error);
|
console.error("[AgentInput] Failed to start recording:", error);
|
||||||
}
|
}
|
||||||
@@ -228,152 +294,66 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function handleCancelRecording() {
|
async function handleCancelRecording() {
|
||||||
|
console.log("[AgentInput] handleCancelRecording", {
|
||||||
|
isDictating,
|
||||||
|
isDictationProcessing,
|
||||||
|
});
|
||||||
|
if (!isDictating && !isDictationProcessing) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
// Stop recording without sending
|
await cancelDictation();
|
||||||
await audioRecorder.stop();
|
|
||||||
setIsRecording(false);
|
|
||||||
overlayTransition.value = withTiming(0, { duration: 250 });
|
|
||||||
|
|
||||||
// Clear timer
|
|
||||||
if (recordingIntervalRef.current) {
|
|
||||||
clearInterval(recordingIntervalRef.current);
|
|
||||||
recordingIntervalRef.current = null;
|
|
||||||
}
|
|
||||||
setRecordingDuration(0);
|
|
||||||
setRecordingVolume(0);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("[AgentInput] Failed to cancel recording:", error);
|
console.error("[AgentInput] Failed to cancel recording:", error);
|
||||||
setIsRecording(false);
|
|
||||||
overlayTransition.value = withTiming(0, { duration: 250 });
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleSendRecording() {
|
async function handleSendRecording() {
|
||||||
|
console.log("[AgentInput] handleSendRecording", {
|
||||||
|
isDictating,
|
||||||
|
isDictationProcessing,
|
||||||
|
});
|
||||||
|
if (!isDictating) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
const audioData = await audioRecorder.stop();
|
await confirmDictation();
|
||||||
setIsRecording(false);
|
|
||||||
|
|
||||||
// Clear timer
|
|
||||||
if (recordingIntervalRef.current) {
|
|
||||||
clearInterval(recordingIntervalRef.current);
|
|
||||||
recordingIntervalRef.current = null;
|
|
||||||
}
|
|
||||||
setRecordingDuration(0);
|
|
||||||
setRecordingVolume(0);
|
|
||||||
|
|
||||||
if (audioData) {
|
|
||||||
// Generate request ID for tracking transcription
|
|
||||||
const requestId = generateMessageId();
|
|
||||||
setTranscribingRequestId(requestId);
|
|
||||||
pendingTranscriptionRef.current = {
|
|
||||||
requestId,
|
|
||||||
};
|
|
||||||
console.log("[AgentInput] Audio recorded:", audioData.size, "bytes", "requestId:", requestId);
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Send audio to agent for transcription and processing
|
|
||||||
await sendAgentAudio(agentId, audioData, requestId, { mode: "transcribe_only" });
|
|
||||||
console.log("[AgentInput] Audio sent to agent");
|
|
||||||
} catch (error) {
|
|
||||||
console.error("[AgentInput] Failed to send audio:", error);
|
|
||||||
// Clear transcribing state on error
|
|
||||||
setTranscribingRequestId(null);
|
|
||||||
pendingTranscriptionRef.current = null;
|
|
||||||
overlayTransition.value = withTiming(0, { duration: 250 });
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// No audio data, dismiss overlay immediately
|
|
||||||
overlayTransition.value = withTiming(0, { duration: 250 });
|
|
||||||
}
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("[AgentInput] Failed to stop recording:", error);
|
console.error("[AgentInput] Failed to send recording:", error);
|
||||||
setIsRecording(false);
|
|
||||||
setTranscribingRequestId(null);
|
|
||||||
pendingTranscriptionRef.current = null;
|
|
||||||
overlayTransition.value = withTiming(0, { duration: 250 });
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
if (!shouldShowAudioDebug) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const unsubscribe = ws.on("transcription_result", (message: SessionOutboundMessage) => {
|
const unsubscribe = ws.on("transcription_result", (message: SessionOutboundMessage) => {
|
||||||
if (message.type !== "transcription_result") {
|
if (message.type !== "transcription_result") {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const pending = pendingTranscriptionRef.current;
|
const pendingRequestId = dictationRequestIdRef.current;
|
||||||
if (!pending || !message.payload.requestId) {
|
if (!pendingRequestId || message.payload.requestId !== pendingRequestId) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (message.payload.requestId !== pending.requestId) {
|
dictationRequestIdRef.current = null;
|
||||||
return;
|
setAudioDebugInfo({
|
||||||
}
|
requestId: pendingRequestId,
|
||||||
|
transcript: message.payload.text?.trim(),
|
||||||
console.log("[AgentInput] Transcription completed for requestId:", pending.requestId);
|
debugRecordingPath: message.payload.debugRecordingPath ?? undefined,
|
||||||
pendingTranscriptionRef.current = null;
|
format: message.payload.format,
|
||||||
setTranscribingRequestId(null);
|
byteLength: message.payload.byteLength,
|
||||||
overlayTransition.value = withTiming(0, { duration: 250 });
|
duration: message.payload.duration,
|
||||||
|
avgLogprob: message.payload.avgLogprob,
|
||||||
if (shouldShowAudioDebug) {
|
isLowConfidence: message.payload.isLowConfidence,
|
||||||
setAudioDebugInfo({
|
});
|
||||||
requestId: pending.requestId,
|
|
||||||
transcript: message.payload.text?.trim(),
|
|
||||||
debugRecordingPath: message.payload.debugRecordingPath ?? undefined,
|
|
||||||
format: message.payload.format,
|
|
||||||
byteLength: message.payload.byteLength,
|
|
||||||
duration: message.payload.duration,
|
|
||||||
avgLogprob: message.payload.avgLogprob,
|
|
||||||
isLowConfidence: message.payload.isLowConfidence,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const transcriptText = message.payload.text?.trim();
|
|
||||||
if (!transcriptText) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const shouldQueue = agentStatusRef.current === "running";
|
|
||||||
if (shouldQueue) {
|
|
||||||
updateQueueRef.current?.((current) => [
|
|
||||||
...current,
|
|
||||||
{
|
|
||||||
id: generateMessageId(),
|
|
||||||
text: transcriptText,
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
void (async () => {
|
|
||||||
try {
|
|
||||||
await sendAgentMessageRef.current?.(agentIdRef.current, transcriptText);
|
|
||||||
} catch (error) {
|
|
||||||
console.error("[AgentInput] Failed to send transcribed message:", error);
|
|
||||||
updateQueueRef.current?.((current) => [
|
|
||||||
...current,
|
|
||||||
{
|
|
||||||
id: generateMessageId(),
|
|
||||||
text: transcriptText,
|
|
||||||
},
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
unsubscribe();
|
unsubscribe();
|
||||||
};
|
};
|
||||||
}, [ws, overlayTransition, shouldShowAudioDebug]);
|
}, [shouldShowAudioDebug, ws]);
|
||||||
|
|
||||||
// Cleanup timer on unmount
|
|
||||||
useEffect(() => {
|
|
||||||
return () => {
|
|
||||||
if (recordingIntervalRef.current) {
|
|
||||||
clearInterval(recordingIntervalRef.current);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
|
|
||||||
function isTextAreaLike(value: unknown): value is TextAreaHandle {
|
function isTextAreaLike(value: unknown): value is TextAreaHandle {
|
||||||
if (!value || typeof value !== "object") {
|
if (!value || typeof value !== "object") {
|
||||||
@@ -600,7 +580,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
event.stopPropagation();
|
event.stopPropagation();
|
||||||
|
|
||||||
if (isRecording) {
|
if (isDictating) {
|
||||||
void handleSendRecording();
|
void handleSendRecording();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -611,7 +591,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (key === "escape" && isRecording) {
|
if (key === "escape" && isDictating) {
|
||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
event.stopPropagation();
|
event.stopPropagation();
|
||||||
void handleCancelRecording();
|
void handleCancelRecording();
|
||||||
@@ -623,7 +603,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
window.removeEventListener("keydown", listener, true);
|
window.removeEventListener("keydown", listener, true);
|
||||||
};
|
};
|
||||||
}, [
|
}, [
|
||||||
isRecording,
|
isDictating,
|
||||||
isRealtimeMode,
|
isRealtimeMode,
|
||||||
shouldShowVoiceControls,
|
shouldShowVoiceControls,
|
||||||
ws,
|
ws,
|
||||||
@@ -768,10 +748,10 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
style={[
|
style={[
|
||||||
styles.voiceButton as any,
|
styles.voiceButton as any,
|
||||||
(!ws.isConnected || isRealtimeMode ? styles.buttonDisabled : undefined) as any,
|
(!ws.isConnected || isRealtimeMode ? styles.buttonDisabled : undefined) as any,
|
||||||
(isRecording ? styles.voiceButtonRecording : undefined) as any,
|
(isDictating ? styles.voiceButtonRecording : undefined) as any,
|
||||||
]}
|
]}
|
||||||
>
|
>
|
||||||
{isRecording ? (
|
{isDictating ? (
|
||||||
<Square size={14} color="white" fill="white" />
|
<Square size={14} color="white" fill="white" />
|
||||||
) : (
|
) : (
|
||||||
<Mic size={20} color={theme.colors.foreground} />
|
<Mic size={20} color={theme.colors.foreground} />
|
||||||
@@ -858,7 +838,7 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
multiline
|
multiline
|
||||||
scrollEnabled={inputHeight >= MAX_INPUT_HEIGHT}
|
scrollEnabled={inputHeight >= MAX_INPUT_HEIGHT}
|
||||||
onContentSizeChange={handleContentSizeChange}
|
onContentSizeChange={handleContentSizeChange}
|
||||||
editable={!isRecording && ws.isConnected}
|
editable={!isDictating && ws.isConnected}
|
||||||
onKeyPress={shouldHandleDesktopSubmit ? handleDesktopSubmitKeyPress : undefined}
|
onKeyPress={shouldHandleDesktopSubmit ? handleDesktopSubmitKeyPress : undefined}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
@@ -942,11 +922,11 @@ export function AgentInputArea({ agentId, serverId }: AgentInputAreaProps) {
|
|||||||
<Animated.View style={[styles.overlayContainer, overlayAnimatedStyle]}>
|
<Animated.View style={[styles.overlayContainer, overlayAnimatedStyle]}>
|
||||||
<View style={styles.inputAreaContent}>
|
<View style={styles.inputAreaContent}>
|
||||||
<VoiceNoteRecordingOverlay
|
<VoiceNoteRecordingOverlay
|
||||||
volume={recordingVolume}
|
volume={dictationVolume}
|
||||||
duration={recordingDuration}
|
duration={dictationDuration}
|
||||||
onCancel={handleCancelRecording}
|
onCancel={handleCancelRecording}
|
||||||
onSend={handleSendRecording}
|
onSend={handleSendRecording}
|
||||||
isTranscribing={transcribingRequestId !== null}
|
isTranscribing={isDictationProcessing}
|
||||||
/>
|
/>
|
||||||
</View>
|
</View>
|
||||||
</Animated.View>
|
</Animated.View>
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ import { theme as defaultTheme } from "@/styles/theme";
|
|||||||
import { useRecentPaths } from "@/hooks/use-recent-paths";
|
import { useRecentPaths } from "@/hooks/use-recent-paths";
|
||||||
import { useRouter } from "expo-router";
|
import { useRouter } from "expo-router";
|
||||||
import { generateMessageId } from "@/types/stream";
|
import { generateMessageId } from "@/types/stream";
|
||||||
import { useAudioRecorder } from "@/hooks/use-audio-recorder";
|
import { useDictation } from "@/hooks/use-dictation";
|
||||||
import { VolumeMeter } from "@/components/volume-meter";
|
import { VolumeMeter } from "@/components/volume-meter";
|
||||||
import { AUDIO_DEBUG_ENABLED } from "@/config/audio-debug";
|
import { AUDIO_DEBUG_ENABLED } from "@/config/audio-debug";
|
||||||
import { AudioDebugNotice, type AudioDebugInfo } from "./audio-debug-notice";
|
import { AudioDebugNotice, type AudioDebugInfo } from "./audio-debug-notice";
|
||||||
@@ -280,7 +280,10 @@ function AgentFlowModal({
|
|||||||
const effectiveWs = ws ?? inertWebSocket;
|
const effectiveWs = ws ?? inertWebSocket;
|
||||||
const createAgent = session?.createAgent;
|
const createAgent = session?.createAgent;
|
||||||
const resumeAgent = session?.resumeAgent;
|
const resumeAgent = session?.resumeAgent;
|
||||||
const sendAgentAudio = session?.sendAgentAudio;
|
const sessionSendAgentAudio = session?.sendAgentAudio;
|
||||||
|
const noopSendAgentAudio = useCallback<SessionContextValue["sendAgentAudio"]>(async () => {}, []);
|
||||||
|
const sendAgentAudio = sessionSendAgentAudio ?? noopSendAgentAudio;
|
||||||
|
const hasSendAgentAudio = Boolean(sessionSendAgentAudio);
|
||||||
const agents = session?.agents;
|
const agents = session?.agents;
|
||||||
const providerModels = session?.providerModels;
|
const providerModels = session?.providerModels;
|
||||||
const requestProviderModels = session?.requestProviderModels;
|
const requestProviderModels = session?.requestProviderModels;
|
||||||
@@ -385,9 +388,6 @@ function AgentFlowModal({
|
|||||||
);
|
);
|
||||||
const [isImportLoading, setIsImportLoading] = useState(false);
|
const [isImportLoading, setIsImportLoading] = useState(false);
|
||||||
const [importError, setImportError] = useState<string | null>(null);
|
const [importError, setImportError] = useState<string | null>(null);
|
||||||
const [isDictating, setIsDictating] = useState(false);
|
|
||||||
const [isDictationProcessing, setIsDictationProcessing] = useState(false);
|
|
||||||
const [dictationVolume, setDictationVolume] = useState(0);
|
|
||||||
const [dictationDebugInfo, setDictationDebugInfo] = useState<AudioDebugInfo | null>(null);
|
const [dictationDebugInfo, setDictationDebugInfo] = useState<AudioDebugInfo | null>(null);
|
||||||
const [openDropdown, setOpenDropdown] = useState<DropdownKey | null>(null);
|
const [openDropdown, setOpenDropdown] = useState<DropdownKey | null>(null);
|
||||||
const pendingRequestIdRef = useRef<string | null>(null);
|
const pendingRequestIdRef = useRef<string | null>(null);
|
||||||
@@ -395,6 +395,7 @@ function AgentFlowModal({
|
|||||||
const promptInputRef = useRef<
|
const promptInputRef = useRef<
|
||||||
TextInput | (TextInput & { getNativeRef?: () => unknown }) | null
|
TextInput | (TextInput & { getNativeRef?: () => unknown }) | null
|
||||||
>(null);
|
>(null);
|
||||||
|
const focusPromptInputRef = useRef<() => void>(() => {});
|
||||||
const promptInputHeightRef = useRef(PROMPT_MIN_HEIGHT);
|
const promptInputHeightRef = useRef(PROMPT_MIN_HEIGHT);
|
||||||
const promptBaselineHeightRef = useRef<number | null>(null);
|
const promptBaselineHeightRef = useRef<number | null>(null);
|
||||||
const formPreferencesHydratedRef = useRef(false);
|
const formPreferencesHydratedRef = useRef(false);
|
||||||
@@ -407,17 +408,80 @@ function AgentFlowModal({
|
|||||||
const prevVisibilityRef = useRef(isVisible);
|
const prevVisibilityRef = useRef(isVisible);
|
||||||
const dictationRequestIdRef = useRef<string | null>(null);
|
const dictationRequestIdRef = useRef<string | null>(null);
|
||||||
|
|
||||||
// Stabilize audio level callback to prevent recorder recreation on every render
|
const handleDictationTranscript = useCallback(
|
||||||
const handleDictationAudioLevel = useCallback((level: number) => {
|
(text: string, _meta: { requestId: string }) => {
|
||||||
setDictationVolume(level);
|
if (!isCreateFlow || !text) {
|
||||||
}, []);
|
return;
|
||||||
|
}
|
||||||
|
setInitialPrompt((prev) => {
|
||||||
|
if (!prev) {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
const needsSpace = /\s$/.test(prev);
|
||||||
|
return `${prev}${needsSpace ? "" : " "}${text}`;
|
||||||
|
});
|
||||||
|
focusPromptInputRef.current?.();
|
||||||
|
},
|
||||||
|
[isCreateFlow, setInitialPrompt]
|
||||||
|
);
|
||||||
|
|
||||||
const dictationRecorder = useAudioRecorder({
|
const handleDictationError = useCallback(
|
||||||
onAudioLevel: handleDictationAudioLevel,
|
(dictationError: Error) => {
|
||||||
|
setErrorMessage(dictationError.message);
|
||||||
|
},
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
|
||||||
|
const canStartDictation = useCallback(() => {
|
||||||
|
const allowed = isCreateFlow && !isLoading && isTargetDaemonReady && isWsConnected;
|
||||||
|
console.log("[CreateAgentModal] canStartDictation", {
|
||||||
|
allowed,
|
||||||
|
isCreateFlow,
|
||||||
|
isLoading,
|
||||||
|
isTargetDaemonReady,
|
||||||
|
isWsConnected,
|
||||||
|
});
|
||||||
|
return allowed;
|
||||||
|
}, [isCreateFlow, isLoading, isTargetDaemonReady, isWsConnected]);
|
||||||
|
|
||||||
|
const canConfirmDictation = useCallback(() => {
|
||||||
|
const allowed = isTargetDaemonReady && hasSendAgentAudio;
|
||||||
|
console.log("[CreateAgentModal] canConfirmDictation", {
|
||||||
|
allowed,
|
||||||
|
isTargetDaemonReady,
|
||||||
|
hasSendAgentAudio,
|
||||||
|
});
|
||||||
|
return allowed;
|
||||||
|
}, [hasSendAgentAudio, isTargetDaemonReady]);
|
||||||
|
|
||||||
|
const {
|
||||||
|
isRecording: isDictating,
|
||||||
|
isProcessing: isDictationProcessing,
|
||||||
|
volume: dictationVolume,
|
||||||
|
pendingRequestId: dictationPendingRequestId,
|
||||||
|
startDictation,
|
||||||
|
cancelDictation,
|
||||||
|
confirmDictation,
|
||||||
|
reset: resetDictation,
|
||||||
|
} = useDictation({
|
||||||
|
agentId: DICTATION_AGENT_ID,
|
||||||
|
sendAgentAudio,
|
||||||
|
ws: effectiveWs,
|
||||||
|
mode: "transcribe_only",
|
||||||
|
onTranscript: handleDictationTranscript,
|
||||||
|
onError: handleDictationError,
|
||||||
|
canStart: canStartDictation,
|
||||||
|
canConfirm: canConfirmDictation,
|
||||||
|
autoStopWhenHidden: { isVisible },
|
||||||
});
|
});
|
||||||
const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED;
|
const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
dictationRequestIdRef.current = dictationPendingRequestId;
|
||||||
|
}, [dictationPendingRequestId]);
|
||||||
|
|
||||||
const hasPendingCreateOrResume = pendingRequestIdRef.current !== null;
|
const hasPendingCreateOrResume = pendingRequestIdRef.current !== null;
|
||||||
const hasPendingDictation = dictationRequestIdRef.current !== null;
|
const hasPendingDictation = dictationPendingRequestId !== null;
|
||||||
const shouldListenForStatus = isVisible || hasPendingCreateOrResume;
|
const shouldListenForStatus = isVisible || hasPendingCreateOrResume;
|
||||||
const shouldListenForDictation =
|
const shouldListenForDictation =
|
||||||
isCreateFlow && (isVisible || hasPendingDictation);
|
isCreateFlow && (isVisible || hasPendingDictation);
|
||||||
@@ -800,6 +864,9 @@ function AgentFlowModal({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, [shouldAutoFocusPrompt]);
|
}, [shouldAutoFocusPrompt]);
|
||||||
|
useEffect(() => {
|
||||||
|
focusPromptInputRef.current = focusPromptInput;
|
||||||
|
}, [focusPromptInput]);
|
||||||
const activeSessionIds = useMemo(() => {
|
const activeSessionIds = useMemo(() => {
|
||||||
const ids = new Set<string>();
|
const ids = new Set<string>();
|
||||||
if (!agents) {
|
if (!agents) {
|
||||||
@@ -916,17 +983,21 @@ function AgentFlowModal({
|
|||||||
pendingNavigationServerIdRef.current = null;
|
pendingNavigationServerIdRef.current = null;
|
||||||
shouldSyncBaseBranchRef.current = true;
|
shouldSyncBaseBranchRef.current = true;
|
||||||
dictationRequestIdRef.current = null;
|
dictationRequestIdRef.current = null;
|
||||||
setIsDictating(false);
|
|
||||||
setIsDictationProcessing(false);
|
|
||||||
setDictationVolume(0);
|
|
||||||
setDictationDebugInfo(null);
|
setDictationDebugInfo(null);
|
||||||
if (isDictating) {
|
void cancelDictation();
|
||||||
void dictationRecorder.stop().catch(() => {});
|
resetDictation();
|
||||||
}
|
|
||||||
cancelRepoInfo();
|
cancelRepoInfo();
|
||||||
}, [cancelRepoInfo, dictationRecorder, isDictating, resetRepoInfo, setPromptHeight]);
|
}, [cancelRepoInfo, cancelDictation, resetDictation, resetRepoInfo, setPromptHeight]);
|
||||||
|
|
||||||
const handleDictationStart = useCallback(async () => {
|
const handleDictationStart = useCallback(async () => {
|
||||||
|
console.log("[CreateAgentModal] handleDictationStart", {
|
||||||
|
isCreateFlow,
|
||||||
|
isLoading,
|
||||||
|
isDictating,
|
||||||
|
isDictationProcessing,
|
||||||
|
isTargetDaemonReady,
|
||||||
|
isWsConnected,
|
||||||
|
});
|
||||||
if (!isCreateFlow || isLoading || isDictating || isDictationProcessing) {
|
if (!isCreateFlow || isLoading || isDictating || isDictationProcessing) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -941,21 +1012,17 @@ function AgentFlowModal({
|
|||||||
if (shouldShowAudioDebug) {
|
if (shouldShowAudioDebug) {
|
||||||
setDictationDebugInfo(null);
|
setDictationDebugInfo(null);
|
||||||
}
|
}
|
||||||
setIsDictating(true);
|
|
||||||
setDictationVolume(0);
|
|
||||||
setErrorMessage("");
|
setErrorMessage("");
|
||||||
await dictationRecorder.start();
|
await startDictation();
|
||||||
|
console.log("[CreateAgentModal] startDictation invoked");
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const isCancelled = error instanceof Error && error.message.includes("Recording cancelled");
|
const isCancelled = error instanceof Error && error.message.includes("Recording cancelled");
|
||||||
if (!isCancelled) {
|
if (!isCancelled) {
|
||||||
console.error("[CreateAgentModal] Failed to start dictation:", error);
|
console.error("[CreateAgentModal] Failed to start dictation:", error);
|
||||||
}
|
}
|
||||||
setIsDictating(false);
|
|
||||||
setDictationVolume(0);
|
|
||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
daemonAvailabilityError,
|
daemonAvailabilityError,
|
||||||
dictationRecorder,
|
|
||||||
isCreateFlow,
|
isCreateFlow,
|
||||||
isDictating,
|
isDictating,
|
||||||
isDictationProcessing,
|
isDictationProcessing,
|
||||||
@@ -964,27 +1031,34 @@ function AgentFlowModal({
|
|||||||
isWsConnected,
|
isWsConnected,
|
||||||
logOfflineDaemonAction,
|
logOfflineDaemonAction,
|
||||||
shouldShowAudioDebug,
|
shouldShowAudioDebug,
|
||||||
|
startDictation,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const handleDictationCancel = useCallback(async () => {
|
const handleDictationCancel = useCallback(async () => {
|
||||||
|
console.log("[CreateAgentModal] handleDictationCancel", {
|
||||||
|
isDictating,
|
||||||
|
});
|
||||||
if (!isDictating) {
|
if (!isDictating) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
await dictationRecorder.stop();
|
await cancelDictation();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("[CreateAgentModal] Failed to cancel dictation:", error);
|
console.error("[CreateAgentModal] Failed to cancel dictation:", error);
|
||||||
} finally {
|
|
||||||
setIsDictating(false);
|
|
||||||
setDictationVolume(0);
|
|
||||||
}
|
}
|
||||||
}, [dictationRecorder, isDictating]);
|
}, [cancelDictation, isDictating]);
|
||||||
|
|
||||||
const handleDictationConfirm = useCallback(async () => {
|
const handleDictationConfirm = useCallback(async () => {
|
||||||
|
console.log("[CreateAgentModal] handleDictationConfirm", {
|
||||||
|
isDictating,
|
||||||
|
isDictationProcessing,
|
||||||
|
isTargetDaemonReady,
|
||||||
|
hasSendAgentAudio,
|
||||||
|
});
|
||||||
if (!isDictating || isDictationProcessing) {
|
if (!isDictating || isDictationProcessing) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!sendAgentAudio || !isTargetDaemonReady) {
|
if (!isTargetDaemonReady || !hasSendAgentAudio) {
|
||||||
logOfflineDaemonAction("dictation");
|
logOfflineDaemonAction("dictation");
|
||||||
setErrorMessage(
|
setErrorMessage(
|
||||||
daemonAvailabilityError ??
|
daemonAvailabilityError ??
|
||||||
@@ -992,44 +1066,19 @@ function AgentFlowModal({
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set processing state BEFORE stopping to ensure UI updates properly
|
|
||||||
setIsDictationProcessing(true);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const audioData = await dictationRecorder.stop();
|
await confirmDictation();
|
||||||
setDictationVolume(0);
|
|
||||||
if (!audioData) {
|
|
||||||
setIsDictating(false);
|
|
||||||
setIsDictationProcessing(false);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const requestId = generateMessageId();
|
|
||||||
dictationRequestIdRef.current = requestId;
|
|
||||||
await sendAgentAudio(
|
|
||||||
DICTATION_AGENT_ID,
|
|
||||||
audioData,
|
|
||||||
requestId,
|
|
||||||
{ mode: "transcribe_only" }
|
|
||||||
);
|
|
||||||
// Only clear isDictating after successful transcription send
|
|
||||||
setIsDictating(false);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("[CreateAgentModal] Failed to complete dictation:", error);
|
console.error("[CreateAgentModal] Failed to complete dictation:", error);
|
||||||
dictationRequestIdRef.current = null;
|
|
||||||
setIsDictationProcessing(false);
|
|
||||||
// CRITICAL FIX: Always reset dictation state on error
|
|
||||||
setIsDictating(false);
|
|
||||||
setDictationVolume(0);
|
|
||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
daemonAvailabilityError,
|
daemonAvailabilityError,
|
||||||
dictationRecorder,
|
confirmDictation,
|
||||||
|
hasSendAgentAudio,
|
||||||
isDictating,
|
isDictating,
|
||||||
isDictationProcessing,
|
isDictationProcessing,
|
||||||
isTargetDaemonReady,
|
isTargetDaemonReady,
|
||||||
logOfflineDaemonAction,
|
logOfflineDaemonAction,
|
||||||
sendAgentAudio,
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const navigateToAgentIfNeeded = useCallback(() => {
|
const navigateToAgentIfNeeded = useCallback(() => {
|
||||||
@@ -1165,39 +1214,6 @@ function AgentFlowModal({
|
|||||||
handleCloseAnimationComplete,
|
handleCloseAnimationComplete,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
if (isVisible || !isDictating) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let cancelled = false;
|
|
||||||
|
|
||||||
const stopRecording = async () => {
|
|
||||||
try {
|
|
||||||
await dictationRecorder.stop();
|
|
||||||
} catch (error) {
|
|
||||||
console.error("[CreateAgentModal] Failed to stop dictation while hidden:", error);
|
|
||||||
} finally {
|
|
||||||
if (!cancelled) {
|
|
||||||
setIsDictating(false);
|
|
||||||
setDictationVolume(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
void stopRecording();
|
|
||||||
|
|
||||||
return () => {
|
|
||||||
cancelled = true;
|
|
||||||
};
|
|
||||||
}, [dictationRecorder, isDictating, isVisible]);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
return () => {
|
|
||||||
if (isDictating) {
|
|
||||||
void dictationRecorder.stop().catch(() => {});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}, [dictationRecorder, isDictating]);
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
return () => {
|
return () => {
|
||||||
@@ -1656,7 +1672,7 @@ function AgentFlowModal({
|
|||||||
}, [handleClose, shouldListenForStatus, ws]);
|
}, [handleClose, shouldListenForStatus, ws]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!shouldListenForDictation || !ws) {
|
if (!shouldListenForDictation || !ws || !shouldShowAudioDebug) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const unsubscribe = ws.on("transcription_result", (message) => {
|
const unsubscribe = ws.on("transcription_result", (message) => {
|
||||||
@@ -1668,37 +1684,22 @@ function AgentFlowModal({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
dictationRequestIdRef.current = null;
|
dictationRequestIdRef.current = null;
|
||||||
setIsDictationProcessing(false);
|
setDictationDebugInfo({
|
||||||
if (shouldShowAudioDebug) {
|
requestId: pendingId,
|
||||||
setDictationDebugInfo({
|
transcript: message.payload.text?.trim(),
|
||||||
requestId: pendingId,
|
debugRecordingPath: message.payload.debugRecordingPath ?? undefined,
|
||||||
transcript: message.payload.text?.trim(),
|
format: message.payload.format,
|
||||||
debugRecordingPath: message.payload.debugRecordingPath ?? undefined,
|
byteLength: message.payload.byteLength,
|
||||||
format: message.payload.format,
|
duration: message.payload.duration,
|
||||||
byteLength: message.payload.byteLength,
|
avgLogprob: message.payload.avgLogprob,
|
||||||
duration: message.payload.duration,
|
isLowConfidence: message.payload.isLowConfidence,
|
||||||
avgLogprob: message.payload.avgLogprob,
|
|
||||||
isLowConfidence: message.payload.isLowConfidence,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
const transcriptText = message.payload.text?.trim();
|
|
||||||
if (!transcriptText) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
setInitialPrompt((prev) => {
|
|
||||||
if (!prev) {
|
|
||||||
return transcriptText;
|
|
||||||
}
|
|
||||||
const needsSpace = /\s$/.test(prev);
|
|
||||||
return `${prev}${needsSpace ? "" : " "}${transcriptText}`;
|
|
||||||
});
|
});
|
||||||
focusPromptInput();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
unsubscribe();
|
unsubscribe();
|
||||||
};
|
};
|
||||||
}, [focusPromptInput, shouldListenForDictation, ws, shouldShowAudioDebug]);
|
}, [shouldListenForDictation, shouldShowAudioDebug, ws]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!isVisible || !isImportFlow) {
|
if (!isVisible || !isImportFlow) {
|
||||||
|
|||||||
@@ -133,14 +133,28 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
throw new Error("Already recording");
|
throw new Error("Already recording");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
const missingNavigator =
|
||||||
typeof navigator === "undefined" ||
|
typeof navigator === "undefined" ||
|
||||||
!navigator.mediaDevices ||
|
!navigator.mediaDevices ||
|
||||||
typeof navigator.mediaDevices.getUserMedia !== "function"
|
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||||
) {
|
|
||||||
|
const secureContext =
|
||||||
|
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||||
|
? window.isSecureContext
|
||||||
|
: true;
|
||||||
|
const currentOrigin =
|
||||||
|
typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
|
||||||
|
|
||||||
|
if (missingNavigator) {
|
||||||
throw new Error("Microphone capture is not supported in this environment");
|
throw new Error("Microphone capture is not supported in this environment");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!secureContext) {
|
||||||
|
throw new Error(
|
||||||
|
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const options = configRef.current;
|
const options = configRef.current;
|
||||||
const constraints: MediaStreamConstraints = {
|
const constraints: MediaStreamConstraints = {
|
||||||
audio: {
|
audio: {
|
||||||
|
|||||||
@@ -0,0 +1,388 @@
|
|||||||
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
|
|
||||||
|
import type { SessionContextValue } from "@/contexts/session-context";
|
||||||
|
import { useAudioRecorder } from "@/hooks/use-audio-recorder";
|
||||||
|
import type { UseWebSocketReturn } from "@/hooks/use-websocket";
|
||||||
|
import { generateMessageId } from "@/types/stream";
|
||||||
|
import type { SessionOutboundMessage } from "@server/server/messages";
|
||||||
|
|
||||||
|
export type UseDictationOptions = {
|
||||||
|
agentId: string;
|
||||||
|
sendAgentAudio: SessionContextValue["sendAgentAudio"];
|
||||||
|
ws: UseWebSocketReturn;
|
||||||
|
mode?: "transcribe_only" | "auto_run";
|
||||||
|
onTranscript: (text: string, meta: { requestId: string }) => void;
|
||||||
|
onError?: (error: Error) => void;
|
||||||
|
canStart?: () => boolean;
|
||||||
|
canConfirm?: () => boolean;
|
||||||
|
autoStopWhenHidden?: { isVisible: boolean };
|
||||||
|
enableDuration?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type UseDictationResult = {
|
||||||
|
isRecording: boolean;
|
||||||
|
isProcessing: boolean;
|
||||||
|
volume: number;
|
||||||
|
duration: number;
|
||||||
|
pendingRequestId: string | null;
|
||||||
|
error: string | null;
|
||||||
|
startDictation: () => Promise<void>;
|
||||||
|
cancelDictation: () => Promise<void>;
|
||||||
|
confirmDictation: () => Promise<void>;
|
||||||
|
reset: () => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DURATION_TICK_MS = 1000;
|
||||||
|
|
||||||
|
const toError = (error: unknown): Error => {
|
||||||
|
if (error instanceof Error) {
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
if (typeof error === "string" && error.trim().length > 0) {
|
||||||
|
return new Error(error);
|
||||||
|
}
|
||||||
|
return new Error("An unexpected error occurred while handling dictation.");
|
||||||
|
};
|
||||||
|
|
||||||
|
export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||||
|
const {
|
||||||
|
agentId,
|
||||||
|
sendAgentAudio,
|
||||||
|
ws,
|
||||||
|
mode = "transcribe_only",
|
||||||
|
onTranscript,
|
||||||
|
onError,
|
||||||
|
canStart,
|
||||||
|
canConfirm,
|
||||||
|
autoStopWhenHidden,
|
||||||
|
enableDuration = false,
|
||||||
|
} = options;
|
||||||
|
|
||||||
|
const [isRecording, setIsRecording] = useState(false);
|
||||||
|
const [isProcessing, setIsProcessing] = useState(false);
|
||||||
|
const [volume, setVolume] = useState(0);
|
||||||
|
const [duration, setDuration] = useState(0);
|
||||||
|
const [pendingRequestId, setPendingRequestId] = useState<string | null>(null);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const handleAudioLevel = useCallback((level: number) => {
|
||||||
|
setVolume(level);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const recorder = useAudioRecorder({ onAudioLevel: handleAudioLevel });
|
||||||
|
const recorderRef = useRef(recorder);
|
||||||
|
useEffect(() => {
|
||||||
|
recorderRef.current = recorder;
|
||||||
|
}, [recorder]);
|
||||||
|
|
||||||
|
const onTranscriptRef = useRef(onTranscript);
|
||||||
|
useEffect(() => {
|
||||||
|
onTranscriptRef.current = onTranscript;
|
||||||
|
}, [onTranscript]);
|
||||||
|
|
||||||
|
const onErrorRef = useRef(onError);
|
||||||
|
useEffect(() => {
|
||||||
|
onErrorRef.current = onError;
|
||||||
|
}, [onError]);
|
||||||
|
|
||||||
|
const isRecordingRef = useRef(isRecording);
|
||||||
|
useEffect(() => {
|
||||||
|
isRecordingRef.current = isRecording;
|
||||||
|
}, [isRecording]);
|
||||||
|
|
||||||
|
const pendingRequestIdRef = useRef<string | null>(null);
|
||||||
|
const activeStopPromiseRef = useRef<Promise<Blob> | null>(null);
|
||||||
|
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
|
||||||
|
const stopDurationTracking = useCallback(() => {
|
||||||
|
if (durationIntervalRef.current) {
|
||||||
|
clearInterval(durationIntervalRef.current);
|
||||||
|
durationIntervalRef.current = null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const startDurationTracking = useCallback(() => {
|
||||||
|
if (!enableDuration) {
|
||||||
|
console.log("[useDictation] startDictation blocked by canStart()");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (durationIntervalRef.current) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
durationIntervalRef.current = setInterval(() => {
|
||||||
|
setDuration((prev) => prev + 1);
|
||||||
|
}, DURATION_TICK_MS);
|
||||||
|
}, [enableDuration]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!enableDuration) {
|
||||||
|
stopDurationTracking();
|
||||||
|
setDuration(0);
|
||||||
|
}
|
||||||
|
}, [enableDuration, stopDurationTracking]);
|
||||||
|
|
||||||
|
const reportError = useCallback(
|
||||||
|
(err: unknown, context?: string) => {
|
||||||
|
const normalized = toError(err);
|
||||||
|
if (context) {
|
||||||
|
console.error(`[useDictation] ${context}`, normalized);
|
||||||
|
} else {
|
||||||
|
console.error("[useDictation]", normalized);
|
||||||
|
}
|
||||||
|
setError(normalized.message);
|
||||||
|
onErrorRef.current?.(normalized);
|
||||||
|
},
|
||||||
|
[setError]
|
||||||
|
);
|
||||||
|
|
||||||
|
const stopRecorder = useCallback(async (): Promise<Blob> => {
|
||||||
|
if (activeStopPromiseRef.current) {
|
||||||
|
return activeStopPromiseRef.current;
|
||||||
|
}
|
||||||
|
const recorderInstance = recorderRef.current;
|
||||||
|
if (!recorderInstance) {
|
||||||
|
throw new Error("Recorder unavailable");
|
||||||
|
}
|
||||||
|
const stopPromise = (async () => {
|
||||||
|
try {
|
||||||
|
return await recorderInstance.stop();
|
||||||
|
} finally {
|
||||||
|
activeStopPromiseRef.current = null;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
activeStopPromiseRef.current = stopPromise;
|
||||||
|
return stopPromise;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const startDictation = useCallback(async () => {
|
||||||
|
console.log("[useDictation] startDictation requested", {
|
||||||
|
isRecording: isRecordingRef.current,
|
||||||
|
isProcessing,
|
||||||
|
});
|
||||||
|
if (isRecordingRef.current || isProcessing) {
|
||||||
|
console.log("[useDictation] startDictation aborted: already recording/processing", {
|
||||||
|
isRecording: isRecordingRef.current,
|
||||||
|
isProcessing,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const startAllowed = canStart ? canStart() : true;
|
||||||
|
if (!startAllowed) {
|
||||||
|
console.log("[useDictation] startDictation blocked by canStart()");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setError(null);
|
||||||
|
setVolume(0);
|
||||||
|
setDuration(0);
|
||||||
|
setIsProcessing(false);
|
||||||
|
pendingRequestIdRef.current = null;
|
||||||
|
setPendingRequestId(null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const recorderInstance = recorderRef.current;
|
||||||
|
if (!recorderInstance) {
|
||||||
|
throw new Error("Recorder unavailable");
|
||||||
|
}
|
||||||
|
await recorderInstance.start();
|
||||||
|
console.log("[useDictation] recorder.start succeeded");
|
||||||
|
isRecordingRef.current = true;
|
||||||
|
setIsRecording(true);
|
||||||
|
if (enableDuration) {
|
||||||
|
startDurationTracking();
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
stopDurationTracking();
|
||||||
|
isRecordingRef.current = false;
|
||||||
|
setIsRecording(false);
|
||||||
|
reportError(err, "Failed to start dictation");
|
||||||
|
}
|
||||||
|
}, [
|
||||||
|
canStart,
|
||||||
|
enableDuration,
|
||||||
|
isProcessing,
|
||||||
|
reportError,
|
||||||
|
startDurationTracking,
|
||||||
|
stopDurationTracking,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const cancelDictation = useCallback(async () => {
|
||||||
|
console.log("[useDictation] cancelDictation requested", {
|
||||||
|
isRecording: isRecordingRef.current,
|
||||||
|
hasActiveStop: Boolean(activeStopPromiseRef.current),
|
||||||
|
});
|
||||||
|
if (!isRecordingRef.current && !activeStopPromiseRef.current) {
|
||||||
|
console.log("[useDictation] cancelDictation ignored: nothing to cancel");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
stopDurationTracking();
|
||||||
|
setDuration(0);
|
||||||
|
setError(null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await stopRecorder();
|
||||||
|
} catch (err) {
|
||||||
|
reportError(err, "Failed to cancel dictation");
|
||||||
|
} finally {
|
||||||
|
isRecordingRef.current = false;
|
||||||
|
setIsRecording(false);
|
||||||
|
setIsProcessing(false);
|
||||||
|
setVolume(0);
|
||||||
|
}
|
||||||
|
}, [reportError, stopDurationTracking, stopRecorder]);
|
||||||
|
|
||||||
|
const confirmDictation = useCallback(async () => {
|
||||||
|
console.log("[useDictation] confirmDictation requested", {
|
||||||
|
isRecording: isRecordingRef.current,
|
||||||
|
isProcessing,
|
||||||
|
});
|
||||||
|
if (!isRecordingRef.current || isProcessing) {
|
||||||
|
console.log("[useDictation] confirmDictation ignored: recording flag", {
|
||||||
|
isRecording: isRecordingRef.current,
|
||||||
|
isProcessing,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const confirmAllowed = canConfirm ? canConfirm() : true;
|
||||||
|
if (!confirmAllowed) {
|
||||||
|
console.log("[useDictation] confirmDictation blocked by canConfirm()");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setError(null);
|
||||||
|
stopDurationTracking();
|
||||||
|
setDuration(0);
|
||||||
|
setIsProcessing(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const audioData = await stopRecorder();
|
||||||
|
isRecordingRef.current = false;
|
||||||
|
setIsRecording(false);
|
||||||
|
setVolume(0);
|
||||||
|
|
||||||
|
const requestId = generateMessageId();
|
||||||
|
pendingRequestIdRef.current = requestId;
|
||||||
|
setPendingRequestId(requestId);
|
||||||
|
|
||||||
|
await sendAgentAudio(agentId, audioData, requestId, { mode });
|
||||||
|
console.log("[useDictation] sendAgentAudio resolved", { requestId });
|
||||||
|
} catch (err) {
|
||||||
|
pendingRequestIdRef.current = null;
|
||||||
|
setPendingRequestId(null);
|
||||||
|
setIsProcessing(false);
|
||||||
|
isRecordingRef.current = false;
|
||||||
|
setIsRecording(false);
|
||||||
|
setVolume(0);
|
||||||
|
reportError(err, "Failed to complete dictation");
|
||||||
|
}
|
||||||
|
}, [
|
||||||
|
agentId,
|
||||||
|
canConfirm,
|
||||||
|
isProcessing,
|
||||||
|
mode,
|
||||||
|
reportError,
|
||||||
|
sendAgentAudio,
|
||||||
|
stopDurationTracking,
|
||||||
|
stopRecorder,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const reset = useCallback(() => {
|
||||||
|
pendingRequestIdRef.current = null;
|
||||||
|
setPendingRequestId(null);
|
||||||
|
setIsRecording(false);
|
||||||
|
isRecordingRef.current = false;
|
||||||
|
setIsProcessing(false);
|
||||||
|
stopDurationTracking();
|
||||||
|
setDuration(0);
|
||||||
|
setVolume(0);
|
||||||
|
setError(null);
|
||||||
|
}, [stopDurationTracking]);
|
||||||
|
|
||||||
|
const cancelRef = useRef<(() => void) | null>(null);
|
||||||
|
useEffect(() => {
|
||||||
|
cancelRef.current = () => {
|
||||||
|
void cancelDictation();
|
||||||
|
};
|
||||||
|
}, [cancelDictation]);
|
||||||
|
|
||||||
|
const visibilityRef = useRef<boolean | null>(
|
||||||
|
typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null
|
||||||
|
);
|
||||||
|
useEffect(() => {
|
||||||
|
const nextVisible =
|
||||||
|
typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null;
|
||||||
|
const prevVisible = visibilityRef.current;
|
||||||
|
visibilityRef.current = nextVisible;
|
||||||
|
|
||||||
|
if (prevVisible === true && nextVisible === false && isRecordingRef.current) {
|
||||||
|
cancelRef.current?.();
|
||||||
|
}
|
||||||
|
}, [autoStopWhenHidden?.isVisible]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
pendingRequestIdRef.current = null;
|
||||||
|
setPendingRequestId(null);
|
||||||
|
setIsProcessing(false);
|
||||||
|
}, [agentId]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const unsubscribe = ws.on("transcription_result", (message: SessionOutboundMessage) => {
|
||||||
|
if (message.type !== "transcription_result") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const expectedId = pendingRequestIdRef.current;
|
||||||
|
if (!expectedId || message.payload.requestId !== expectedId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
pendingRequestIdRef.current = null;
|
||||||
|
setPendingRequestId(null);
|
||||||
|
setIsProcessing(false);
|
||||||
|
|
||||||
|
const transcriptText = message.payload.text?.trim();
|
||||||
|
if (!transcriptText) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("[useDictation] transcription_result received", {
|
||||||
|
requestId: expectedId,
|
||||||
|
textLength: transcriptText.length,
|
||||||
|
});
|
||||||
|
onTranscriptRef.current?.(transcriptText, {
|
||||||
|
requestId: message.payload.requestId ?? expectedId,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
unsubscribe();
|
||||||
|
};
|
||||||
|
}, [ws]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
return () => {
|
||||||
|
stopDurationTracking();
|
||||||
|
const activeStop = activeStopPromiseRef.current;
|
||||||
|
if (activeStop) {
|
||||||
|
void activeStop.catch(() => undefined);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const recorderInstance = recorderRef.current;
|
||||||
|
if (recorderInstance?.isRecording?.()) {
|
||||||
|
void recorderInstance.stop().catch(() => undefined);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}, [stopDurationTracking]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
isRecording,
|
||||||
|
isProcessing,
|
||||||
|
volume,
|
||||||
|
duration,
|
||||||
|
pendingRequestId,
|
||||||
|
error,
|
||||||
|
startDictation,
|
||||||
|
cancelDictation,
|
||||||
|
confirmDictation,
|
||||||
|
reset,
|
||||||
|
};
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user