jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix dictation lifecycle and add real STT e2e

Browse Source
This commit is contained in:
Mohamed Boudra
2026-01-20 13:09:22 +07:00
parent 687ea706b1
commit facd99ab13
9 changed files with 306 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/hooks/use-audio-recorder.native.ts
+67 -73
View File
@@ -1,6 +1,7 @@
import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
import { Paths, File, Directory, FileInfo } from 'expo-file-system';
import { useState, useEffect, useRef, useMemo, useCallback } from 'react';
import { AttemptCancelledError, AttemptGuard } from '@/utils/attempt-guard';
export interface AudioCaptureConfig {
sampleRate?: number;
@@ -127,8 +128,8 @@ async function uriToBlob(uri: string): Promise<Blob> {
*/
export function useAudioRecorder(config?: AudioCaptureConfig) {
const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
const recordingIntentRef = useRef(false);
const stopRequestedRef = useRef(false);
const attemptGuardRef = useRef(new AttemptGuard());
const startStopMutexRef = useRef<Promise<unknown> | null>(null);
// Store config callbacks in refs so they can update without recreating the recorder
const configRef = useRef(config);
@@ -203,19 +204,13 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
}
try {
recordingIntentRef.current = true;
const ensureNotCancelled = () => {
if (stopRequestedRef.current) {
throw new Error('Recording cancelled');
}
};
ensureNotCancelled();
const attemptId = attemptGuardRef.current.next();
attemptGuardRef.current.assertCurrent(attemptId);
// Request microphone permissions
console.log('[AudioRecorder] Requesting recording permissions...');
const permissionResponse = await requestRecordingPermissionsAsync();
ensureNotCancelled();
attemptGuardRef.current.assertCurrent(attemptId);
if (!permissionResponse.granted) {
throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
@@ -227,7 +222,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
playsInSilentMode: true,
allowsRecording: true,
});
ensureNotCancelled();
attemptGuardRef.current.assertCurrent(attemptId);
console.log('[AudioRecorder] Starting recording with options:', {
sampleRate: recordingOptions.sampleRate,
@@ -237,25 +232,26 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
const startTime = new Date();
setRecordingStartTime(startTime);
ensureNotCancelled();
attemptGuardRef.current.assertCurrent(attemptId);
// Prepare the recorder before recording (required step)
console.log('[AudioRecorder] Preparing recorder...');
await recorder.prepareToRecordAsync();
ensureNotCancelled();
attemptGuardRef.current.assertCurrent(attemptId);
console.log('[AudioRecorder] Starting recording...');
await recorder.record();
ensureNotCancelled();
attemptGuardRef.current.assertCurrent(attemptId);
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
console.log('[AudioRecorder] Recorder isRecording:', recorder.isRecording);
} catch (error: any) {
recordingIntentRef.current = false;
setRecordingStartTime(null);
if (error?.message === 'Recording cancelled') {
if (error instanceof AttemptCancelledError) {
console.log('[AudioRecorder] Recording start cancelled.');
} else {
return;
}
if (error?.message !== 'Recording cancelled') {
console.error('[AudioRecorder] Failed to start recording:', error);
}
throw new Error(`Failed to start audio recording: ${error.message}`);
@@ -265,76 +261,75 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
const stop = useCallback(async (): Promise<Blob> => {
const recorder = recorderRef.current;
// Guard against duplicate stop calls
if (!recorder.isRecording && !recordingIntentRef.current) {
console.error('[AudioRecorder] Attempted to stop when not recording. Recorder state:', {
isRecording: recorder.isRecording,
hasRecordingIntent: recordingIntentRef.current,
hasRecordingStartTime: !!recordingStartTime,
});
throw new Error('Not recording');
// Cancel any in-flight start attempt and serialize stop/cleanup.
attemptGuardRef.current.cancel();
if (startStopMutexRef.current) {
await startStopMutexRef.current.catch(() => undefined);
}
stopRequestedRef.current = true;
try {
// Stop recording
if (recorder.isRecording) {
await recorder.stop();
} else {
console.warn('[AudioRecorder] Recorder already stopped before stop() call, continuing cleanup.');
}
// Get URI from recorder
let uri = recorder.uri;
console.log('[AudioRecorder] Initial URI from recorder:', uri);
// Workaround for Expo SDK 54 Android bug - find actual recording file
if (recordingStartTime && (!uri || uri === '')) {
console.log('[AudioRecorder] Using workaround to find actual recording file...');
const actualUri = await getActualRecordingUri(recordingStartTime);
if (actualUri) {
uri = actualUri;
console.log('[AudioRecorder] Found actual recording URI:', uri);
const stopPromise = (async () => {
// Stop recording
if (recorder.isRecording) {
await recorder.stop();
} else {
console.warn('[AudioRecorder] Recorder already stopped before stop() call, continuing cleanup.');
}
}
if (!uri || uri === '') {
throw new Error('No recording URI found');
}
// Get URI from recorder
let uri = recorder.uri;
console.log('[AudioRecorder] Initial URI from recorder:', uri);
// Get file info
const file = new File(uri);
const exists = file.exists;
console.log('[AudioRecorder] File exists:', exists);
// Workaround for Expo SDK 54 Android bug - find actual recording file
if (recordingStartTime && (!uri || uri === '')) {
console.log('[AudioRecorder] Using workaround to find actual recording file...');
const actualUri = await getActualRecordingUri(recordingStartTime);
if (actualUri) {
uri = actualUri;
console.log('[AudioRecorder] Found actual recording URI:', uri);
}
}
if (!exists) {
throw new Error('Recording file does not exist');
}
if (!uri || uri === '') {
// Cancellation / early stop: return an empty blob, but guarantee cleanup.
setRecordingStartTime(null);
return new Blob([], { type: 'audio/m4a' });
}
// Convert URI to Blob
const audioBlob = await uriToBlob(uri);
// Get file info
const file = new File(uri);
const exists = file.exists;
console.log('[AudioRecorder] File exists:', exists);
console.log('[AudioRecorder] Recording converted to blob:', {
size: audioBlob.size,
type: audioBlob.type,
});
if (!exists) {
setRecordingStartTime(null);
return new Blob([], { type: 'audio/m4a' });
}
// Clean up the temporary file
file.delete();
// Convert URI to Blob
const audioBlob = await uriToBlob(uri);
// Reset start time and clear recording intent
setRecordingStartTime(null);
recordingIntentRef.current = false;
console.log('[AudioRecorder] Recording converted to blob:', {
size: audioBlob.size,
type: audioBlob.type,
});
return audioBlob;
// Clean up the temporary file
file.delete();
// Reset start time
setRecordingStartTime(null);
return audioBlob;
})();
startStopMutexRef.current = stopPromise;
return await stopPromise;
} catch (error: any) {
recordingIntentRef.current = false;
setRecordingStartTime(null);
console.error('[AudioRecorder] Failed to stop recording:', error);
throw new Error(`Failed to stop audio recording: ${error.message}`);
} finally {
stopRequestedRef.current = false;
startStopMutexRef.current = null;
}
}, [recordingStartTime]);
@@ -345,8 +340,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
}, []);
const isRecording = useCallback(() => {
// Treat local intent as backup so callers can detect pending sessions
return Boolean(recorderState.isRecording) || recordingIntentRef.current;
return Boolean(recorderState.isRecording);
}, [recorderState.isRecording]);
// Return stable object using useMemo
packages/app/src/hooks/use-audio-recorder.web.ts
+50 -10
View File
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { AttemptCancelledError, AttemptGuard } from "@/utils/attempt-guard";
export interface AudioCaptureConfig {
sampleRate?: number;
@@ -21,6 +22,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const mediaStreamRef = useRef<MediaStream | null>(null);
const attemptGuardRef = useRef(new AttemptGuard());
const chunksRef = useRef<Blob[]>([]);
const supportedMimeTypeRef = useRef<string | null | undefined>(undefined);
const analyserRef = useRef<AnalyserNode | null>(null);
@@ -79,6 +81,14 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
}
}, []);
const hardReset = useCallback(() => {
stopMetering();
cleanupStream();
mediaRecorderRef.current = null;
chunksRef.current = [];
setIsRecording(false);
}, [cleanupStream, stopMetering]);
const startMetering = useCallback((stream: MediaStream) => {
const onAudioLevel = configRef.current?.onAudioLevel;
if (!onAudioLevel) {
@@ -129,6 +139,8 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
}, []);
const start = useCallback(async () => {
const attemptId = attemptGuardRef.current.next();
if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") {
throw new Error("Already recording");
}
@@ -173,6 +185,22 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
throw new Error(`Failed to access microphone: ${error?.message ?? error}`);
}
try {
attemptGuardRef.current.assertCurrent(attemptId);
} catch (err) {
if (err instanceof AttemptCancelledError) {
stream.getTracks().forEach((track) => {
try {
track.stop();
} catch {
// Ignore track stop errors.
}
});
return;
}
throw err;
}
mediaStreamRef.current = stream;
const recorderOptions: MediaRecorderOptions = {};
@@ -222,11 +250,25 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
}, [cleanupStream, detectSupportedMimeType, startMetering]);
const stop = useCallback(async (): Promise<Blob> => {
attemptGuardRef.current.cancel();
const recorder = mediaRecorderRef.current;
if (!recorder || recorder.state === "inactive") {
if (!recorder) {
throw new Error("Not recording");
}
if (recorder.state === "inactive") {
stopMetering();
cleanupStream();
mediaRecorderRef.current = null;
setIsRecording(false);
const mimeType = recorder.mimeType || detectSupportedMimeType() || "audio/webm";
const blob = new Blob(chunksRef.current, { type: mimeType });
chunksRef.current = [];
return blob;
}
return await new Promise<Blob>((resolve, reject) => {
const finalize = () => {
recorder.removeEventListener("stop", finalize);
@@ -275,17 +317,15 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
useEffect(() => {
return () => {
if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") {
try {
mediaRecorderRef.current.stop();
} catch {
// Ignore stop during unmount.
}
attemptGuardRef.current.cancel();
try {
void mediaRecorderRef.current?.stop();
} catch {
// Ignore stop during unmount.
}
stopMetering();
cleanupStream();
hardReset();
};
}, [cleanupStream, stopMetering]);
}, [hardReset]);
const getSupportedMimeType = useCallback(() => detectSupportedMimeType(), [
detectSupportedMimeType,
packages/app/src/hooks/use-dictation.ts
+35 -3
View File
@@ -5,6 +5,7 @@ import { useAudioRecorder } from "@/hooks/use-audio-recorder";
import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
import type { TranscriptionResultMessage } from "@server/shared/messages";
import { generateMessageId } from "@/types/stream";
import { AttemptGuard } from "@/utils/attempt-guard";
export type DictationStatus = "idle" | "recording" | "uploading" | "retrying" | "failed";
@@ -310,9 +311,15 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
isRecordingRef.current = isRecording;
}, [isRecording]);
const isProcessingRef = useRef(isProcessing);
useEffect(() => {
isProcessingRef.current = isProcessing;
}, [isProcessing]);
const pendingRequestIdRef = useRef<string | null>(null);
const activeStopPromiseRef = useRef<Promise<Blob> | null>(null);
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const attemptGuardRef = useRef(new AttemptGuard());
const stopDurationTracking = useCallback(() => {
if (durationIntervalRef.current) {
@@ -510,6 +517,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
isRecording: isRecordingRef.current,
hasActiveStop: Boolean(activeStopPromiseRef.current),
});
attemptGuardRef.current.cancel();
if (!isRecordingRef.current && !activeStopPromiseRef.current) {
console.log("[useDictation] cancelDictation ignored: nothing to cancel");
return;
@@ -561,8 +569,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
setRetryAttempt(0);
setLastOutcome(null);
const attemptId = attemptGuardRef.current.next();
try {
const audioData = await stopRecorder();
attemptGuardRef.current.assertCurrent(attemptId);
const recordedDurationSeconds = durationRef.current;
pendingAudioRef.current = buildCapturedAudioPayload(audioData, recordedDurationSeconds);
setStatus("uploading");
@@ -571,6 +582,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
setVolume(0);
const transcription = await transmitDictation();
attemptGuardRef.current.assertCurrent(attemptId);
handleTranscriptionSuccess(transcription);
} catch (err) {
resetTranscriptionMutation();
@@ -661,14 +673,34 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const prevVisible = visibilityRef.current;
visibilityRef.current = nextVisible;
if (prevVisible === true && nextVisible === false && isRecordingRef.current) {
cancelRef.current?.();
if (prevVisible === true && nextVisible === false) {
attemptGuardRef.current.cancel();
if (isRecordingRef.current) {
cancelRef.current?.();
return;
}
if (isProcessingRef.current) {
stopDurationTracking();
setDuration(0);
setIsProcessing(false);
setVolume(0);
setError(null);
setStatus("idle");
setRetryAttempt(0);
setRetryInfo(null);
pendingAudioRef.current = null;
setFailedRecording(null);
setLastOutcome(null);
}
}
}, [autoStopWhenHidden?.isVisible]);
}, [autoStopWhenHidden?.isVisible, stopDurationTracking]);
useEffect(() => {
return () => {
attemptGuardRef.current.cancel();
stopDurationTracking();
const activeStop = activeStopPromiseRef.current;
if (activeStop) {