Fix dictation lifecycle and add real STT e2e
This commit is contained in:
@@ -2,9 +2,15 @@ import { test, expect } from './fixtures';
|
|||||||
import { createAgent, ensureHostSelected, gotoHome, setWorkingDirectory } from './helpers/app';
|
import { createAgent, ensureHostSelected, gotoHome, setWorkingDirectory } from './helpers/app';
|
||||||
import { createTempGitRepo } from './helpers/workspace';
|
import { createTempGitRepo } from './helpers/workspace';
|
||||||
import type { Page } from '@playwright/test';
|
import type { Page } from '@playwright/test';
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import path from 'node:path';
|
||||||
|
|
||||||
function addFakeMicrophone(page: Page) {
|
async function addFakeMicrophone(page: Page) {
|
||||||
return page.addInitScript(() => {
|
const fixturePath = path.resolve(__dirname, 'fixtures', 'recording.webm');
|
||||||
|
const base64Audio = (await readFile(fixturePath)).toString('base64');
|
||||||
|
const mimeType = 'audio/webm;codecs=opus';
|
||||||
|
|
||||||
|
return page.addInitScript(({ base64Audio, mimeType }) => {
|
||||||
const mic = {
|
const mic = {
|
||||||
active: 0,
|
active: 0,
|
||||||
getUserMediaCalls: 0,
|
getUserMediaCalls: 0,
|
||||||
@@ -13,29 +19,34 @@ function addFakeMicrophone(page: Page) {
|
|||||||
};
|
};
|
||||||
(window as any).__mic = mic;
|
(window as any).__mic = mic;
|
||||||
|
|
||||||
const ensureMediaDevices = () => {
|
(window as any).isSecureContext = true;
|
||||||
const nav = navigator as any;
|
|
||||||
if (!nav.mediaDevices) {
|
const nav = navigator as any;
|
||||||
nav.mediaDevices = {};
|
if (!nav.mediaDevices) {
|
||||||
}
|
nav.mediaDevices = {};
|
||||||
if (typeof nav.mediaDevices.getUserMedia !== 'function') {
|
}
|
||||||
nav.mediaDevices.getUserMedia = async () => {
|
nav.mediaDevices.getUserMedia = async () => {
|
||||||
mic.getUserMediaCalls += 1;
|
mic.getUserMediaCalls += 1;
|
||||||
mic.active += 1;
|
mic.active += 1;
|
||||||
const track = {
|
const track = {
|
||||||
stop: () => {
|
stop: () => {
|
||||||
mic.stopCalls += 1;
|
mic.stopCalls += 1;
|
||||||
mic.active = Math.max(0, mic.active - 1);
|
mic.active = Math.max(0, mic.active - 1);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
return {
|
return {
|
||||||
getTracks: () => [track],
|
getTracks: () => [track],
|
||||||
};
|
};
|
||||||
};
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
ensureMediaDevices();
|
const blobFromBase64 = (base64: string, mimeType: string): Blob => {
|
||||||
|
const binaryString = atob(base64);
|
||||||
|
const bytes = new Uint8Array(binaryString.length);
|
||||||
|
for (let i = 0; i < binaryString.length; i++) {
|
||||||
|
bytes[i] = binaryString.charCodeAt(i);
|
||||||
|
}
|
||||||
|
return new Blob([bytes], { type: mimeType });
|
||||||
|
};
|
||||||
|
|
||||||
class FakeMediaRecorder extends EventTarget {
|
class FakeMediaRecorder extends EventTarget {
|
||||||
public static isTypeSupported() {
|
public static isTypeSupported() {
|
||||||
@@ -64,7 +75,7 @@ function addFakeMicrophone(page: Page) {
|
|||||||
this.state = 'inactive';
|
this.state = 'inactive';
|
||||||
try {
|
try {
|
||||||
this.ondataavailable?.({
|
this.ondataavailable?.({
|
||||||
data: new Blob(['paseo-e2e-audio'], { type: this.mimeType }),
|
data: blobFromBase64(base64Audio, mimeType),
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.onerror?.(err);
|
this.onerror?.(err);
|
||||||
@@ -74,7 +85,7 @@ function addFakeMicrophone(page: Page) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
(window as any).MediaRecorder = FakeMediaRecorder;
|
(window as any).MediaRecorder = FakeMediaRecorder;
|
||||||
});
|
}, { base64Audio, mimeType });
|
||||||
}
|
}
|
||||||
|
|
||||||
test('dictation hotkeys do not trigger on background screens', async ({ page }) => {
|
test('dictation hotkeys do not trigger on background screens', async ({ page }) => {
|
||||||
@@ -88,6 +99,7 @@ test('dictation hotkeys do not trigger on background screens', async ({ page })
|
|||||||
await createAgent(page, 'Respond with exactly: Hello');
|
await createAgent(page, 'Respond with exactly: Hello');
|
||||||
|
|
||||||
await expect(page).toHaveURL(/\/agent\//);
|
await expect(page).toHaveURL(/\/agent\//);
|
||||||
|
await expect(page.getByRole('textbox', { name: 'Message agent...' })).toBeEditable();
|
||||||
|
|
||||||
await page.keyboard.press('Control+d');
|
await page.keyboard.press('Control+d');
|
||||||
await page.waitForTimeout(200);
|
await page.waitForTimeout(200);
|
||||||
@@ -107,6 +119,33 @@ test('dictation hotkeys do not trigger on background screens', async ({ page })
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('dictation transcribes fixture via real STT', async ({ page }) => {
|
||||||
|
await addFakeMicrophone(page);
|
||||||
|
|
||||||
|
const repo = await createTempGitRepo();
|
||||||
|
try {
|
||||||
|
await gotoHome(page);
|
||||||
|
await setWorkingDirectory(page, repo.path);
|
||||||
|
await ensureHostSelected(page);
|
||||||
|
await createAgent(page, 'Respond with exactly: Hello');
|
||||||
|
|
||||||
|
await expect(page).toHaveURL(/\/agent\//);
|
||||||
|
await expect(page.getByRole('textbox', { name: 'Message agent...' })).toBeEditable();
|
||||||
|
|
||||||
|
await page.keyboard.press('Control+d');
|
||||||
|
await expect
|
||||||
|
.poll(async () => page.evaluate(() => (window as any).__mic.active as number))
|
||||||
|
.toBe(1);
|
||||||
|
|
||||||
|
await page.keyboard.press('Control+d');
|
||||||
|
|
||||||
|
const transcriptLocator = page.getByText(/voice note/i);
|
||||||
|
await expect(transcriptLocator).toBeVisible({ timeout: 60_000 });
|
||||||
|
} finally {
|
||||||
|
await repo.cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('cancel stops mic even if recorder is already inactive', async ({ page }) => {
|
test('cancel stops mic even if recorder is already inactive', async ({ page }) => {
|
||||||
await addFakeMicrophone(page);
|
await addFakeMicrophone(page);
|
||||||
|
|
||||||
@@ -118,6 +157,7 @@ test('cancel stops mic even if recorder is already inactive', async ({ page }) =
|
|||||||
await createAgent(page, 'Respond with exactly: Hello');
|
await createAgent(page, 'Respond with exactly: Hello');
|
||||||
|
|
||||||
await expect(page).toHaveURL(/\/agent\//);
|
await expect(page).toHaveURL(/\/agent\//);
|
||||||
|
await expect(page.getByRole('textbox', { name: 'Message agent...' })).toBeEditable();
|
||||||
|
|
||||||
await page.keyboard.press('Control+d');
|
await page.keyboard.press('Control+d');
|
||||||
await expect
|
await expect
|
||||||
@@ -139,3 +179,41 @@ test('cancel stops mic even if recorder is already inactive', async ({ page }) =
|
|||||||
await repo.cleanup();
|
await repo.cleanup();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('dictation confirm+send does not dispatch after navigating away', async ({ page }) => {
|
||||||
|
await addFakeMicrophone(page);
|
||||||
|
|
||||||
|
const repo = await createTempGitRepo();
|
||||||
|
try {
|
||||||
|
await gotoHome(page);
|
||||||
|
await setWorkingDirectory(page, repo.path);
|
||||||
|
await ensureHostSelected(page);
|
||||||
|
await createAgent(page, 'Respond with exactly: Hello');
|
||||||
|
|
||||||
|
await expect(page).toHaveURL(/\/agent\//);
|
||||||
|
await expect(page.getByRole('textbox', { name: 'Message agent...' })).toBeEditable();
|
||||||
|
|
||||||
|
await page.keyboard.press('Control+d');
|
||||||
|
await expect
|
||||||
|
.poll(async () => page.evaluate(() => (window as any).__mic.active as number))
|
||||||
|
.toBe(1);
|
||||||
|
|
||||||
|
await page.keyboard.press('Control+d');
|
||||||
|
|
||||||
|
const newAgentButton = page.getByText('New Agent', { exact: true }).first();
|
||||||
|
await expect(newAgentButton).toBeVisible();
|
||||||
|
await newAgentButton.click();
|
||||||
|
await expect(page).toHaveURL(/\/$/);
|
||||||
|
|
||||||
|
await page.waitForTimeout(10_000);
|
||||||
|
|
||||||
|
const agentEntry = page.getByText(repo.path).first();
|
||||||
|
await expect(agentEntry).toBeVisible();
|
||||||
|
await agentEntry.click();
|
||||||
|
await expect(page).toHaveURL(/\/agent\//);
|
||||||
|
|
||||||
|
await expect(page.getByText(/voice note/i)).not.toBeVisible();
|
||||||
|
} finally {
|
||||||
|
await repo.cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
@@ -16,6 +16,7 @@ import Animated, {
|
|||||||
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
|
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
|
||||||
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
||||||
import { useRealtime } from "@/contexts/realtime-context";
|
import { useRealtime } from "@/contexts/realtime-context";
|
||||||
|
import { useIsFocused } from "@react-navigation/native";
|
||||||
import { FOOTER_HEIGHT, MAX_CONTENT_WIDTH } from "@/constants/layout";
|
import { FOOTER_HEIGHT, MAX_CONTENT_WIDTH } from "@/constants/layout";
|
||||||
import { generateMessageId } from "@/types/stream";
|
import { generateMessageId } from "@/types/stream";
|
||||||
import { AgentStatusBar } from "./agent-status-bar";
|
import { AgentStatusBar } from "./agent-status-bar";
|
||||||
@@ -77,6 +78,7 @@ export function AgentInputArea({
|
|||||||
const { theme } = useUnistyles();
|
const { theme } = useUnistyles();
|
||||||
const insets = useSafeAreaInsets();
|
const insets = useSafeAreaInsets();
|
||||||
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
|
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
|
||||||
|
const isScreenFocused = useIsFocused();
|
||||||
|
|
||||||
const client = useSessionStore(
|
const client = useSessionStore(
|
||||||
(state) => state.sessions[serverId]?.client ?? null
|
(state) => state.sessions[serverId]?.client ?? null
|
||||||
@@ -568,6 +570,7 @@ export function AgentInputArea({
|
|||||||
placeholder="Message agent..."
|
placeholder="Message agent..."
|
||||||
autoFocus={autoFocus}
|
autoFocus={autoFocus}
|
||||||
disabled={isRealtimeMode}
|
disabled={isRealtimeMode}
|
||||||
|
isScreenFocused={isScreenFocused}
|
||||||
leftContent={leftContent}
|
leftContent={leftContent}
|
||||||
rightContent={rightContent}
|
rightContent={rightContent}
|
||||||
isAgentRunning={isAgentRunning}
|
isAgentRunning={isAgentRunning}
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ export interface MessageInputProps {
|
|||||||
placeholder?: string;
|
placeholder?: string;
|
||||||
autoFocus?: boolean;
|
autoFocus?: boolean;
|
||||||
disabled?: boolean;
|
disabled?: boolean;
|
||||||
|
/** True when the containing screen is focused (React Navigation). Used to disable global hotkeys and cancel dictation when unfocused. */
|
||||||
|
isScreenFocused?: boolean;
|
||||||
/** Content to render on the left side of the button row (e.g., AgentStatusBar) */
|
/** Content to render on the left side of the button row (e.g., AgentStatusBar) */
|
||||||
leftContent?: React.ReactNode;
|
leftContent?: React.ReactNode;
|
||||||
/** Content to render on the right side after voice button (e.g., realtime button, cancel button) */
|
/** Content to render on the right side after voice button (e.g., realtime button, cancel button) */
|
||||||
@@ -105,6 +107,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
placeholder = "Message...",
|
placeholder = "Message...",
|
||||||
autoFocus = false,
|
autoFocus = false,
|
||||||
disabled = false,
|
disabled = false,
|
||||||
|
isScreenFocused = true,
|
||||||
leftContent,
|
leftContent,
|
||||||
rightContent,
|
rightContent,
|
||||||
isAgentRunning = false,
|
isAgentRunning = false,
|
||||||
@@ -202,12 +205,20 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
onError: handleDictationError,
|
onError: handleDictationError,
|
||||||
canStart: canStartDictation,
|
canStart: canStartDictation,
|
||||||
canConfirm: canConfirmDictation,
|
canConfirm: canConfirmDictation,
|
||||||
|
autoStopWhenHidden: { isVisible: isScreenFocused },
|
||||||
enableDuration: true,
|
enableDuration: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (isDictating || isDictationProcessing || dictationStatus === "retrying") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
sendAfterTranscriptRef.current = false;
|
||||||
|
}, [dictationStatus, isDictating, isDictationProcessing]);
|
||||||
|
|
||||||
// Cmd+D to start/submit dictation, Escape to cancel
|
// Cmd+D to start/submit dictation, Escape to cancel
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!IS_WEB) return;
|
if (!IS_WEB || !isScreenFocused) return;
|
||||||
function handleKeyDown(event: KeyboardEvent) {
|
function handleKeyDown(event: KeyboardEvent) {
|
||||||
// Cmd+D: start dictation or submit if already dictating
|
// Cmd+D: start dictation or submit if already dictating
|
||||||
if ((event.metaKey || event.ctrlKey) && event.key === "d") {
|
if ((event.metaKey || event.ctrlKey) && event.key === "d") {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
|
import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
|
||||||
import { Paths, File, Directory, FileInfo } from 'expo-file-system';
|
import { Paths, File, Directory, FileInfo } from 'expo-file-system';
|
||||||
import { useState, useEffect, useRef, useMemo, useCallback } from 'react';
|
import { useState, useEffect, useRef, useMemo, useCallback } from 'react';
|
||||||
|
import { AttemptCancelledError, AttemptGuard } from '@/utils/attempt-guard';
|
||||||
|
|
||||||
export interface AudioCaptureConfig {
|
export interface AudioCaptureConfig {
|
||||||
sampleRate?: number;
|
sampleRate?: number;
|
||||||
@@ -127,8 +128,8 @@ async function uriToBlob(uri: string): Promise<Blob> {
|
|||||||
*/
|
*/
|
||||||
export function useAudioRecorder(config?: AudioCaptureConfig) {
|
export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||||
const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
|
const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
|
||||||
const recordingIntentRef = useRef(false);
|
const attemptGuardRef = useRef(new AttemptGuard());
|
||||||
const stopRequestedRef = useRef(false);
|
const startStopMutexRef = useRef<Promise<unknown> | null>(null);
|
||||||
|
|
||||||
// Store config callbacks in refs so they can update without recreating the recorder
|
// Store config callbacks in refs so they can update without recreating the recorder
|
||||||
const configRef = useRef(config);
|
const configRef = useRef(config);
|
||||||
@@ -203,19 +204,13 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
recordingIntentRef.current = true;
|
const attemptId = attemptGuardRef.current.next();
|
||||||
const ensureNotCancelled = () => {
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
if (stopRequestedRef.current) {
|
|
||||||
throw new Error('Recording cancelled');
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
ensureNotCancelled();
|
|
||||||
|
|
||||||
// Request microphone permissions
|
// Request microphone permissions
|
||||||
console.log('[AudioRecorder] Requesting recording permissions...');
|
console.log('[AudioRecorder] Requesting recording permissions...');
|
||||||
const permissionResponse = await requestRecordingPermissionsAsync();
|
const permissionResponse = await requestRecordingPermissionsAsync();
|
||||||
ensureNotCancelled();
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
|
|
||||||
if (!permissionResponse.granted) {
|
if (!permissionResponse.granted) {
|
||||||
throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
|
throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
|
||||||
@@ -227,7 +222,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
playsInSilentMode: true,
|
playsInSilentMode: true,
|
||||||
allowsRecording: true,
|
allowsRecording: true,
|
||||||
});
|
});
|
||||||
ensureNotCancelled();
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
|
|
||||||
console.log('[AudioRecorder] Starting recording with options:', {
|
console.log('[AudioRecorder] Starting recording with options:', {
|
||||||
sampleRate: recordingOptions.sampleRate,
|
sampleRate: recordingOptions.sampleRate,
|
||||||
@@ -237,25 +232,26 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
|
|
||||||
const startTime = new Date();
|
const startTime = new Date();
|
||||||
setRecordingStartTime(startTime);
|
setRecordingStartTime(startTime);
|
||||||
ensureNotCancelled();
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
|
|
||||||
// Prepare the recorder before recording (required step)
|
// Prepare the recorder before recording (required step)
|
||||||
console.log('[AudioRecorder] Preparing recorder...');
|
console.log('[AudioRecorder] Preparing recorder...');
|
||||||
await recorder.prepareToRecordAsync();
|
await recorder.prepareToRecordAsync();
|
||||||
ensureNotCancelled();
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
|
|
||||||
console.log('[AudioRecorder] Starting recording...');
|
console.log('[AudioRecorder] Starting recording...');
|
||||||
await recorder.record();
|
await recorder.record();
|
||||||
ensureNotCancelled();
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
|
|
||||||
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
|
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
|
||||||
console.log('[AudioRecorder] Recorder isRecording:', recorder.isRecording);
|
console.log('[AudioRecorder] Recorder isRecording:', recorder.isRecording);
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
recordingIntentRef.current = false;
|
|
||||||
setRecordingStartTime(null);
|
setRecordingStartTime(null);
|
||||||
if (error?.message === 'Recording cancelled') {
|
if (error instanceof AttemptCancelledError) {
|
||||||
console.log('[AudioRecorder] Recording start cancelled.');
|
console.log('[AudioRecorder] Recording start cancelled.');
|
||||||
} else {
|
return;
|
||||||
|
}
|
||||||
|
if (error?.message !== 'Recording cancelled') {
|
||||||
console.error('[AudioRecorder] Failed to start recording:', error);
|
console.error('[AudioRecorder] Failed to start recording:', error);
|
||||||
}
|
}
|
||||||
throw new Error(`Failed to start audio recording: ${error.message}`);
|
throw new Error(`Failed to start audio recording: ${error.message}`);
|
||||||
@@ -265,76 +261,75 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
const stop = useCallback(async (): Promise<Blob> => {
|
const stop = useCallback(async (): Promise<Blob> => {
|
||||||
const recorder = recorderRef.current;
|
const recorder = recorderRef.current;
|
||||||
|
|
||||||
// Guard against duplicate stop calls
|
// Cancel any in-flight start attempt and serialize stop/cleanup.
|
||||||
if (!recorder.isRecording && !recordingIntentRef.current) {
|
attemptGuardRef.current.cancel();
|
||||||
console.error('[AudioRecorder] Attempted to stop when not recording. Recorder state:', {
|
if (startStopMutexRef.current) {
|
||||||
isRecording: recorder.isRecording,
|
await startStopMutexRef.current.catch(() => undefined);
|
||||||
hasRecordingIntent: recordingIntentRef.current,
|
|
||||||
hasRecordingStartTime: !!recordingStartTime,
|
|
||||||
});
|
|
||||||
throw new Error('Not recording');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
stopRequestedRef.current = true;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Stop recording
|
const stopPromise = (async () => {
|
||||||
if (recorder.isRecording) {
|
// Stop recording
|
||||||
await recorder.stop();
|
if (recorder.isRecording) {
|
||||||
} else {
|
await recorder.stop();
|
||||||
console.warn('[AudioRecorder] Recorder already stopped before stop() call, continuing cleanup.');
|
} else {
|
||||||
}
|
console.warn('[AudioRecorder] Recorder already stopped before stop() call, continuing cleanup.');
|
||||||
|
|
||||||
// Get URI from recorder
|
|
||||||
let uri = recorder.uri;
|
|
||||||
console.log('[AudioRecorder] Initial URI from recorder:', uri);
|
|
||||||
|
|
||||||
// Workaround for Expo SDK 54 Android bug - find actual recording file
|
|
||||||
if (recordingStartTime && (!uri || uri === '')) {
|
|
||||||
console.log('[AudioRecorder] Using workaround to find actual recording file...');
|
|
||||||
const actualUri = await getActualRecordingUri(recordingStartTime);
|
|
||||||
if (actualUri) {
|
|
||||||
uri = actualUri;
|
|
||||||
console.log('[AudioRecorder] Found actual recording URI:', uri);
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (!uri || uri === '') {
|
// Get URI from recorder
|
||||||
throw new Error('No recording URI found');
|
let uri = recorder.uri;
|
||||||
}
|
console.log('[AudioRecorder] Initial URI from recorder:', uri);
|
||||||
|
|
||||||
// Get file info
|
// Workaround for Expo SDK 54 Android bug - find actual recording file
|
||||||
const file = new File(uri);
|
if (recordingStartTime && (!uri || uri === '')) {
|
||||||
const exists = file.exists;
|
console.log('[AudioRecorder] Using workaround to find actual recording file...');
|
||||||
console.log('[AudioRecorder] File exists:', exists);
|
const actualUri = await getActualRecordingUri(recordingStartTime);
|
||||||
|
if (actualUri) {
|
||||||
|
uri = actualUri;
|
||||||
|
console.log('[AudioRecorder] Found actual recording URI:', uri);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!exists) {
|
if (!uri || uri === '') {
|
||||||
throw new Error('Recording file does not exist');
|
// Cancellation / early stop: return an empty blob, but guarantee cleanup.
|
||||||
}
|
setRecordingStartTime(null);
|
||||||
|
return new Blob([], { type: 'audio/m4a' });
|
||||||
|
}
|
||||||
|
|
||||||
// Convert URI to Blob
|
// Get file info
|
||||||
const audioBlob = await uriToBlob(uri);
|
const file = new File(uri);
|
||||||
|
const exists = file.exists;
|
||||||
|
console.log('[AudioRecorder] File exists:', exists);
|
||||||
|
|
||||||
console.log('[AudioRecorder] Recording converted to blob:', {
|
if (!exists) {
|
||||||
size: audioBlob.size,
|
setRecordingStartTime(null);
|
||||||
type: audioBlob.type,
|
return new Blob([], { type: 'audio/m4a' });
|
||||||
});
|
}
|
||||||
|
|
||||||
// Clean up the temporary file
|
// Convert URI to Blob
|
||||||
file.delete();
|
const audioBlob = await uriToBlob(uri);
|
||||||
|
|
||||||
// Reset start time and clear recording intent
|
console.log('[AudioRecorder] Recording converted to blob:', {
|
||||||
setRecordingStartTime(null);
|
size: audioBlob.size,
|
||||||
recordingIntentRef.current = false;
|
type: audioBlob.type,
|
||||||
|
});
|
||||||
|
|
||||||
return audioBlob;
|
// Clean up the temporary file
|
||||||
|
file.delete();
|
||||||
|
|
||||||
|
// Reset start time
|
||||||
|
setRecordingStartTime(null);
|
||||||
|
|
||||||
|
return audioBlob;
|
||||||
|
})();
|
||||||
|
startStopMutexRef.current = stopPromise;
|
||||||
|
return await stopPromise;
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
recordingIntentRef.current = false;
|
|
||||||
setRecordingStartTime(null);
|
setRecordingStartTime(null);
|
||||||
console.error('[AudioRecorder] Failed to stop recording:', error);
|
console.error('[AudioRecorder] Failed to stop recording:', error);
|
||||||
throw new Error(`Failed to stop audio recording: ${error.message}`);
|
throw new Error(`Failed to stop audio recording: ${error.message}`);
|
||||||
} finally {
|
} finally {
|
||||||
stopRequestedRef.current = false;
|
startStopMutexRef.current = null;
|
||||||
}
|
}
|
||||||
}, [recordingStartTime]);
|
}, [recordingStartTime]);
|
||||||
|
|
||||||
@@ -345,8 +340,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const isRecording = useCallback(() => {
|
const isRecording = useCallback(() => {
|
||||||
// Treat local intent as backup so callers can detect pending sessions
|
return Boolean(recorderState.isRecording);
|
||||||
return Boolean(recorderState.isRecording) || recordingIntentRef.current;
|
|
||||||
}, [recorderState.isRecording]);
|
}, [recorderState.isRecording]);
|
||||||
|
|
||||||
// Return stable object using useMemo
|
// Return stable object using useMemo
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { useCallback, useEffect, useRef, useState } from "react";
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
|
import { AttemptCancelledError, AttemptGuard } from "@/utils/attempt-guard";
|
||||||
|
|
||||||
export interface AudioCaptureConfig {
|
export interface AudioCaptureConfig {
|
||||||
sampleRate?: number;
|
sampleRate?: number;
|
||||||
@@ -21,6 +22,7 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
|
|
||||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||||
const mediaStreamRef = useRef<MediaStream | null>(null);
|
const mediaStreamRef = useRef<MediaStream | null>(null);
|
||||||
|
const attemptGuardRef = useRef(new AttemptGuard());
|
||||||
const chunksRef = useRef<Blob[]>([]);
|
const chunksRef = useRef<Blob[]>([]);
|
||||||
const supportedMimeTypeRef = useRef<string | null | undefined>(undefined);
|
const supportedMimeTypeRef = useRef<string | null | undefined>(undefined);
|
||||||
const analyserRef = useRef<AnalyserNode | null>(null);
|
const analyserRef = useRef<AnalyserNode | null>(null);
|
||||||
@@ -79,6 +81,14 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const hardReset = useCallback(() => {
|
||||||
|
stopMetering();
|
||||||
|
cleanupStream();
|
||||||
|
mediaRecorderRef.current = null;
|
||||||
|
chunksRef.current = [];
|
||||||
|
setIsRecording(false);
|
||||||
|
}, [cleanupStream, stopMetering]);
|
||||||
|
|
||||||
const startMetering = useCallback((stream: MediaStream) => {
|
const startMetering = useCallback((stream: MediaStream) => {
|
||||||
const onAudioLevel = configRef.current?.onAudioLevel;
|
const onAudioLevel = configRef.current?.onAudioLevel;
|
||||||
if (!onAudioLevel) {
|
if (!onAudioLevel) {
|
||||||
@@ -129,6 +139,8 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const start = useCallback(async () => {
|
const start = useCallback(async () => {
|
||||||
|
const attemptId = attemptGuardRef.current.next();
|
||||||
|
|
||||||
if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") {
|
if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") {
|
||||||
throw new Error("Already recording");
|
throw new Error("Already recording");
|
||||||
}
|
}
|
||||||
@@ -173,6 +185,22 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
throw new Error(`Failed to access microphone: ${error?.message ?? error}`);
|
throw new Error(`Failed to access microphone: ${error?.message ?? error}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof AttemptCancelledError) {
|
||||||
|
stream.getTracks().forEach((track) => {
|
||||||
|
try {
|
||||||
|
track.stop();
|
||||||
|
} catch {
|
||||||
|
// Ignore track stop errors.
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
mediaStreamRef.current = stream;
|
mediaStreamRef.current = stream;
|
||||||
|
|
||||||
const recorderOptions: MediaRecorderOptions = {};
|
const recorderOptions: MediaRecorderOptions = {};
|
||||||
@@ -222,11 +250,25 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
}, [cleanupStream, detectSupportedMimeType, startMetering]);
|
}, [cleanupStream, detectSupportedMimeType, startMetering]);
|
||||||
|
|
||||||
const stop = useCallback(async (): Promise<Blob> => {
|
const stop = useCallback(async (): Promise<Blob> => {
|
||||||
|
attemptGuardRef.current.cancel();
|
||||||
|
|
||||||
const recorder = mediaRecorderRef.current;
|
const recorder = mediaRecorderRef.current;
|
||||||
if (!recorder || recorder.state === "inactive") {
|
if (!recorder) {
|
||||||
throw new Error("Not recording");
|
throw new Error("Not recording");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (recorder.state === "inactive") {
|
||||||
|
stopMetering();
|
||||||
|
cleanupStream();
|
||||||
|
mediaRecorderRef.current = null;
|
||||||
|
setIsRecording(false);
|
||||||
|
|
||||||
|
const mimeType = recorder.mimeType || detectSupportedMimeType() || "audio/webm";
|
||||||
|
const blob = new Blob(chunksRef.current, { type: mimeType });
|
||||||
|
chunksRef.current = [];
|
||||||
|
return blob;
|
||||||
|
}
|
||||||
|
|
||||||
return await new Promise<Blob>((resolve, reject) => {
|
return await new Promise<Blob>((resolve, reject) => {
|
||||||
const finalize = () => {
|
const finalize = () => {
|
||||||
recorder.removeEventListener("stop", finalize);
|
recorder.removeEventListener("stop", finalize);
|
||||||
@@ -275,17 +317,15 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
return () => {
|
return () => {
|
||||||
if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") {
|
attemptGuardRef.current.cancel();
|
||||||
try {
|
try {
|
||||||
mediaRecorderRef.current.stop();
|
void mediaRecorderRef.current?.stop();
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore stop during unmount.
|
// Ignore stop during unmount.
|
||||||
}
|
|
||||||
}
|
}
|
||||||
stopMetering();
|
hardReset();
|
||||||
cleanupStream();
|
|
||||||
};
|
};
|
||||||
}, [cleanupStream, stopMetering]);
|
}, [hardReset]);
|
||||||
|
|
||||||
const getSupportedMimeType = useCallback(() => detectSupportedMimeType(), [
|
const getSupportedMimeType = useCallback(() => detectSupportedMimeType(), [
|
||||||
detectSupportedMimeType,
|
detectSupportedMimeType,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { useAudioRecorder } from "@/hooks/use-audio-recorder";
|
|||||||
import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
|
import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
|
||||||
import type { TranscriptionResultMessage } from "@server/shared/messages";
|
import type { TranscriptionResultMessage } from "@server/shared/messages";
|
||||||
import { generateMessageId } from "@/types/stream";
|
import { generateMessageId } from "@/types/stream";
|
||||||
|
import { AttemptGuard } from "@/utils/attempt-guard";
|
||||||
|
|
||||||
export type DictationStatus = "idle" | "recording" | "uploading" | "retrying" | "failed";
|
export type DictationStatus = "idle" | "recording" | "uploading" | "retrying" | "failed";
|
||||||
|
|
||||||
@@ -310,9 +311,15 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
isRecordingRef.current = isRecording;
|
isRecordingRef.current = isRecording;
|
||||||
}, [isRecording]);
|
}, [isRecording]);
|
||||||
|
|
||||||
|
const isProcessingRef = useRef(isProcessing);
|
||||||
|
useEffect(() => {
|
||||||
|
isProcessingRef.current = isProcessing;
|
||||||
|
}, [isProcessing]);
|
||||||
|
|
||||||
const pendingRequestIdRef = useRef<string | null>(null);
|
const pendingRequestIdRef = useRef<string | null>(null);
|
||||||
const activeStopPromiseRef = useRef<Promise<Blob> | null>(null);
|
const activeStopPromiseRef = useRef<Promise<Blob> | null>(null);
|
||||||
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
const attemptGuardRef = useRef(new AttemptGuard());
|
||||||
|
|
||||||
const stopDurationTracking = useCallback(() => {
|
const stopDurationTracking = useCallback(() => {
|
||||||
if (durationIntervalRef.current) {
|
if (durationIntervalRef.current) {
|
||||||
@@ -510,6 +517,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
isRecording: isRecordingRef.current,
|
isRecording: isRecordingRef.current,
|
||||||
hasActiveStop: Boolean(activeStopPromiseRef.current),
|
hasActiveStop: Boolean(activeStopPromiseRef.current),
|
||||||
});
|
});
|
||||||
|
attemptGuardRef.current.cancel();
|
||||||
if (!isRecordingRef.current && !activeStopPromiseRef.current) {
|
if (!isRecordingRef.current && !activeStopPromiseRef.current) {
|
||||||
console.log("[useDictation] cancelDictation ignored: nothing to cancel");
|
console.log("[useDictation] cancelDictation ignored: nothing to cancel");
|
||||||
return;
|
return;
|
||||||
@@ -561,8 +569,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
setRetryAttempt(0);
|
setRetryAttempt(0);
|
||||||
setLastOutcome(null);
|
setLastOutcome(null);
|
||||||
|
|
||||||
|
const attemptId = attemptGuardRef.current.next();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const audioData = await stopRecorder();
|
const audioData = await stopRecorder();
|
||||||
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
const recordedDurationSeconds = durationRef.current;
|
const recordedDurationSeconds = durationRef.current;
|
||||||
pendingAudioRef.current = buildCapturedAudioPayload(audioData, recordedDurationSeconds);
|
pendingAudioRef.current = buildCapturedAudioPayload(audioData, recordedDurationSeconds);
|
||||||
setStatus("uploading");
|
setStatus("uploading");
|
||||||
@@ -571,6 +582,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
setVolume(0);
|
setVolume(0);
|
||||||
|
|
||||||
const transcription = await transmitDictation();
|
const transcription = await transmitDictation();
|
||||||
|
attemptGuardRef.current.assertCurrent(attemptId);
|
||||||
handleTranscriptionSuccess(transcription);
|
handleTranscriptionSuccess(transcription);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
resetTranscriptionMutation();
|
resetTranscriptionMutation();
|
||||||
@@ -661,14 +673,34 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
const prevVisible = visibilityRef.current;
|
const prevVisible = visibilityRef.current;
|
||||||
visibilityRef.current = nextVisible;
|
visibilityRef.current = nextVisible;
|
||||||
|
|
||||||
if (prevVisible === true && nextVisible === false && isRecordingRef.current) {
|
if (prevVisible === true && nextVisible === false) {
|
||||||
cancelRef.current?.();
|
attemptGuardRef.current.cancel();
|
||||||
|
|
||||||
|
if (isRecordingRef.current) {
|
||||||
|
cancelRef.current?.();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isProcessingRef.current) {
|
||||||
|
stopDurationTracking();
|
||||||
|
setDuration(0);
|
||||||
|
setIsProcessing(false);
|
||||||
|
setVolume(0);
|
||||||
|
setError(null);
|
||||||
|
setStatus("idle");
|
||||||
|
setRetryAttempt(0);
|
||||||
|
setRetryInfo(null);
|
||||||
|
pendingAudioRef.current = null;
|
||||||
|
setFailedRecording(null);
|
||||||
|
setLastOutcome(null);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}, [autoStopWhenHidden?.isVisible]);
|
}, [autoStopWhenHidden?.isVisible, stopDurationTracking]);
|
||||||
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
return () => {
|
return () => {
|
||||||
|
attemptGuardRef.current.cancel();
|
||||||
stopDurationTracking();
|
stopDurationTracking();
|
||||||
const activeStop = activeStopPromiseRef.current;
|
const activeStop = activeStopPromiseRef.current;
|
||||||
if (activeStop) {
|
if (activeStop) {
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
export class AttemptCancelledError extends Error {
|
||||||
|
constructor(message = "Attempt cancelled") {
|
||||||
|
super(message);
|
||||||
|
this.name = "AttemptCancelledError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tiny helper for async lifecycles:
|
||||||
|
* - Call `next()` to start a new attempt and invalidate previous ones
|
||||||
|
* - Call `cancel()` to invalidate any in-flight attempt
|
||||||
|
* - Call `assertCurrent(id)` after awaits to enforce a single active attempt
|
||||||
|
*/
|
||||||
|
export class AttemptGuard {
|
||||||
|
private attemptId = 0;
|
||||||
|
|
||||||
|
public next(): number {
|
||||||
|
this.attemptId += 1;
|
||||||
|
return this.attemptId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public cancel(): void {
|
||||||
|
this.attemptId += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public isCurrent(id: number): boolean {
|
||||||
|
return id === this.attemptId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public assertCurrent(id: number): void {
|
||||||
|
if (!this.isCurrent(id)) {
|
||||||
|
throw new AttemptCancelledError();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Reference in New Issue
Block a user