Stop Android audio interruptions crashing voice mode (#1941)

* fix(android): handle denied voice audio focus

Android can deny voice-mode audio focus while another system audio owner, such as an incoming call, is active. Treat that as an interruption so resume does not crash the app and JS stops voice mode coherently.

* fix(voice): keep interruption state consistent

Make native interruption handling payload-aware so iOS resume events do not stop the shared JS runtime, avoid ending voice mode for duck-only Android focus changes, and roll host voice mode back if capture fails after startup enabled it.

* fix(voice): avoid duplicate interruption handling

Keep exactly one Android audio-focus request active at a time and leave JS capture state untouched for non-terminal interruption events.

* fix(android): abandon blocked audio focus requests

When Android denies or blocks voice audio focus, abandon the pending focus request so delayed focus gain cannot arrive after voice mode has already stopped.

* fix(voice): handle interrupted dictation capture

Stop Android resume from replaying after recording restart fails, and propagate native blocked interruptions into dictation so users do not remain in a stale recording state.
This commit is contained in:
Mohamed Boudra
2026-07-08 19:45:02 +08:00
committed by GitHub
parent c72bcb49e5
commit 251d88a37a
10 changed files with 152 additions and 25 deletions
+13
View File
@@ -198,6 +198,19 @@ for i in $(seq 1 "$ITERATIONS"); do
done done
``` ```
### Android audio focus interruptions
Voice mode uses the custom `expo-two-way-audio` Android module, so incoming calls and other system audio owners must be tested with emulator/system commands, not a JS-only test. To verify that voice resume handles denied audio focus without crashing:
```bash
adb shell am start -n sh.paseo/.MainActivity
# Start voice mode in an existing composer, then background Paseo with Home.
adb emu gsm call 5551234
# Foreground Paseo while the call is still ringing.
```
Expected result: Paseo does not throw `RuntimeException: Audio focus request failed`; native audio reports an interruption and voice mode stops or pauses coherently.
## Unistyles + Reanimated ## Unistyles + Reanimated
### The crash ### The crash
@@ -125,6 +125,11 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
onVolumeLevel: (level) => { onVolumeLevel: (level) => {
runtime?.handleCaptureVolume(level); runtime?.handleCaptureVolume(level);
}, },
onInterruption: () => {
void runtime?.stopVoice().catch((error) => {
console.error("[VoiceEngine] Failed to stop after audio interruption:", error);
});
},
onError: (error) => { onError: (error) => {
console.error("[VoiceEngine] Capture error:", error); console.error("[VoiceEngine] Capture error:", error);
}, },
@@ -12,6 +12,7 @@ import type {
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource { export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
const onPcmSegmentRef = useRef(config.onPcmSegment); const onPcmSegmentRef = useRef(config.onPcmSegment);
const onErrorRef = useRef(config.onError); const onErrorRef = useRef(config.onError);
const onInterruptionRef = useRef(config.onInterruption);
const [volume, setVolume] = useState(0); const [volume, setVolume] = useState(0);
const engineRef = useRef<ReturnType<typeof createAudioEngine> | null>(null); const engineRef = useRef<ReturnType<typeof createAudioEngine> | null>(null);
@@ -30,6 +31,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
onError: (error) => { onError: (error) => {
onErrorRef.current?.(error); onErrorRef.current?.(error);
}, },
onInterruption: () => {
onInterruptionRef.current?.();
},
}); });
return engineRef.current; return engineRef.current;
}, []); }, []);
@@ -37,7 +41,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
useEffect(() => { useEffect(() => {
onPcmSegmentRef.current = config.onPcmSegment; onPcmSegmentRef.current = config.onPcmSegment;
onErrorRef.current = config.onError; onErrorRef.current = config.onError;
}, [config.onPcmSegment, config.onError]); onInterruptionRef.current = config.onInterruption;
}, [config.onPcmSegment, config.onError, config.onInterruption]);
const start = useCallback(async () => { const start = useCallback(async () => {
const engine = getOrCreateEngine(); const engine = getOrCreateEngine();
@@ -1,6 +1,7 @@
export interface DictationAudioSourceConfig { export interface DictationAudioSourceConfig {
onPcmSegment: (pcm16Base64: string) => void; onPcmSegment: (pcm16Base64: string) => void;
onError?: (error: Error) => void; onError?: (error: Error) => void;
onInterruption?: () => void;
} }
export interface DictationAudioSource { export interface DictationAudioSource {
+23 -14
View File
@@ -184,19 +184,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
}); });
}, [client]); }, [client]);
const audio = useDictationAudioSource({
onPcmSegment: (audioData) => {
senderRef.current?.enqueueSegment(audioData);
},
onError: (err) => {
onErrorRef.current?.(err);
},
});
const audioStopRef = useRef(audio.stop);
useEffect(() => {
audioStopRef.current = audio.stop;
}, [audio.stop]);
const handleStreamingTranscriptionSuccess = useCallback( const handleStreamingTranscriptionSuccess = useCallback(
(text: string, requestId: string) => { (text: string, requestId: string) => {
setIsProcessing(false); setIsProcessing(false);
@@ -220,6 +207,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
(failure: unknown) => { (failure: unknown) => {
const normalized = toError(failure); const normalized = toError(failure);
const failureId = generateMessageId(); const failureId = generateMessageId();
stopDurationTracking();
setIsProcessing(false); setIsProcessing(false);
isProcessingRef.current = false; isProcessingRef.current = false;
isRecordingRef.current = false; isRecordingRef.current = false;
@@ -234,9 +222,30 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
reportError(normalized, "Failed to complete dictation"); reportError(normalized, "Failed to complete dictation");
}, },
[reportError], [reportError, stopDurationTracking],
); );
const audio = useDictationAudioSource({
onPcmSegment: (audioData) => {
senderRef.current?.enqueueSegment(audioData);
},
onError: (err) => {
onErrorRef.current?.(err);
},
onInterruption: () => {
try {
senderRef.current?.cancel();
} catch {
// no-op
}
handleDictationFailure(new Error("Dictation was interrupted by another audio source."));
},
});
const audioStopRef = useRef(audio.stop);
useEffect(() => {
audioStopRef.current = audio.stop;
}, [audio.stop]);
const startDictation = useCallback(async () => { const startDictation = useCallback(async () => {
if ( if (
actionGateRef.current.starting || actionGateRef.current.starting ||
@@ -1,6 +1,7 @@
export interface AudioEngineCallbacks { export interface AudioEngineCallbacks {
onCaptureData(pcm: Uint8Array): void; onCaptureData(pcm: Uint8Array): void;
onVolumeLevel(level: number): void; onVolumeLevel(level: number): void;
onInterruption?(): void;
onError?(error: Error): void; onError?(error: Error): void;
} }
+22 -1
View File
@@ -116,6 +116,21 @@ export function createAudioEngine(
callbacks.onVolumeLevel(level); callbacks.onVolumeLevel(level);
}, },
); );
const interruptionSubscription = native.addExpoTwoWayAudioEventListener(
"onAudioInterruption",
(event: { data: string }) => {
if (event.data !== "blocked") {
return;
}
const wasCaptureActive = refs.captureActive;
refs.captureActive = false;
refs.muted = false;
callbacks.onVolumeLevel(0);
if (wasCaptureActive) {
callbacks.onInterruption?.();
}
},
);
async function ensureInitialized(): Promise<void> { async function ensureInitialized(): Promise<void> {
if (refs.initialized) { if (refs.initialized) {
@@ -234,6 +249,7 @@ export function createAudioEngine(
} }
microphoneSubscription.remove(); microphoneSubscription.remove();
volumeSubscription.remove(); volumeSubscription.remove();
interruptionSubscription.remove();
}, },
async startCapture() { async startCapture() {
@@ -244,7 +260,12 @@ export function createAudioEngine(
try { try {
await ensureMicrophonePermission(); await ensureMicrophonePermission();
await ensureInitialized(); await ensureInitialized();
native.toggleRecording(true); const isRecording = native.toggleRecording(true);
if (!isRecording) {
throw new Error(
"Microphone capture could not start because Android audio focus is unavailable.",
);
}
refs.captureActive = true; refs.captureActive = true;
} catch (error) { } catch (error) {
const wrapped = error instanceof Error ? error : new Error(String(error)); const wrapped = error instanceof Error ? error : new Error(String(error));
@@ -423,6 +423,22 @@ describe("voice runtime", () => {
expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false); expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false);
}); });
it("rolls daemon voice mode back when capture fails after enabling it", async () => {
const adapter = createSessionAdapter();
const engine = createAudioEngineMock();
vi.mocked(engine.startCapture).mockRejectedValue(new Error("audio focus unavailable"));
const { runtime } = createRuntime({ engine });
runtime.registerSession(adapter);
await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
"audio focus unavailable",
);
expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(1, true, "agent-1");
expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(2, false);
expect(runtime.getSnapshot().phase).toBe("disabled");
});
it("returns an explicit not-ready error when the adapter is missing", async () => { it("returns an explicit not-ready error when the adapter is missing", async () => {
const { runtime } = createRuntime(); const { runtime } = createRuntime();
await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow( await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
+5
View File
@@ -730,6 +730,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const previousServerId = state.snapshot.activeServerId; const previousServerId = state.snapshot.activeServerId;
const previousAgentId = state.snapshot.activeAgentId; const previousAgentId = state.snapshot.activeAgentId;
const generation = state.generation + 1; const generation = state.generation + 1;
let enabledCurrentVoiceMode = false;
state.generation = generation; state.generation = generation;
state.transportReady = false; state.transportReady = false;
patchSnapshot((prev) => ({ patchSnapshot((prev) => ({
@@ -759,6 +760,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
await deps.engine.initialize(); await deps.engine.initialize();
await session.adapter.setVoiceMode(true, agentId); await session.adapter.setVoiceMode(true, agentId);
enabledCurrentVoiceMode = true;
await deps.engine.startCapture(); await deps.engine.startCapture();
if (state.generation !== generation) { if (state.generation !== generation) {
return; return;
@@ -776,6 +778,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
isMuted: deps.engine.isMuted(), isMuted: deps.engine.isMuted(),
})); }));
} catch (error) { } catch (error) {
if (enabledCurrentVoiceMode) {
await session.adapter.setVoiceMode(false).catch(() => undefined);
}
await performLocalStop(); await performLocalStop();
throw error; throw error;
} }
@@ -83,7 +83,9 @@ class AudioEngine (context: Context) {
private fun initializeAudio(context:Context) { private fun initializeAudio(context:Context) {
audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
audioManager.mode = AudioManager.MODE_IN_COMMUNICATION audioManager.mode = AudioManager.MODE_IN_COMMUNICATION
requestAudioFocus() if (!requestAudioFocus()) {
handleAudioFocusBlocked()
}
// Route audio to external device if connected, otherwise route to speaker // Route audio to external device if connected, otherwise route to speaker
updateAudioRouting() updateAudioRouting()
@@ -168,7 +170,12 @@ class AudioEngine (context: Context) {
} }
@SuppressLint("NewApi") @SuppressLint("NewApi")
private fun requestAudioFocus() { private fun requestAudioFocus(): Boolean {
audioFocusRequest?.let { request ->
audioManager.abandonAudioFocusRequest(request)
audioFocusRequest = null
}
val focusRequest = val focusRequest =
AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE)
.setAudioAttributes( .setAudioAttributes(
@@ -180,9 +187,19 @@ class AudioEngine (context: Context) {
.setAcceptsDelayedFocusGain(true) .setAcceptsDelayedFocusGain(true)
.setOnAudioFocusChangeListener { focusChange -> .setOnAudioFocusChangeListener { focusChange ->
when (focusChange) { when (focusChange) {
AudioManager.AUDIOFOCUS_GAIN -> {
Log.d("AudioEngine", "Audio focus gained")
}
AudioManager.AUDIOFOCUS_LOSS -> { AudioManager.AUDIOFOCUS_LOSS -> {
Log.d("AudioEngine", "Audio focus lost") Log.d("AudioEngine", "Audio focus lost")
onAudioInterruptionCallback?.let { it("blocked") } handleAudioFocusBlocked()
}
AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> {
Log.d("AudioEngine", "Audio focus lost transiently")
handleAudioFocusBlocked()
}
AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> {
Log.d("AudioEngine", "Audio focus duck requested")
} }
} }
} }
@@ -191,14 +208,40 @@ class AudioEngine (context: Context) {
audioFocusRequest = focusRequest audioFocusRequest = focusRequest
val result = audioManager.requestAudioFocus(focusRequest) val result = audioManager.requestAudioFocus(focusRequest)
if (result != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { if (result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
throw RuntimeException("Audio focus request failed") return true
} }
Log.w("AudioEngine", "Audio focus request was not granted: $result")
return false
}
private fun handleAudioFocusBlocked() {
if (isRecording) {
stopRecording()
}
audioFocusRequest?.let { request ->
audioManager.abandonAudioFocusRequest(request)
audioFocusRequest = null
}
if (::audioTrack.isInitialized) {
audioTrack.pause()
}
audioSampleQueue.clear()
isPlaying = false
isRecordingBeforePause = false
onOutputVolumeCallback?.invoke(0.0F)
onAudioInterruptionCallback?.let { it("blocked") }
} }
@RequiresApi(Build.VERSION_CODES.Q) @RequiresApi(Build.VERSION_CODES.Q)
@SuppressLint("MissingPermission") @SuppressLint("MissingPermission")
private fun startRecording(){ private fun startRecording(): Boolean {
if (!requestAudioFocus()) {
handleAudioFocusBlocked()
return false
}
val bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) val bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT)
audioRecord = AudioRecord( audioRecord = AudioRecord(
MediaRecorder.AudioSource.VOICE_COMMUNICATION, MediaRecorder.AudioSource.VOICE_COMMUNICATION,
@@ -231,6 +274,7 @@ class AudioEngine (context: Context) {
audioRecord.startRecording() audioRecord.startRecording()
isRecording = true isRecording = true
startMicSampleTap() startMicSampleTap()
return true
} }
private fun startMicSampleTap(){ private fun startMicSampleTap(){
@@ -274,7 +318,7 @@ class AudioEngine (context: Context) {
if (value == isRecording) return isRecording if (value == isRecording) return isRecording
if (value) { if (value) {
startRecording() return startRecording()
} else { } else {
stopRecording() stopRecording()
} }
@@ -353,8 +397,15 @@ class AudioEngine (context: Context) {
@RequiresApi(Build.VERSION_CODES.Q) @RequiresApi(Build.VERSION_CODES.Q)
fun resumeRecordingAndPlayer() { fun resumeRecordingAndPlayer() {
requestAudioFocus() if (!requestAudioFocus()) {
isRecording = toggleRecording(isRecordingBeforePause) handleAudioFocusBlocked()
return
}
val wasRecordingBeforePause = isRecordingBeforePause
isRecording = toggleRecording(wasRecordingBeforePause)
if (wasRecordingBeforePause && !isRecording) {
return
}
audioTrack.play() audioTrack.play()
} }