jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(voice): defer speech confirmation and add diagnostics

Browse Source
This commit is contained in:
Mohamed Boudra
2026-03-14 20:05:48 +07:00
parent cad4110e24
commit d1632d80e3
13 changed files with 223 additions and 790 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/components/dictation-controls.tsx
-2
View File
@@ -69,7 +69,6 @@ export function DictationControls({
<VolumeMeter
volume={volume}
isMuted={false}
isDetecting
isSpeaking={false}
orientation="horizontal"
/>
@@ -177,7 +176,6 @@ export function DictationOverlay({
<VolumeMeter
volume={volume}
isMuted={false}
isDetecting
isSpeaking={false}
orientation="horizontal"
color={theme.colors.palette.white}
packages/app/src/components/realtime-voice-overlay.tsx
+1 -2
View File
@@ -22,14 +22,13 @@ export function RealtimeVoiceOverlay({
onStop,
}: RealtimeVoiceOverlayProps) {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const { volume, isSpeaking } = useVoiceTelemetry();
return (
<View style={styles.container}>
<View style={styles.meterContainer}>
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
/>
packages/app/src/components/voice-compact-indicator.tsx
+1 -2
View File
@@ -6,7 +6,7 @@ import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
export function VoiceCompactIndicator() {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const { volume, isSpeaking } = useVoiceTelemetry();
const {
isVoiceMode,
isVoiceSwitching,
@@ -24,7 +24,6 @@ export function VoiceCompactIndicator() {
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
variant="compact"
packages/app/src/components/voice-panel.tsx
+1 -2
View File
@@ -8,7 +8,7 @@ import { useHosts } from "@/runtime/host-runtime";
export function VoicePanel() {
const { theme } = useUnistyles();
const daemons = useHosts();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const { volume, isSpeaking } = useVoiceTelemetry();
const {
isMuted,
stopVoice,
@@ -27,7 +27,6 @@ export function VoicePanel() {
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
variant="compact"
packages/app/src/components/volume-meter.tsx
-1
View File
@@ -13,7 +13,6 @@ import { StyleSheet, useUnistyles } from "react-native-unistyles";
interface VolumeMeterProps {
volume: number;
isMuted?: boolean;
isDetecting?: boolean;
isSpeaking?: boolean;
orientation?: "vertical" | "horizontal";
variant?: "default" | "compact";
packages/app/src/contexts/voice-context.tsx
-1
View File
@@ -35,7 +35,6 @@ const EMPTY_SNAPSHOT: VoiceRuntimeSnapshot = {
const EMPTY_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
packages/app/src/voice/audio-engine.native.ts
+44 -2
View File
@@ -20,6 +20,8 @@ interface BridgeStats {
windowStartedAtMs: number;
captureEvents: number;
captureBytes: number;
volumeEvents: number;
volumeMax: number;
playbackEvents: number;
playbackInputBytes: number;
playbackResampledBytes: number;
@@ -88,6 +90,8 @@ export function createAudioEngine(
windowStartedAtMs: Date.now(),
captureEvents: 0,
captureBytes: 0,
volumeEvents: 0,
volumeMax: 0,
playbackEvents: 0,
playbackInputBytes: 0,
playbackResampledBytes: 0,
@@ -108,6 +112,7 @@ export function createAudioEngine(
console.log(
`[AudioEngine.native#${instanceId}][bridge] ${reason} ` +
`capture=${bridgeStats.captureEvents}ev/${bridgeStats.captureBytes}B ` +
`volume=${bridgeStats.volumeEvents}ev max=${bridgeStats.volumeMax.toFixed(3)} ` +
`play=${bridgeStats.playbackEvents}ev/${bridgeStats.playbackInputBytes}B->${bridgeStats.playbackResampledBytes}B ` +
`playMs=${bridgeStats.playbackDurationMs.toFixed(1)} ` +
`windowMs=${elapsedMs}`
@@ -115,6 +120,8 @@ export function createAudioEngine(
bridgeStats.windowStartedAtMs = now;
bridgeStats.captureEvents = 0;
bridgeStats.captureBytes = 0;
bridgeStats.volumeEvents = 0;
bridgeStats.volumeMax = 0;
bridgeStats.playbackEvents = 0;
bridgeStats.playbackInputBytes = 0;
bridgeStats.playbackResampledBytes = 0;
@@ -133,6 +140,8 @@ export function createAudioEngine(
reject: (error: Error) => void;
settled: boolean;
} | null;
sawFirstMicChunk: boolean;
sawFirstVolumeEvent: boolean;
destroyed: boolean;
} = {
initialized: false,
@@ -142,6 +151,8 @@ export function createAudioEngine(
processingQueue: false,
playbackTimeout: null,
activePlayback: null,
sawFirstMicChunk: false,
sawFirstVolumeEvent: false,
destroyed: false,
};
@@ -152,6 +163,12 @@ export function createAudioEngine(
return;
}
const pcm = event.data as Uint8Array;
if (!refs.sawFirstMicChunk) {
refs.sawFirstMicChunk = true;
console.log(
`[AudioEngine.native#${instanceId}] firstMicChunk bytes=${pcm.byteLength} head=${toHexPreview(pcm)}`
);
}
bridgeStats.captureEvents += 1;
bridgeStats.captureBytes += pcm.byteLength;
maybeFlushBridgeStats("capture");
@@ -165,6 +182,15 @@ export function createAudioEngine(
return;
}
const level = refs.muted ? 0 : event.data;
bridgeStats.volumeEvents += 1;
bridgeStats.volumeMax = Math.max(bridgeStats.volumeMax, level);
if (!refs.sawFirstVolumeEvent) {
refs.sawFirstVolumeEvent = true;
console.log(
`[AudioEngine.native#${instanceId}] firstInputVolume level=${level.toFixed(3)} muted=${refs.muted}`
);
}
maybeFlushBridgeStats("volume");
callbacks.onVolumeLevel(level);
}
);
@@ -190,8 +216,14 @@ export function createAudioEngine(
async function ensureMicrophonePermission(): Promise<void> {
let permission = await native.getMicrophonePermissionsAsync().catch(() => null);
console.log(
`[AudioEngine.native#${instanceId}] microphonePermission initial=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}`
);
if (!permission?.granted) {
permission = await native.requestMicrophonePermissionsAsync().catch(() => null);
console.log(
`[AudioEngine.native#${instanceId}] microphonePermission requested=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}`
);
}
if (!permission?.granted) {
throw new Error(
@@ -307,14 +339,21 @@ export function createAudioEngine(
async startCapture() {
if (refs.captureActive) {
console.log(`[AudioEngine.native#${instanceId}] startCapture skipped: already active`);
return;
}
try {
console.log(`[AudioEngine.native#${instanceId}] startCapture begin`);
await ensureMicrophonePermission();
await ensureInitialized();
native.toggleRecording(true);
refs.sawFirstMicChunk = false;
refs.sawFirstVolumeEvent = false;
const isRecording = native.toggleRecording(true);
refs.captureActive = true;
console.log(
`[AudioEngine.native#${instanceId}] startCapture toggleRecording(true) => ${String(isRecording)}`
);
} catch (error) {
const wrapped = error instanceof Error ? error : new Error(String(error));
callbacks.onError?.(wrapped);
@@ -324,7 +363,10 @@ export function createAudioEngine(
async stopCapture() {
if (refs.captureActive) {
native.toggleRecording(false);
const isRecording = native.toggleRecording(false);
console.log(
`[AudioEngine.native#${instanceId}] stopCapture toggleRecording(false) => ${String(isRecording)}`
);
}
refs.captureActive = false;
refs.muted = false;
packages/app/src/voice/voice-runtime.test.ts
+34 -1
View File
@@ -268,6 +268,40 @@ describe("voice runtime", () => {
expect(engine.play).toHaveBeenCalled();
});
it("does not restart the thinking tone from local detection jitter while waiting", async () => {
const adapter = createSessionAdapter();
let resolvePlay!: (duration: number) => void;
const engine = createAudioEngineMock();
vi.mocked(engine.play).mockImplementation(
() =>
new Promise<number>((resolve) => {
resolvePlay = resolve;
})
);
const { runtime } = createRuntime({ engine });
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
expect(runtime.getSnapshot().phase).toBe("waiting");
expect(engine.play).toHaveBeenCalledTimes(1);
runtime.handleCaptureVolume(REALTIME_VOICE_VAD_CONFIG.volumeThreshold + 0.05);
runtime.handleCaptureVolume(0);
expect(engine.stop).not.toHaveBeenCalled();
expect(engine.clearQueue).not.toHaveBeenCalled();
expect(engine.play).toHaveBeenCalledTimes(1);
runtime.onServerSpeechStateChanged("server-1", true);
expect(engine.stop).toHaveBeenCalledTimes(1);
expect(engine.clearQueue).toHaveBeenCalledTimes(1);
resolvePlay(0.1);
});
it("returns to listening after assistant playback once the turn is complete", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
@@ -308,7 +342,6 @@ describe("voice runtime", () => {
expect(runtime.getTelemetrySnapshot()).toMatchObject({
volume: expect.any(Number),
isDetecting: true,
isSpeaking: false,
});
packages/app/src/voice/voice-runtime.ts
+41 -21
View File
@@ -45,7 +45,6 @@ export interface VoiceRuntimeSnapshot {
export interface VoiceRuntimeTelemetrySnapshot {
volume: number;
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
}
@@ -85,7 +84,9 @@ interface RuntimeState {
generation: number;
segmentDurationTimer: ReturnType<typeof setInterval> | null;
lastDisplayVolumePublishMs: number;
localSpeechStartedAt: number | null;
serverSpeechStartedAt: number | null;
lastNoServerSpeechLogMs: number;
localAboveThresholdActive: boolean;
}
type AudioOutputPayload = Extract<
@@ -149,7 +150,6 @@ const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
@@ -176,7 +176,6 @@ function telemetryEqual(
): boolean {
return (
left.volume === right.volume &&
left.isDetecting === right.isDetecting &&
left.isSpeaking === right.isSpeaking &&
left.segmentDuration === right.segmentDuration
);
@@ -219,7 +218,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
generation: 0,
segmentDurationTimer: null,
lastDisplayVolumePublishMs: 0,
localSpeechStartedAt: null,
serverSpeechStartedAt: null,
lastNoServerSpeechLogMs: 0,
localAboveThresholdActive: false,
};
const playback: RuntimePlaybackState = {
groups: new Map(),
@@ -502,7 +503,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
function reconcileSegmentDurationTimer(): void {
if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) {
if (!state.serverSpeechDetected) {
clearSegmentDurationTimer();
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
return;
@@ -513,7 +514,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
state.segmentDurationTimer = setInterval(() => {
const startedAt = state.localSpeechStartedAt;
const startedAt = state.serverSpeechStartedAt;
patchTelemetry((prev) => ({
...prev,
segmentDuration: startedAt ? Date.now() - startedAt : 0,
@@ -525,7 +526,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return (
state.snapshot.isVoiceMode &&
state.snapshot.phase === "waiting" &&
!state.telemetry.isDetecting &&
!state.telemetry.isSpeaking
);
}
@@ -546,7 +546,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
function resetCaptureTelemetry(): void {
clearSegmentDurationTimer();
state.localSpeechStartedAt = null;
state.serverSpeechStartedAt = null;
patchTelemetry({ ...INITIAL_TELEMETRY });
}
@@ -621,6 +621,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
state.turnInProgress = false;
state.serverSpeechDetected = false;
state.lastDisplayVolumePublishMs = 0;
state.lastNoServerSpeechLogMs = 0;
state.localAboveThresholdActive = false;
uploader.reset();
resetCaptureTelemetry();
patchSnapshot({ ...INITIAL_SNAPSHOT });
@@ -744,6 +746,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
if (bridgeStats.captureEvents === 0) {
console.log(
`[VoiceRuntime#${instanceId}] firstCapturePcm bytes=${chunk.byteLength} phase=${state.snapshot.phase} transportReady=${state.transportReady}`
);
}
uploader.pushPcmChunk(chunk);
},
@@ -752,10 +759,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const displayLevel = state.snapshot.isMuted ? 0 : level;
publishDisplayVolume(displayLevel, nowMs);
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
state.localSpeechStartedAt = null;
patchTelemetry((prev) => ({
...prev,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
}));
@@ -763,22 +768,34 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
const isActive = level > REALTIME_VOICE_VAD_CONFIG.volumeThreshold;
if (isActive && state.localSpeechStartedAt === null) {
state.localSpeechStartedAt = nowMs;
if (isActive && !state.localAboveThresholdActive) {
state.localAboveThresholdActive = true;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechActive level=${level.toFixed(3)} threshold=${REALTIME_VOICE_VAD_CONFIG.volumeThreshold.toFixed(3)} phase=${state.snapshot.phase} transportReady=${state.transportReady}`
);
}
if (!isActive) {
state.localSpeechStartedAt = null;
if (!isActive && state.localAboveThresholdActive) {
state.localAboveThresholdActive = false;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechInactive phase=${state.snapshot.phase} serverSpeaking=${state.serverSpeechDetected}`
);
}
if (
isActive &&
!state.serverSpeechDetected &&
nowMs - state.lastNoServerSpeechLogMs >= 1500
) {
state.lastNoServerSpeechLogMs = nowMs;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechWithoutServerSpeech level=${level.toFixed(3)} phase=${state.snapshot.phase} turnInProgress=${state.turnInProgress} transportReady=${state.transportReady}`
);
}
patchTelemetry((prev) => ({
...prev,
isDetecting: isActive,
isSpeaking: state.serverSpeechDetected,
}));
reconcileSegmentDurationTimer();
if (!isActive) {
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
}
reconcileCue();
},
@@ -808,7 +825,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
let group = playback.groups.get(groupId);
if (!group) {
const shouldPlay = api.shouldPlayVoiceAudio(serverId);
console.log(`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isDetecting=${state.telemetry.isDetecting} isSpeaking=${state.telemetry.isSpeaking}`);
console.log(`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isSpeaking=${state.telemetry.isSpeaking}`);
group = {
groupId,
isVoiceMode: payload.isVoiceMode,
@@ -1040,8 +1057,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return;
}
console.log(`[VoiceRuntime] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase}`);
console.log(
`[VoiceRuntime#${instanceId}] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase} volume=${state.telemetry.volume}`
);
state.serverSpeechDetected = isSpeaking;
state.serverSpeechStartedAt = isSpeaking ? (state.serverSpeechStartedAt ?? Date.now()) : null;
if (isSpeaking) {
resetPlaybackState();
deps.engine.stop();