jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(voice): defer speech confirmation and add diagnostics

Browse Source
This commit is contained in:
Mohamed Boudra
2026-03-14 20:05:48 +07:00
parent cad4110e24
commit d1632d80e3
13 changed files with 223 additions and 790 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/components/dictation-controls.tsx
-2
View File
@@ -69,7 +69,6 @@ export function DictationControls({
<VolumeMeter
volume={volume}
isMuted={false}
isDetecting
isSpeaking={false}
orientation="horizontal"
/>
@@ -177,7 +176,6 @@ export function DictationOverlay({
<VolumeMeter
volume={volume}
isMuted={false}
isDetecting
isSpeaking={false}
orientation="horizontal"
color={theme.colors.palette.white}
packages/app/src/components/realtime-voice-overlay.tsx
+1 -2
View File
@@ -22,14 +22,13 @@ export function RealtimeVoiceOverlay({
onStop,
}: RealtimeVoiceOverlayProps) {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const { volume, isSpeaking } = useVoiceTelemetry();
return (
<View style={styles.container}>
<View style={styles.meterContainer}>
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
/>
packages/app/src/components/voice-compact-indicator.tsx
+1 -2
View File
@@ -6,7 +6,7 @@ import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
export function VoiceCompactIndicator() {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const { volume, isSpeaking } = useVoiceTelemetry();
const {
isVoiceMode,
isVoiceSwitching,
@@ -24,7 +24,6 @@ export function VoiceCompactIndicator() {
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
variant="compact"
packages/app/src/components/voice-panel.tsx
+1 -2
View File
@@ -8,7 +8,7 @@ import { useHosts } from "@/runtime/host-runtime";
export function VoicePanel() {
const { theme } = useUnistyles();
const daemons = useHosts();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const { volume, isSpeaking } = useVoiceTelemetry();
const {
isMuted,
stopVoice,
@@ -27,7 +27,6 @@ export function VoicePanel() {
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
variant="compact"
packages/app/src/components/volume-meter.tsx
-1
View File
@@ -13,7 +13,6 @@ import { StyleSheet, useUnistyles } from "react-native-unistyles";
interface VolumeMeterProps {
volume: number;
isMuted?: boolean;
isDetecting?: boolean;
isSpeaking?: boolean;
orientation?: "vertical" | "horizontal";
variant?: "default" | "compact";
packages/app/src/contexts/voice-context.tsx
-1
View File
@@ -35,7 +35,6 @@ const EMPTY_SNAPSHOT: VoiceRuntimeSnapshot = {
const EMPTY_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
packages/app/src/voice/audio-engine.native.ts
+44 -2
View File
@@ -20,6 +20,8 @@ interface BridgeStats {
windowStartedAtMs: number;
captureEvents: number;
captureBytes: number;
volumeEvents: number;
volumeMax: number;
playbackEvents: number;
playbackInputBytes: number;
playbackResampledBytes: number;
@@ -88,6 +90,8 @@ export function createAudioEngine(
windowStartedAtMs: Date.now(),
captureEvents: 0,
captureBytes: 0,
volumeEvents: 0,
volumeMax: 0,
playbackEvents: 0,
playbackInputBytes: 0,
playbackResampledBytes: 0,
@@ -108,6 +112,7 @@ export function createAudioEngine(
console.log(
`[AudioEngine.native#${instanceId}][bridge] ${reason} ` +
`capture=${bridgeStats.captureEvents}ev/${bridgeStats.captureBytes}B ` +
`volume=${bridgeStats.volumeEvents}ev max=${bridgeStats.volumeMax.toFixed(3)} ` +
`play=${bridgeStats.playbackEvents}ev/${bridgeStats.playbackInputBytes}B->${bridgeStats.playbackResampledBytes}B ` +
`playMs=${bridgeStats.playbackDurationMs.toFixed(1)} ` +
`windowMs=${elapsedMs}`
@@ -115,6 +120,8 @@ export function createAudioEngine(
bridgeStats.windowStartedAtMs = now;
bridgeStats.captureEvents = 0;
bridgeStats.captureBytes = 0;
bridgeStats.volumeEvents = 0;
bridgeStats.volumeMax = 0;
bridgeStats.playbackEvents = 0;
bridgeStats.playbackInputBytes = 0;
bridgeStats.playbackResampledBytes = 0;
@@ -133,6 +140,8 @@ export function createAudioEngine(
reject: (error: Error) => void;
settled: boolean;
} | null;
sawFirstMicChunk: boolean;
sawFirstVolumeEvent: boolean;
destroyed: boolean;
} = {
initialized: false,
@@ -142,6 +151,8 @@ export function createAudioEngine(
processingQueue: false,
playbackTimeout: null,
activePlayback: null,
sawFirstMicChunk: false,
sawFirstVolumeEvent: false,
destroyed: false,
};
@@ -152,6 +163,12 @@ export function createAudioEngine(
return;
}
const pcm = event.data as Uint8Array;
if (!refs.sawFirstMicChunk) {
refs.sawFirstMicChunk = true;
console.log(
`[AudioEngine.native#${instanceId}] firstMicChunk bytes=${pcm.byteLength} head=${toHexPreview(pcm)}`
);
}
bridgeStats.captureEvents += 1;
bridgeStats.captureBytes += pcm.byteLength;
maybeFlushBridgeStats("capture");
@@ -165,6 +182,15 @@ export function createAudioEngine(
return;
}
const level = refs.muted ? 0 : event.data;
bridgeStats.volumeEvents += 1;
bridgeStats.volumeMax = Math.max(bridgeStats.volumeMax, level);
if (!refs.sawFirstVolumeEvent) {
refs.sawFirstVolumeEvent = true;
console.log(
`[AudioEngine.native#${instanceId}] firstInputVolume level=${level.toFixed(3)} muted=${refs.muted}`
);
}
maybeFlushBridgeStats("volume");
callbacks.onVolumeLevel(level);
}
);
@@ -190,8 +216,14 @@ export function createAudioEngine(
async function ensureMicrophonePermission(): Promise<void> {
let permission = await native.getMicrophonePermissionsAsync().catch(() => null);
console.log(
`[AudioEngine.native#${instanceId}] microphonePermission initial=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}`
);
if (!permission?.granted) {
permission = await native.requestMicrophonePermissionsAsync().catch(() => null);
console.log(
`[AudioEngine.native#${instanceId}] microphonePermission requested=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}`
);
}
if (!permission?.granted) {
throw new Error(
@@ -307,14 +339,21 @@ export function createAudioEngine(
async startCapture() {
if (refs.captureActive) {
console.log(`[AudioEngine.native#${instanceId}] startCapture skipped: already active`);
return;
}
try {
console.log(`[AudioEngine.native#${instanceId}] startCapture begin`);
await ensureMicrophonePermission();
await ensureInitialized();
native.toggleRecording(true);
refs.sawFirstMicChunk = false;
refs.sawFirstVolumeEvent = false;
const isRecording = native.toggleRecording(true);
refs.captureActive = true;
console.log(
`[AudioEngine.native#${instanceId}] startCapture toggleRecording(true) => ${String(isRecording)}`
);
} catch (error) {
const wrapped = error instanceof Error ? error : new Error(String(error));
callbacks.onError?.(wrapped);
@@ -324,7 +363,10 @@ export function createAudioEngine(
async stopCapture() {
if (refs.captureActive) {
native.toggleRecording(false);
const isRecording = native.toggleRecording(false);
console.log(
`[AudioEngine.native#${instanceId}] stopCapture toggleRecording(false) => ${String(isRecording)}`
);
}
refs.captureActive = false;
refs.muted = false;
packages/app/src/voice/voice-runtime.test.ts
+34 -1
View File
@@ -268,6 +268,40 @@ describe("voice runtime", () => {
expect(engine.play).toHaveBeenCalled();
});
it("does not restart the thinking tone from local detection jitter while waiting", async () => {
const adapter = createSessionAdapter();
let resolvePlay!: (duration: number) => void;
const engine = createAudioEngineMock();
vi.mocked(engine.play).mockImplementation(
() =>
new Promise<number>((resolve) => {
resolvePlay = resolve;
})
);
const { runtime } = createRuntime({ engine });
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
expect(runtime.getSnapshot().phase).toBe("waiting");
expect(engine.play).toHaveBeenCalledTimes(1);
runtime.handleCaptureVolume(REALTIME_VOICE_VAD_CONFIG.volumeThreshold + 0.05);
runtime.handleCaptureVolume(0);
expect(engine.stop).not.toHaveBeenCalled();
expect(engine.clearQueue).not.toHaveBeenCalled();
expect(engine.play).toHaveBeenCalledTimes(1);
runtime.onServerSpeechStateChanged("server-1", true);
expect(engine.stop).toHaveBeenCalledTimes(1);
expect(engine.clearQueue).toHaveBeenCalledTimes(1);
resolvePlay(0.1);
});
it("returns to listening after assistant playback once the turn is complete", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
@@ -308,7 +342,6 @@ describe("voice runtime", () => {
expect(runtime.getTelemetrySnapshot()).toMatchObject({
volume: expect.any(Number),
isDetecting: true,
isSpeaking: false,
});
packages/app/src/voice/voice-runtime.ts
+41 -21
View File
@@ -45,7 +45,6 @@ export interface VoiceRuntimeSnapshot {
export interface VoiceRuntimeTelemetrySnapshot {
volume: number;
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
}
@@ -85,7 +84,9 @@ interface RuntimeState {
generation: number;
segmentDurationTimer: ReturnType<typeof setInterval> | null;
lastDisplayVolumePublishMs: number;
localSpeechStartedAt: number | null;
serverSpeechStartedAt: number | null;
lastNoServerSpeechLogMs: number;
localAboveThresholdActive: boolean;
}
type AudioOutputPayload = Extract<
@@ -149,7 +150,6 @@ const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
@@ -176,7 +176,6 @@ function telemetryEqual(
): boolean {
return (
left.volume === right.volume &&
left.isDetecting === right.isDetecting &&
left.isSpeaking === right.isSpeaking &&
left.segmentDuration === right.segmentDuration
);
@@ -219,7 +218,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
generation: 0,
segmentDurationTimer: null,
lastDisplayVolumePublishMs: 0,
localSpeechStartedAt: null,
serverSpeechStartedAt: null,
lastNoServerSpeechLogMs: 0,
localAboveThresholdActive: false,
};
const playback: RuntimePlaybackState = {
groups: new Map(),
@@ -502,7 +503,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
function reconcileSegmentDurationTimer(): void {
if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) {
if (!state.serverSpeechDetected) {
clearSegmentDurationTimer();
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
return;
@@ -513,7 +514,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
state.segmentDurationTimer = setInterval(() => {
const startedAt = state.localSpeechStartedAt;
const startedAt = state.serverSpeechStartedAt;
patchTelemetry((prev) => ({
...prev,
segmentDuration: startedAt ? Date.now() - startedAt : 0,
@@ -525,7 +526,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return (
state.snapshot.isVoiceMode &&
state.snapshot.phase === "waiting" &&
!state.telemetry.isDetecting &&
!state.telemetry.isSpeaking
);
}
@@ -546,7 +546,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
function resetCaptureTelemetry(): void {
clearSegmentDurationTimer();
state.localSpeechStartedAt = null;
state.serverSpeechStartedAt = null;
patchTelemetry({ ...INITIAL_TELEMETRY });
}
@@ -621,6 +621,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
state.turnInProgress = false;
state.serverSpeechDetected = false;
state.lastDisplayVolumePublishMs = 0;
state.lastNoServerSpeechLogMs = 0;
state.localAboveThresholdActive = false;
uploader.reset();
resetCaptureTelemetry();
patchSnapshot({ ...INITIAL_SNAPSHOT });
@@ -744,6 +746,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
if (bridgeStats.captureEvents === 0) {
console.log(
`[VoiceRuntime#${instanceId}] firstCapturePcm bytes=${chunk.byteLength} phase=${state.snapshot.phase} transportReady=${state.transportReady}`
);
}
uploader.pushPcmChunk(chunk);
},
@@ -752,10 +759,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const displayLevel = state.snapshot.isMuted ? 0 : level;
publishDisplayVolume(displayLevel, nowMs);
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
state.localSpeechStartedAt = null;
patchTelemetry((prev) => ({
...prev,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
}));
@@ -763,22 +768,34 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
}
const isActive = level > REALTIME_VOICE_VAD_CONFIG.volumeThreshold;
if (isActive && state.localSpeechStartedAt === null) {
state.localSpeechStartedAt = nowMs;
if (isActive && !state.localAboveThresholdActive) {
state.localAboveThresholdActive = true;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechActive level=${level.toFixed(3)} threshold=${REALTIME_VOICE_VAD_CONFIG.volumeThreshold.toFixed(3)} phase=${state.snapshot.phase} transportReady=${state.transportReady}`
);
}
if (!isActive) {
state.localSpeechStartedAt = null;
if (!isActive && state.localAboveThresholdActive) {
state.localAboveThresholdActive = false;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechInactive phase=${state.snapshot.phase} serverSpeaking=${state.serverSpeechDetected}`
);
}
if (
isActive &&
!state.serverSpeechDetected &&
nowMs - state.lastNoServerSpeechLogMs >= 1500
) {
state.lastNoServerSpeechLogMs = nowMs;
console.log(
`[VoiceRuntime#${instanceId}] localSpeechWithoutServerSpeech level=${level.toFixed(3)} phase=${state.snapshot.phase} turnInProgress=${state.turnInProgress} transportReady=${state.transportReady}`
);
}
patchTelemetry((prev) => ({
...prev,
isDetecting: isActive,
isSpeaking: state.serverSpeechDetected,
}));
reconcileSegmentDurationTimer();
if (!isActive) {
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
}
reconcileCue();
},
@@ -808,7 +825,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
let group = playback.groups.get(groupId);
if (!group) {
const shouldPlay = api.shouldPlayVoiceAudio(serverId);
console.log(`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isDetecting=${state.telemetry.isDetecting} isSpeaking=${state.telemetry.isSpeaking}`);
console.log(`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isSpeaking=${state.telemetry.isSpeaking}`);
group = {
groupId,
isVoiceMode: payload.isVoiceMode,
@@ -1040,8 +1057,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
return;
}
console.log(`[VoiceRuntime] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase}`);
console.log(
`[VoiceRuntime#${instanceId}] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase} volume=${state.telemetry.volume}`
);
state.serverSpeechDetected = isSpeaking;
state.serverSpeechStartedAt = isSpeaking ? (state.serverSpeechStartedAt ?? Date.now()) : null;
if (isSpeaking) {
resetPlaybackState();
deps.engine.stop();
packages/server/src/server/session.ts
+100 -13
View File
@@ -339,6 +339,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(PCM_BYTES_PER_MS * MIN_STREAMING_
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/
const AgentIdSchema = z.string().uuid()
const VOICE_MCP_SERVER_NAME = 'paseo_voice'
const VOICE_INTERRUPT_CONFIRMATION_MS = 500
type VoiceModeBaseConfig = {
systemPrompt?: string
@@ -530,10 +531,15 @@ export class Session {
// Voice mode state
private isVoiceMode = false
private speechInProgress = false
private pendingVoiceSpeechStartAt: number | null = null
private pendingVoiceSpeechTimer: NodeJS.Timeout | null = null
private readonly dictationStreamManager: DictationStreamManager
private readonly resolveVoiceTurnDetection: () => TurnDetectionProvider | null
private voiceTurnController: VoiceTurnController | null = null
private voiceInputChunkCount = 0
private voiceInputBytes = 0
private voiceInputWindowStartedAt = Date.now()
// Audio buffering for interruption handling
private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = []
@@ -2419,21 +2425,10 @@ export class Session {
},
callbacks: {
onSpeechStarted: async () => {
this.emit({
type: 'voice_input_state',
payload: {
isSpeaking: true,
},
})
await this.handleVoiceSpeechStart()
this.handleProvisionalVoiceSpeechStarted()
},
onSpeechStopped: async () => {
this.emit({
type: 'voice_input_state',
payload: {
isSpeaking: false,
},
})
this.handleVoiceSpeechStopped()
},
onError: (error) => {
this.sessionLogger.error({ err: error }, 'Voice turn controller failed')
@@ -2452,11 +2447,72 @@ export class Session {
return
}
this.clearPendingVoiceSpeechStart('turn-controller-stop')
const controller = this.voiceTurnController
this.voiceTurnController = null
await controller.stop()
}
private clearPendingVoiceSpeechStart(reason: string): void {
if (this.pendingVoiceSpeechTimer) {
clearTimeout(this.pendingVoiceSpeechTimer)
this.pendingVoiceSpeechTimer = null
}
if (this.pendingVoiceSpeechStartAt !== null) {
this.sessionLogger.debug({ reason }, 'Clearing provisional voice speech start')
this.pendingVoiceSpeechStartAt = null
}
}
private handleProvisionalVoiceSpeechStarted(): void {
if (this.speechInProgress || this.pendingVoiceSpeechTimer) {
return
}
const startedAt = Date.now()
this.pendingVoiceSpeechStartAt = startedAt
this.sessionLogger.info(
{ confirmationMs: VOICE_INTERRUPT_CONFIRMATION_MS },
'Silero VAD provisional speech_started'
)
this.pendingVoiceSpeechTimer = setTimeout(() => {
this.pendingVoiceSpeechTimer = null
if (this.pendingVoiceSpeechStartAt !== startedAt || this.speechInProgress) {
return
}
this.pendingVoiceSpeechStartAt = null
this.sessionLogger.info('voice_input_state emitting isSpeaking=true')
this.emit({
type: 'voice_input_state',
payload: {
isSpeaking: true,
},
})
void this.handleVoiceSpeechStart()
}, VOICE_INTERRUPT_CONFIRMATION_MS)
}
private handleVoiceSpeechStopped(): void {
if (this.pendingVoiceSpeechStartAt !== null) {
const durationMs = Date.now() - this.pendingVoiceSpeechStartAt
this.clearPendingVoiceSpeechStart('speech-stopped-before-confirmation')
this.sessionLogger.info(
{ durationMs, confirmationMs: VOICE_INTERRUPT_CONFIRMATION_MS },
'Ignoring provisional voice speech start that ended before confirmation'
)
return
}
this.sessionLogger.info('voice_input_state emitting isSpeaking=false')
this.emit({
type: 'voice_input_state',
payload: {
isSpeaking: false,
},
})
}
/**
* Handle text message to agent (with optional image attachments)
*/
@@ -6317,6 +6373,36 @@ export class Session {
if (!this.voiceTurnController) {
throw new Error('Voice mode is enabled but the voice turn controller is not running')
}
const chunkBytes = Buffer.byteLength(msg.audio, 'base64')
this.voiceInputChunkCount += 1
this.voiceInputBytes += chunkBytes
if (this.voiceInputChunkCount === 1) {
this.sessionLogger.info(
{
format: chunkFormat,
audioBytes: chunkBytes,
},
'Received first voice_audio_chunk for active voice mode'
)
}
const now = Date.now()
if (
this.voiceInputChunkCount % 50 === 0 ||
now - this.voiceInputWindowStartedAt >= 1000
) {
this.sessionLogger.info(
{
chunkCount: this.voiceInputChunkCount,
audioBytes: this.voiceInputBytes,
windowMs: now - this.voiceInputWindowStartedAt,
format: chunkFormat,
},
'Voice input chunk summary'
)
this.voiceInputWindowStartedAt = now
this.voiceInputChunkCount = 0
this.voiceInputBytes = 0
}
await this.voiceTurnController.appendClientChunk({
audioBase64: msg.audio,
format: chunkFormat,
@@ -6772,6 +6858,7 @@ export class Session {
* Clear speech-in-progress flag once the user turn has completed
*/
private clearSpeechInProgress(reason: string): void {
this.clearPendingVoiceSpeechStart(`clear-speech-in-progress:${reason}`)
if (!this.speechInProgress) {
return
}
packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts
+1 -1
View File
@@ -9,7 +9,7 @@ const DEFAULT_SAMPLE_RATE = 16000;
const DEFAULT_BUFFER_SIZE_SECONDS = 60;
const DEFAULT_SILERO_THRESHOLD = 0.5;
const DEFAULT_MIN_SILENCE_DURATION = 1.2;
const DEFAULT_MIN_SPEECH_DURATION = 0.12;
const DEFAULT_MIN_SPEECH_DURATION = 0.2;
const DEFAULT_WINDOW_SIZE = 512;
type SherpaVadHandle = {