From d1632d80e3fd72d3c9af4693d5547356cb4c691a Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sat, 14 Mar 2026 20:05:48 +0700 Subject: [PATCH] refactor(voice): defer speech confirmation and add diagnostics --- docs/VOICE_TURN_DETECTION_PLAN.md | 581 ------------------ .../app/src/components/dictation-controls.tsx | 2 - .../src/components/realtime-voice-overlay.tsx | 3 +- .../components/voice-compact-indicator.tsx | 3 +- packages/app/src/components/voice-panel.tsx | 3 +- packages/app/src/components/volume-meter.tsx | 1 - packages/app/src/contexts/voice-context.tsx | 1 - packages/app/src/voice/audio-engine.native.ts | 46 +- packages/app/src/voice/voice-runtime.test.ts | 35 +- packages/app/src/voice/voice-runtime.ts | 62 +- packages/server/src/server/session.ts | 113 +++- .../local/sherpa/silero-vad-session.ts | 2 +- windows-startup-refactor-plan.md | 161 ----- 13 files changed, 223 insertions(+), 790 deletions(-) delete mode 100644 docs/VOICE_TURN_DETECTION_PLAN.md delete mode 100644 windows-startup-refactor-plan.md diff --git a/docs/VOICE_TURN_DETECTION_PLAN.md b/docs/VOICE_TURN_DETECTION_PLAN.md deleted file mode 100644 index d916d5af1..000000000 --- a/docs/VOICE_TURN_DETECTION_PLAN.md +++ /dev/null @@ -1,581 +0,0 @@ -# Local Voice Turn Detection Plan - -## Goal - -For local realtime voice, the client should stream PCM continuously to the server. The server should own turn detection using bundled Sherpa + Silero VAD. Client-side logic stays only for meter/activation visuals and is not authoritative for turn boundaries. - -The server should start buffering immediately so we do not clip the beginning of speech. - -## Scope - -This plan is intentionally local-only for now. - -- Turn detection: local only -- STT: local only -- TTS: unchanged -- Client VAD: removed as an authority, retained only as UI metering/activation - -## Design Summary - -### Boundaries - -- Client - - Captures PCM - - Streams fixed-size chunks continuously during voice mode - - Shows volume meter / activation visuals - - Does not decide turn start or turn end -- Server turn detection - - Consumes live PCM stream - - Runs local Silero VAD through Sherpa - - Emits normalized events: - - `speech_started` - - `speech_stopped` -- Server voice turn controller - - Owns barge-in - - Owns rolling pre-speech buffer - - Owns utterance lifecycle - - Feeds utterance audio into the existing STT path -- Server STT - - Transcribes utterance audio selected by turn detection - -## Core Principle - -Do not hide VAD inside STT. - -If STT owns VAD, the abstraction gets coupled again. Turn detection and transcription are different responsibilities and should be modeled separately. - -Bad shape: - -```ts -interface SpeechToTextProvider { - createSession(): { - appendPcm16(chunk: Buffer): void; - commit(): void; - on(event: "speech_started" | "speech_stopped" | "transcript", handler: unknown): unknown; - }; -} -``` - -Better shape: - -```ts -interface TurnDetectionProvider { - createSession(input: { - logger: pino.Logger; - }): TurnDetectionSession; -} - -interface TurnDetectionSession { - requiredSampleRate: number; - connect(): Promise; - appendPcm16(chunk: Buffer): void; - close(): void; - - on(event: "speech_started", handler: () => void): unknown; - on(event: "speech_stopped", handler: () => void): unknown; - on(event: "error", handler: (err: unknown) => void): unknown; -} -``` - -And keep STT separate: - -```ts -interface SpeechToTextProvider { - id: "local" | "openai" | (string & {}); - createSession(input: { - logger: pino.Logger; - language?: string; - prompt?: string; - }): StreamingTranscriptionSession; -} -``` - -## Config Shape - -Even in a local-only rollout, the boundary should be explicit now so it does not need a second refactor later. - -Current shape: - -```ts -type RequestedSpeechProviders = { - dictationStt: RequestedSpeechProvider; - voiceStt: RequestedSpeechProvider; - voiceTts: RequestedSpeechProvider; -}; -``` - -Proposed shape: - -```ts -type RequestedSpeechProviders = { - dictationStt: RequestedSpeechProvider; - voiceTurnDetection: RequestedSpeechProvider; - voiceStt: RequestedSpeechProvider; - voiceTts: RequestedSpeechProvider; -}; -``` - -For the first implementation, resolution can still force local: - -```ts -const providers = { - dictationStt: { provider: "local", explicit: false, enabled: true }, - voiceTurnDetection: { provider: "local", explicit: false, enabled: true }, - voiceStt: { provider: "local", explicit: false, enabled: true }, - voiceTts: { provider: "local", explicit: false, enabled: true }, -}; -``` - -## Runtime Shape - -### 1. Local Turn Detection Provider - -Add a local provider parallel to local STT/TTS runtime wiring. - -Suggested files: - -- `packages/server/src/server/speech/turn-detection-provider.ts` -- `packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts` -- `packages/server/src/server/speech/providers/local/sherpa/silero-vad-provider.ts` - -Expanded local runtime output: - -```ts -interface InitializedLocalSpeech { - sttService: SpeechToTextProvider | null; - ttsService: TextToSpeechProvider | null; - dictationSttService: SpeechToTextProvider | null; - turnDetectionService: TurnDetectionProvider | null; - cleanup: () => void; -} -``` - -Provider construction in local runtime: - -```ts -if ( - providers.voiceTurnDetection.enabled !== false && - providers.voiceTurnDetection.provider === "local" -) { - turnDetectionService = new SherpaSileroTurnDetectionProvider( - { - modelPath: bundledSileroModelPath, - sampleRate: 16000, - }, - logger - ); -} -``` - -### 2. Voice Turn Controller - -This should sit in the server session layer and own the utterance state machine. - -Suggested file: - -- `packages/server/src/server/voice/voice-turn-controller.ts` - -Responsibilities: - -- receive continuous PCM chunks -- write chunks into a rolling pre-speech buffer -- feed chunks into detector -- on `speech_started` - - perform barge-in - - start a new utterance - - flush pre-speech buffer into utterance -- while speech is active - - append live chunks into utterance -- on `speech_stopped` - - finalize utterance into STT - -Public shape: - -```ts -interface VoiceTurnController { - start(): Promise; - stop(): Promise; - appendClientChunk(input: { - audioBase64: string; - format: string; - }): Promise; -} -``` - -Internal state should be explicit: - -```ts -type VoiceInputState = - | { status: "idle" } - | { - status: "listening"; - rollingPrefixBytes: number; - } - | { - status: "capturing"; - utteranceId: string; - rollingPrefixBytes: number; - utteranceBytes: number; - }; -``` - -That is better than several booleans like `speechInProgress`, `hasStarted`, or `awaitingStop`. - -## Immediate Buffering Strategy - -Because buffering starts immediately, the detector does not decide whether audio is kept at all. It decides when buffered audio becomes an utterance. - -### Rolling Prefix Buffer - -Maintain a small ring buffer of recent PCM before speech start. - -Suggested defaults: - -- sample rate: `16000` -- mono PCM16 -- prefix duration: `300ms` to `500ms` - -Example utility: - -```ts -interface RollingPcmBuffer { - append(chunk: Buffer): void; - drain(): Buffer; - clear(): void; -} -``` - -Example implementation shape: - -```ts -class FixedDurationPcmRingBuffer { - private chunks: Buffer[] = []; - private totalBytes = 0; - - constructor(private readonly maxBytes: number) {} - - append(chunk: Buffer): void { - if (chunk.length === 0) { - return; - } - - this.chunks.push(chunk); - this.totalBytes += chunk.length; - - while (this.totalBytes > this.maxBytes && this.chunks.length > 0) { - const removed = this.chunks.shift()!; - this.totalBytes -= removed.length; - } - } - - drain(): Buffer { - return Buffer.concat(this.chunks); - } - - clear(): void { - this.chunks = []; - this.totalBytes = 0; - } -} -``` - -### Flow - -Per incoming chunk: - -```ts -prefixBuffer.append(chunk); -detector.appendPcm16(chunk); - -if (state.status === "capturing") { - utteranceChunks.push(chunk); -} -``` - -On speech start: - -```ts -async function onSpeechStarted(): Promise { - await interruptPlaybackAndAgentRun(); - - const prefix = prefixBuffer.drain(); - const utteranceChunks = prefix.length > 0 ? [prefix] : []; - - state = { - status: "capturing", - utteranceId: uuidv4(), - rollingPrefixBytes: prefix.length, - utteranceBytes: prefix.length, - }; - - activeUtteranceChunks = utteranceChunks; -} -``` - -On speech stop: - -```ts -async function onSpeechStopped(): Promise { - if (state.status !== "capturing") { - return; - } - - const pcm16 = Buffer.concat(activeUtteranceChunks); - - await submitUtterance({ - pcm16, - sampleRate: 16000, - format: "audio/pcm;rate=16000;bits=16", - }); - - activeUtteranceChunks = []; - state = { - status: "listening", - rollingPrefixBytes: 0, - }; -} -``` - -## STT Integration - -Do not make the turn controller talk to STT one chunk at a time unless there is a strong need. - -For the first version, the simplest shape is: - -- detector runs on the continuous stream -- utterance audio is accumulated in memory -- on stop, the utterance goes through the existing internal voice dictation path - -That minimizes refactor risk and keeps the first migration focused on authority and layering. - -Boundary: - -```ts -interface VoiceUtteranceSink { - submitUtterance(input: { - pcm16: Buffer; - sampleRate: number; - format: string; - }): Promise; -} -``` - -Then `session.ts` can implement that sink using the existing voice dictation internals. - -## Session Refactor - -Current problem: - -- raw `voice_audio_chunk` handling triggers speech-start behavior too early -- the client remains the authority for `isLast` - -Target shape: - -```ts -private async handleAudioChunk( - msg: Extract -): Promise { - if (this.isVoiceMode) { - await this.voiceTurnController.appendClientChunk({ - audioBase64: msg.audio, - format: msg.format || "audio/pcm;rate=16000;bits=16", - }); - return; - } - - // existing non-voice path remains -} -``` - -And the controller calls back into the session: - -```ts -interface VoiceTurnControllerCallbacks { - onSpeechStarted(): Promise; - onUtteranceReady(input: { - pcm16: Buffer; - sampleRate: number; - format: string; - }): Promise; - onError(error: Error): void; -} -``` - -Session remains responsible for: - -- aborting agent/TTS -- submitting transcript text to the agent -- emitting logs - -The controller is responsible for: - -- audio and detector state -- prefix buffering -- utterance assembly - -## Client Refactor - -The app should stop using `SpeechSegmenter` as authoritative segmentation in voice mode. - -Instead, voice mode uses a continuous chunk sender: - -```ts -interface ContinuousVoiceUploader { - appendPcmChunk(chunk: Uint8Array): void; - flush(): Promise; -} -``` - -In `voice-runtime.ts`, the path becomes: - -```ts -handleCapturePcm(chunk) { - if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) { - return; - } - - continuousUploader.appendPcmChunk(chunk); -} -``` - -And UI activation remains independent: - -```ts -handleCaptureVolume(level) { - const nowMs = Date.now(); - publishDisplayVolume(level, nowMs); - - const isActive = level > DISPLAY_ACTIVE_THRESHOLD; - patchTelemetry((prev) => ({ - ...prev, - isSpeaking: isActive, - })); -} -``` - -The client should no longer use a “segmenter” abstraction for voice mode. The concept is now wrong for that layer. - -## Bundling Silero - -Silero VAD should be treated as a bundled runtime asset, not a user-downloaded speech model. - -Implications: - -- not listed in `paseo speech download` -- not part of user-facing missing-model readiness -- shipped with `packages/server` - -Example: - -```ts -function resolveBundledSileroVadModelPath(): string { - return fileURLToPath( - new URL("./assets/silero_vad.onnx", import.meta.url) - ); -} -``` - -That keeps the model an implementation detail of local realtime voice infrastructure. - -## What Not To Do In V1 - -- No semantic turn detection -- No mixed authorities -- No client-side fallback turn detection -- No detector logic buried inside dictation manager -- No detector-specific branching spread through session logic - -If local VAD is the authority, then only local VAD decides `speech_started` and `speech_stopped`. - -## Verification Plan - -### Unit Tests - -`silero-vad-session.test.ts` - -- emits `speech_started` after speech frames -- emits `speech_stopped` after silence frames -- does not emit duplicate starts/stops - -`fixed-duration-pcm-ring-buffer.test.ts` - -- retains only configured prefix window -- drains prefix in correct order - -`voice-turn-controller.test.ts` - -- buffers before speech start -- includes prefix in utterance on start -- finalizes one utterance on stop -- does not barge in on silence-only chunks - -### Integration Tests - -`session.voice-turn-control.test.ts` - -- continuous voice chunks do not immediately interrupt agent output -- detector start triggers barge-in -- detector stop triggers one finalized transcript submission - -`voice-local-agent.e2e.test.ts` - -- local voice mode works with continuous client upload -- leading words are preserved because prefix buffering is included - -### Regression - -- dictation stream tests remain unchanged -- TTS playback tests remain unchanged -- app voice UI tests still verify meter/activation behavior - -## Ordered Implementation Plan - -1. Add server-side turn detection interfaces and config type. -2. Add bundled local Silero provider via Sherpa. -3. Add rolling prefix buffer utility. -4. Add `VoiceTurnController`. -5. Refactor `session.ts` voice path to use controller callbacks instead of client `isLast`. -6. Replace app voice segmentation with continuous PCM upload. -7. Keep app meter/activation visuals only. -8. Add tests bottom-up. -9. Run `npm run typecheck`. - -## Suggested Canonical Types - -```ts -interface DetectedVoiceUtterance { - pcm16: Buffer; - sampleRate: number; - startedAt: number; - endedAt: number; -} -``` - -```ts -interface VoiceTurnControllerCallbacks { - onSpeechStarted(): Promise; - onUtteranceReady(utterance: DetectedVoiceUtterance): Promise; - onError(error: Error): void; -} -``` - -```ts -interface TurnDetectionProvider { - id: "local" | "openai" | (string & {}); - createSession(input: { - logger: pino.Logger; - }): TurnDetectionSession; -} -``` - -## Recommendation - -For v1, optimize for correctness and clean layering rather than minimum diff size: - -- continuous upload from client -- bundled local Silero detector on server -- rolling prefix buffer -- detector-owned turn boundaries -- existing STT pipeline reused as utterance sink - -That keeps responsibilities clear and avoids inventing a leaky “smart STT session” abstraction. diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index 091b9c462..37f27b032 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -69,7 +69,6 @@ export function DictationControls({ @@ -177,7 +176,6 @@ export function DictationOverlay({ diff --git a/packages/app/src/components/voice-compact-indicator.tsx b/packages/app/src/components/voice-compact-indicator.tsx index fc14cb72e..e70a8da4f 100644 --- a/packages/app/src/components/voice-compact-indicator.tsx +++ b/packages/app/src/components/voice-compact-indicator.tsx @@ -6,7 +6,7 @@ import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context"; export function VoiceCompactIndicator() { const { theme } = useUnistyles(); - const { volume, isDetecting, isSpeaking } = useVoiceTelemetry(); + const { volume, isSpeaking } = useVoiceTelemetry(); const { isVoiceMode, isVoiceSwitching, @@ -24,7 +24,6 @@ export function VoiceCompactIndicator() { ${bridgeStats.playbackResampledBytes}B ` + `playMs=${bridgeStats.playbackDurationMs.toFixed(1)} ` + `windowMs=${elapsedMs}` @@ -115,6 +120,8 @@ export function createAudioEngine( bridgeStats.windowStartedAtMs = now; bridgeStats.captureEvents = 0; bridgeStats.captureBytes = 0; + bridgeStats.volumeEvents = 0; + bridgeStats.volumeMax = 0; bridgeStats.playbackEvents = 0; bridgeStats.playbackInputBytes = 0; bridgeStats.playbackResampledBytes = 0; @@ -133,6 +140,8 @@ export function createAudioEngine( reject: (error: Error) => void; settled: boolean; } | null; + sawFirstMicChunk: boolean; + sawFirstVolumeEvent: boolean; destroyed: boolean; } = { initialized: false, @@ -142,6 +151,8 @@ export function createAudioEngine( processingQueue: false, playbackTimeout: null, activePlayback: null, + sawFirstMicChunk: false, + sawFirstVolumeEvent: false, destroyed: false, }; @@ -152,6 +163,12 @@ export function createAudioEngine( return; } const pcm = event.data as Uint8Array; + if (!refs.sawFirstMicChunk) { + refs.sawFirstMicChunk = true; + console.log( + `[AudioEngine.native#${instanceId}] firstMicChunk bytes=${pcm.byteLength} head=${toHexPreview(pcm)}` + ); + } bridgeStats.captureEvents += 1; bridgeStats.captureBytes += pcm.byteLength; maybeFlushBridgeStats("capture"); @@ -165,6 +182,15 @@ export function createAudioEngine( return; } const level = refs.muted ? 0 : event.data; + bridgeStats.volumeEvents += 1; + bridgeStats.volumeMax = Math.max(bridgeStats.volumeMax, level); + if (!refs.sawFirstVolumeEvent) { + refs.sawFirstVolumeEvent = true; + console.log( + `[AudioEngine.native#${instanceId}] firstInputVolume level=${level.toFixed(3)} muted=${refs.muted}` + ); + } + maybeFlushBridgeStats("volume"); callbacks.onVolumeLevel(level); } ); @@ -190,8 +216,14 @@ export function createAudioEngine( async function ensureMicrophonePermission(): Promise { let permission = await native.getMicrophonePermissionsAsync().catch(() => null); + console.log( + `[AudioEngine.native#${instanceId}] microphonePermission initial=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}` + ); if (!permission?.granted) { permission = await native.requestMicrophonePermissionsAsync().catch(() => null); + console.log( + `[AudioEngine.native#${instanceId}] microphonePermission requested=${permission?.status ?? "unknown"} granted=${String(permission?.granted ?? false)}` + ); } if (!permission?.granted) { throw new Error( @@ -307,14 +339,21 @@ export function createAudioEngine( async startCapture() { if (refs.captureActive) { + console.log(`[AudioEngine.native#${instanceId}] startCapture skipped: already active`); return; } try { + console.log(`[AudioEngine.native#${instanceId}] startCapture begin`); await ensureMicrophonePermission(); await ensureInitialized(); - native.toggleRecording(true); + refs.sawFirstMicChunk = false; + refs.sawFirstVolumeEvent = false; + const isRecording = native.toggleRecording(true); refs.captureActive = true; + console.log( + `[AudioEngine.native#${instanceId}] startCapture toggleRecording(true) => ${String(isRecording)}` + ); } catch (error) { const wrapped = error instanceof Error ? error : new Error(String(error)); callbacks.onError?.(wrapped); @@ -324,7 +363,10 @@ export function createAudioEngine( async stopCapture() { if (refs.captureActive) { - native.toggleRecording(false); + const isRecording = native.toggleRecording(false); + console.log( + `[AudioEngine.native#${instanceId}] stopCapture toggleRecording(false) => ${String(isRecording)}` + ); } refs.captureActive = false; refs.muted = false; diff --git a/packages/app/src/voice/voice-runtime.test.ts b/packages/app/src/voice/voice-runtime.test.ts index 7d6c79737..a6a2eb787 100644 --- a/packages/app/src/voice/voice-runtime.test.ts +++ b/packages/app/src/voice/voice-runtime.test.ts @@ -268,6 +268,40 @@ describe("voice runtime", () => { expect(engine.play).toHaveBeenCalled(); }); + it("does not restart the thinking tone from local detection jitter while waiting", async () => { + const adapter = createSessionAdapter(); + let resolvePlay!: (duration: number) => void; + const engine = createAudioEngineMock(); + vi.mocked(engine.play).mockImplementation( + () => + new Promise((resolve) => { + resolvePlay = resolve; + }) + ); + const { runtime } = createRuntime({ engine }); + runtime.registerSession(adapter); + + await runtime.startVoice("server-1", "agent-1"); + runtime.onTurnEvent("server-1", "agent-1", "turn_started"); + + expect(runtime.getSnapshot().phase).toBe("waiting"); + expect(engine.play).toHaveBeenCalledTimes(1); + + runtime.handleCaptureVolume(REALTIME_VOICE_VAD_CONFIG.volumeThreshold + 0.05); + runtime.handleCaptureVolume(0); + + expect(engine.stop).not.toHaveBeenCalled(); + expect(engine.clearQueue).not.toHaveBeenCalled(); + expect(engine.play).toHaveBeenCalledTimes(1); + + runtime.onServerSpeechStateChanged("server-1", true); + + expect(engine.stop).toHaveBeenCalledTimes(1); + expect(engine.clearQueue).toHaveBeenCalledTimes(1); + + resolvePlay(0.1); + }); + it("returns to listening after assistant playback once the turn is complete", async () => { const adapter = createSessionAdapter(); const { runtime, engine } = createRuntime(); @@ -308,7 +342,6 @@ describe("voice runtime", () => { expect(runtime.getTelemetrySnapshot()).toMatchObject({ volume: expect.any(Number), - isDetecting: true, isSpeaking: false, }); diff --git a/packages/app/src/voice/voice-runtime.ts b/packages/app/src/voice/voice-runtime.ts index 45455ef63..dd9186f32 100644 --- a/packages/app/src/voice/voice-runtime.ts +++ b/packages/app/src/voice/voice-runtime.ts @@ -45,7 +45,6 @@ export interface VoiceRuntimeSnapshot { export interface VoiceRuntimeTelemetrySnapshot { volume: number; - isDetecting: boolean; isSpeaking: boolean; segmentDuration: number; } @@ -85,7 +84,9 @@ interface RuntimeState { generation: number; segmentDurationTimer: ReturnType | null; lastDisplayVolumePublishMs: number; - localSpeechStartedAt: number | null; + serverSpeechStartedAt: number | null; + lastNoServerSpeechLogMs: number; + localAboveThresholdActive: boolean; } type AudioOutputPayload = Extract< @@ -149,7 +150,6 @@ const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = { const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = { volume: 0, - isDetecting: false, isSpeaking: false, segmentDuration: 0, }; @@ -176,7 +176,6 @@ function telemetryEqual( ): boolean { return ( left.volume === right.volume && - left.isDetecting === right.isDetecting && left.isSpeaking === right.isSpeaking && left.segmentDuration === right.segmentDuration ); @@ -219,7 +218,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { generation: 0, segmentDurationTimer: null, lastDisplayVolumePublishMs: 0, - localSpeechStartedAt: null, + serverSpeechStartedAt: null, + lastNoServerSpeechLogMs: 0, + localAboveThresholdActive: false, }; const playback: RuntimePlaybackState = { groups: new Map(), @@ -502,7 +503,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { } function reconcileSegmentDurationTimer(): void { - if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) { + if (!state.serverSpeechDetected) { clearSegmentDurationTimer(); patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 })); return; @@ -513,7 +514,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { } state.segmentDurationTimer = setInterval(() => { - const startedAt = state.localSpeechStartedAt; + const startedAt = state.serverSpeechStartedAt; patchTelemetry((prev) => ({ ...prev, segmentDuration: startedAt ? Date.now() - startedAt : 0, @@ -525,7 +526,6 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { return ( state.snapshot.isVoiceMode && state.snapshot.phase === "waiting" && - !state.telemetry.isDetecting && !state.telemetry.isSpeaking ); } @@ -546,7 +546,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { function resetCaptureTelemetry(): void { clearSegmentDurationTimer(); - state.localSpeechStartedAt = null; + state.serverSpeechStartedAt = null; patchTelemetry({ ...INITIAL_TELEMETRY }); } @@ -621,6 +621,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { state.turnInProgress = false; state.serverSpeechDetected = false; state.lastDisplayVolumePublishMs = 0; + state.lastNoServerSpeechLogMs = 0; + state.localAboveThresholdActive = false; uploader.reset(); resetCaptureTelemetry(); patchSnapshot({ ...INITIAL_SNAPSHOT }); @@ -744,6 +746,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) { return; } + if (bridgeStats.captureEvents === 0) { + console.log( + `[VoiceRuntime#${instanceId}] firstCapturePcm bytes=${chunk.byteLength} phase=${state.snapshot.phase} transportReady=${state.transportReady}` + ); + } uploader.pushPcmChunk(chunk); }, @@ -752,10 +759,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { const displayLevel = state.snapshot.isMuted ? 0 : level; publishDisplayVolume(displayLevel, nowMs); if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) { - state.localSpeechStartedAt = null; patchTelemetry((prev) => ({ ...prev, - isDetecting: false, isSpeaking: false, segmentDuration: 0, })); @@ -763,22 +768,34 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { } const isActive = level > REALTIME_VOICE_VAD_CONFIG.volumeThreshold; - if (isActive && state.localSpeechStartedAt === null) { - state.localSpeechStartedAt = nowMs; + if (isActive && !state.localAboveThresholdActive) { + state.localAboveThresholdActive = true; + console.log( + `[VoiceRuntime#${instanceId}] localSpeechActive level=${level.toFixed(3)} threshold=${REALTIME_VOICE_VAD_CONFIG.volumeThreshold.toFixed(3)} phase=${state.snapshot.phase} transportReady=${state.transportReady}` + ); } - if (!isActive) { - state.localSpeechStartedAt = null; + if (!isActive && state.localAboveThresholdActive) { + state.localAboveThresholdActive = false; + console.log( + `[VoiceRuntime#${instanceId}] localSpeechInactive phase=${state.snapshot.phase} serverSpeaking=${state.serverSpeechDetected}` + ); + } + if ( + isActive && + !state.serverSpeechDetected && + nowMs - state.lastNoServerSpeechLogMs >= 1500 + ) { + state.lastNoServerSpeechLogMs = nowMs; + console.log( + `[VoiceRuntime#${instanceId}] localSpeechWithoutServerSpeech level=${level.toFixed(3)} phase=${state.snapshot.phase} turnInProgress=${state.turnInProgress} transportReady=${state.transportReady}` + ); } patchTelemetry((prev) => ({ ...prev, - isDetecting: isActive, isSpeaking: state.serverSpeechDetected, })); reconcileSegmentDurationTimer(); - if (!isActive) { - patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 })); - } reconcileCue(); }, @@ -808,7 +825,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { let group = playback.groups.get(groupId); if (!group) { const shouldPlay = api.shouldPlayVoiceAudio(serverId); - console.log(`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isDetecting=${state.telemetry.isDetecting} isSpeaking=${state.telemetry.isSpeaking}`); + console.log(`[VoiceRuntime] new group=${groupId} shouldPlay=${shouldPlay} phase=${state.snapshot.phase} isSpeaking=${state.telemetry.isSpeaking}`); group = { groupId, isVoiceMode: payload.isVoiceMode, @@ -1040,8 +1057,11 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { return; } - console.log(`[VoiceRuntime] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase}`); + console.log( + `[VoiceRuntime#${instanceId}] onServerSpeechStateChanged isSpeaking=${isSpeaking} phase=${state.snapshot.phase} volume=${state.telemetry.volume}` + ); state.serverSpeechDetected = isSpeaking; + state.serverSpeechStartedAt = isSpeaking ? (state.serverSpeechStartedAt ?? Date.now()) : null; if (isSpeaking) { resetPlaybackState(); deps.engine.stop(); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 6bafc1295..6446ab0dd 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -339,6 +339,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(PCM_BYTES_PER_MS * MIN_STREAMING_ const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/ const AgentIdSchema = z.string().uuid() const VOICE_MCP_SERVER_NAME = 'paseo_voice' +const VOICE_INTERRUPT_CONFIRMATION_MS = 500 type VoiceModeBaseConfig = { systemPrompt?: string @@ -530,10 +531,15 @@ export class Session { // Voice mode state private isVoiceMode = false private speechInProgress = false + private pendingVoiceSpeechStartAt: number | null = null + private pendingVoiceSpeechTimer: NodeJS.Timeout | null = null private readonly dictationStreamManager: DictationStreamManager private readonly resolveVoiceTurnDetection: () => TurnDetectionProvider | null private voiceTurnController: VoiceTurnController | null = null + private voiceInputChunkCount = 0 + private voiceInputBytes = 0 + private voiceInputWindowStartedAt = Date.now() // Audio buffering for interruption handling private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [] @@ -2419,21 +2425,10 @@ export class Session { }, callbacks: { onSpeechStarted: async () => { - this.emit({ - type: 'voice_input_state', - payload: { - isSpeaking: true, - }, - }) - await this.handleVoiceSpeechStart() + this.handleProvisionalVoiceSpeechStarted() }, onSpeechStopped: async () => { - this.emit({ - type: 'voice_input_state', - payload: { - isSpeaking: false, - }, - }) + this.handleVoiceSpeechStopped() }, onError: (error) => { this.sessionLogger.error({ err: error }, 'Voice turn controller failed') @@ -2452,11 +2447,72 @@ export class Session { return } + this.clearPendingVoiceSpeechStart('turn-controller-stop') const controller = this.voiceTurnController this.voiceTurnController = null await controller.stop() } + private clearPendingVoiceSpeechStart(reason: string): void { + if (this.pendingVoiceSpeechTimer) { + clearTimeout(this.pendingVoiceSpeechTimer) + this.pendingVoiceSpeechTimer = null + } + if (this.pendingVoiceSpeechStartAt !== null) { + this.sessionLogger.debug({ reason }, 'Clearing provisional voice speech start') + this.pendingVoiceSpeechStartAt = null + } + } + + private handleProvisionalVoiceSpeechStarted(): void { + if (this.speechInProgress || this.pendingVoiceSpeechTimer) { + return + } + + const startedAt = Date.now() + this.pendingVoiceSpeechStartAt = startedAt + this.sessionLogger.info( + { confirmationMs: VOICE_INTERRUPT_CONFIRMATION_MS }, + 'Silero VAD provisional speech_started' + ) + this.pendingVoiceSpeechTimer = setTimeout(() => { + this.pendingVoiceSpeechTimer = null + if (this.pendingVoiceSpeechStartAt !== startedAt || this.speechInProgress) { + return + } + + this.pendingVoiceSpeechStartAt = null + this.sessionLogger.info('voice_input_state emitting isSpeaking=true') + this.emit({ + type: 'voice_input_state', + payload: { + isSpeaking: true, + }, + }) + void this.handleVoiceSpeechStart() + }, VOICE_INTERRUPT_CONFIRMATION_MS) + } + + private handleVoiceSpeechStopped(): void { + if (this.pendingVoiceSpeechStartAt !== null) { + const durationMs = Date.now() - this.pendingVoiceSpeechStartAt + this.clearPendingVoiceSpeechStart('speech-stopped-before-confirmation') + this.sessionLogger.info( + { durationMs, confirmationMs: VOICE_INTERRUPT_CONFIRMATION_MS }, + 'Ignoring provisional voice speech start that ended before confirmation' + ) + return + } + + this.sessionLogger.info('voice_input_state emitting isSpeaking=false') + this.emit({ + type: 'voice_input_state', + payload: { + isSpeaking: false, + }, + }) + } + /** * Handle text message to agent (with optional image attachments) */ @@ -6317,6 +6373,36 @@ export class Session { if (!this.voiceTurnController) { throw new Error('Voice mode is enabled but the voice turn controller is not running') } + const chunkBytes = Buffer.byteLength(msg.audio, 'base64') + this.voiceInputChunkCount += 1 + this.voiceInputBytes += chunkBytes + if (this.voiceInputChunkCount === 1) { + this.sessionLogger.info( + { + format: chunkFormat, + audioBytes: chunkBytes, + }, + 'Received first voice_audio_chunk for active voice mode' + ) + } + const now = Date.now() + if ( + this.voiceInputChunkCount % 50 === 0 || + now - this.voiceInputWindowStartedAt >= 1000 + ) { + this.sessionLogger.info( + { + chunkCount: this.voiceInputChunkCount, + audioBytes: this.voiceInputBytes, + windowMs: now - this.voiceInputWindowStartedAt, + format: chunkFormat, + }, + 'Voice input chunk summary' + ) + this.voiceInputWindowStartedAt = now + this.voiceInputChunkCount = 0 + this.voiceInputBytes = 0 + } await this.voiceTurnController.appendClientChunk({ audioBase64: msg.audio, format: chunkFormat, @@ -6772,6 +6858,7 @@ export class Session { * Clear speech-in-progress flag once the user turn has completed */ private clearSpeechInProgress(reason: string): void { + this.clearPendingVoiceSpeechStart(`clear-speech-in-progress:${reason}`) if (!this.speechInProgress) { return } diff --git a/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts b/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts index dbe021d66..b5f82dc77 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts @@ -9,7 +9,7 @@ const DEFAULT_SAMPLE_RATE = 16000; const DEFAULT_BUFFER_SIZE_SECONDS = 60; const DEFAULT_SILERO_THRESHOLD = 0.5; const DEFAULT_MIN_SILENCE_DURATION = 1.2; -const DEFAULT_MIN_SPEECH_DURATION = 0.12; +const DEFAULT_MIN_SPEECH_DURATION = 0.2; const DEFAULT_WINDOW_SIZE = 512; type SherpaVadHandle = { diff --git a/windows-startup-refactor-plan.md b/windows-startup-refactor-plan.md deleted file mode 100644 index 663b9b151..000000000 --- a/windows-startup-refactor-plan.md +++ /dev/null @@ -1,161 +0,0 @@ -# Desktop Startup Refactor Plan - -## Goal - -1. Tauri runs `paseo start`, then polls `paseo daemon status --json` until ready -2. `daemon status --json` returns ALL needed data: serverId, listen address, hostname, status, pid -3. App takes that data and adds it to the registry like any other host -4. App blocks on startup splash until any host runtime goes online (race) -5. 30s timeout → redirect to welcome/add-connection screen - -## Core Principles - -- The daemon is autonomous. Tauri does not pass `--home`, `--listen`, or read any files. -- `daemon status --json` is the single source of truth. All data comes from it. -- The host registry has no concept of "managed" daemons. A host is just a serverId + connections. - ---- - -## Implementation Plan - -### Phase 1: Add serverId to `daemon status --json` - -**File: `packages/cli/src/commands/daemon/status.ts`** - -`DaemonStatus` currently has: `status`, `home`, `listen`, `pid`, `startedAt`, `owner`, `logPath`, etc. - -Add: -- `serverId: string | null` -- `hostname: string | null` (already available from pidInfo, just not in the output) - -The status command already calls `resolveLocalDaemonState()` which reads the pid file (has hostname) and resolves home. From home it can call `getOrCreateServerId(home)` to get the serverId. All internal to the status command — Tauri never touches these files. - -### Phase 2: Tauri — just start + poll status - -**File: `packages/desktop/src-tauri/src/runtime_manager.rs`** - -#### 2a. `start_managed_daemon_internal` becomes: - -1. Run `paseo start` (no `--home`, no `--listen`) -2. Poll `paseo daemon status --json` in a loop until `status == "running"` and `serverId` is present -3. Return the parsed JSON to the app - -#### 2b. Remove all daemon plumbing from Rust - -Delete: -- `managed_transport_target()`, `ManagedTransportTarget` -- `default_transport_path()`, `default_transport_type()` -- `default_managed_home()` -- `resolve_paths()` / `ManagedPaths` -- `read_server_id()`, `read_hostname_from_home()`, `read_pid_from_home()` -- `cli_env()` — no `PASEO_HOME` or `PASEO_HOST` injection -- State file reading/writing for transport config -- All direct file I/O for daemon state - -Keep: -- `bundled_runtime_root()` — to find bundled Node.js + CLI -- `cli_command()` — to invoke the bundled CLI -- `ensure_runtime_ready_internal()` — to verify runtime extraction - -#### 2c. `ManagedDaemonStatus` mirrors `daemon status --json` - -- `serverId: string` -- `status: string` -- `listen: string` -- `hostname: string | null` -- `pid: number | null` -- `home: string` -- Plus: `runtimeId`, `runtimeVersion` (from bundled runtime manifest) - -App parses `listen` to determine transport type: -- `\\.\pipe\*` or `pipe://` → pipe -- `/` or `unix://` → socket -- `host:port` → tcp - -### Phase 3: App — remove "managed" concept from registry - -**File: `packages/app/src/contexts/daemon-registry-context.tsx`** - -#### 3a. Delete managed-specific code - -- `ManagedHostReconciliationInput` -- `DesktopStartupReconciliationInput` -- `reconcileManagedHostInProfiles()` -- `reconcileDesktopStartupRegistry()` -- `probeManagedConnectionUntilReady()` -- `probeManagedStartupTarget()` -- `resolveManagedDesktopStartupStatus()` -- `DEFAULT_LOCAL_TRANSPORT_BOOTSTRAP_*` constants - -#### 3b. Remove `managed` from `HostLifecycle` - -Remove `managed`, `managedRuntimeId`, `managedRuntimeVersion`, `associatedServerId`. Remove all code that checks `lifecycle.managed`. - -#### 3c. Desktop startup effect becomes: - -```typescript -const bootstrap = async () => { - const daemon = await startManagedDaemon() - if (cancelled || !daemon.serverId) return - - const connection = connectionFromListen(daemon.listen) - if (!connection) return - - await upsertHostConnection({ - serverId: daemon.serverId, - label: daemon.hostname ?? undefined, - connection, - }) -} -``` - -#### 3d. Remove `isReconciling` - -No longer needed. Registry update triggers `MultiDaemonSessionHost` → `syncHosts()` → `HostRuntimeController` connects. - -### Phase 4: Index screen — race until online - -**File: `packages/app/src/app/index.tsx`** - -- New hook: `useAnyHostOnline(serverIds)` — subscribes to `HostRuntimeStore` -- Show splash while: registry loading OR (hosts exist, none online, within 30s) -- Any host goes online → navigate to it -- 30s timeout → WelcomeScreen - -Both managed daemon and localhost:6767 race. First to connect wins. - -### Phase 5: Localhost bootstrap - -Non-desktop: keep the lightweight probe (2.5s, needs serverId from handshake). -Desktop: localhost:6767 added as a race participant alongside the daemon connection. - -### Phase 6: Metro must ignore app test files - -**Files: `packages/app/metro.config.js` and any directly related Metro resolver helpers/tests** - -The app already contains many `*.test.ts` and `*.test.tsx` files under `packages/app/src`. Those files must remain in the repo and must NOT be deleted to make Metro happy. - -Implement Metro-side protection so Expo/Metro does not try to bundle app test files from `packages/app/src` during normal app/web startup: - -- Treat `*.test.ts`, `*.test.tsx`, `*.spec.ts`, and `*.spec.tsx` under `packages/app/src` as ignored/non-bundle inputs for Metro app resolution -- If the current Metro resolver hook needs to special-case local imports or extensions, update it so test files are skipped cleanly instead of causing bundle resolution crashes -- Preserve Vitest usage for those files; the change is only about Metro bundling behavior, not removing or renaming tests -- If a test index/helper file is needed for app tests, keep it and make Metro ignore it rather than deleting it - ---- - -## What stays the same - -- `HostRuntimeController`, `HostRuntimeStore`, `MultiDaemonSessionHost` — no changes -- `DaemonClient` transports — no changes -- The daemon itself — no changes (already autonomous) - -## Acceptance Criteria - -1. `paseo daemon status --json` returns `serverId`, `listen`, `hostname`, `status`, and `pid`, and desktop startup consumes that shape as the source of truth -2. The app registry no longer contains managed-daemon lifecycle concepts; desktop startup upserts the managed daemon like any other host -3. Desktop startup blocks on the startup splash during the connection race and only leaves it when a host goes online or the 30s timeout expires -4. Desktop still races the managed daemon and `localhost:6767`; non-desktop keeps the lightweight localhost probe -5. Metro does not try to bundle `packages/app/src/**/*.test.*` or `packages/app/src/**/*.spec.*` during normal app startup -6. Existing app test files, including any `index.test.ts` helper needed by tests, remain in the repo and are not deleted as a workaround -7. `npm run typecheck` passes