Abort realtime playback as soon as speech arrives

packages/server/src/server/agent/tts-manager.ts

@@ -125,6 +125,27 @@ export class TTSManager {
     this.pendingPlaybacks.delete(audioId);
   }
 
+  /**
+   * Cancel all pending playbacks (e.g., user interrupted audio)
+   */
+  public cancelPendingPlaybacks(reason: string): void {
+    if (this.pendingPlaybacks.size === 0) {
+      return;
+    }
+
+    console.log(
+      `[TTS-Manager ${this.sessionId}] Cancelling ${this.pendingPlaybacks.size} pending playback(s): ${reason}`
+    );
+
+    for (const [audioId, pending] of this.pendingPlaybacks.entries()) {
+      pending.resolve();
+      this.pendingPlaybacks.delete(audioId);
+      console.log(
+        `[TTS-Manager ${this.sessionId}] Cleared pending playback ${audioId}`
+      );
+    }
+  }
+
   /**
    * Cleanup all pending playbacks
    */

packages/server/src/server/session.ts

@@ -86,6 +86,7 @@ export class Session {
 
   // Realtime mode state
   private isRealtimeMode = false;
+  private speechInProgress = false;
 
   // Audio buffering for interruption handling
   private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
@@ -1496,6 +1497,8 @@ export class Session {
   private async handleAudioChunk(
     msg: Extract<SessionInboundMessage, { type: "realtime_audio_chunk" }>
   ): Promise<void> {
+    await this.handleRealtimeSpeechStart();
+
     // Decode base64
     const audioBuffer = Buffer.from(msg.audio, "base64");
 
@@ -1590,6 +1593,7 @@ export class Session {
           `[Session ${this.clientId}] Empty transcription (false positive), not aborting`
         );
         this.setPhase("idle");
+        this.clearSpeechInProgress("empty transcription");
         return;
       }
 
@@ -1633,12 +1637,14 @@ export class Session {
       this.messages.push({ role: "user", content: result.text });
 
       // Set phase to LLM and process (TTS enabled in realtime mode for voice conversations)
+      this.clearSpeechInProgress("transcription complete");
       this.setPhase("llm");
       this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
       await this.currentStreamPromise;
       this.setPhase("idle");
     } catch (error: any) {
       this.setPhase("idle");
+      this.clearSpeechInProgress("transcription error");
       this.emit({
         type: "activity_log",
         payload: {
@@ -1663,7 +1669,7 @@ export class Session {
     const flushTextBuffer = () => {
       if (textBuffer.length > 0) {
         // TTS handling (capture mode at generation time for drift protection)
-        if (enableTTS) {
+        if (enableTTS && !this.speechInProgress) {
           const modeAtGeneration = this.isRealtimeMode;
           pendingTTS = this.ttsManager.generateAndWaitForPlayback(
             textBuffer,
@@ -1671,6 +1677,10 @@ export class Session {
             this.abortController.signal,
             modeAtGeneration
           );
+        } else if (enableTTS && this.speechInProgress) {
+          console.log(
+            `[Session ${this.clientId}] Skipping TTS chunk while speech in progress`
+          );
         }
 
         // Emit activity log
@@ -2049,6 +2059,36 @@ export class Session {
     this.ttsManager.confirmAudioPlayed(id);
   }
 
+  /**
+   * Mark speech detection start and abort any active playback/LLM
+   */
+  private async handleRealtimeSpeechStart(): Promise<void> {
+    if (this.speechInProgress) {
+      return;
+    }
+
+    this.speechInProgress = true;
+    console.log(
+      `[Session ${this.clientId}] Realtime speech chunk detected – aborting playback and LLM`
+    );
+    this.ttsManager.cancelPendingPlaybacks("realtime speech detected");
+    await this.handleAbort();
+  }
+
+  /**
+   * Clear speech-in-progress flag once the user turn has completed
+   */
+  private clearSpeechInProgress(reason: string): void {
+    if (!this.speechInProgress) {
+      return;
+    }
+
+    this.speechInProgress = false;
+    console.log(
+      `[Session ${this.clientId}] Speech turn complete (${reason}) – resuming TTS`
+    );
+  }
+
   /**
    * Create new AbortController, aborting the previous one
    */

plan.md

@@ -56,7 +56,8 @@
 
 - [x] Wire interrupt signal on speech start so VAD immediately issues `abort_request` to the server before any audio segment is sent.
   - Realtime speech detection now sends a `session/abort_request` as soon as VAD confirms speech, ensuring any in-flight LLM turn is interrupted before the buffered audio is uploaded (implemented in `packages/app/src/contexts/realtime-context.tsx` with logging/error handling).
-- [ ] Abort server-side playback/LLM as soon as a realtime audio chunk arrives; set a speech-in-progress flag that pauses new TTS until the turn completes.
+- [x] Abort server-side playback/LLM as soon as a realtime audio chunk arrives; set a speech-in-progress flag that pauses new TTS until the turn completes.
+  - Added a `speechInProgress` guard in `packages/server/src/server/session.ts` that triggers on the first realtime chunk, cancels pending TTS playback via the new `TTSManager.cancelPendingPlaybacks`, and immediately routes through the abort flow before buffering audio; the flag is cleared when transcription finishes so the next assistant reply can synthesize speech. TTS generation now skips while the flag is set, and `npm run typecheck --workspace=@voice-dev/server` still fails in pre-existing cross-workspace `packages/app/src/types/stream.ts` imports (unchanged by this fix).
 - [ ] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh.
 - [ ] Harden playback stop/queue clearing so speech detection purges suppressed audio and the server stops emitting TTS while the user is talking.
 - [ ] Prototype chunked audio input: emit smaller PCM frames with `isLast=false`, update `handleAudioChunk` to buffer and trigger STT once the minimum duration is reached.