From ee315bb9a004a4ae141efe6b57c51c6e16183231 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 20 Oct 2025 18:14:08 +0200 Subject: [PATCH] ``` refactor: improve session management, audio handling, and MCP configuration - Change default session name from 'voice-dev' to '__voice-dev' - Change default server port from 3000 to 6767 - Add conversation history persistence to .conversations directory - Defer stream abort until after transcription completes - Return empty content arrays from MCP tool handlers (use structuredContent only) - Add pause/resume support to audio player for handling false positive speech detection - Add .conversations to .gitignore ``` --- packages/voice-assistant/.gitignore | 1 + .../src/server/agent/llm-openai.ts | 2 +- packages/voice-assistant/src/server/index.ts | 2 +- .../voice-assistant/src/server/session.ts | 85 ++++++++---- .../src/server/terminal-mcp/server.ts | 14 +- packages/voice-assistant/src/ui/App.tsx | 126 ++++++++++++------ .../src/ui/lib/audio-playback.ts | 34 +++++ 7 files changed, 189 insertions(+), 75 deletions(-) diff --git a/packages/voice-assistant/.gitignore b/packages/voice-assistant/.gitignore index fa5d7b6ba..5370e8dc2 100644 --- a/packages/voice-assistant/.gitignore +++ b/packages/voice-assistant/.gitignore @@ -3,3 +3,4 @@ dist .env .DS_Store *.log +.conversations diff --git a/packages/voice-assistant/src/server/agent/llm-openai.ts b/packages/voice-assistant/src/server/agent/llm-openai.ts index 07e73ad47..1ed94dfaf 100644 --- a/packages/voice-assistant/src/server/agent/llm-openai.ts +++ b/packages/voice-assistant/src/server/agent/llm-openai.ts @@ -24,7 +24,7 @@ async function getTerminalMcpClient() { } // Create Terminal MCP server - const server = await createTerminalMcpServer({ sessionName: "voice-dev" }); + const server = await createTerminalMcpServer({ sessionName: "__voice-dev" }); // Create linked transport pair const [clientTransport, serverTransport] = diff --git a/packages/voice-assistant/src/server/index.ts b/packages/voice-assistant/src/server/index.ts index f004094bc..86ebabf9a 100644 --- a/packages/voice-assistant/src/server/index.ts +++ b/packages/voice-assistant/src/server/index.ts @@ -82,7 +82,7 @@ async function createServer(httpServer: HttpServer, config: ServerConfig) { } async function main() { - const port = parseInt(process.env.PORT || "3000", 10); + const port = parseInt(process.env.PORT || "6767", 10); const isDev = process.env.NODE_ENV !== "production"; const config: ServerConfig = { port, isDev }; diff --git a/packages/voice-assistant/src/server/session.ts b/packages/voice-assistant/src/server/session.ts index a9c90b461..14a449f21 100644 --- a/packages/voice-assistant/src/server/session.ts +++ b/packages/voice-assistant/src/server/session.ts @@ -1,7 +1,8 @@ import { v4 as uuidv4 } from "uuid"; -import { readFile } from "fs/promises"; +import { readFile, mkdir, writeFile } from "fs/promises"; import { exec } from "child_process"; import { promisify } from "util"; +import { join } from "path"; import invariant from "tiny-invariant"; import { streamText, stepCountIs } from "ai"; import type { ModelMessage } from "@ai-sdk/provider-utils"; @@ -53,6 +54,7 @@ export class Session { // Conversation history private messages: ModelMessage[] = []; + private turnIndex = 0; // Per-session managers private readonly ttsManager: TTSManager; @@ -236,17 +238,6 @@ export class Session { * Process audio through STT and then LLM */ private async processAudio(audio: Buffer, format: string): Promise { - // Abort any in-progress stream immediately - this.createAbortController(); - - // Wait for aborted stream to finish cleanup (save partial response) - if (this.currentStreamPromise) { - console.log( - `[Session ${this.clientId}] Waiting for aborted stream to finish cleanup` - ); - await this.currentStreamPromise; - } - this.setPhase("transcribing"); this.emit({ @@ -265,12 +256,23 @@ export class Session { const transcriptText = result.text.trim(); if (!transcriptText) { console.log( - `[Session ${this.clientId}] Empty transcription, skipping LLM` + `[Session ${this.clientId}] Empty transcription (false positive), not aborting` ); this.setPhase("idle"); return; } + // Has content - abort any in-progress stream now + this.createAbortController(); + + // Wait for aborted stream to finish cleanup (save partial response) + if (this.currentStreamPromise) { + console.log( + `[Session ${this.clientId}] Waiting for aborted stream to finish cleanup` + ); + await this.currentStreamPromise; + } + // Emit transcription result this.emit({ type: "transcription_result", @@ -353,6 +355,9 @@ export class Session { }; try { + // Debug: dump conversation before LLM call + await this.dumpConversation(); + invariant( process.env.OPENROUTER_API_KEY, "OPENROUTER_API_KEY is required" @@ -371,20 +376,13 @@ export class Session { tools: allTools, abortSignal: this.abortController.signal, onFinish: async (event) => { - for (const step of event.steps) { - this.messages.push(...step.response.messages); + const newMessages = event.response.messages; + if (newMessages.length > 0) { + this.messages.push(...newMessages); + console.log( + `[Session ${this.clientId}] onFinish - saved message with ${newMessages.length} steps` + ); } - console.log( - `[Session ${this.clientId}] onFinish - saved message with ${event.steps.length} steps` - ); - }, - onAbort: async (event) => { - for (const step of event.steps) { - this.messages.push(...step.response.messages); - } - console.log( - `[Session ${this.clientId}] onAbort - saved message with ${event.steps.length} steps` - ); }, onChunk: async ({ chunk }) => { if (chunk.type === "text-delta") { @@ -451,6 +449,8 @@ export class Session { } }, onError: async (error) => { + console.error(error); + this.emit({ type: "activity_log", payload: { @@ -539,6 +539,9 @@ export class Session { // Re-throw unexpected errors throw error; } finally { + // Increment turn index for next LLM call + this.turnIndex++; + // Clear the stream promise tracker this.currentStreamPromise = null; } @@ -708,6 +711,36 @@ export class Session { this.onMessage(msg); } + /** + * Debug helper: dump conversation to disk + */ + private async dumpConversation(): Promise { + try { + const dumpDir = join(process.cwd(), ".conversations"); + await mkdir(dumpDir, { recursive: true }); + + const filename = `${this.conversationId}-${this.turnIndex}.json`; + const filepath = join(dumpDir, filename); + + const dump = { + conversationId: this.conversationId, + turnIndex: this.turnIndex, + timestamp: new Date().toISOString(), + messages: this.messages, + }; + + await writeFile(filepath, JSON.stringify(dump, null, 2), "utf-8"); + console.log( + `[Session ${this.clientId}] Dumped conversation to ${filepath}` + ); + } catch (error) { + console.error( + `[Session ${this.clientId}] Failed to dump conversation:`, + error + ); + } + } + /** * Clean up session resources */ diff --git a/packages/voice-assistant/src/server/terminal-mcp/server.ts b/packages/voice-assistant/src/server/terminal-mcp/server.ts index 294600dd6..a33dc5a1d 100644 --- a/packages/voice-assistant/src/server/terminal-mcp/server.ts +++ b/packages/voice-assistant/src/server/terminal-mcp/server.ts @@ -47,7 +47,7 @@ export async function createTerminalMcpServer( const terminals = await terminalManager.listTerminals(); const output = { terminals }; return { - content: [{ type: "text", text: JSON.stringify(output) }], + content: [], structuredContent: output, }; } @@ -95,7 +95,7 @@ export async function createTerminalMcpServer( }); const output = { terminal }; return { - content: [{ type: "text", text: JSON.stringify(output) }], + content: [], structuredContent: output, }; } @@ -133,7 +133,7 @@ export async function createTerminalMcpServer( ); const result = { output }; return { - content: [{ type: "text", text: JSON.stringify(result) }], + content: [], structuredContent: result, }; } @@ -196,7 +196,7 @@ export async function createTerminalMcpServer( ); const result = { output: output || undefined }; return { - content: [{ type: "text", text: JSON.stringify(result) }], + content: [], structuredContent: result, }; } @@ -258,7 +258,7 @@ export async function createTerminalMcpServer( ); const result = { output: output || undefined }; return { - content: [{ type: "text", text: JSON.stringify(result) }], + content: [], structuredContent: result, }; } @@ -287,7 +287,7 @@ export async function createTerminalMcpServer( await terminalManager.renameTerminal(terminalName, newName); const output = { success: true }; return { - content: [{ type: "text", text: JSON.stringify(output) }], + content: [], structuredContent: output, }; } @@ -315,7 +315,7 @@ export async function createTerminalMcpServer( await terminalManager.killTerminal(terminalName); const output = { success: true }; return { - content: [{ type: "text", text: JSON.stringify(output) }], + content: [], structuredContent: output, }; } diff --git a/packages/voice-assistant/src/ui/App.tsx b/packages/voice-assistant/src/ui/App.tsx index 4b46ec23e..a84c53295 100644 --- a/packages/voice-assistant/src/ui/App.tsx +++ b/packages/voice-assistant/src/ui/App.tsx @@ -3,7 +3,11 @@ import { Mic, Send, Radio } from "lucide-react"; import { useWebSocket } from "./hooks/useWebSocket"; import { createAudioPlayer } from "./lib/audio-playback"; import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture"; -import { createRealtimeVAD, float32ArrayToBlob, type RealtimeVAD } from "./lib/audio-realtime"; +import { + createRealtimeVAD, + float32ArrayToBlob, + type RealtimeVAD, +} from "./lib/audio-realtime"; import { ToolCallCard } from "./components/ToolCallCard"; import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer"; import "./App.css"; @@ -80,7 +84,7 @@ function App() { useEffect(() => { if (textareaRef.current) { - textareaRef.current.style.height = 'auto'; + textareaRef.current.style.height = "auto"; const newHeight = Math.max(textareaRef.current.scrollHeight, 44); textareaRef.current.style.height = `${newHeight}px`; } @@ -106,19 +110,26 @@ function App() { useEffect(() => { // Listen for status messages const unsubStatus = ws.on("status", (message) => { - if (message.type !== 'status') return; - const msg = 'message' in message.payload ? String(message.payload.message) : undefined; + if (message.type !== "status") return; + const msg = + "message" in message.payload + ? String(message.payload.message) + : undefined; addLog("info", msg || `Status: ${message.payload.status}`); }); // Listen for activity log messages const unsubActivity = ws.on("activity_log", (message) => { - if (message.type !== 'activity_log') return; + if (message.type !== "activity_log") return; const data = message.payload; // Handle tool calls if (data.type === "tool_call" && data.metadata) { - const { toolCallId, toolName, arguments: args } = data.metadata as { + const { + toolCallId, + toolName, + arguments: args, + } = data.metadata as { toolCallId: string; toolName: string; arguments: unknown; @@ -155,7 +166,11 @@ function App() { } // Handle tool errors - update tool call status to failed - if (data.type === "error" && data.metadata && 'toolCallId' in data.metadata) { + if ( + data.type === "error" && + data.metadata && + "toolCallId" in data.metadata + ) { const { toolCallId, error } = data.metadata as { toolCallId: string; error: unknown; @@ -172,7 +187,13 @@ function App() { } // Map activity log types to UI log types - let logType: "system" | "info" | "success" | "error" | "user" | "assistant" = "info"; + let logType: + | "system" + | "info" + | "success" + | "error" + | "user" + | "assistant" = "info"; if (data.type === "transcript") logType = "user"; else if (data.type === "assistant") logType = "assistant"; else if (data.type === "error") logType = "error"; @@ -187,20 +208,37 @@ function App() { // Listen for streaming assistant chunks const unsubChunk = ws.on("assistant_chunk", (message) => { - if (message.type !== 'assistant_chunk') return; + if (message.type !== "assistant_chunk") return; setCurrentAssistantMessage((prev) => prev + message.payload.chunk); }); // Listen for transcription results - const unsubTranscription = ws.on("transcription_result", () => { - // Note: Transcription is already broadcast as activity_log with type "transcript" - // No need to log it again here to avoid duplication + const unsubTranscription = ws.on("transcription_result", (message) => { + if (message.type !== "transcription_result") return; + setIsProcessingAudio(false); + + const transcriptText = message.payload.text.trim(); + + if (!transcriptText) { + // Empty transcription - false positive, resume playback + console.log( + "[App] Empty transcription (false positive) - resuming playback" + ); + audioPlayerRef.current.resume(); + } else { + // Has content - real speech detected + // Server will handle abort, just stop playback on client + console.log("[App] Transcription received - stopping playback"); + audioPlayerRef.current.stop(); + setIsPlayingAudio(false); + setCurrentAssistantMessage(""); + } }); // Listen for artifacts const unsubArtifact = ws.on("artifact", (message) => { - if (message.type !== 'artifact') return; + if (message.type !== "artifact") return; const artifact = message.payload; // Add to artifacts map @@ -229,7 +267,7 @@ function App() { // Listen for audio output (TTS) const unsubAudioOutput = ws.on("audio_output", async (message) => { - if (message.type !== 'audio_output') return; + if (message.type !== "audio_output") return; const data = message.payload; try { @@ -311,12 +349,14 @@ function App() { try { if (isRecording) { - console.log('[App] Stopping recording...'); + console.log("[App] Stopping recording..."); const audioBlob = await recorder.stop(); setIsRecording(false); - const format = audioBlob.type || 'audio/webm'; - console.log(`[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`); + const format = audioBlob.type || "audio/webm"; + console.log( + `[App] Recording complete: ${audioBlob.size} bytes, format: ${format}` + ); setIsProcessingAudio(true); addLog("info", "Sending audio to server..."); @@ -339,9 +379,11 @@ function App() { }, }); - console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`); + console.log( + `[App] Sent audio: ${audioBlob.size} bytes, format: ${format}` + ); } else { - console.log('[App] Starting recording...'); + console.log("[App] Starting recording..."); // Stop any currently playing audio when starting to record audioPlayerRef.current.stop(); @@ -351,7 +393,7 @@ function App() { setIsRecording(true); } } catch (error: any) { - console.error('[App] Recording error:', error); + console.error("[App] Recording error:", error); addLog("error", `Failed to record audio: ${error.message}`); setIsRecording(false); } @@ -369,9 +411,9 @@ function App() { const handleKeyDown = (e: React.KeyboardEvent) => { // Detect if device is desktop (not mobile/tablet) - const isDesktop = window.matchMedia('(pointer: fine)').matches; + const isDesktop = window.matchMedia("(pointer: fine)").matches; - if (e.key === 'Enter') { + if (e.key === "Enter") { if (isDesktop && !e.shiftKey && userInput.trim()) { // Desktop: plain Enter sends message e.preventDefault(); @@ -425,18 +467,9 @@ function App() { console.log("[App] Speech detected!"); setIsSpeechDetected(true); - // Interrupt playback when speech is detected - audioPlayerRef.current.stop(); - setIsPlayingAudio(false); - setCurrentAssistantMessage(""); - - // Send abort request to server immediately - ws.send({ - type: "session", - message: { - type: "abort_request", - }, - }); + // Pause playback when speech is detected + // Will resume if false positive, or stop completely if real speech + audioPlayerRef.current.pause(); }, onSpeechEnd: async (audioData: Float32Array) => { console.log("[App] Speech ended, processing..."); @@ -475,8 +508,11 @@ function App() { } }, onVADMisfire: () => { - console.log("[App] VAD misfire"); + console.log("[App] VAD misfire - resuming playback"); setIsSpeechDetected(false); + + // Resume playback since it was a false positive + audioPlayerRef.current.resume(); }, onError: (error: Error) => { console.error("[App] VAD error:", error); @@ -506,7 +542,9 @@ function App() {

Voice Assistant

- {isPlayingAudio &&
} + {isPlayingAudio && ( +
+ )}
setUserInput(e.target.value)} onKeyDown={handleKeyDown} - placeholder="Type a message or tap mic to talk..." + placeholder="Type a message" disabled={!ws.isConnected || isRecording || isRealtimeMode} className="message-input" rows={1} @@ -593,8 +631,14 @@ function App() { type="button" onClick={handleToggleRealtimeMode} disabled={!ws.isConnected || isRecording || isVADLoading} - className={`send-button realtime-button ${isRealtimeMode ? 'active' : ''} ${isSpeechDetected ? 'speech-detected' : ''} ${isVADLoading ? 'loading' : ''}`} - title={isRealtimeMode ? "Stop realtime mode" : "Start realtime mode"} + className={`send-button realtime-button ${ + isRealtimeMode ? "active" : "" + } ${isSpeechDetected ? "speech-detected" : ""} ${ + isVADLoading ? "loading" : "" + }`} + title={ + isRealtimeMode ? "Stop realtime mode" : "Start realtime mode" + } > {isVADLoading ? ( @@ -606,7 +650,9 @@ function App() { type="button" onClick={handleButtonClick} disabled={!ws.isConnected || isProcessingAudio || isRealtimeMode} - className={`send-button ${isRecording ? 'recording' : ''} ${isProcessingAudio ? 'processing' : ''} ${userInput.trim() ? 'has-text' : ''}`} + className={`send-button ${isRecording ? "recording" : ""} ${ + isProcessingAudio ? "processing" : "" + } ${userInput.trim() ? "has-text" : ""}`} > {isRecording ? ( diff --git a/packages/voice-assistant/src/ui/lib/audio-playback.ts b/packages/voice-assistant/src/ui/lib/audio-playback.ts index 98f237b73..b7e94b04c 100644 --- a/packages/voice-assistant/src/ui/lib/audio-playback.ts +++ b/packages/voice-assistant/src/ui/lib/audio-playback.ts @@ -1,7 +1,10 @@ export interface AudioPlayer { play(audioData: Blob): Promise; stop(): void; + pause(): void; + resume(): void; isPlaying(): boolean; + isPaused(): boolean; clearQueue(): void; } @@ -14,6 +17,7 @@ interface QueuedAudio { export function createAudioPlayer(): AudioPlayer { let currentAudio: HTMLAudioElement | null = null; let playing = false; + let paused = false; let queue: QueuedAudio[] = []; let isProcessingQueue = false; let currentReject: ((error: Error) => void) | null = null; @@ -124,6 +128,7 @@ export function createAudioPlayer(): AudioPlayer { currentAudio = null; } playing = false; + paused = false; // Reject the current playing promise if it exists if (currentReject) { @@ -140,10 +145,36 @@ export function createAudioPlayer(): AudioPlayer { isProcessingQueue = false; } + function pausePlayback(): void { + if (currentAudio && playing && !paused) { + currentAudio.pause(); + paused = true; + console.log("[AudioPlayer] Playback paused"); + } + } + + function resumePlayback(): void { + if (currentAudio && paused) { + currentAudio.play().catch((error) => { + console.error("[AudioPlayer] Failed to resume playback:", error); + if (currentReject) { + currentReject(error); + currentReject = null; + } + }); + paused = false; + console.log("[AudioPlayer] Playback resumed"); + } + } + function isPlayingFunc(): boolean { return playing; } + function isPausedFunc(): boolean { + return paused; + } + function clearQueue(): void { // Reject all pending promises in the queue while (queue.length > 0) { @@ -155,7 +186,10 @@ export function createAudioPlayer(): AudioPlayer { return { play, stop, + pause: pausePlayback, + resume: resumePlayback, isPlaying: isPlayingFunc, + isPaused: isPausedFunc, clearQueue, }; }