jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

```

Browse Source 
refactor: improve session management, audio handling, and MCP configuration

- Change default session name from 'voice-dev' to '__voice-dev'
- Change default server port from 3000 to 6767
- Add conversation history persistence to .conversations directory
- Defer stream abort until after transcription completes
- Return empty content arrays from MCP tool handlers (use structuredContent only)
- Add pause/resume support to audio player for handling false positive speech detection
- Add .conversations to .gitignore
```
This commit is contained in:
Mohamed Boudra
2025-10-20 18:14:08 +02:00
parent d2de52c57d
commit ee315bb9a0
7 changed files with 189 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-assistant/.gitignore
+1
View File
@@ -3,3 +3,4 @@ dist
.env
.DS_Store
*.log
.conversations
packages/voice-assistant/src/server/agent/llm-openai.ts
+1 -1
View File
@@ -24,7 +24,7 @@ async function getTerminalMcpClient() {
}
// Create Terminal MCP server
const server = await createTerminalMcpServer({ sessionName: "voice-dev" });
const server = await createTerminalMcpServer({ sessionName: "__voice-dev" });
// Create linked transport pair
const [clientTransport, serverTransport] =
packages/voice-assistant/src/server/index.ts
+1 -1
View File
@@ -82,7 +82,7 @@ async function createServer(httpServer: HttpServer, config: ServerConfig) {
}
async function main() {
const port = parseInt(process.env.PORT || "3000", 10);
const port = parseInt(process.env.PORT || "6767", 10);
const isDev = process.env.NODE_ENV !== "production";
const config: ServerConfig = { port, isDev };
packages/voice-assistant/src/server/session.ts
+59 -26
View File
@@ -1,7 +1,8 @@
import { v4 as uuidv4 } from "uuid";
import { readFile } from "fs/promises";
import { readFile, mkdir, writeFile } from "fs/promises";
import { exec } from "child_process";
import { promisify } from "util";
import { join } from "path";
import invariant from "tiny-invariant";
import { streamText, stepCountIs } from "ai";
import type { ModelMessage } from "@ai-sdk/provider-utils";
@@ -53,6 +54,7 @@ export class Session {
// Conversation history
private messages: ModelMessage[] = [];
private turnIndex = 0;
// Per-session managers
private readonly ttsManager: TTSManager;
@@ -236,17 +238,6 @@ export class Session {
* Process audio through STT and then LLM
*/
private async processAudio(audio: Buffer, format: string): Promise<void> {
// Abort any in-progress stream immediately
this.createAbortController();
// Wait for aborted stream to finish cleanup (save partial response)
if (this.currentStreamPromise) {
console.log(
`[Session ${this.clientId}] Waiting for aborted stream to finish cleanup`
);
await this.currentStreamPromise;
}
this.setPhase("transcribing");
this.emit({
@@ -265,12 +256,23 @@ export class Session {
const transcriptText = result.text.trim();
if (!transcriptText) {
console.log(
`[Session ${this.clientId}] Empty transcription, skipping LLM`
`[Session ${this.clientId}] Empty transcription (false positive), not aborting`
);
this.setPhase("idle");
return;
}
// Has content - abort any in-progress stream now
this.createAbortController();
// Wait for aborted stream to finish cleanup (save partial response)
if (this.currentStreamPromise) {
console.log(
`[Session ${this.clientId}] Waiting for aborted stream to finish cleanup`
);
await this.currentStreamPromise;
}
// Emit transcription result
this.emit({
type: "transcription_result",
@@ -353,6 +355,9 @@ export class Session {
};
try {
// Debug: dump conversation before LLM call
await this.dumpConversation();
invariant(
process.env.OPENROUTER_API_KEY,
"OPENROUTER_API_KEY is required"
@@ -371,20 +376,13 @@ export class Session {
tools: allTools,
abortSignal: this.abortController.signal,
onFinish: async (event) => {
for (const step of event.steps) {
this.messages.push(...step.response.messages);
const newMessages = event.response.messages;
if (newMessages.length > 0) {
this.messages.push(...newMessages);
console.log(
`[Session ${this.clientId}] onFinish - saved message with ${newMessages.length} steps`
);
}
console.log(
`[Session ${this.clientId}] onFinish - saved message with ${event.steps.length} steps`
);
},
onAbort: async (event) => {
for (const step of event.steps) {
this.messages.push(...step.response.messages);
}
console.log(
`[Session ${this.clientId}] onAbort - saved message with ${event.steps.length} steps`
);
},
onChunk: async ({ chunk }) => {
if (chunk.type === "text-delta") {
@@ -451,6 +449,8 @@ export class Session {
}
},
onError: async (error) => {
console.error(error);
this.emit({
type: "activity_log",
payload: {
@@ -539,6 +539,9 @@ export class Session {
// Re-throw unexpected errors
throw error;
} finally {
// Increment turn index for next LLM call
this.turnIndex++;
// Clear the stream promise tracker
this.currentStreamPromise = null;
}
@@ -708,6 +711,36 @@ export class Session {
this.onMessage(msg);
}
/**
* Debug helper: dump conversation to disk
*/
private async dumpConversation(): Promise<void> {
try {
const dumpDir = join(process.cwd(), ".conversations");
await mkdir(dumpDir, { recursive: true });
const filename = `${this.conversationId}-${this.turnIndex}.json`;
const filepath = join(dumpDir, filename);
const dump = {
conversationId: this.conversationId,
turnIndex: this.turnIndex,
timestamp: new Date().toISOString(),
messages: this.messages,
};
await writeFile(filepath, JSON.stringify(dump, null, 2), "utf-8");
console.log(
`[Session ${this.clientId}] Dumped conversation to ${filepath}`
);
} catch (error) {
console.error(
`[Session ${this.clientId}] Failed to dump conversation:`,
error
);
}
}
/**
* Clean up session resources
*/
packages/voice-assistant/src/server/terminal-mcp/server.ts
+7 -7
View File
@@ -47,7 +47,7 @@ export async function createTerminalMcpServer(
const terminals = await terminalManager.listTerminals();
const output = { terminals };
return {
content: [{ type: "text", text: JSON.stringify(output) }],
content: [],
structuredContent: output,
};
}
@@ -95,7 +95,7 @@ export async function createTerminalMcpServer(
});
const output = { terminal };
return {
content: [{ type: "text", text: JSON.stringify(output) }],
content: [],
structuredContent: output,
};
}
@@ -133,7 +133,7 @@ export async function createTerminalMcpServer(
);
const result = { output };
return {
content: [{ type: "text", text: JSON.stringify(result) }],
content: [],
structuredContent: result,
};
}
@@ -196,7 +196,7 @@ export async function createTerminalMcpServer(
);
const result = { output: output || undefined };
return {
content: [{ type: "text", text: JSON.stringify(result) }],
content: [],
structuredContent: result,
};
}
@@ -258,7 +258,7 @@ export async function createTerminalMcpServer(
);
const result = { output: output || undefined };
return {
content: [{ type: "text", text: JSON.stringify(result) }],
content: [],
structuredContent: result,
};
}
@@ -287,7 +287,7 @@ export async function createTerminalMcpServer(
await terminalManager.renameTerminal(terminalName, newName);
const output = { success: true };
return {
content: [{ type: "text", text: JSON.stringify(output) }],
content: [],
structuredContent: output,
};
}
@@ -315,7 +315,7 @@ export async function createTerminalMcpServer(
await terminalManager.killTerminal(terminalName);
const output = { success: true };
return {
content: [{ type: "text", text: JSON.stringify(output) }],
content: [],
structuredContent: output,
};
}
packages/voice-assistant/src/ui/App.tsx
+86 -40
View File
@@ -3,7 +3,11 @@ import { Mic, Send, Radio } from "lucide-react";
import { useWebSocket } from "./hooks/useWebSocket";
import { createAudioPlayer } from "./lib/audio-playback";
import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture";
import { createRealtimeVAD, float32ArrayToBlob, type RealtimeVAD } from "./lib/audio-realtime";
import {
createRealtimeVAD,
float32ArrayToBlob,
type RealtimeVAD,
} from "./lib/audio-realtime";
import { ToolCallCard } from "./components/ToolCallCard";
import { ArtifactDrawer, type Artifact } from "./components/ArtifactDrawer";
import "./App.css";
@@ -80,7 +84,7 @@ function App() {
useEffect(() => {
if (textareaRef.current) {
textareaRef.current.style.height = 'auto';
textareaRef.current.style.height = "auto";
const newHeight = Math.max(textareaRef.current.scrollHeight, 44);
textareaRef.current.style.height = `${newHeight}px`;
}
@@ -106,19 +110,26 @@ function App() {
useEffect(() => {
// Listen for status messages
const unsubStatus = ws.on("status", (message) => {
if (message.type !== 'status') return;
const msg = 'message' in message.payload ? String(message.payload.message) : undefined;
if (message.type !== "status") return;
const msg =
"message" in message.payload
? String(message.payload.message)
: undefined;
addLog("info", msg || `Status: ${message.payload.status}`);
});
// Listen for activity log messages
const unsubActivity = ws.on("activity_log", (message) => {
if (message.type !== 'activity_log') return;
if (message.type !== "activity_log") return;
const data = message.payload;
// Handle tool calls
if (data.type === "tool_call" && data.metadata) {
const { toolCallId, toolName, arguments: args } = data.metadata as {
const {
toolCallId,
toolName,
arguments: args,
} = data.metadata as {
toolCallId: string;
toolName: string;
arguments: unknown;
@@ -155,7 +166,11 @@ function App() {
}
// Handle tool errors - update tool call status to failed
if (data.type === "error" && data.metadata && 'toolCallId' in data.metadata) {
if (
data.type === "error" &&
data.metadata &&
"toolCallId" in data.metadata
) {
const { toolCallId, error } = data.metadata as {
toolCallId: string;
error: unknown;
@@ -172,7 +187,13 @@ function App() {
}
// Map activity log types to UI log types
let logType: "system" | "info" | "success" | "error" | "user" | "assistant" = "info";
let logType:
| "system"
| "info"
| "success"
| "error"
| "user"
| "assistant" = "info";
if (data.type === "transcript") logType = "user";
else if (data.type === "assistant") logType = "assistant";
else if (data.type === "error") logType = "error";
@@ -187,20 +208,37 @@ function App() {
// Listen for streaming assistant chunks
const unsubChunk = ws.on("assistant_chunk", (message) => {
if (message.type !== 'assistant_chunk') return;
if (message.type !== "assistant_chunk") return;
setCurrentAssistantMessage((prev) => prev + message.payload.chunk);
});
// Listen for transcription results
const unsubTranscription = ws.on("transcription_result", () => {
// Note: Transcription is already broadcast as activity_log with type "transcript"
// No need to log it again here to avoid duplication
const unsubTranscription = ws.on("transcription_result", (message) => {
if (message.type !== "transcription_result") return;
setIsProcessingAudio(false);
const transcriptText = message.payload.text.trim();
if (!transcriptText) {
// Empty transcription - false positive, resume playback
console.log(
"[App] Empty transcription (false positive) - resuming playback"
);
audioPlayerRef.current.resume();
} else {
// Has content - real speech detected
// Server will handle abort, just stop playback on client
console.log("[App] Transcription received - stopping playback");
audioPlayerRef.current.stop();
setIsPlayingAudio(false);
setCurrentAssistantMessage("");
}
});
// Listen for artifacts
const unsubArtifact = ws.on("artifact", (message) => {
if (message.type !== 'artifact') return;
if (message.type !== "artifact") return;
const artifact = message.payload;
// Add to artifacts map
@@ -229,7 +267,7 @@ function App() {
// Listen for audio output (TTS)
const unsubAudioOutput = ws.on("audio_output", async (message) => {
if (message.type !== 'audio_output') return;
if (message.type !== "audio_output") return;
const data = message.payload;
try {
@@ -311,12 +349,14 @@ function App() {
try {
if (isRecording) {
console.log('[App] Stopping recording...');
console.log("[App] Stopping recording...");
const audioBlob = await recorder.stop();
setIsRecording(false);
const format = audioBlob.type || 'audio/webm';
console.log(`[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`);
const format = audioBlob.type || "audio/webm";
console.log(
`[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`
);
setIsProcessingAudio(true);
addLog("info", "Sending audio to server...");
@@ -339,9 +379,11 @@ function App() {
},
});
console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`);
console.log(
`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`
);
} else {
console.log('[App] Starting recording...');
console.log("[App] Starting recording...");
// Stop any currently playing audio when starting to record
audioPlayerRef.current.stop();
@@ -351,7 +393,7 @@ function App() {
setIsRecording(true);
}
} catch (error: any) {
console.error('[App] Recording error:', error);
console.error("[App] Recording error:", error);
addLog("error", `Failed to record audio: ${error.message}`);
setIsRecording(false);
}
@@ -369,9 +411,9 @@ function App() {
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
// Detect if device is desktop (not mobile/tablet)
const isDesktop = window.matchMedia('(pointer: fine)').matches;
const isDesktop = window.matchMedia("(pointer: fine)").matches;
if (e.key === 'Enter') {
if (e.key === "Enter") {
if (isDesktop && !e.shiftKey && userInput.trim()) {
// Desktop: plain Enter sends message
e.preventDefault();
@@ -425,18 +467,9 @@ function App() {
console.log("[App] Speech detected!");
setIsSpeechDetected(true);
// Interrupt playback when speech is detected
audioPlayerRef.current.stop();
setIsPlayingAudio(false);
setCurrentAssistantMessage("");
// Send abort request to server immediately
ws.send({
type: "session",
message: {
type: "abort_request",
},
});
// Pause playback when speech is detected
// Will resume if false positive, or stop completely if real speech
audioPlayerRef.current.pause();
},
onSpeechEnd: async (audioData: Float32Array) => {
console.log("[App] Speech ended, processing...");
@@ -475,8 +508,11 @@ function App() {
}
},
onVADMisfire: () => {
console.log("[App] VAD misfire");
console.log("[App] VAD misfire - resuming playback");
setIsSpeechDetected(false);
// Resume playback since it was a false positive
audioPlayerRef.current.resume();
},
onError: (error: Error) => {
console.error("[App] VAD error:", error);
@@ -506,7 +542,9 @@ function App() {
<header className="header">
<h1>Voice Assistant</h1>
<div className="header-status">
{isPlayingAudio && <div className="audio-playing-indicator" title="Playing audio" />}
{isPlayingAudio && (
<div className="audio-playing-indicator" title="Playing audio" />
)}
<div
className={`status-indicator ${
ws.isConnected ? "connected" : "disconnected"
@@ -584,7 +622,7 @@ function App() {
value={userInput}
onChange={(e) => setUserInput(e.target.value)}
onKeyDown={handleKeyDown}
placeholder="Type a message or tap mic to talk..."
placeholder="Type a message"
disabled={!ws.isConnected || isRecording || isRealtimeMode}
className="message-input"
rows={1}
@@ -593,8 +631,14 @@ function App() {
type="button"
onClick={handleToggleRealtimeMode}
disabled={!ws.isConnected || isRecording || isVADLoading}
className={`send-button realtime-button ${isRealtimeMode ? 'active' : ''} ${isSpeechDetected ? 'speech-detected' : ''} ${isVADLoading ? 'loading' : ''}`}
title={isRealtimeMode ? "Stop realtime mode" : "Start realtime mode"}
className={`send-button realtime-button ${
isRealtimeMode ? "active" : ""
} ${isSpeechDetected ? "speech-detected" : ""} ${
isVADLoading ? "loading" : ""
}`}
title={
isRealtimeMode ? "Stop realtime mode" : "Start realtime mode"
}
>
{isVADLoading ? (
<span className="loading-indicator" />
@@ -606,7 +650,9 @@ function App() {
type="button"
onClick={handleButtonClick}
disabled={!ws.isConnected || isProcessingAudio || isRealtimeMode}
className={`send-button ${isRecording ? 'recording' : ''} ${isProcessingAudio ? 'processing' : ''} ${userInput.trim() ? 'has-text' : ''}`}
className={`send-button ${isRecording ? "recording" : ""} ${
isProcessingAudio ? "processing" : ""
} ${userInput.trim() ? "has-text" : ""}`}
>
{isRecording ? (
<span className="recording-indicator" />
packages/voice-assistant/src/ui/lib/audio-playback.ts
+34
View File
@@ -1,7 +1,10 @@
export interface AudioPlayer {
play(audioData: Blob): Promise<number>;
stop(): void;
pause(): void;
resume(): void;
isPlaying(): boolean;
isPaused(): boolean;
clearQueue(): void;
}
@@ -14,6 +17,7 @@ interface QueuedAudio {
export function createAudioPlayer(): AudioPlayer {
let currentAudio: HTMLAudioElement | null = null;
let playing = false;
let paused = false;
let queue: QueuedAudio[] = [];
let isProcessingQueue = false;
let currentReject: ((error: Error) => void) | null = null;
@@ -124,6 +128,7 @@ export function createAudioPlayer(): AudioPlayer {
currentAudio = null;
}
playing = false;
paused = false;
// Reject the current playing promise if it exists
if (currentReject) {
@@ -140,10 +145,36 @@ export function createAudioPlayer(): AudioPlayer {
isProcessingQueue = false;
}
function pausePlayback(): void {
if (currentAudio && playing && !paused) {
currentAudio.pause();
paused = true;
console.log("[AudioPlayer] Playback paused");
}
}
function resumePlayback(): void {
if (currentAudio && paused) {
currentAudio.play().catch((error) => {
console.error("[AudioPlayer] Failed to resume playback:", error);
if (currentReject) {
currentReject(error);
currentReject = null;
}
});
paused = false;
console.log("[AudioPlayer] Playback resumed");
}
}
function isPlayingFunc(): boolean {
return playing;
}
function isPausedFunc(): boolean {
return paused;
}
function clearQueue(): void {
// Reject all pending promises in the queue
while (queue.length > 0) {
@@ -155,7 +186,10 @@ export function createAudioPlayer(): AudioPlayer {
return {
play,
stop,
pause: pausePlayback,
resume: resumePlayback,
isPlaying: isPlayingFunc,
isPaused: isPausedFunc,
clearQueue,
};
}