From 81bdba0d48ee05c6291905ceeb9ddf869b862492 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sun, 19 Oct 2025 16:57:25 +0200 Subject: [PATCH] ``` chore: upgrade Vite to v7 and update OpenAI services - Upgrade vite from ^5.0.0 to ^7.1.10 with new Node.js requirements - Add dotenv package for environment variable management - Update STT to use gpt-4o-transcribe model - Remove verbose_json format from STT response - Format code with consistent style (quotes, spacing) - Add HMR WebSocket configuration for development - Improve UI layout with overflow handling - Reorganize chat interface component order - Add user message broadcasting to activity log ``` --- packages/voice-assistant/package.json | 3 +- .../src/server/agent/llm-openai.ts | 2 + .../src/server/agent/orchestrator.ts | 39 ++++++++------- .../src/server/agent/stt-openai.ts | 49 +++++++++--------- .../src/server/agent/tts-openai.ts | 50 +++++++++++-------- packages/voice-assistant/src/server/index.ts | 19 +++++-- packages/voice-assistant/src/ui/App.css | 5 +- packages/voice-assistant/src/ui/App.tsx | 23 ++------- 8 files changed, 105 insertions(+), 85 deletions(-) diff --git a/packages/voice-assistant/package.json b/packages/voice-assistant/package.json index a79b1e45f..8b77ce5b1 100644 --- a/packages/voice-assistant/package.json +++ b/packages/voice-assistant/package.json @@ -16,6 +16,7 @@ "@deepgram/sdk": "^3.4.0", "@openrouter/ai-sdk-provider": "^1.2.0", "ai": "^5.0.76", + "dotenv": "^17.2.3", "express": "^4.18.2", "openai": "^4.20.0", "tiny-invariant": "^1.3.3", @@ -36,6 +37,6 @@ "react-dom": "^18.2.0", "tsx": "^4.6.0", "typescript": "^5.2.2", - "vite": "^5.0.0" + "vite": "^7.1.10" } } diff --git a/packages/voice-assistant/src/server/agent/llm-openai.ts b/packages/voice-assistant/src/server/agent/llm-openai.ts index 86f1c6378..bcdd93f0f 100644 --- a/packages/voice-assistant/src/server/agent/llm-openai.ts +++ b/packages/voice-assistant/src/server/agent/llm-openai.ts @@ -173,6 +173,8 @@ export async function streamLLM(params: StreamLLMParams): Promise { let fullText = ""; + result.fullStream; + // Stream text chunks to the caller for await (const chunk of result.textStream) { fullText += chunk; diff --git a/packages/voice-assistant/src/server/agent/orchestrator.ts b/packages/voice-assistant/src/server/agent/orchestrator.ts index 4fbe6b0f0..36fcdccd6 100644 --- a/packages/voice-assistant/src/server/agent/orchestrator.ts +++ b/packages/voice-assistant/src/server/agent/orchestrator.ts @@ -1,7 +1,7 @@ -import { v4 as uuidv4 } from 'uuid'; -import { getSystemPrompt } from './system-prompt.js'; -import { streamLLM, type Message } from './llm-openai.js'; -import type { VoiceAssistantWebSocketServer } from '../websocket-server.js'; +import { v4 as uuidv4 } from "uuid"; +import { getSystemPrompt } from "./system-prompt.js"; +import { streamLLM, type Message } from "./llm-openai.js"; +import type { VoiceAssistantWebSocketServer } from "../websocket-server.js"; interface ConversationContext { id: string; @@ -23,7 +23,7 @@ export function createConversation(): string { const id = uuidv4(); conversations.set(id, { id, - messages: [{ role: 'system', content: getSystemPrompt() }], + messages: [{ role: "system", content: getSystemPrompt() }], createdAt: new Date(), lastActivity: new Date(), }); @@ -49,12 +49,12 @@ export async function processUserMessage(params: { }): Promise { const conversation = conversations.get(params.conversationId); if (!conversation) { - throw new Error('Conversation not found'); + throw new Error("Conversation not found"); } // Add user message to context conversation.messages.push({ - role: 'user', + role: "user", content: params.message, }); conversation.lastActivity = new Date(); @@ -62,7 +62,7 @@ export async function processUserMessage(params: { // Note: User message is already broadcast by the caller (e.g., after STT in index.ts) // No need to broadcast again here to avoid duplication - let assistantResponse = ''; + let assistantResponse = ""; try { // Stream LLM response with tool execution @@ -72,7 +72,7 @@ export async function processUserMessage(params: { // Broadcast streaming chunks to WebSocket if (params.wsServer) { params.wsServer.broadcast({ - type: 'assistant_chunk', + type: "assistant_chunk", payload: { chunk }, }); } @@ -83,7 +83,7 @@ export async function processUserMessage(params: { params.wsServer.broadcastActivityLog({ id: uuidv4(), timestamp: new Date(), - type: 'tool_call', + type: "tool_call", content: `Calling ${toolName}`, metadata: { toolName, arguments: args }, }); @@ -95,7 +95,7 @@ export async function processUserMessage(params: { params.wsServer.broadcastActivityLog({ id: uuidv4(), timestamp: new Date(), - type: 'tool_result', + type: "tool_result", content: `Tool ${toolName} completed`, metadata: { toolName, result }, }); @@ -107,7 +107,7 @@ export async function processUserMessage(params: { params.wsServer.broadcastActivityLog({ id: uuidv4(), timestamp: new Date(), - type: 'assistant', + type: "assistant", content: fullText, }); } @@ -116,16 +116,16 @@ export async function processUserMessage(params: { // Add assistant response to context conversation.messages.push({ - role: 'assistant', + role: "assistant", content: assistantResponse, }); - } catch (error: any) { + } catch (error) { // Broadcast error to WebSocket if (params.wsServer) { params.wsServer.broadcastActivityLog({ id: uuidv4(), timestamp: new Date(), - type: 'error', + type: "error", content: `Error: ${error.message}`, }); } @@ -141,7 +141,8 @@ export async function processUserMessage(params: { export function cleanupConversations(maxAgeMinutes: number = 60): void { const now = new Date(); for (const [id, conv] of conversations.entries()) { - const ageMinutes = (now.getTime() - conv.lastActivity.getTime()) / (1000 * 60); + const ageMinutes = + (now.getTime() - conv.lastActivity.getTime()) / (1000 * 60); if (ageMinutes > maxAgeMinutes) { conversations.delete(id); } @@ -153,7 +154,11 @@ export function cleanupConversations(maxAgeMinutes: number = 60): void { */ export function getConversationStats(): { total: number; - conversations: Array<{ id: string; messageCount: number; lastActivity: Date }>; + conversations: Array<{ + id: string; + messageCount: number; + lastActivity: Date; + }>; } { return { total: conversations.size, diff --git a/packages/voice-assistant/src/server/agent/stt-openai.ts b/packages/voice-assistant/src/server/agent/stt-openai.ts index 9a6d9e7c1..a66794c82 100644 --- a/packages/voice-assistant/src/server/agent/stt-openai.ts +++ b/packages/voice-assistant/src/server/agent/stt-openai.ts @@ -1,12 +1,12 @@ -import OpenAI from 'openai'; -import { writeFile, unlink } from 'fs/promises'; -import { join } from 'path'; -import { tmpdir } from 'os'; -import { v4 as uuidv4 } from 'uuid'; +import OpenAI from "openai"; +import { writeFile, unlink } from "fs/promises"; +import { join } from "path"; +import { tmpdir } from "os"; +import { v4 as uuidv4 } from "uuid"; export interface STTConfig { apiKey: string; - model?: 'whisper-1'; + model?: "whisper-1"; } export interface TranscriptionResult { @@ -23,7 +23,7 @@ export function initializeSTT(sttConfig: STTConfig): void { openaiClient = new OpenAI({ apiKey: sttConfig.apiKey, }); - console.log('✓ STT (OpenAI Whisper) initialized'); + console.log("✓ STT (OpenAI Whisper) initialized"); } export async function transcribeAudio( @@ -31,7 +31,7 @@ export async function transcribeAudio( format: string ): Promise { if (!openaiClient || !config) { - throw new Error('STT not initialized. Call initializeSTT() first.'); + throw new Error("STT not initialized. Call initializeSTT() first."); } const startTime = Date.now(); @@ -46,26 +46,29 @@ export async function transcribeAudio( tempFilePath = join(tmpdir(), `audio-${uuidv4()}.${ext}`); await writeFile(tempFilePath, audioBuffer); - console.log(`[STT] Transcribing audio file: ${tempFilePath} (${audioBuffer.length} bytes)`); + console.log( + `[STT] Transcribing audio file: ${tempFilePath} (${audioBuffer.length} bytes)` + ); // Call OpenAI Whisper API const response = await openaiClient.audio.transcriptions.create({ - file: await import('fs').then((fs) => fs.createReadStream(tempFilePath!)), - model: config.model || 'whisper-1', - response_format: 'verbose_json', // Get language and duration info + file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)), + model: config.model || "gpt-4o-transcribe", + response_format: "json", // Get language and duration info }); const duration = Date.now() - startTime; - console.log(`[STT] Transcription complete in ${duration}ms: "${response.text}"`); + console.log( + `[STT] Transcription complete in ${duration}ms: "${response.text}"` + ); return { text: response.text, - language: response.language, duration: duration, }; } catch (error: any) { - console.error('[STT] Transcription error:', error); + console.error("[STT] Transcription error:", error); throw new Error(`STT transcription failed: ${error.message}`); } finally { // Clean up temporary file @@ -83,16 +86,16 @@ function getFileExtension(format: string): string { // Map mime types or format strings to file extensions const formatLower = format.toLowerCase(); - if (formatLower.includes('webm')) return 'webm'; - if (formatLower.includes('ogg')) return 'ogg'; - if (formatLower.includes('mp3')) return 'mp3'; - if (formatLower.includes('wav')) return 'wav'; - if (formatLower.includes('m4a')) return 'm4a'; - if (formatLower.includes('mp4')) return 'mp4'; - if (formatLower.includes('flac')) return 'flac'; + if (formatLower.includes("webm")) return "webm"; + if (formatLower.includes("ogg")) return "ogg"; + if (formatLower.includes("mp3")) return "mp3"; + if (formatLower.includes("wav")) return "wav"; + if (formatLower.includes("m4a")) return "m4a"; + if (formatLower.includes("mp4")) return "mp4"; + if (formatLower.includes("flac")) return "flac"; // Default to webm (common browser format) - return 'webm'; + return "webm"; } export function isSTTInitialized(): boolean { diff --git a/packages/voice-assistant/src/server/agent/tts-openai.ts b/packages/voice-assistant/src/server/agent/tts-openai.ts index ee5ccac85..91f0b2e1d 100644 --- a/packages/voice-assistant/src/server/agent/tts-openai.ts +++ b/packages/voice-assistant/src/server/agent/tts-openai.ts @@ -1,11 +1,11 @@ -import OpenAI from 'openai'; -import { Readable } from 'stream'; +import OpenAI from "openai"; +import { Readable } from "stream"; export interface TTSConfig { apiKey: string; - model?: 'tts-1' | 'tts-1-hd'; - voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer'; - responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm'; + model?: "tts-1" | "tts-1-hd"; + voice?: "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer"; + responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm"; } export interface SpeechResult { @@ -18,51 +18,61 @@ let config: TTSConfig | null = null; export function initializeTTS(ttsConfig: TTSConfig): void { config = { - model: 'tts-1', - voice: 'alloy', - responseFormat: 'mp3', + model: "tts-1", + voice: "alloy", + responseFormat: "mp3", ...ttsConfig, }; openaiClient = new OpenAI({ apiKey: ttsConfig.apiKey, }); - console.log(`✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}`); + console.log( + `✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}` + ); } export async function synthesizeSpeech(text: string): Promise { if (!openaiClient || !config) { - throw new Error('TTS not initialized. Call initializeTTS() first.'); + throw new Error("TTS not initialized. Call initializeTTS() first."); } if (!text || text.trim().length === 0) { - throw new Error('Cannot synthesize empty text'); + throw new Error("Cannot synthesize empty text"); } const startTime = Date.now(); try { - console.log(`[TTS] Synthesizing speech: "${text.substring(0, 50)}${text.length > 50 ? '...' : ''}"`); + console.log( + `[TTS] Synthesizing speech: "${text.substring(0, 50)}${ + text.length > 50 ? "..." : "" + }"` + ); // Call OpenAI TTS API with streaming const response = await openaiClient.audio.speech.create({ model: config.model!, voice: config.voice!, input: text, - response_format: config.responseFormat as 'mp3' | 'opus' | 'aac' | 'flac', + response_format: config.responseFormat as "mp3" | "opus" | "aac" | "flac", }); // Response is a stream, convert to buffer - const audioBuffer = await streamToBuffer(response.body as unknown as Readable); + const audioBuffer = await streamToBuffer( + response.body as unknown as Readable + ); const duration = Date.now() - startTime; - console.log(`[TTS] Speech synthesis complete in ${duration}ms (${audioBuffer.length} bytes)`); + console.log( + `[TTS] Speech synthesis complete in ${duration}ms (${audioBuffer.length} bytes)` + ); return { audio: audioBuffer, - format: config.responseFormat || 'mp3', + format: config.responseFormat || "mp3", }; } catch (error: any) { - console.error('[TTS] Speech synthesis error:', error); + console.error("[TTS] Speech synthesis error:", error); throw new Error(`TTS synthesis failed: ${error.message}`); } } @@ -71,15 +81,15 @@ async function streamToBuffer(stream: Readable): Promise { const chunks: Buffer[] = []; return new Promise((resolve, reject) => { - stream.on('data', (chunk) => { + stream.on("data", (chunk) => { chunks.push(Buffer.from(chunk)); }); - stream.on('end', () => { + stream.on("end", () => { resolve(Buffer.concat(chunks)); }); - stream.on('error', (error) => { + stream.on("error", (error) => { reject(error); }); }); diff --git a/packages/voice-assistant/src/server/index.ts b/packages/voice-assistant/src/server/index.ts index a77104a81..f86d13543 100644 --- a/packages/voice-assistant/src/server/index.ts +++ b/packages/voice-assistant/src/server/index.ts @@ -1,3 +1,4 @@ +import "dotenv/config"; import express from "express"; import path from "path"; import { fileURLToPath } from "url"; @@ -20,7 +21,6 @@ import { captureTerminal, killTerminal, } from "./daemon/terminal-manager.js"; -import { initializeOpenAI } from "./agent/llm-openai.js"; import { initializeSTT, transcribeAudio } from "./agent/stt-openai.js"; import { initializeTTS, synthesizeSpeech } from "./agent/tts-openai.js"; import { @@ -137,8 +137,12 @@ async function createServer(httpServer: HttpServer, config: ServerConfig) { server: { middlewareMode: true, allowedHosts: ["localhost", "mohameds-macbook-pro.tail8fe838.ts.net"], - port: 3000, - hmr: false, + hmr: { + server: httpServer, + port: config.port, + protocol: "wss", + path: "/hmr", + }, }, appType: "custom", }); @@ -201,7 +205,6 @@ async function main() { // Initialize OpenAI client const apiKey = process.env.OPENAI_API_KEY; if (apiKey) { - initializeOpenAI(apiKey); console.log("✓ OpenAI client initialized"); // Initialize STT (Whisper) @@ -282,6 +285,14 @@ async function main() { // Wire orchestrator to WebSocket for text messages wsServer.setMessageHandler(async (message: string) => { try { + // Broadcast user's text message as activity log + wsServer.broadcastActivityLog({ + id: uuidv4(), + timestamp: new Date(), + type: "transcript", + content: message, + }); + await processUserMessage({ conversationId: defaultConversationId, message, diff --git a/packages/voice-assistant/src/ui/App.css b/packages/voice-assistant/src/ui/App.css index c44e4f545..361ab04f7 100644 --- a/packages/voice-assistant/src/ui/App.css +++ b/packages/voice-assistant/src/ui/App.css @@ -55,8 +55,9 @@ .main { display: flex; flex: 1; - padding: 2rem; gap: 2rem; +max-height: 100vh; +overflow: hidden; } .chat-interface { @@ -67,7 +68,7 @@ display: flex; flex-direction: column; gap: 1rem; - max-height: calc(100vh - 120px); + height: 100%; } .chat-interface h2 { diff --git a/packages/voice-assistant/src/ui/App.tsx b/packages/voice-assistant/src/ui/App.tsx index 86c66cc03..ddc071803 100644 --- a/packages/voice-assistant/src/ui/App.tsx +++ b/packages/voice-assistant/src/ui/App.tsx @@ -40,7 +40,6 @@ function App() { message: string, metadata?: Record ) => { - console.trace("addLog", type, message, metadata); setLogs((prev) => [ ...prev, { @@ -231,14 +230,6 @@ function App() {
-

Chat with Assistant

- - -
{logs.map((log) => ( @@ -286,15 +277,11 @@ function App() { -
- -
+