jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

```

Browse Source 
chore: upgrade Vite to v7 and update OpenAI services

- Upgrade vite from ^5.0.0 to ^7.1.10 with new Node.js requirements
- Add dotenv package for environment variable management
- Update STT to use gpt-4o-transcribe model
- Remove verbose_json format from STT response
- Format code with consistent style (quotes, spacing)
- Add HMR WebSocket configuration for development
- Improve UI layout with overflow handling
- Reorganize chat interface component order
- Add user message broadcasting to activity log
```
This commit is contained in:
Mohamed Boudra
2025-10-19 16:57:25 +02:00
parent c3049b8fa0
commit 81bdba0d48
8 changed files with 105 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-assistant/src/server/agent/llm-openai.ts
+2
View File
@@ -173,6 +173,8 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
let fullText = "";
result.fullStream;
// Stream text chunks to the caller
for await (const chunk of result.textStream) {
fullText += chunk;
packages/voice-assistant/src/server/agent/orchestrator.ts
+22 -17
View File
@@ -1,7 +1,7 @@
import { v4 as uuidv4 } from 'uuid';
import { getSystemPrompt } from './system-prompt.js';
import { streamLLM, type Message } from './llm-openai.js';
import type { VoiceAssistantWebSocketServer } from '../websocket-server.js';
import { v4 as uuidv4 } from "uuid";
import { getSystemPrompt } from "./system-prompt.js";
import { streamLLM, type Message } from "./llm-openai.js";
import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
interface ConversationContext {
id: string;
@@ -23,7 +23,7 @@ export function createConversation(): string {
const id = uuidv4();
conversations.set(id, {
id,
messages: [{ role: 'system', content: getSystemPrompt() }],
messages: [{ role: "system", content: getSystemPrompt() }],
createdAt: new Date(),
lastActivity: new Date(),
});
@@ -49,12 +49,12 @@ export async function processUserMessage(params: {
}): Promise<string> {
const conversation = conversations.get(params.conversationId);
if (!conversation) {
throw new Error('Conversation not found');
throw new Error("Conversation not found");
}
// Add user message to context
conversation.messages.push({
role: 'user',
role: "user",
content: params.message,
});
conversation.lastActivity = new Date();
@@ -62,7 +62,7 @@ export async function processUserMessage(params: {
// Note: User message is already broadcast by the caller (e.g., after STT in index.ts)
// No need to broadcast again here to avoid duplication
let assistantResponse = '';
let assistantResponse = "";
try {
// Stream LLM response with tool execution
@@ -72,7 +72,7 @@ export async function processUserMessage(params: {
// Broadcast streaming chunks to WebSocket
if (params.wsServer) {
params.wsServer.broadcast({
type: 'assistant_chunk',
type: "assistant_chunk",
payload: { chunk },
});
}
@@ -83,7 +83,7 @@ export async function processUserMessage(params: {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'tool_call',
type: "tool_call",
content: `Calling ${toolName}`,
metadata: { toolName, arguments: args },
});
@@ -95,7 +95,7 @@ export async function processUserMessage(params: {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'tool_result',
type: "tool_result",
content: `Tool ${toolName} completed`,
metadata: { toolName, result },
});
@@ -107,7 +107,7 @@ export async function processUserMessage(params: {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'assistant',
type: "assistant",
content: fullText,
});
}
@@ -116,16 +116,16 @@ export async function processUserMessage(params: {
// Add assistant response to context
conversation.messages.push({
role: 'assistant',
role: "assistant",
content: assistantResponse,
});
} catch (error: any) {
} catch (error) {
// Broadcast error to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'error',
type: "error",
content: `Error: ${error.message}`,
});
}
@@ -141,7 +141,8 @@ export async function processUserMessage(params: {
export function cleanupConversations(maxAgeMinutes: number = 60): void {
const now = new Date();
for (const [id, conv] of conversations.entries()) {
const ageMinutes = (now.getTime() - conv.lastActivity.getTime()) / (1000 * 60);
const ageMinutes =
(now.getTime() - conv.lastActivity.getTime()) / (1000 * 60);
if (ageMinutes > maxAgeMinutes) {
conversations.delete(id);
}
@@ -153,7 +154,11 @@ export function cleanupConversations(maxAgeMinutes: number = 60): void {
*/
export function getConversationStats(): {
total: number;
conversations: Array<{ id: string; messageCount: number; lastActivity: Date }>;
conversations: Array<{
id: string;
messageCount: number;
lastActivity: Date;
}>;
} {
return {
total: conversations.size,
packages/voice-assistant/src/server/agent/stt-openai.ts
+26 -23
View File
@@ -1,12 +1,12 @@
import OpenAI from 'openai';
import { writeFile, unlink } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
import { v4 as uuidv4 } from 'uuid';
import OpenAI from "openai";
import { writeFile, unlink } from "fs/promises";
import { join } from "path";
import { tmpdir } from "os";
import { v4 as uuidv4 } from "uuid";
export interface STTConfig {
apiKey: string;
model?: 'whisper-1';
model?: "whisper-1";
}
export interface TranscriptionResult {
@@ -23,7 +23,7 @@ export function initializeSTT(sttConfig: STTConfig): void {
openaiClient = new OpenAI({
apiKey: sttConfig.apiKey,
});
console.log('✓ STT (OpenAI Whisper) initialized');
console.log("✓ STT (OpenAI Whisper) initialized");
}
export async function transcribeAudio(
@@ -31,7 +31,7 @@ export async function transcribeAudio(
format: string
): Promise<TranscriptionResult> {
if (!openaiClient || !config) {
throw new Error('STT not initialized. Call initializeSTT() first.');
throw new Error("STT not initialized. Call initializeSTT() first.");
}
const startTime = Date.now();
@@ -46,26 +46,29 @@ export async function transcribeAudio(
tempFilePath = join(tmpdir(), `audio-${uuidv4()}.${ext}`);
await writeFile(tempFilePath, audioBuffer);
console.log(`[STT] Transcribing audio file: ${tempFilePath} (${audioBuffer.length} bytes)`);
console.log(
`[STT] Transcribing audio file: ${tempFilePath} (${audioBuffer.length} bytes)`
);
// Call OpenAI Whisper API
const response = await openaiClient.audio.transcriptions.create({
file: await import('fs').then((fs) => fs.createReadStream(tempFilePath!)),
model: config.model || 'whisper-1',
response_format: 'verbose_json', // Get language and duration info
file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)),
model: config.model || "gpt-4o-transcribe",
response_format: "json", // Get language and duration info
});
const duration = Date.now() - startTime;
console.log(`[STT] Transcription complete in ${duration}ms: "${response.text}"`);
console.log(
`[STT] Transcription complete in ${duration}ms: "${response.text}"`
);
return {
text: response.text,
language: response.language,
duration: duration,
};
} catch (error: any) {
console.error('[STT] Transcription error:', error);
console.error("[STT] Transcription error:", error);
throw new Error(`STT transcription failed: ${error.message}`);
} finally {
// Clean up temporary file
@@ -83,16 +86,16 @@ function getFileExtension(format: string): string {
// Map mime types or format strings to file extensions
const formatLower = format.toLowerCase();
if (formatLower.includes('webm')) return 'webm';
if (formatLower.includes('ogg')) return 'ogg';
if (formatLower.includes('mp3')) return 'mp3';
if (formatLower.includes('wav')) return 'wav';
if (formatLower.includes('m4a')) return 'm4a';
if (formatLower.includes('mp4')) return 'mp4';
if (formatLower.includes('flac')) return 'flac';
if (formatLower.includes("webm")) return "webm";
if (formatLower.includes("ogg")) return "ogg";
if (formatLower.includes("mp3")) return "mp3";
if (formatLower.includes("wav")) return "wav";
if (formatLower.includes("m4a")) return "m4a";
if (formatLower.includes("mp4")) return "mp4";
if (formatLower.includes("flac")) return "flac";
// Default to webm (common browser format)
return 'webm';
return "webm";
}
export function isSTTInitialized(): boolean {
packages/voice-assistant/src/server/agent/tts-openai.ts
+30 -20
View File
@@ -1,11 +1,11 @@
import OpenAI from 'openai';
import { Readable } from 'stream';
import OpenAI from "openai";
import { Readable } from "stream";
export interface TTSConfig {
apiKey: string;
model?: 'tts-1' | 'tts-1-hd';
voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';
model?: "tts-1" | "tts-1-hd";
voice?: "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer";
responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm";
}
export interface SpeechResult {
@@ -18,51 +18,61 @@ let config: TTSConfig | null = null;
export function initializeTTS(ttsConfig: TTSConfig): void {
config = {
model: 'tts-1',
voice: 'alloy',
responseFormat: 'mp3',
model: "tts-1",
voice: "alloy",
responseFormat: "mp3",
...ttsConfig,
};
openaiClient = new OpenAI({
apiKey: ttsConfig.apiKey,
});
console.log(`✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}`);
console.log(
`✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}`
);
}
export async function synthesizeSpeech(text: string): Promise<SpeechResult> {
if (!openaiClient || !config) {
throw new Error('TTS not initialized. Call initializeTTS() first.');
throw new Error("TTS not initialized. Call initializeTTS() first.");
}
if (!text || text.trim().length === 0) {
throw new Error('Cannot synthesize empty text');
throw new Error("Cannot synthesize empty text");
}
const startTime = Date.now();
try {
console.log(`[TTS] Synthesizing speech: "${text.substring(0, 50)}${text.length > 50 ? '...' : ''}"`);
console.log(
`[TTS] Synthesizing speech: "${text.substring(0, 50)}${
text.length > 50 ? "..." : ""
}"`
);
// Call OpenAI TTS API with streaming
const response = await openaiClient.audio.speech.create({
model: config.model!,
voice: config.voice!,
input: text,
response_format: config.responseFormat as 'mp3' | 'opus' | 'aac' | 'flac',
response_format: config.responseFormat as "mp3" | "opus" | "aac" | "flac",
});
// Response is a stream, convert to buffer
const audioBuffer = await streamToBuffer(response.body as unknown as Readable);
const audioBuffer = await streamToBuffer(
response.body as unknown as Readable
);
const duration = Date.now() - startTime;
console.log(`[TTS] Speech synthesis complete in ${duration}ms (${audioBuffer.length} bytes)`);
console.log(
`[TTS] Speech synthesis complete in ${duration}ms (${audioBuffer.length} bytes)`
);
return {
audio: audioBuffer,
format: config.responseFormat || 'mp3',
format: config.responseFormat || "mp3",
};
} catch (error: any) {
console.error('[TTS] Speech synthesis error:', error);
console.error("[TTS] Speech synthesis error:", error);
throw new Error(`TTS synthesis failed: ${error.message}`);
}
}
@@ -71,15 +81,15 @@ async function streamToBuffer(stream: Readable): Promise<Buffer> {
const chunks: Buffer[] = [];
return new Promise((resolve, reject) => {
stream.on('data', (chunk) => {
stream.on("data", (chunk) => {
chunks.push(Buffer.from(chunk));
});
stream.on('end', () => {
stream.on("end", () => {
resolve(Buffer.concat(chunks));
});
stream.on('error', (error) => {
stream.on("error", (error) => {
reject(error);
});
});