jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Prompt is too long

Browse Source
This commit is contained in:
Mohamed Boudra
2025-10-19 16:23:13 +02:00
parent 40b199e329
commit c3049b8fa0
67 changed files with 504 additions and 17776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-assistant/src/server/agent/llm-openai.ts
+65 -49
View File
@@ -1,6 +1,6 @@
import { streamText, tool } from 'ai';
import { createOpenAI } from '@ai-sdk/openai';
import { z } from 'zod';
import { stepCountIs, streamText, tool } from "ai";
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
import { z } from "zod";
import {
listTerminals,
createTerminal,
@@ -8,16 +8,8 @@ import {
sendText,
renameTerminal,
killTerminal,
} from '../daemon/terminal-manager.js';
let openaiClient: ReturnType<typeof createOpenAI> | null = null;
/**
* Initialize OpenAI client with API key
*/
export function initializeOpenAI(apiKey: string): void {
openaiClient = createOpenAI({ apiKey });
}
} from "../daemon/terminal-manager.js";
import invariant from "tiny-invariant";
/**
* Terminal tools using Vercel AI SDK tool() function
@@ -25,8 +17,8 @@ export function initializeOpenAI(apiKey: string): void {
*/
export const terminalTools = {
list_terminals: tool({
description: 'List all available terminals in the voice-dev session',
parameters: z.object({}),
description: "List all available terminals in the voice-dev session",
inputSchema: z.object({}),
execute: async () => {
const terminals = await listTerminals();
return { terminals };
@@ -34,24 +26,38 @@ export const terminalTools = {
}),
create_terminal: tool({
description: 'Create a new terminal with specified name and working directory',
parameters: z.object({
name: z.string().describe('Unique name for the terminal'),
workingDirectory: z.string().describe('Working directory path'),
initialCommand: z.string().optional().describe('Optional command to run on creation'),
description:
"Create a new terminal with specified name and working directory",
inputSchema: z.object({
name: z.string().describe("Unique name for the terminal"),
workingDirectory: z.string().describe("Working directory path"),
initialCommand: z
.string()
.optional()
.describe("Optional command to run on creation"),
}),
execute: async ({ name, workingDirectory, initialCommand }) => {
const terminal = await createTerminal({ name, workingDirectory, initialCommand });
const terminal = await createTerminal({
name,
workingDirectory,
initialCommand,
});
return { terminal };
},
}),
capture_terminal: tool({
description: 'Capture and return the output from a terminal',
parameters: z.object({
terminalId: z.string().describe('Terminal ID (e.g., @123)'),
lines: z.number().optional().describe('Number of lines to capture (default: 200)'),
wait: z.number().optional().describe('Milliseconds to wait before capture'),
description: "Capture and return the output from a terminal",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID (e.g., @123)"),
lines: z
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
.number()
.optional()
.describe("Milliseconds to wait before capture"),
}),
execute: async ({ terminalId, lines, wait }) => {
const output = await captureTerminal(terminalId, lines, wait);
@@ -60,30 +66,38 @@ export const terminalTools = {
}),
send_text: tool({
description: 'Send text or commands to a terminal',
parameters: z.object({
terminalId: z.string().describe('Terminal ID (e.g., @123)'),
text: z.string().describe('Text to send to the terminal'),
pressEnter: z.boolean().optional().describe('Whether to press Enter after text'),
description: "Send text or commands to a terminal",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID (e.g., @123)"),
text: z.string().describe("Text to send to the terminal"),
pressEnter: z
.boolean()
.optional()
.describe("Whether to press Enter after text"),
return_output: z
.object({
lines: z.number().optional(),
wait: z.number().optional(),
})
.optional()
.describe('Capture output after sending'),
.describe("Capture output after sending"),
}),
execute: async ({ terminalId, text, pressEnter, return_output }) => {
const output = await sendText(terminalId, text, pressEnter, return_output);
const output = await sendText(
terminalId,
text,
pressEnter,
return_output
);
return { output };
},
}),
rename_terminal: tool({
description: 'Rename an existing terminal',
parameters: z.object({
terminalId: z.string().describe('Terminal ID to rename'),
newName: z.string().describe('New unique name for the terminal'),
description: "Rename an existing terminal",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID to rename"),
newName: z.string().describe("New unique name for the terminal"),
}),
execute: async ({ terminalId, newName }) => {
await renameTerminal(terminalId, newName);
@@ -92,9 +106,9 @@ export const terminalTools = {
}),
kill_terminal: tool({
description: 'Close and destroy a terminal',
parameters: z.object({
terminalId: z.string().describe('Terminal ID to kill'),
description: "Close and destroy a terminal",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID to kill"),
}),
execute: async ({ terminalId }) => {
await killTerminal(terminalId);
@@ -107,7 +121,7 @@ export const terminalTools = {
* Message interface for conversation
*/
export interface Message {
role: 'system' | 'user' | 'assistant';
role: "system" | "user" | "assistant";
content: string;
}
@@ -126,21 +140,23 @@ export interface StreamLLMParams {
* Stream LLM response with automatic tool execution
*/
export async function streamLLM(params: StreamLLMParams): Promise<string> {
if (!openaiClient) {
throw new Error('OpenAI client not initialized. Call initializeOpenAI() first.');
}
invariant(process.env.OPENROUTER_API_KEY, "OPENROUTER_API_KEY is required");
const openrouter = createOpenRouter({
apiKey: process.env.OPENROUTER_API_KEY,
});
const result = await streamText({
model: openaiClient('gpt-4o') as any, // Type assertion to work around AI SDK version conflicts
model: openrouter("anthropic/claude-haiku-4.5"),
messages: params.messages,
tools: terminalTools,
maxSteps: 5, // Allow up to 5 tool executions per turn
stopWhen: stepCountIs(10),
onStepFinish: async (event) => {
// Called after each step (tool call or text generation)
if (event.toolCalls && event.toolCalls.length > 0) {
for (const toolCall of event.toolCalls) {
if (params.onToolCall) {
params.onToolCall(toolCall.toolName, toolCall.args);
params.onToolCall(toolCall.toolName, toolCall.input);
}
}
}
@@ -148,14 +164,14 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
if (event.toolResults && event.toolResults.length > 0) {
for (const toolResult of event.toolResults) {
if (params.onToolResult) {
params.onToolResult(toolResult.toolName, toolResult.result);
params.onToolResult(toolResult.toolName, toolResult.output);
}
}
}
},
});
let fullText = '';
let fullText = "";
// Stream text chunks to the caller
for await (const chunk of result.textStream) {
packages/voice-assistant/src/server/agent/orchestrator.ts
+2 -9
View File
@@ -59,15 +59,8 @@ export async function processUserMessage(params: {
});
conversation.lastActivity = new Date();
// Broadcast user message to WebSocket clients
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'transcript',
content: params.message,
});
}
// Note: User message is already broadcast by the caller (e.g., after STT in index.ts)
// No need to broadcast again here to avoid duplication
let assistantResponse = '';
packages/voice-assistant/src/server/index.ts
+145 -104
View File
@@ -1,13 +1,17 @@
import express from 'express';
import path from 'path';
import { fileURLToPath } from 'url';
import { createServer as createHTTPServer } from 'http';
import { readFile } from 'fs/promises';
import { v4 as uuidv4 } from 'uuid';
import { createServer as createViteServer } from 'vite';
import type { ViteDevServer } from 'vite';
import type { ServerConfig } from './types.js';
import { VoiceAssistantWebSocketServer } from './websocket-server.js';
import express from "express";
import path from "path";
import { fileURLToPath } from "url";
import {
createServer as createHTTPServer,
Server as HttpServer,
RequestListener,
} from "http";
import { readFile } from "fs/promises";
import { v4 as uuidv4 } from "uuid";
import { createServer as createViteServer } from "vite";
import type { ViteDevServer } from "vite";
import type { ServerConfig } from "./types.js";
import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
import {
initializeDefaultSession,
listTerminals,
@@ -15,94 +19,105 @@ import {
sendText,
captureTerminal,
killTerminal,
} from './daemon/terminal-manager.js';
import { initializeOpenAI } from './agent/llm-openai.js';
import { initializeSTT, transcribeAudio } from './agent/stt-openai.js';
import { initializeTTS, synthesizeSpeech } from './agent/tts-openai.js';
} from "./daemon/terminal-manager.js";
import { initializeOpenAI } from "./agent/llm-openai.js";
import { initializeSTT, transcribeAudio } from "./agent/stt-openai.js";
import { initializeTTS, synthesizeSpeech } from "./agent/tts-openai.js";
import {
createConversation,
processUserMessage,
cleanupConversations,
} from './agent/orchestrator.js';
} from "./agent/orchestrator.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
async function createServer(config: ServerConfig) {
async function createServer(httpServer: HttpServer, config: ServerConfig) {
const app = express();
// Middleware
app.use(express.json());
// Health check endpoint
app.get('/api/health', (_req, res) => {
res.json({ status: 'ok', timestamp: new Date().toISOString() });
app.get("/api/health", (_req, res) => {
res.json({ status: "ok", timestamp: new Date().toISOString() });
});
// Test endpoint for LLM integration (removed - using orchestrator now)
// Test endpoint for terminal operations
app.get('/api/test-terminal', async (_req, res) => {
app.get("/api/test-terminal", async (_req, res) => {
try {
const testResults: string[] = [];
// 1. Initialize default session
testResults.push('1. Initializing default session...');
testResults.push("1. Initializing default session...");
await initializeDefaultSession();
testResults.push(' ✓ Session initialized');
testResults.push(" ✓ Session initialized");
// 2. List existing terminals
testResults.push('2. Listing existing terminals...');
testResults.push("2. Listing existing terminals...");
const beforeTerminals = await listTerminals();
testResults.push(` ✓ Found ${beforeTerminals.length} existing terminals`);
testResults.push(
` ✓ Found ${beforeTerminals.length} existing terminals`
);
// 3. Create a test terminal
testResults.push('3. Creating test terminal...');
testResults.push("3. Creating test terminal...");
const terminal = await createTerminal({
name: 'test-terminal',
workingDirectory: '~',
name: "test-terminal",
workingDirectory: "~",
initialCommand: undefined,
});
testResults.push(` ✓ Created terminal: ${terminal.id} (${terminal.name})`);
testResults.push(
` ✓ Created terminal: ${terminal.id} (${terminal.name})`
);
// 4. Send a command to the terminal
testResults.push('4. Sending command "echo hello world"...');
await sendText(terminal.id, 'echo "hello world"', true, { lines: 20, wait: 500 });
testResults.push(' ✓ Command sent');
await sendText(terminal.id, 'echo "hello world"', true, {
lines: 20,
wait: 500,
});
testResults.push(" ✓ Command sent");
// 5. Capture output
testResults.push('5. Capturing terminal output...');
testResults.push("5. Capturing terminal output...");
const output = await captureTerminal(terminal.id, 20);
testResults.push(` ✓ Captured ${output.split('\n').length} lines`);
testResults.push(` ✓ Captured ${output.split("\n").length} lines`);
testResults.push(` Output preview: ${output.substring(0, 100)}...`);
// 6. List terminals again
testResults.push('6. Listing terminals again...');
testResults.push("6. Listing terminals again...");
const afterTerminals = await listTerminals();
testResults.push(` ✓ Found ${afterTerminals.length} terminals`);
const testTerminal = afterTerminals.find((t) => t.name === 'test-terminal');
const testTerminal = afterTerminals.find(
(t) => t.name === "test-terminal"
);
if (testTerminal) {
testResults.push(` ✓ Found test-terminal: ${testTerminal.id}`);
}
// 7. Kill the test terminal
testResults.push('7. Killing test terminal...');
testResults.push("7. Killing test terminal...");
await killTerminal(terminal.id);
testResults.push(' ✓ Terminal killed');
testResults.push(" ✓ Terminal killed");
// 8. Verify it's gone
testResults.push('8. Verifying terminal was removed...');
testResults.push("8. Verifying terminal was removed...");
const finalTerminals = await listTerminals();
const stillExists = finalTerminals.find((t) => t.name === 'test-terminal');
const stillExists = finalTerminals.find(
(t) => t.name === "test-terminal"
);
if (!stillExists) {
testResults.push(' ✓ Terminal successfully removed');
testResults.push(" ✓ Terminal successfully removed");
} else {
testResults.push(' ✗ Terminal still exists!');
testResults.push(" ✗ Terminal still exists!");
}
res.json({
success: true,
message: 'All terminal operations completed successfully',
message: "All terminal operations completed successfully",
results: testResults,
});
} catch (error: any) {
@@ -119,37 +134,42 @@ async function createServer(config: ServerConfig) {
if (config.isDev) {
// Development: Create Vite server in middleware mode
vite = await createViteServer({
server: { middlewareMode: true },
appType: 'custom',
server: {
middlewareMode: true,
allowedHosts: ["localhost", "mohameds-macbook-pro.tail8fe838.ts.net"],
port: 3000,
hmr: false,
},
appType: "custom",
});
// Use Vite's middleware to handle HMR, module transformation, etc.
app.use(vite.middlewares);
} else {
// Production: Serve static files from dist/ui
const uiPath = path.join(__dirname, '../ui');
const uiPath = path.join(__dirname, "../ui");
app.use(express.static(uiPath));
}
// SPA fallback - serve index.html for all non-API routes
app.use('*', async (req, res, next) => {
if (req.originalUrl.startsWith('/api')) {
return res.status(404).json({ error: 'Not found' });
app.use("*", async (req, res, next) => {
if (req.originalUrl.startsWith("/api")) {
return res.status(404).json({ error: "Not found" });
}
try {
const indexHtmlPath = config.isDev
? path.resolve(__dirname, '../../src/ui/index.html')
: path.join(__dirname, '../ui/index.html');
? path.resolve(__dirname, "../../src/ui/index.html")
: path.join(__dirname, "../ui/index.html");
let html = await readFile(indexHtmlPath, 'utf-8');
let html = await readFile(indexHtmlPath, "utf-8");
// In development, transform the HTML with Vite
if (config.isDev && vite) {
html = await vite.transformIndexHtml(req.originalUrl, html);
}
res.status(200).set({ 'Content-Type': 'text/html' }).end(html);
res.status(200).set({ "Content-Type": "text/html" }).end(html);
} catch (e) {
if (config.isDev && vite) {
vite.ssrFixStacktrace(e as Error);
@@ -162,14 +182,18 @@ async function createServer(config: ServerConfig) {
}
async function main() {
const port = parseInt(process.env.PORT || '3000', 10);
const isDev = process.env.NODE_ENV !== 'production';
const port = parseInt(process.env.PORT || "3000", 10);
const isDev = process.env.NODE_ENV !== "production";
const config: ServerConfig = { port, isDev };
const { app, vite } = await createServer(config);
const httpServer = createHTTPServer((req, res) => {
app(req, res);
});
const { app, vite } = await createServer(httpServer, config);
// Create HTTP server
const httpServer = createHTTPServer(app);
// Initialize WebSocket server
const wsServer = new VoiceAssistantWebSocketServer(httpServer);
@@ -178,22 +202,30 @@ async function main() {
const apiKey = process.env.OPENAI_API_KEY;
if (apiKey) {
initializeOpenAI(apiKey);
console.log('✓ OpenAI client initialized');
console.log("✓ OpenAI client initialized");
// Initialize STT (Whisper)
initializeSTT({ apiKey });
// Initialize TTS
const ttsVoice = (process.env.TTS_VOICE || 'alloy') as 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
const ttsModel = (process.env.TTS_MODEL || 'tts-1') as 'tts-1' | 'tts-1-hd';
const ttsVoice = (process.env.TTS_VOICE || "alloy") as
| "alloy"
| "echo"
| "fable"
| "onyx"
| "nova"
| "shimmer";
const ttsModel = (process.env.TTS_MODEL || "tts-1") as "tts-1" | "tts-1-hd";
initializeTTS({
apiKey,
voice: ttsVoice,
model: ttsModel,
responseFormat: 'mp3',
responseFormat: "mp3",
});
} else {
console.warn('⚠ OPENAI_API_KEY not set - LLM, STT, and TTS features will not work');
console.warn(
"⚠ OPENAI_API_KEY not set - LLM, STT, and TTS features will not work"
);
}
// Create default conversation
@@ -201,7 +233,10 @@ async function main() {
console.log(`✓ Default conversation created: ${defaultConversationId}`);
// Helper function to process message and optionally generate TTS
async function processMessageWithOptionalTTS(message: string, enableTTS: boolean = false): Promise<string> {
async function processMessageWithOptionalTTS(
message: string,
enableTTS: boolean = false
): Promise<string> {
const response = await processUserMessage({
conversationId: defaultConversationId,
message,
@@ -211,15 +246,15 @@ async function main() {
// Generate TTS for the response if enabled
if (enableTTS && apiKey) {
try {
console.log('[TTS] Generating speech for assistant response...');
console.log("[TTS] Generating speech for assistant response...");
const speechResult = await synthesizeSpeech(response);
// Convert audio buffer to base64
const base64Audio = speechResult.audio.toString('base64');
const base64Audio = speechResult.audio.toString("base64");
// Send audio to client via WebSocket
wsServer.broadcast({
type: 'audio_output',
type: "audio_output",
payload: {
id: uuidv4(),
audio: base64Audio,
@@ -227,13 +262,15 @@ async function main() {
},
});
console.log(`[TTS] Sent audio to client via WebSocket (${speechResult.audio.length} bytes)`);
console.log(
`[TTS] Sent audio to client via WebSocket (${speechResult.audio.length} bytes)`
);
} catch (error: any) {
console.error('[TTS] Error generating speech:', error);
console.error("[TTS] Error generating speech:", error);
wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'error',
type: "error",
content: `TTS error: ${error.message}`,
});
}
@@ -251,50 +288,52 @@ async function main() {
wsServer,
});
} catch (error: any) {
console.error('[Orchestrator] Error processing message:', error);
console.error("[Orchestrator] Error processing message:", error);
wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'error',
type: "error",
content: `Error: ${error.message}`,
});
}
});
// Wire audio handler to WebSocket for voice input (STT)
wsServer.setAudioHandler(async (audio: Buffer, format: string): Promise<string> => {
try {
// Transcribe audio using OpenAI Whisper
const result = await transcribeAudio(audio, format);
wsServer.setAudioHandler(
async (audio: Buffer, format: string): Promise<string> => {
try {
// Transcribe audio using OpenAI Whisper
const result = await transcribeAudio(audio, format);
// Broadcast transcription result as activity log
wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'transcript',
content: result.text,
metadata: {
language: result.language,
duration: result.duration,
},
});
// Broadcast transcription result as activity log
wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "transcript",
content: result.text,
metadata: {
language: result.language,
duration: result.duration,
},
});
// Process the transcribed text through the orchestrator WITH TTS enabled
// Since this came from voice input, respond with voice output
await processMessageWithOptionalTTS(result.text, true);
// Process the transcribed text through the orchestrator WITH TTS enabled
// Since this came from voice input, respond with voice output
await processMessageWithOptionalTTS(result.text, true);
return result.text;
} catch (error: any) {
console.error('[STT] Error transcribing audio:', error);
wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: 'error',
content: `Transcription error: ${error.message}`,
});
throw error;
return result.text;
} catch (error: any) {
console.error("[STT] Error transcribing audio:", error);
wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "error",
content: `Transcription error: ${error.message}`,
});
throw error;
}
}
});
);
// Start conversation cleanup interval (every 10 minutes)
setInterval(() => {
@@ -304,33 +343,35 @@ async function main() {
// Initialize default tmux session for terminal control
try {
await initializeDefaultSession();
console.log('✓ Default tmux session initialized');
console.log("✓ Default tmux session initialized");
} catch (error) {
console.error('Failed to initialize tmux session:', error);
console.error("Failed to initialize tmux session:", error);
}
httpServer.listen(port, () => {
console.log(`\n✓ Voice Assistant server running on http://localhost:${port}`);
console.log(
`\n✓ Voice Assistant server running on http://localhost:${port}`
);
if (isDev) {
console.log(`✓ Vite dev server integrated (HMR enabled)\n`);
}
});
// Graceful shutdown
process.on('SIGTERM', async () => {
console.log('\nSIGTERM received, shutting down gracefully...');
process.on("SIGTERM", async () => {
console.log("\nSIGTERM received, shutting down gracefully...");
wsServer.close();
if (vite) {
await vite.close();
}
httpServer.close(() => {
console.log('Server closed');
console.log("Server closed");
process.exit(0);
});
});
}
main().catch((error) => {
console.error('Failed to start server:', error);
console.error("Failed to start server:", error);
process.exit(1);
});
packages/voice-assistant/src/server/websocket-server.ts
+71 -41
View File
@@ -1,55 +1,63 @@
import { WebSocketServer, WebSocket } from 'ws';
import { Server as HTTPServer } from 'http';
import { WebSocketServer, WebSocket } from "ws";
import { Server as HTTPServer } from "http";
import type {
WebSocketMessage,
ActivityLogEntry,
AudioChunkPayload,
} from './types.js';
} from "./types.js";
export class VoiceAssistantWebSocketServer {
private wss: WebSocketServer;
private clients: Map<WebSocket, string> = new Map(); // Map ws to client ID
private messageHandler?: (message: string) => Promise<void>;
private audioHandler?: (audio: Buffer, format: string) => Promise<string>;
private audioBuffers: Map<string, { chunks: Buffer[]; format: string }> = new Map();
private audioBuffers: Map<string, { chunks: Buffer[]; format: string }> =
new Map();
private clientIdCounter: number = 0;
constructor(server: HTTPServer) {
this.wss = new WebSocketServer({ server, path: '/ws' });
this.wss = new WebSocketServer({ server, path: "/ws" });
this.wss.on('connection', (ws) => {
this.wss.on("connection", (ws) => {
this.handleConnection(ws);
});
console.log('✓ WebSocket server initialized on /ws');
console.log("✓ WebSocket server initialized on /ws");
}
private handleConnection(ws: WebSocket): void {
// Generate unique client ID
const clientId = `client-${++this.clientIdCounter}`;
this.clients.set(ws, clientId);
console.log(`[WS] Client connected: ${clientId} (total: ${this.clients.size})`);
console.log(
`[WS] Client connected: ${clientId} (total: ${this.clients.size})`
);
// Send welcome message
this.sendToClient(ws, {
type: 'status',
payload: { status: 'connected', message: 'WebSocket connection established' },
type: "status",
payload: {
status: "connected",
message: "WebSocket connection established",
},
});
ws.on('message', (data) => {
ws.on("message", (data) => {
this.handleMessage(ws, data);
});
ws.on('close', () => {
ws.on("close", () => {
const clientId = this.clients.get(ws);
if (clientId) {
this.clients.delete(ws);
console.log(`[WS] Client disconnected: ${clientId} (total: ${this.clients.size})`);
console.log(
`[WS] Client disconnected: ${clientId} (total: ${this.clients.size})`
);
}
});
ws.on('error', (error) => {
console.error('[WS] Client error:', error);
ws.on("error", (error) => {
console.error("[WS] Client error:", error);
const clientId = this.clients.get(ws);
if (clientId) {
this.clients.delete(ws);
@@ -57,28 +65,31 @@ export class VoiceAssistantWebSocketServer {
});
}
private async handleMessage(ws: WebSocket, data: Buffer | ArrayBuffer | Buffer[]): Promise<void> {
private async handleMessage(
ws: WebSocket,
data: Buffer | ArrayBuffer | Buffer[]
): Promise<void> {
try {
const message = JSON.parse(data.toString()) as WebSocketMessage;
console.log(`[WS] Received message type: ${message.type}`);
switch (message.type) {
case 'ping':
this.sendToClient(ws, { type: 'pong', payload: {} });
case "ping":
this.sendToClient(ws, { type: "pong", payload: {} });
break;
case 'user_message':
case "user_message":
// Handle user message through orchestrator
const payload = message.payload as { message: string };
if (this.messageHandler) {
await this.messageHandler(payload.message);
} else {
console.warn('[WS] No message handler registered');
console.warn("[WS] No message handler registered");
}
break;
case 'audio_chunk':
case "audio_chunk":
// Handle audio chunk for STT
await this.handleAudioChunk(ws, message.payload as AudioChunkPayload);
break;
@@ -87,12 +98,13 @@ export class VoiceAssistantWebSocketServer {
console.warn(`[WS] Unknown message type: ${message.type}`);
}
} catch (error) {
console.error('[WS] Failed to parse message:', error);
console.error("[WS] Failed to parse message:", error);
}
}
private sendToClient(ws: WebSocket, message: WebSocketMessage): void {
if (ws.readyState === 1) { // WebSocket.OPEN = 1
if (ws.readyState === 1) {
// WebSocket.OPEN = 1
ws.send(JSON.stringify(message));
}
}
@@ -100,7 +112,8 @@ export class VoiceAssistantWebSocketServer {
public broadcast(message: WebSocketMessage): void {
const payload = JSON.stringify(message);
this.clients.forEach((_clientId, client) => {
if (client.readyState === 1) { // WebSocket.OPEN = 1
if (client.readyState === 1) {
// WebSocket.OPEN = 1
client.send(payload);
}
});
@@ -108,14 +121,17 @@ export class VoiceAssistantWebSocketServer {
public broadcastActivityLog(entry: ActivityLogEntry): void {
this.broadcast({
type: 'activity_log',
type: "activity_log",
payload: entry,
});
}
public broadcastStatus(status: string, metadata?: Record<string, unknown>): void {
public broadcastStatus(
status: string,
metadata?: Record<string, unknown>
): void {
this.broadcast({
type: 'status',
type: "status",
payload: { status, ...metadata },
});
}
@@ -124,35 +140,49 @@ export class VoiceAssistantWebSocketServer {
this.messageHandler = handler;
}
public setAudioHandler(handler: (audio: Buffer, format: string) => Promise<string>): void {
public setAudioHandler(
handler: (audio: Buffer, format: string) => Promise<string>
): void {
this.audioHandler = handler;
}
private async handleAudioChunk(ws: WebSocket, payload: AudioChunkPayload): Promise<void> {
private async handleAudioChunk(
ws: WebSocket,
payload: AudioChunkPayload
): Promise<void> {
try {
// Use a client-specific key for buffering (in case multiple clients)
const clientId = 'default'; // Could be enhanced with per-client tracking
const clientId = "default"; // Could be enhanced with per-client tracking
// Decode base64 audio data
const audioBuffer = Buffer.from(payload.audio, 'base64');
const audioBuffer = Buffer.from(payload.audio, "base64");
if (!payload.isLast) {
// Buffer the chunk
if (!this.audioBuffers.has(clientId)) {
this.audioBuffers.set(clientId, { chunks: [], format: payload.format });
this.audioBuffers.set(clientId, {
chunks: [],
format: payload.format,
});
}
const buffer = this.audioBuffers.get(clientId)!;
buffer.chunks.push(audioBuffer);
console.log(`[WS] Buffered audio chunk (${audioBuffer.length} bytes, total chunks: ${buffer.chunks.length})`);
console.log(
`[WS] Buffered audio chunk (${audioBuffer.length} bytes, total chunks: ${buffer.chunks.length})`
);
} else {
// Last chunk - process complete audio
const buffer = this.audioBuffers.get(clientId);
const allChunks = buffer ? [...buffer.chunks, audioBuffer] : [audioBuffer];
const allChunks = buffer
? [...buffer.chunks, audioBuffer]
: [audioBuffer];
const format = buffer?.format || payload.format;
// Concatenate all chunks
const completeAudio = Buffer.concat(allChunks);
console.log(`[WS] Complete audio received (${completeAudio.length} bytes, ${allChunks.length} chunks)`);
console.log(
`[WS] Complete audio received (${completeAudio.length} bytes, ${allChunks.length} chunks)`
);
// Clear buffer
this.audioBuffers.delete(clientId);
@@ -162,27 +192,27 @@ export class VoiceAssistantWebSocketServer {
this.broadcastActivityLog({
id: Date.now().toString(),
timestamp: new Date(),
type: 'system',
content: 'Transcribing audio...',
type: "system",
content: "Transcribing audio...",
});
const transcript = await this.audioHandler(completeAudio, format);
// Send transcription result back to client
this.sendToClient(ws, {
type: 'transcription_result',
type: "transcription_result",
payload: { text: transcript },
});
} else {
console.warn('[WS] No audio handler registered');
console.warn("[WS] No audio handler registered");
}
}
} catch (error: any) {
console.error('[WS] Audio chunk handling error:', error);
console.error("[WS] Audio chunk handling error:", error);
this.broadcastActivityLog({
id: Date.now().toString(),
timestamp: new Date(),
type: 'error',
type: "error",
content: `Audio processing error: ${error.message}`,
});
}
packages/voice-assistant/src/ui/App.tsx
+97 -53
View File
@@ -1,59 +1,72 @@
import { useState, useEffect, useRef } from 'react';
import { useWebSocket } from './hooks/useWebSocket';
import { VoiceControls } from './components/VoiceControls';
import { createAudioPlayer } from './lib/audio-playback';
import './App.css';
import { useState, useEffect, useRef } from "react";
import { useWebSocket } from "./hooks/useWebSocket";
import { VoiceControls } from "./components/VoiceControls";
import { createAudioPlayer } from "./lib/audio-playback";
import "./App.css";
interface LogEntry {
id: string;
timestamp: number;
type: 'system' | 'info' | 'success' | 'error' | 'user' | 'assistant' | 'tool';
type: "system" | "info" | "success" | "error" | "user" | "assistant" | "tool";
message: string;
metadata?: Record<string, unknown>;
}
function App() {
const [logs, setLogs] = useState<LogEntry[]>([
{ id: '1', timestamp: Date.now(), type: 'system', message: 'System initialized' },
{
id: "1",
timestamp: Date.now(),
type: "system",
message: "System initialized",
},
]);
const [userInput, setUserInput] = useState('');
const [currentAssistantMessage, setCurrentAssistantMessage] = useState('');
const [userInput, setUserInput] = useState("");
const [currentAssistantMessage, setCurrentAssistantMessage] = useState("");
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
const logEndRef = useRef<HTMLDivElement>(null);
const audioPlayerRef = useRef(createAudioPlayer());
// WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
const wsUrl = `ws://${window.location.host}/ws`;
const wsUrl = `${window.location.protocol === "https:" ? "wss" : "ws"}://${
window.location.host
}/ws`;
const ws = useWebSocket(wsUrl);
const addLog = (
type: LogEntry['type'],
type: LogEntry["type"],
message: string,
metadata?: Record<string, unknown>
) => {
console.trace("addLog", type, message, metadata);
setLogs((prev) => [
...prev,
{ id: Date.now().toString(), timestamp: Date.now(), type, message, metadata },
{
id: Date.now().toString(),
timestamp: Date.now(),
type,
message,
metadata,
},
]);
};
useEffect(() => {
// Listen for status messages
const unsubStatus = ws.on('status', (payload: unknown) => {
const unsubStatus = ws.on("status", (payload: unknown) => {
const data = payload as { status: string; message?: string };
addLog('info', data.message || `Status: ${data.status}`);
addLog("info", data.message || `Status: ${data.status}`);
});
// Listen for pong messages
const unsubPong = ws.on('pong', () => {
addLog('success', 'Received pong from server');
const unsubPong = ws.on("pong", () => {
addLog("success", "Received pong from server");
});
// Listen for activity log messages
const unsubActivity = ws.on('activity_log', (payload: unknown) => {
const unsubActivity = ws.on("activity_log", (payload: unknown) => {
const data = payload as {
type: string;
content: string;
@@ -61,34 +74,43 @@ function App() {
};
// Map activity log types to UI log types
let logType: LogEntry['type'] = 'info';
if (data.type === 'transcript') logType = 'user';
else if (data.type === 'assistant') logType = 'assistant';
else if (data.type === 'tool_call' || data.type === 'tool_result') logType = 'tool';
else if (data.type === 'error') logType = 'error';
let logType: LogEntry["type"] = "info";
if (data.type === "transcript") logType = "user";
else if (data.type === "assistant") logType = "assistant";
else if (data.type === "tool_call" || data.type === "tool_result")
logType = "tool";
else if (data.type === "error") logType = "error";
addLog(logType, data.content, data.metadata);
// Clear streaming state when complete assistant message arrives
if (data.type === "assistant") {
setCurrentAssistantMessage("");
}
});
// Listen for streaming assistant chunks
const unsubChunk = ws.on('assistant_chunk', (payload: unknown) => {
const unsubChunk = ws.on("assistant_chunk", (payload: unknown) => {
const data = payload as { chunk: string };
setCurrentAssistantMessage((prev) => prev + data.chunk);
});
// Listen for transcription results
const unsubTranscription = ws.on('transcription_result', (payload: unknown) => {
const data = payload as { text: string };
addLog('info', `Transcription: ${data.text}`);
setIsProcessingAudio(false);
});
const unsubTranscription = ws.on(
"transcription_result",
(payload: unknown) => {
// Note: Transcription is already broadcast as activity_log with type "transcript"
// No need to log it again here to avoid duplication
setIsProcessingAudio(false);
}
);
// Listen for audio output (TTS)
const unsubAudioOutput = ws.on('audio_output', async (payload: unknown) => {
const unsubAudioOutput = ws.on("audio_output", async (payload: unknown) => {
const data = payload as { audio: string; format: string; id: string };
try {
addLog('info', 'Playing assistant audio response...');
addLog("info", "Playing assistant audio response...");
setIsPlayingAudio(true);
// Decode base64 audio
@@ -99,17 +121,18 @@ function App() {
}
// Create blob with correct mime type
const mimeType = data.format === 'mp3' ? 'audio/mpeg' : `audio/${data.format}`;
const mimeType =
data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`;
const audioBlob = new Blob([bytes], { type: mimeType });
// Play audio
await audioPlayerRef.current.play(audioBlob);
setIsPlayingAudio(false);
addLog('success', 'Audio playback complete');
addLog("success", "Audio playback complete");
} catch (error: any) {
console.error('[App] Audio playback error:', error);
addLog('error', `Audio playback failed: ${error.message}`);
console.error("[App] Audio playback error:", error);
addLog("error", `Audio playback failed: ${error.message}`);
setIsPlayingAudio(false);
}
});
@@ -126,20 +149,20 @@ function App() {
useEffect(() => {
if (ws.isConnected) {
addLog('success', 'WebSocket connected');
addLog("success", "WebSocket connected");
} else {
addLog('system', 'WebSocket disconnected');
addLog("system", "WebSocket disconnected");
}
}, [ws.isConnected]);
// Auto-scroll to bottom when new logs are added
useEffect(() => {
logEndRef.current?.scrollIntoView({ behavior: 'smooth' });
logEndRef.current?.scrollIntoView({ behavior: "smooth" });
}, [logs, currentAssistantMessage]);
const handlePing = () => {
ws.sendPing();
addLog('info', 'Sent ping to server');
addLog("info", "Sent ping to server");
};
const handleSendMessage = (e: React.FormEvent) => {
@@ -150,29 +173,32 @@ function App() {
ws.sendUserMessage(userInput);
// Clear input and reset streaming state
setUserInput('');
setCurrentAssistantMessage('');
setUserInput("");
setCurrentAssistantMessage("");
};
const handleAudioRecorded = async (audioBlob: Blob, format: string) => {
if (!ws.isConnected) {
addLog('error', 'Cannot send audio - not connected to server');
addLog("error", "Cannot send audio - not connected to server");
return;
}
try {
setIsProcessingAudio(true);
addLog('info', 'Sending audio to server...');
addLog("info", "Sending audio to server...");
// Convert blob to base64
const arrayBuffer = await audioBlob.arrayBuffer();
const base64Audio = btoa(
new Uint8Array(arrayBuffer).reduce((data, byte) => data + String.fromCharCode(byte), '')
new Uint8Array(arrayBuffer).reduce(
(data, byte) => data + String.fromCharCode(byte),
""
)
);
// Send audio chunk via WebSocket
ws.send({
type: 'audio_chunk',
type: "audio_chunk",
payload: {
audio: base64Audio,
format: format,
@@ -180,10 +206,12 @@ function App() {
},
});
console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`);
console.log(
`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`
);
} catch (error: any) {
console.error('[App] Error sending audio:', error);
addLog('error', `Failed to send audio: ${error.message}`);
console.error("[App] Error sending audio:", error);
addLog("error", `Failed to send audio: ${error.message}`);
setIsProcessingAudio(false);
}
};
@@ -192,8 +220,12 @@ function App() {
<div className="app">
<header className="header">
<h1>Voice Assistant</h1>
<div className={`status-indicator ${ws.isConnected ? 'connected' : 'disconnected'}`}>
{ws.isConnected ? 'connected' : 'disconnected'}
<div
className={`status-indicator ${
ws.isConnected ? "connected" : "disconnected"
}`}
>
{ws.isConnected ? "connected" : "disconnected"}
</div>
</header>
@@ -211,7 +243,9 @@ function App() {
<div className="log-entries">
{logs.map((log) => (
<div key={log.id} className={`log-entry ${log.type}`}>
<span className="log-time">{new Date(log.timestamp).toLocaleTimeString()}</span>
<span className="log-time">
{new Date(log.timestamp).toLocaleTimeString()}
</span>
<span className="log-message">{log.message}</span>
{log.metadata && (
<details className="log-metadata">
@@ -223,7 +257,9 @@ function App() {
))}
{currentAssistantMessage && (
<div className="log-entry assistant streaming">
<span className="log-time">{new Date().toLocaleTimeString()}</span>
<span className="log-time">
{new Date().toLocaleTimeString()}
</span>
<span className="log-message">{currentAssistantMessage}</span>
<span className="streaming-indicator">...</span>
</div>
@@ -241,13 +277,21 @@ function App() {
disabled={!ws.isConnected}
className="message-input"
/>
<button type="submit" disabled={!ws.isConnected || !userInput.trim()} className="send-button">
<button
type="submit"
disabled={!ws.isConnected || !userInput.trim()}
className="send-button"
>
Send
</button>
</form>
<div className="test-controls">
<button onClick={handlePing} disabled={!ws.isConnected} className="ping-button">
<button
onClick={handlePing}
disabled={!ws.isConnected}
className="ping-button"
>
Send Ping (Test)
</button>
</div>