Based on the git diff showing extensive package-lock.json changes and modifications to the agent system prompt and configuration, here's the commit message:
``` feat: add AI SDK dependencies and refine agent voice interaction config - Add @ai-sdk packages for enhanced AI capabilities - Install Deepgram SDK, livekit-client, and werift for voice/WebRTC - Expand agent system prompt with multi-tool workflows and Claude Code interaction patterns - Adjust VAD timing (1.2s silence threshold) and turn detection for better voice UX - Switch TTS to inworld/inworld-tts-1 with Olivia voice - Configure interruption timing (0.2s detection, no word requirement) - Add comprehensive guidelines for conversational responses vs structured output - Document plan mode triggers and Claude response type handling 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> ```
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,173 @@
|
||||
import { streamText, tool } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
listTerminals,
|
||||
createTerminal,
|
||||
captureTerminal,
|
||||
sendText,
|
||||
renameTerminal,
|
||||
killTerminal,
|
||||
} from '../daemon/terminal-manager.js';
|
||||
|
||||
let openaiClient: ReturnType<typeof createOpenAI> | null = null;
|
||||
|
||||
/**
|
||||
* Initialize OpenAI client with API key
|
||||
*/
|
||||
export function initializeOpenAI(apiKey: string): void {
|
||||
openaiClient = createOpenAI({ apiKey });
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminal tools using Vercel AI SDK tool() function
|
||||
* These are automatically executed when the LLM calls them
|
||||
*/
|
||||
export const terminalTools = {
|
||||
list_terminals: tool({
|
||||
description: 'List all available terminals in the voice-dev session',
|
||||
parameters: z.object({}),
|
||||
execute: async () => {
|
||||
const terminals = await listTerminals();
|
||||
return { terminals };
|
||||
},
|
||||
}),
|
||||
|
||||
create_terminal: tool({
|
||||
description: 'Create a new terminal with specified name and working directory',
|
||||
parameters: z.object({
|
||||
name: z.string().describe('Unique name for the terminal'),
|
||||
workingDirectory: z.string().describe('Working directory path'),
|
||||
initialCommand: z.string().optional().describe('Optional command to run on creation'),
|
||||
}),
|
||||
execute: async ({ name, workingDirectory, initialCommand }) => {
|
||||
const terminal = await createTerminal({ name, workingDirectory, initialCommand });
|
||||
return { terminal };
|
||||
},
|
||||
}),
|
||||
|
||||
capture_terminal: tool({
|
||||
description: 'Capture and return the output from a terminal',
|
||||
parameters: z.object({
|
||||
terminalId: z.string().describe('Terminal ID (e.g., @123)'),
|
||||
lines: z.number().optional().describe('Number of lines to capture (default: 200)'),
|
||||
wait: z.number().optional().describe('Milliseconds to wait before capture'),
|
||||
}),
|
||||
execute: async ({ terminalId, lines, wait }) => {
|
||||
const output = await captureTerminal(terminalId, lines, wait);
|
||||
return { output };
|
||||
},
|
||||
}),
|
||||
|
||||
send_text: tool({
|
||||
description: 'Send text or commands to a terminal',
|
||||
parameters: z.object({
|
||||
terminalId: z.string().describe('Terminal ID (e.g., @123)'),
|
||||
text: z.string().describe('Text to send to the terminal'),
|
||||
pressEnter: z.boolean().optional().describe('Whether to press Enter after text'),
|
||||
return_output: z
|
||||
.object({
|
||||
lines: z.number().optional(),
|
||||
wait: z.number().optional(),
|
||||
})
|
||||
.optional()
|
||||
.describe('Capture output after sending'),
|
||||
}),
|
||||
execute: async ({ terminalId, text, pressEnter, return_output }) => {
|
||||
const output = await sendText(terminalId, text, pressEnter, return_output);
|
||||
return { output };
|
||||
},
|
||||
}),
|
||||
|
||||
rename_terminal: tool({
|
||||
description: 'Rename an existing terminal',
|
||||
parameters: z.object({
|
||||
terminalId: z.string().describe('Terminal ID to rename'),
|
||||
newName: z.string().describe('New unique name for the terminal'),
|
||||
}),
|
||||
execute: async ({ terminalId, newName }) => {
|
||||
await renameTerminal(terminalId, newName);
|
||||
return { success: true };
|
||||
},
|
||||
}),
|
||||
|
||||
kill_terminal: tool({
|
||||
description: 'Close and destroy a terminal',
|
||||
parameters: z.object({
|
||||
terminalId: z.string().describe('Terminal ID to kill'),
|
||||
}),
|
||||
execute: async ({ terminalId }) => {
|
||||
await killTerminal(terminalId);
|
||||
return { success: true };
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
/**
|
||||
* Message interface for conversation
|
||||
*/
|
||||
export interface Message {
|
||||
role: 'system' | 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming LLM parameters
|
||||
*/
|
||||
export interface StreamLLMParams {
|
||||
messages: Message[];
|
||||
onChunk?: (chunk: string) => void;
|
||||
onToolCall?: (toolName: string, args: any) => void;
|
||||
onToolResult?: (toolName: string, result: any) => void;
|
||||
onFinish?: (fullText: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream LLM response with automatic tool execution
|
||||
*/
|
||||
export async function streamLLM(params: StreamLLMParams): Promise<string> {
|
||||
if (!openaiClient) {
|
||||
throw new Error('OpenAI client not initialized. Call initializeOpenAI() first.');
|
||||
}
|
||||
|
||||
const result = await streamText({
|
||||
model: openaiClient('gpt-4o') as any, // Type assertion to work around AI SDK version conflicts
|
||||
messages: params.messages,
|
||||
tools: terminalTools,
|
||||
maxSteps: 5, // Allow up to 5 tool executions per turn
|
||||
onStepFinish: async (event) => {
|
||||
// Called after each step (tool call or text generation)
|
||||
if (event.toolCalls && event.toolCalls.length > 0) {
|
||||
for (const toolCall of event.toolCalls) {
|
||||
if (params.onToolCall) {
|
||||
params.onToolCall(toolCall.toolName, toolCall.args);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (event.toolResults && event.toolResults.length > 0) {
|
||||
for (const toolResult of event.toolResults) {
|
||||
if (params.onToolResult) {
|
||||
params.onToolResult(toolResult.toolName, toolResult.result);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
let fullText = '';
|
||||
|
||||
// Stream text chunks to the caller
|
||||
for await (const chunk of result.textStream) {
|
||||
fullText += chunk;
|
||||
if (params.onChunk) {
|
||||
params.onChunk(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
if (params.onFinish) {
|
||||
params.onFinish(fullText);
|
||||
}
|
||||
|
||||
return fullText;
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { getSystemPrompt } from './system-prompt.js';
|
||||
import { streamLLM, type Message } from './llm-openai.js';
|
||||
import type { VoiceAssistantWebSocketServer } from '../websocket-server.js';
|
||||
|
||||
interface ConversationContext {
|
||||
id: string;
|
||||
messages: Message[];
|
||||
createdAt: Date;
|
||||
lastActivity: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store active conversations (in-memory for now)
|
||||
* In production, this could be persisted to a database
|
||||
*/
|
||||
const conversations = new Map<string, ConversationContext>();
|
||||
|
||||
/**
|
||||
* Create a new conversation with system prompt
|
||||
*/
|
||||
export function createConversation(): string {
|
||||
const id = uuidv4();
|
||||
conversations.set(id, {
|
||||
id,
|
||||
messages: [{ role: 'system', content: getSystemPrompt() }],
|
||||
createdAt: new Date(),
|
||||
lastActivity: new Date(),
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get conversation by ID
|
||||
*/
|
||||
export function getConversation(id: string): ConversationContext | null {
|
||||
return conversations.get(id) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process user message through the LLM orchestrator
|
||||
* Handles streaming, tool calls, and WebSocket broadcasting
|
||||
*/
|
||||
export async function processUserMessage(params: {
|
||||
conversationId: string;
|
||||
message: string;
|
||||
wsServer?: VoiceAssistantWebSocketServer;
|
||||
enableTTS?: boolean;
|
||||
}): Promise<string> {
|
||||
const conversation = conversations.get(params.conversationId);
|
||||
if (!conversation) {
|
||||
throw new Error('Conversation not found');
|
||||
}
|
||||
|
||||
// Add user message to context
|
||||
conversation.messages.push({
|
||||
role: 'user',
|
||||
content: params.message,
|
||||
});
|
||||
conversation.lastActivity = new Date();
|
||||
|
||||
// Broadcast user message to WebSocket clients
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcastActivityLog({
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: 'transcript',
|
||||
content: params.message,
|
||||
});
|
||||
}
|
||||
|
||||
let assistantResponse = '';
|
||||
|
||||
try {
|
||||
// Stream LLM response with tool execution
|
||||
assistantResponse = await streamLLM({
|
||||
messages: conversation.messages,
|
||||
onChunk: (chunk) => {
|
||||
// Broadcast streaming chunks to WebSocket
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcast({
|
||||
type: 'assistant_chunk',
|
||||
payload: { chunk },
|
||||
});
|
||||
}
|
||||
},
|
||||
onToolCall: (toolName, args) => {
|
||||
// Broadcast tool call to WebSocket
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcastActivityLog({
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: 'tool_call',
|
||||
content: `Calling ${toolName}`,
|
||||
metadata: { toolName, arguments: args },
|
||||
});
|
||||
}
|
||||
},
|
||||
onToolResult: (toolName, result) => {
|
||||
// Broadcast tool result to WebSocket
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcastActivityLog({
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: 'tool_result',
|
||||
content: `Tool ${toolName} completed`,
|
||||
metadata: { toolName, result },
|
||||
});
|
||||
}
|
||||
},
|
||||
onFinish: (fullText) => {
|
||||
// Broadcast complete assistant response
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcastActivityLog({
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: 'assistant',
|
||||
content: fullText,
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Add assistant response to context
|
||||
conversation.messages.push({
|
||||
role: 'assistant',
|
||||
content: assistantResponse,
|
||||
});
|
||||
} catch (error: any) {
|
||||
// Broadcast error to WebSocket
|
||||
if (params.wsServer) {
|
||||
params.wsServer.broadcastActivityLog({
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: 'error',
|
||||
content: `Error: ${error.message}`,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return assistantResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old conversations (call periodically)
|
||||
*/
|
||||
export function cleanupConversations(maxAgeMinutes: number = 60): void {
|
||||
const now = new Date();
|
||||
for (const [id, conv] of conversations.entries()) {
|
||||
const ageMinutes = (now.getTime() - conv.lastActivity.getTime()) / (1000 * 60);
|
||||
if (ageMinutes > maxAgeMinutes) {
|
||||
conversations.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get conversation statistics
|
||||
*/
|
||||
export function getConversationStats(): {
|
||||
total: number;
|
||||
conversations: Array<{ id: string; messageCount: number; lastActivity: Date }>;
|
||||
} {
|
||||
return {
|
||||
total: conversations.size,
|
||||
conversations: Array.from(conversations.values()).map((conv) => ({
|
||||
id: conv.id,
|
||||
messageCount: conv.messages.length,
|
||||
lastActivity: conv.lastActivity,
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import OpenAI from 'openai';
|
||||
import { writeFile, unlink } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
export interface STTConfig {
|
||||
apiKey: string;
|
||||
model?: 'whisper-1';
|
||||
}
|
||||
|
||||
export interface TranscriptionResult {
|
||||
text: string;
|
||||
language?: string;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
let openaiClient: OpenAI | null = null;
|
||||
let config: STTConfig | null = null;
|
||||
|
||||
export function initializeSTT(sttConfig: STTConfig): void {
|
||||
config = sttConfig;
|
||||
openaiClient = new OpenAI({
|
||||
apiKey: sttConfig.apiKey,
|
||||
});
|
||||
console.log('✓ STT (OpenAI Whisper) initialized');
|
||||
}
|
||||
|
||||
export async function transcribeAudio(
|
||||
audioBuffer: Buffer,
|
||||
format: string
|
||||
): Promise<TranscriptionResult> {
|
||||
if (!openaiClient || !config) {
|
||||
throw new Error('STT not initialized. Call initializeSTT() first.');
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
let tempFilePath: string | null = null;
|
||||
|
||||
try {
|
||||
// Map format to file extension
|
||||
const ext = getFileExtension(format);
|
||||
|
||||
// Write audio buffer to temporary file
|
||||
// OpenAI API requires file upload, not raw buffer
|
||||
tempFilePath = join(tmpdir(), `audio-${uuidv4()}.${ext}`);
|
||||
await writeFile(tempFilePath, audioBuffer);
|
||||
|
||||
console.log(`[STT] Transcribing audio file: ${tempFilePath} (${audioBuffer.length} bytes)`);
|
||||
|
||||
// Call OpenAI Whisper API
|
||||
const response = await openaiClient.audio.transcriptions.create({
|
||||
file: await import('fs').then((fs) => fs.createReadStream(tempFilePath!)),
|
||||
model: config.model || 'whisper-1',
|
||||
response_format: 'verbose_json', // Get language and duration info
|
||||
});
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
console.log(`[STT] Transcription complete in ${duration}ms: "${response.text}"`);
|
||||
|
||||
return {
|
||||
text: response.text,
|
||||
language: response.language,
|
||||
duration: duration,
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[STT] Transcription error:', error);
|
||||
throw new Error(`STT transcription failed: ${error.message}`);
|
||||
} finally {
|
||||
// Clean up temporary file
|
||||
if (tempFilePath) {
|
||||
try {
|
||||
await unlink(tempFilePath);
|
||||
} catch (cleanupError) {
|
||||
console.warn(`[STT] Failed to clean up temp file: ${tempFilePath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getFileExtension(format: string): string {
|
||||
// Map mime types or format strings to file extensions
|
||||
const formatLower = format.toLowerCase();
|
||||
|
||||
if (formatLower.includes('webm')) return 'webm';
|
||||
if (formatLower.includes('ogg')) return 'ogg';
|
||||
if (formatLower.includes('mp3')) return 'mp3';
|
||||
if (formatLower.includes('wav')) return 'wav';
|
||||
if (formatLower.includes('m4a')) return 'm4a';
|
||||
if (formatLower.includes('mp4')) return 'mp4';
|
||||
if (formatLower.includes('flac')) return 'flac';
|
||||
|
||||
// Default to webm (common browser format)
|
||||
return 'webm';
|
||||
}
|
||||
|
||||
export function isSTTInitialized(): boolean {
|
||||
return openaiClient !== null && config !== null;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
import path from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Load system prompt from agent-prompt.md in package root
|
||||
export function loadSystemPrompt(): string {
|
||||
const promptPath = path.join(__dirname, '../../../agent-prompt.md');
|
||||
return readFileSync(promptPath, 'utf-8');
|
||||
}
|
||||
|
||||
// Cache the loaded prompt
|
||||
let cachedPrompt: string | null = null;
|
||||
|
||||
export function getSystemPrompt(): string {
|
||||
if (!cachedPrompt) {
|
||||
cachedPrompt = loadSystemPrompt();
|
||||
}
|
||||
return cachedPrompt;
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import OpenAI from 'openai';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export interface TTSConfig {
|
||||
apiKey: string;
|
||||
model?: 'tts-1' | 'tts-1-hd';
|
||||
voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
|
||||
responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';
|
||||
}
|
||||
|
||||
export interface SpeechResult {
|
||||
audio: Buffer;
|
||||
format: string;
|
||||
}
|
||||
|
||||
let openaiClient: OpenAI | null = null;
|
||||
let config: TTSConfig | null = null;
|
||||
|
||||
export function initializeTTS(ttsConfig: TTSConfig): void {
|
||||
config = {
|
||||
model: 'tts-1',
|
||||
voice: 'alloy',
|
||||
responseFormat: 'mp3',
|
||||
...ttsConfig,
|
||||
};
|
||||
openaiClient = new OpenAI({
|
||||
apiKey: ttsConfig.apiKey,
|
||||
});
|
||||
console.log(`✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}`);
|
||||
}
|
||||
|
||||
export async function synthesizeSpeech(text: string): Promise<SpeechResult> {
|
||||
if (!openaiClient || !config) {
|
||||
throw new Error('TTS not initialized. Call initializeTTS() first.');
|
||||
}
|
||||
|
||||
if (!text || text.trim().length === 0) {
|
||||
throw new Error('Cannot synthesize empty text');
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
console.log(`[TTS] Synthesizing speech: "${text.substring(0, 50)}${text.length > 50 ? '...' : ''}"`);
|
||||
|
||||
// Call OpenAI TTS API with streaming
|
||||
const response = await openaiClient.audio.speech.create({
|
||||
model: config.model!,
|
||||
voice: config.voice!,
|
||||
input: text,
|
||||
response_format: config.responseFormat as 'mp3' | 'opus' | 'aac' | 'flac',
|
||||
});
|
||||
|
||||
// Response is a stream, convert to buffer
|
||||
const audioBuffer = await streamToBuffer(response.body as unknown as Readable);
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`[TTS] Speech synthesis complete in ${duration}ms (${audioBuffer.length} bytes)`);
|
||||
|
||||
return {
|
||||
audio: audioBuffer,
|
||||
format: config.responseFormat || 'mp3',
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[TTS] Speech synthesis error:', error);
|
||||
throw new Error(`TTS synthesis failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function streamToBuffer(stream: Readable): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk) => {
|
||||
chunks.push(Buffer.from(chunk));
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
resolve(Buffer.concat(chunks));
|
||||
});
|
||||
|
||||
stream.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function isTTSInitialized(): boolean {
|
||||
return openaiClient !== null && config !== null;
|
||||
}
|
||||
|
||||
export function getTTSConfig(): TTSConfig | null {
|
||||
return config;
|
||||
}
|
||||
Reference in New Issue
Block a user