jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Playwright MCP integration and TTS improvements

Browse Source 
- Integrate Playwright MCP for browser automation capabilities
- Add TTS manager for improved audio synthesis management
- Add ToolCallCard UI component for better tool call visualization
- Upgrade Vite to v7.1.10 for improved build performance
- Add lucide-react for icon components
- Add dotenv for environment variable management
- Update agent prompt and orchestrator for enhanced interactions
- Improve audio playback and voice controls in UI

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
2025-10-19 23:23:32 +02:00
co-authored by Claude
parent d7a387d911
commit 0bfda11c1f
20 changed files with 1721 additions and 1317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-assistant/src/server/agent/llm-openai.ts
+139 -63
View File
@@ -6,6 +6,7 @@ import {
createTerminal,
captureTerminal,
sendText,
sendKeys,
renameTerminal,
killTerminal,
} from "../daemon/terminal-manager.js";
@@ -17,7 +18,8 @@ import invariant from "tiny-invariant";
*/
export const terminalTools = {
list_terminals: tool({
description: "List all available terminals in the voice-dev session",
description:
"List all terminals (isolated shell environments). Returns terminal name, active status, current working directory, currently running command, and the last 5 lines of output for each terminal.",
inputSchema: z.object({}),
execute: async () => {
const terminals = await listTerminals();
@@ -27,14 +29,24 @@ export const terminalTools = {
create_terminal: tool({
description:
"Create a new terminal with specified name and working directory",
"Create a new terminal (isolated shell environment) at a specific working directory. Optionally execute an initial command after creation. Terminal names must be unique. Always specify workingDirectory based on context - use project paths when working on projects, or the same directory as current terminal when user says 'another terminal here'. Defaults to ~ only if no context.",
inputSchema: z.object({
name: z.string().describe("Unique name for the terminal"),
workingDirectory: z.string().describe("Working directory path"),
name: z
.string()
.describe(
"Unique name for the terminal. Should be descriptive of what the terminal is used for (e.g., 'web-dev', 'api-server', 'tests')."
),
workingDirectory: z
.string()
.describe(
"Absolute path to the working directory for this terminal. Can use ~ for home directory. Required parameter - set contextually based on what the user is working on. Use project paths when working on projects. Defaults to home directory (~) only if no context."
),
initialCommand: z
.string()
.optional()
.describe("Optional command to run on creation"),
.describe(
"Optional command to execute after creating the terminal (e.g., 'npm run dev', 'python -m venv venv'). The command runs after changing to the working directory."
),
}),
execute: async ({ name, workingDirectory, initialCommand }) => {
const terminal = await createTerminal({
@@ -47,9 +59,10 @@ export const terminalTools = {
}),
capture_terminal: tool({
description: "Capture and return the output from a terminal",
description:
"Capture and return the output from a terminal. Returns the last N lines of terminal content. Useful for checking command results, monitoring running processes, or debugging issues.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID (e.g., @123)"),
terminalName: z.string().describe("Name of the terminal"),
lines: z
.number()
.optional()
@@ -57,34 +70,51 @@ export const terminalTools = {
wait: z
.number()
.optional()
.describe("Milliseconds to wait before capture"),
.describe(
"Milliseconds to wait before capturing output. Useful for slow commands."
),
}),
execute: async ({ terminalId, lines, wait }) => {
const output = await captureTerminal(terminalId, lines, wait);
execute: async ({ terminalName, lines, wait }) => {
const output = await captureTerminal(terminalName, lines, wait);
return { output };
},
}),
send_text: tool({
description: "Send text or commands to a terminal",
description:
"Type text into a terminal. This is the PRIMARY way to execute shell commands with bash operators (&&, ||, |, ;, etc.) - set pressEnter=true to run the command. Also use for interactive applications, REPLs, forms, and text entry. For special keys or control sequences, use send_keys instead.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID (e.g., @123)"),
text: z.string().describe("Text to send to the terminal"),
terminalName: z.string().describe("Name of the terminal"),
text: z
.string()
.describe(
"Text to type into the terminal. For shell commands, can use any bash operators: && (chain), || (or), | (pipe), ; (sequential), etc."
),
pressEnter: z
.boolean()
.optional()
.describe("Whether to press Enter after text"),
.describe(
"Press Enter after typing the text (default: false). Set to true to execute shell commands or submit text input."
),
return_output: z
.object({
lines: z.number().optional(),
wait: z.number().optional(),
lines: z
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
.number()
.optional()
.describe("Milliseconds to wait before capturing output"),
})
.optional()
.describe("Capture output after sending"),
.describe(
"Capture terminal output after sending text. Specify 'wait' for slow commands."
),
}),
execute: async ({ terminalId, text, pressEnter, return_output }) => {
execute: async ({ terminalName, text, pressEnter, return_output }) => {
const output = await sendText(
terminalId,
terminalName,
text,
pressEnter,
return_output
@@ -93,25 +123,72 @@ export const terminalTools = {
},
}),
rename_terminal: tool({
description: "Rename an existing terminal",
send_keys: tool({
description:
"Send special keys or key combinations to a terminal. Use for TUI navigation and control sequences. Examples: 'Up', 'Down', 'Enter', 'Escape', 'C-c' (Ctrl+C), 'M-x' (Alt+X). For typing regular text, use send_text instead. Supports repeating key presses and optionally capturing output after sending keys.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID to rename"),
newName: z.string().describe("New unique name for the terminal"),
terminalName: z.string().describe("Name of the terminal"),
keys: z
.string()
.describe(
"Special key name or key combination: 'Up', 'Down', 'Left', 'Right', 'Enter', 'Escape', 'Tab', 'Space', 'C-c', 'M-x', etc."
),
repeat: z
.number()
.min(1)
.optional()
.describe("Number of times to repeat the key press (default: 1)"),
return_output: z
.object({
lines: z
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
.number()
.optional()
.describe("Milliseconds to wait before capturing output"),
})
.optional()
.describe(
"Capture terminal output after sending keys. Specify 'wait' for slow commands."
),
}),
execute: async ({ terminalId, newName }) => {
await renameTerminal(terminalId, newName);
execute: async ({ terminalName, keys, repeat, return_output }) => {
const output = await sendKeys(terminalName, keys, repeat, return_output);
return { output };
},
}),
rename_terminal: tool({
description:
"Rename a terminal to a more descriptive name. The new name must be unique among all terminals.",
inputSchema: z.object({
terminalName: z.string().describe("Current name of the terminal"),
newName: z
.string()
.describe(
"New unique name for the terminal. Should be descriptive of the terminal's purpose."
),
}),
execute: async ({ terminalName, newName }) => {
await renameTerminal(terminalName, newName);
return { success: true };
},
}),
kill_terminal: tool({
description: "Close and destroy a terminal",
description:
"Close and destroy a terminal. This will terminate any running processes in the terminal. Use with caution.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID to kill"),
terminalName: z
.string()
.describe(
"Name of the terminal to kill. Get this from list_terminals."
),
}),
execute: async ({ terminalId }) => {
await killTerminal(terminalId);
execute: async ({ terminalName }) => {
await killTerminal(terminalName);
return { success: true };
},
}),
@@ -129,12 +206,14 @@ export interface Message {
* Streaming LLM parameters
*/
export interface StreamLLMParams {
systemPrompt: string;
messages: Message[];
onChunk?: (chunk: string) => void;
abortSignal?: AbortSignal;
onChunk?: (chunk: string) => void | Promise<void>;
onTextSegment?: (segment: string) => void;
onToolCall?: (toolName: string, args: any) => void;
onToolResult?: (toolName: string, result: any) => void;
onFinish?: (fullText: string) => void;
onToolCall?: (toolCallId: string, toolName: string, args: any) => Promise<void>;
onToolResult?: (toolCallId: string, toolName: string, result: any) => void;
onFinish?: (fullText: string) => void | Promise<void>;
}
/**
@@ -149,12 +228,32 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
const result = await streamText({
model: openrouter("anthropic/claude-haiku-4.5"),
system: params.systemPrompt,
messages: params.messages,
tools: terminalTools,
onChunk: (chuk) => {
// if (params.onChunk) {
// params.onChunk(chunk);
// }
abortSignal: params.abortSignal,
onChunk: async ({ chunk }) => {
// console.log("onChunk", chunk);
if (chunk.type === "text-delta") {
// Accumulate text in buffer
textBuffer += chunk.text;
fullText += chunk.text;
params.onChunk?.(chunk.text);
} else if (chunk.type === "tool-call") {
// Flush accumulated text as a segment before tool call
flushTextBuffer();
// Emit tool call event
if (params.onToolCall) {
await params.onToolCall(chunk.toolCallId, chunk.toolName, chunk.input);
}
} else if (chunk.type === "tool-result") {
// Emit tool result event
if (params.onToolResult) {
params.onToolResult(chunk.toolCallId, chunk.toolName, chunk.output);
}
}
},
stopWhen: stepCountIs(10),
});
@@ -169,38 +268,15 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
textBuffer = "";
}
// Stream all events from fullStream
for await (const part of result.fullStream) {
if (part.type === "text-delta") {
// Accumulate text in buffer
textBuffer += part.text;
fullText += part.text;
// Stream individual deltas to caller
if (params.onChunk) {
params.onChunk(part.text);
}
} else if (part.type === "tool-call") {
// Flush accumulated text as a segment before tool call
flushTextBuffer();
// Emit tool call event
if (params.onToolCall) {
params.onToolCall(part.toolName, part.input);
}
} else if (part.type === "tool-result") {
// Emit tool result event
if (params.onToolResult) {
params.onToolResult(part.toolName, part.output);
}
}
for await (const _part of result.fullStream) {
// console.log("part", _part);
}
// Flush any remaining text at the end
flushTextBuffer();
if (params.onFinish) {
params.onFinish(fullText);
await params.onFinish(fullText);
}
return fullText;
packages/voice-assistant/src/server/agent/orchestrator.ts
+67 -49
View File
@@ -1,6 +1,7 @@
import { v4 as uuidv4 } from "uuid";
import { getSystemPrompt } from "./system-prompt.js";
import { streamLLM, type Message } from "./llm-openai.js";
import { generateTTSAndWaitForPlayback } from "./tts-manager.js";
import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
interface ConversationContext {
@@ -17,13 +18,13 @@ interface ConversationContext {
const conversations = new Map<string, ConversationContext>();
/**
* Create a new conversation with system prompt
* Create a new conversation
*/
export function createConversation(): string {
const id = uuidv4();
conversations.set(id, {
id,
messages: [{ role: "system", content: getSystemPrompt() }],
messages: [],
createdAt: new Date(),
lastActivity: new Date(),
});
@@ -37,6 +38,13 @@ export function getConversation(id: string): ConversationContext | null {
return conversations.get(id) || null;
}
/**
* Delete a conversation by ID
*/
export function deleteConversation(id: string): void {
conversations.delete(id);
}
/**
* Process user message through the LLM orchestrator
* Handles streaming, tool calls, and WebSocket broadcasting
@@ -46,6 +54,7 @@ export async function processUserMessage(params: {
message: string;
wsServer?: VoiceAssistantWebSocketServer;
enableTTS?: boolean;
abortSignal?: AbortSignal;
}): Promise<string> {
const conversation = conversations.get(params.conversationId);
if (!conversation) {
@@ -65,65 +74,72 @@ export async function processUserMessage(params: {
let assistantResponse = "";
try {
// Track pending TTS playback promise
let pendingTTS: Promise<void> | null = null;
// Stream LLM response with tool execution
assistantResponse = await streamLLM({
systemPrompt: getSystemPrompt(),
messages: conversation.messages,
onChunk: (chunk) => {
// Broadcast streaming chunks to WebSocket
if (params.wsServer) {
params.wsServer.broadcast({
type: "assistant_chunk",
payload: { chunk },
});
}
},
abortSignal: params.abortSignal,
onTextSegment: (segment) => {
// Broadcast complete text segments (for future TTS integration)
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "system",
content: `Text segment: ${segment}`,
metadata: { segment, readyForTTS: true },
});
// Create TTS promise (don't await it yet)
if (params.wsServer && params.enableTTS) {
pendingTTS = generateTTSAndWaitForPlayback(segment, params.wsServer);
}
},
onToolCall: (toolName, args) => {
// Broadcast tool call to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "tool_call",
content: `Calling ${toolName}`,
metadata: { toolName, arguments: args },
});
}
},
onToolResult: (toolName, result) => {
// Broadcast tool result to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "tool_result",
content: `Tool ${toolName} completed`,
metadata: { toolName, result },
});
}
},
onFinish: (fullText) => {
// Broadcast complete assistant response
// Broadcast complete text segments
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: fullText,
content: segment,
});
}
},
onChunk: async (chunk) => {
params.wsServer?.broadcast({
type: "assistant_chunk",
payload: { chunk },
});
},
onToolCall: async (toolCallId, toolName, args) => {
if (pendingTTS) {
console.log("Waiting for pending TTS to finish to execute", toolName);
await pendingTTS;
pendingTTS = null;
}
// Broadcast tool call to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: toolCallId,
timestamp: new Date(),
type: "tool_call",
content: `Calling ${toolName}`,
metadata: { toolCallId, toolName, arguments: args },
});
}
},
onToolResult: (toolCallId, toolName, result) => {
// Broadcast tool result to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: toolCallId,
timestamp: new Date(),
type: "tool_result",
content: `Tool ${toolName} completed`,
metadata: { toolCallId, toolName, result },
});
}
},
onFinish: async () => {
// Wait for any final pending TTS to complete
if (pendingTTS) {
await pendingTTS;
pendingTTS = null;
}
},
});
// Add assistant response to context
@@ -138,7 +154,9 @@ export async function processUserMessage(params: {
id: uuidv4(),
timestamp: new Date(),
type: "error",
content: `Error: ${error instanceof Error ? error.message : String(error)}`,
content: `Error: ${
error instanceof Error ? error.message : String(error)
}`,
});
}
throw error;
packages/voice-assistant/src/server/agent/tts-manager.ts
+88
View File
@@ -0,0 +1,88 @@
import { v4 as uuidv4 } from "uuid";
import { synthesizeSpeech } from "./tts-openai.js";
import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
interface PendingPlayback {
resolve: () => void;
reject: (error: Error) => void;
timeout: NodeJS.Timeout;
}
/**
* Store pending playback confirmations
* Maps audio ID -> promise resolve/reject handlers
*/
const pendingPlaybacks = new Map<string, PendingPlayback>();
/**
* Timeout for waiting on client playback confirmation (30 seconds)
*/
const PLAYBACK_TIMEOUT_MS = 30000;
/**
* Generate TTS audio, broadcast to clients, and wait for playback confirmation
* Returns a Promise that resolves when the client confirms playback completed
*/
export async function generateTTSAndWaitForPlayback(
text: string,
wsServer: VoiceAssistantWebSocketServer
): Promise<void> {
// Generate TTS audio
const { audio, format } = await synthesizeSpeech(text);
// Create unique ID for this audio segment
const audioId = uuidv4();
// Create promise that will be resolved when client confirms playback
const playbackPromise = new Promise<void>((resolve, reject) => {
// Set timeout to prevent hanging forever
const timeout = setTimeout(() => {
pendingPlaybacks.delete(audioId);
reject(new Error(`Audio playback timeout for ${audioId}`));
}, PLAYBACK_TIMEOUT_MS);
// Store handlers
pendingPlaybacks.set(audioId, { resolve, reject, timeout });
});
// Broadcast audio to clients
wsServer.broadcast({
type: "audio_output",
payload: {
id: audioId,
audio: audio.toString("base64"),
format,
},
});
console.log(
`[TTS-Manager] ${new Date().toISOString()} Sent audio ${audioId}, waiting for playback...`
);
// Wait for playback confirmation
await playbackPromise;
console.log(
`[TTS-Manager] ${new Date().toISOString()} Audio ${audioId} playback confirmed`
);
}
/**
* Called when client confirms audio playback completed
* Resolves the corresponding promise
*/
export function confirmAudioPlayed(audioId: string): void {
const pending = pendingPlaybacks.get(audioId);
if (!pending) {
console.warn(
`[TTS-Manager] Received confirmation for unknown audio ID: ${audioId}`
);
return;
}
// Clear timeout and resolve promise
clearTimeout(pending.timeout);
pending.resolve();
pendingPlaybacks.delete(audioId);
}
packages/voice-assistant/src/server/agent/tts-openai.ts
+1
View File
@@ -54,6 +54,7 @@ export async function synthesizeSpeech(text: string): Promise<SpeechResult> {
model: config.model!,
voice: config.voice!,
input: text,
// speed: 1.2,
response_format: config.responseFormat as "mp3" | "opus" | "aac" | "flac",
});