jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Playwright MCP integration and TTS improvements

Browse Source 
- Integrate Playwright MCP for browser automation capabilities
- Add TTS manager for improved audio synthesis management
- Add ToolCallCard UI component for better tool call visualization
- Upgrade Vite to v7.1.10 for improved build performance
- Add lucide-react for icon components
- Add dotenv for environment variable management
- Update agent prompt and orchestrator for enhanced interactions
- Improve audio playback and voice controls in UI

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
2025-10-19 23:23:32 +02:00
co-authored by Claude
parent d7a387d911
commit 0bfda11c1f
20 changed files with 1721 additions and 1317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-assistant/src/server/agent/llm-openai.ts
+139 -63
View File
@@ -6,6 +6,7 @@ import {
createTerminal,
captureTerminal,
sendText,
sendKeys,
renameTerminal,
killTerminal,
} from "../daemon/terminal-manager.js";
@@ -17,7 +18,8 @@ import invariant from "tiny-invariant";
*/
export const terminalTools = {
list_terminals: tool({
description: "List all available terminals in the voice-dev session",
description:
"List all terminals (isolated shell environments). Returns terminal name, active status, current working directory, currently running command, and the last 5 lines of output for each terminal.",
inputSchema: z.object({}),
execute: async () => {
const terminals = await listTerminals();
@@ -27,14 +29,24 @@ export const terminalTools = {
create_terminal: tool({
description:
"Create a new terminal with specified name and working directory",
"Create a new terminal (isolated shell environment) at a specific working directory. Optionally execute an initial command after creation. Terminal names must be unique. Always specify workingDirectory based on context - use project paths when working on projects, or the same directory as current terminal when user says 'another terminal here'. Defaults to ~ only if no context.",
inputSchema: z.object({
name: z.string().describe("Unique name for the terminal"),
workingDirectory: z.string().describe("Working directory path"),
name: z
.string()
.describe(
"Unique name for the terminal. Should be descriptive of what the terminal is used for (e.g., 'web-dev', 'api-server', 'tests')."
),
workingDirectory: z
.string()
.describe(
"Absolute path to the working directory for this terminal. Can use ~ for home directory. Required parameter - set contextually based on what the user is working on. Use project paths when working on projects. Defaults to home directory (~) only if no context."
),
initialCommand: z
.string()
.optional()
.describe("Optional command to run on creation"),
.describe(
"Optional command to execute after creating the terminal (e.g., 'npm run dev', 'python -m venv venv'). The command runs after changing to the working directory."
),
}),
execute: async ({ name, workingDirectory, initialCommand }) => {
const terminal = await createTerminal({
@@ -47,9 +59,10 @@ export const terminalTools = {
}),
capture_terminal: tool({
description: "Capture and return the output from a terminal",
description:
"Capture and return the output from a terminal. Returns the last N lines of terminal content. Useful for checking command results, monitoring running processes, or debugging issues.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID (e.g., @123)"),
terminalName: z.string().describe("Name of the terminal"),
lines: z
.number()
.optional()
@@ -57,34 +70,51 @@ export const terminalTools = {
wait: z
.number()
.optional()
.describe("Milliseconds to wait before capture"),
.describe(
"Milliseconds to wait before capturing output. Useful for slow commands."
),
}),
execute: async ({ terminalId, lines, wait }) => {
const output = await captureTerminal(terminalId, lines, wait);
execute: async ({ terminalName, lines, wait }) => {
const output = await captureTerminal(terminalName, lines, wait);
return { output };
},
}),
send_text: tool({
description: "Send text or commands to a terminal",
description:
"Type text into a terminal. This is the PRIMARY way to execute shell commands with bash operators (&&, ||, |, ;, etc.) - set pressEnter=true to run the command. Also use for interactive applications, REPLs, forms, and text entry. For special keys or control sequences, use send_keys instead.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID (e.g., @123)"),
text: z.string().describe("Text to send to the terminal"),
terminalName: z.string().describe("Name of the terminal"),
text: z
.string()
.describe(
"Text to type into the terminal. For shell commands, can use any bash operators: && (chain), || (or), | (pipe), ; (sequential), etc."
),
pressEnter: z
.boolean()
.optional()
.describe("Whether to press Enter after text"),
.describe(
"Press Enter after typing the text (default: false). Set to true to execute shell commands or submit text input."
),
return_output: z
.object({
lines: z.number().optional(),
wait: z.number().optional(),
lines: z
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
.number()
.optional()
.describe("Milliseconds to wait before capturing output"),
})
.optional()
.describe("Capture output after sending"),
.describe(
"Capture terminal output after sending text. Specify 'wait' for slow commands."
),
}),
execute: async ({ terminalId, text, pressEnter, return_output }) => {
execute: async ({ terminalName, text, pressEnter, return_output }) => {
const output = await sendText(
terminalId,
terminalName,
text,
pressEnter,
return_output
@@ -93,25 +123,72 @@ export const terminalTools = {
},
}),
rename_terminal: tool({
description: "Rename an existing terminal",
send_keys: tool({
description:
"Send special keys or key combinations to a terminal. Use for TUI navigation and control sequences. Examples: 'Up', 'Down', 'Enter', 'Escape', 'C-c' (Ctrl+C), 'M-x' (Alt+X). For typing regular text, use send_text instead. Supports repeating key presses and optionally capturing output after sending keys.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID to rename"),
newName: z.string().describe("New unique name for the terminal"),
terminalName: z.string().describe("Name of the terminal"),
keys: z
.string()
.describe(
"Special key name or key combination: 'Up', 'Down', 'Left', 'Right', 'Enter', 'Escape', 'Tab', 'Space', 'C-c', 'M-x', etc."
),
repeat: z
.number()
.min(1)
.optional()
.describe("Number of times to repeat the key press (default: 1)"),
return_output: z
.object({
lines: z
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
.number()
.optional()
.describe("Milliseconds to wait before capturing output"),
})
.optional()
.describe(
"Capture terminal output after sending keys. Specify 'wait' for slow commands."
),
}),
execute: async ({ terminalId, newName }) => {
await renameTerminal(terminalId, newName);
execute: async ({ terminalName, keys, repeat, return_output }) => {
const output = await sendKeys(terminalName, keys, repeat, return_output);
return { output };
},
}),
rename_terminal: tool({
description:
"Rename a terminal to a more descriptive name. The new name must be unique among all terminals.",
inputSchema: z.object({
terminalName: z.string().describe("Current name of the terminal"),
newName: z
.string()
.describe(
"New unique name for the terminal. Should be descriptive of the terminal's purpose."
),
}),
execute: async ({ terminalName, newName }) => {
await renameTerminal(terminalName, newName);
return { success: true };
},
}),
kill_terminal: tool({
description: "Close and destroy a terminal",
description:
"Close and destroy a terminal. This will terminate any running processes in the terminal. Use with caution.",
inputSchema: z.object({
terminalId: z.string().describe("Terminal ID to kill"),
terminalName: z
.string()
.describe(
"Name of the terminal to kill. Get this from list_terminals."
),
}),
execute: async ({ terminalId }) => {
await killTerminal(terminalId);
execute: async ({ terminalName }) => {
await killTerminal(terminalName);
return { success: true };
},
}),
@@ -129,12 +206,14 @@ export interface Message {
* Streaming LLM parameters
*/
export interface StreamLLMParams {
systemPrompt: string;
messages: Message[];
onChunk?: (chunk: string) => void;
abortSignal?: AbortSignal;
onChunk?: (chunk: string) => void | Promise<void>;
onTextSegment?: (segment: string) => void;
onToolCall?: (toolName: string, args: any) => void;
onToolResult?: (toolName: string, result: any) => void;
onFinish?: (fullText: string) => void;
onToolCall?: (toolCallId: string, toolName: string, args: any) => Promise<void>;
onToolResult?: (toolCallId: string, toolName: string, result: any) => void;
onFinish?: (fullText: string) => void | Promise<void>;
}
/**
@@ -149,12 +228,32 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
const result = await streamText({
model: openrouter("anthropic/claude-haiku-4.5"),
system: params.systemPrompt,
messages: params.messages,
tools: terminalTools,
onChunk: (chuk) => {
// if (params.onChunk) {
// params.onChunk(chunk);
// }
abortSignal: params.abortSignal,
onChunk: async ({ chunk }) => {
// console.log("onChunk", chunk);
if (chunk.type === "text-delta") {
// Accumulate text in buffer
textBuffer += chunk.text;
fullText += chunk.text;
params.onChunk?.(chunk.text);
} else if (chunk.type === "tool-call") {
// Flush accumulated text as a segment before tool call
flushTextBuffer();
// Emit tool call event
if (params.onToolCall) {
await params.onToolCall(chunk.toolCallId, chunk.toolName, chunk.input);
}
} else if (chunk.type === "tool-result") {
// Emit tool result event
if (params.onToolResult) {
params.onToolResult(chunk.toolCallId, chunk.toolName, chunk.output);
}
}
},
stopWhen: stepCountIs(10),
});
@@ -169,38 +268,15 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
textBuffer = "";
}
// Stream all events from fullStream
for await (const part of result.fullStream) {
if (part.type === "text-delta") {
// Accumulate text in buffer
textBuffer += part.text;
fullText += part.text;
// Stream individual deltas to caller
if (params.onChunk) {
params.onChunk(part.text);
}
} else if (part.type === "tool-call") {
// Flush accumulated text as a segment before tool call
flushTextBuffer();
// Emit tool call event
if (params.onToolCall) {
params.onToolCall(part.toolName, part.input);
}
} else if (part.type === "tool-result") {
// Emit tool result event
if (params.onToolResult) {
params.onToolResult(part.toolName, part.output);
}
}
for await (const _part of result.fullStream) {
// console.log("part", _part);
}
// Flush any remaining text at the end
flushTextBuffer();
if (params.onFinish) {
params.onFinish(fullText);
await params.onFinish(fullText);
}
return fullText;
packages/voice-assistant/src/server/agent/orchestrator.ts
+67 -49
View File
@@ -1,6 +1,7 @@
import { v4 as uuidv4 } from "uuid";
import { getSystemPrompt } from "./system-prompt.js";
import { streamLLM, type Message } from "./llm-openai.js";
import { generateTTSAndWaitForPlayback } from "./tts-manager.js";
import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
interface ConversationContext {
@@ -17,13 +18,13 @@ interface ConversationContext {
const conversations = new Map<string, ConversationContext>();
/**
* Create a new conversation with system prompt
* Create a new conversation
*/
export function createConversation(): string {
const id = uuidv4();
conversations.set(id, {
id,
messages: [{ role: "system", content: getSystemPrompt() }],
messages: [],
createdAt: new Date(),
lastActivity: new Date(),
});
@@ -37,6 +38,13 @@ export function getConversation(id: string): ConversationContext | null {
return conversations.get(id) || null;
}
/**
* Delete a conversation by ID
*/
export function deleteConversation(id: string): void {
conversations.delete(id);
}
/**
* Process user message through the LLM orchestrator
* Handles streaming, tool calls, and WebSocket broadcasting
@@ -46,6 +54,7 @@ export async function processUserMessage(params: {
message: string;
wsServer?: VoiceAssistantWebSocketServer;
enableTTS?: boolean;
abortSignal?: AbortSignal;
}): Promise<string> {
const conversation = conversations.get(params.conversationId);
if (!conversation) {
@@ -65,65 +74,72 @@ export async function processUserMessage(params: {
let assistantResponse = "";
try {
// Track pending TTS playback promise
let pendingTTS: Promise<void> | null = null;
// Stream LLM response with tool execution
assistantResponse = await streamLLM({
systemPrompt: getSystemPrompt(),
messages: conversation.messages,
onChunk: (chunk) => {
// Broadcast streaming chunks to WebSocket
if (params.wsServer) {
params.wsServer.broadcast({
type: "assistant_chunk",
payload: { chunk },
});
}
},
abortSignal: params.abortSignal,
onTextSegment: (segment) => {
// Broadcast complete text segments (for future TTS integration)
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "system",
content: `Text segment: ${segment}`,
metadata: { segment, readyForTTS: true },
});
// Create TTS promise (don't await it yet)
if (params.wsServer && params.enableTTS) {
pendingTTS = generateTTSAndWaitForPlayback(segment, params.wsServer);
}
},
onToolCall: (toolName, args) => {
// Broadcast tool call to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "tool_call",
content: `Calling ${toolName}`,
metadata: { toolName, arguments: args },
});
}
},
onToolResult: (toolName, result) => {
// Broadcast tool result to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "tool_result",
content: `Tool ${toolName} completed`,
metadata: { toolName, result },
});
}
},
onFinish: (fullText) => {
// Broadcast complete assistant response
// Broadcast complete text segments
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: fullText,
content: segment,
});
}
},
onChunk: async (chunk) => {
params.wsServer?.broadcast({
type: "assistant_chunk",
payload: { chunk },
});
},
onToolCall: async (toolCallId, toolName, args) => {
if (pendingTTS) {
console.log("Waiting for pending TTS to finish to execute", toolName);
await pendingTTS;
pendingTTS = null;
}
// Broadcast tool call to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: toolCallId,
timestamp: new Date(),
type: "tool_call",
content: `Calling ${toolName}`,
metadata: { toolCallId, toolName, arguments: args },
});
}
},
onToolResult: (toolCallId, toolName, result) => {
// Broadcast tool result to WebSocket
if (params.wsServer) {
params.wsServer.broadcastActivityLog({
id: toolCallId,
timestamp: new Date(),
type: "tool_result",
content: `Tool ${toolName} completed`,
metadata: { toolCallId, toolName, result },
});
}
},
onFinish: async () => {
// Wait for any final pending TTS to complete
if (pendingTTS) {
await pendingTTS;
pendingTTS = null;
}
},
});
// Add assistant response to context
@@ -138,7 +154,9 @@ export async function processUserMessage(params: {
id: uuidv4(),
timestamp: new Date(),
type: "error",
content: `Error: ${error instanceof Error ? error.message : String(error)}`,
content: `Error: ${
error instanceof Error ? error.message : String(error)
}`,
});
}
throw error;
packages/voice-assistant/src/server/agent/tts-manager.ts
+88
View File
@@ -0,0 +1,88 @@
import { v4 as uuidv4 } from "uuid";
import { synthesizeSpeech } from "./tts-openai.js";
import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
interface PendingPlayback {
resolve: () => void;
reject: (error: Error) => void;
timeout: NodeJS.Timeout;
}
/**
* Store pending playback confirmations
* Maps audio ID -> promise resolve/reject handlers
*/
const pendingPlaybacks = new Map<string, PendingPlayback>();
/**
* Timeout for waiting on client playback confirmation (30 seconds)
*/
const PLAYBACK_TIMEOUT_MS = 30000;
/**
* Generate TTS audio, broadcast to clients, and wait for playback confirmation
* Returns a Promise that resolves when the client confirms playback completed
*/
export async function generateTTSAndWaitForPlayback(
text: string,
wsServer: VoiceAssistantWebSocketServer
): Promise<void> {
// Generate TTS audio
const { audio, format } = await synthesizeSpeech(text);
// Create unique ID for this audio segment
const audioId = uuidv4();
// Create promise that will be resolved when client confirms playback
const playbackPromise = new Promise<void>((resolve, reject) => {
// Set timeout to prevent hanging forever
const timeout = setTimeout(() => {
pendingPlaybacks.delete(audioId);
reject(new Error(`Audio playback timeout for ${audioId}`));
}, PLAYBACK_TIMEOUT_MS);
// Store handlers
pendingPlaybacks.set(audioId, { resolve, reject, timeout });
});
// Broadcast audio to clients
wsServer.broadcast({
type: "audio_output",
payload: {
id: audioId,
audio: audio.toString("base64"),
format,
},
});
console.log(
`[TTS-Manager] ${new Date().toISOString()} Sent audio ${audioId}, waiting for playback...`
);
// Wait for playback confirmation
await playbackPromise;
console.log(
`[TTS-Manager] ${new Date().toISOString()} Audio ${audioId} playback confirmed`
);
}
/**
* Called when client confirms audio playback completed
* Resolves the corresponding promise
*/
export function confirmAudioPlayed(audioId: string): void {
const pending = pendingPlaybacks.get(audioId);
if (!pending) {
console.warn(
`[TTS-Manager] Received confirmation for unknown audio ID: ${audioId}`
);
return;
}
// Clear timeout and resolve promise
clearTimeout(pending.timeout);
pending.resolve();
pendingPlaybacks.delete(audioId);
}
packages/voice-assistant/src/server/agent/tts-openai.ts
+1
View File
@@ -54,6 +54,7 @@ export async function synthesizeSpeech(text: string): Promise<SpeechResult> {
model: config.model!,
voice: config.voice!,
input: text,
// speed: 1.2,
response_format: config.responseFormat as "mp3" | "opus" | "aac" | "flac",
});
packages/voice-assistant/src/server/daemon/terminal-manager.ts
+125 -26
View File
@@ -3,8 +3,11 @@ import {
createSession,
listWindows,
createWindow,
listPanes,
findWindowByName,
capturePaneContent,
sendText as tmuxSendText,
sendKeys as tmuxSendKeys,
renameWindow,
killWindow,
isWindowNameUnique,
@@ -15,13 +18,14 @@ import {
const DEFAULT_SESSION = "voice-dev";
// Terminal model: session → windows (single pane per window)
// Terminal ID = window ID (format: @123)
// Terminals are identified by their unique names, not IDs
export interface TerminalInfo {
id: string; // window ID
name: string;
active: boolean;
workingDirectory: string;
currentCommand: string;
lastLines?: string;
}
export interface CreateTerminalParams {
@@ -31,7 +35,6 @@ export interface CreateTerminalParams {
}
export interface Terminal extends TerminalInfo {
active: boolean;
sessionId: string;
}
@@ -49,7 +52,7 @@ export async function initializeDefaultSession(): Promise<void> {
/**
* List all terminals in the voice-dev session
* Returns terminal info including ID, name, working directory, and current command
* Returns terminal info including name, active status, working directory, and current command
*/
export async function listTerminals(): Promise<TerminalInfo[]> {
const session = await findSessionByName(DEFAULT_SESSION);
@@ -69,20 +72,23 @@ export async function listTerminals(): Promise<TerminalInfo[]> {
try {
const workingDirectory = await getCurrentWorkingDirectory(paneId);
const currentCommand = await getCurrentCommand(paneId);
const lastLines = await capturePaneContent(paneId, 5, false);
terminals.push({
id: window.id,
name: window.name,
active: window.active,
workingDirectory,
currentCommand,
lastLines,
});
} catch (error) {
// If we can't get pane info, still include the terminal with empty values
terminals.push({
id: window.id,
name: window.name,
active: window.active,
workingDirectory: "",
currentCommand: "",
lastLines: "",
});
}
}
@@ -126,49 +132,86 @@ export async function createTerminal(params: CreateTerminalParams): Promise<Term
const currentCommand = await getCurrentCommand(paneId);
return {
id: windowResult.id,
name: windowResult.name,
active: windowResult.active,
sessionId: session.id,
workingDirectory,
currentCommand,
sessionId: session.id,
};
}
/**
* Capture output from a terminal
* Capture output from a terminal by name
* Returns the last N lines of terminal content
*/
export async function captureTerminal(
terminalId: string,
terminalName: string,
lines: number = 200,
wait?: number
): Promise<string> {
// Terminal ID is window ID, get the first pane
const paneId = `${terminalId}.0`;
const session = await findSessionByName(DEFAULT_SESSION);
if (!session) {
throw new Error(`Session '${DEFAULT_SESSION}' not found.`);
}
// Resolve terminal name to window
const window = await findWindowByName(session.id, terminalName);
if (!window) {
const windows = await listWindows(session.id);
const availableNames = windows.map((w) => w.name).join(", ");
throw new Error(
`Terminal '${terminalName}' not found. Available terminals: ${availableNames}`
);
}
// Get the first pane
const panes = await listPanes(window.id);
const pane = panes[0];
if (!pane) {
throw new Error(`No pane found for terminal ${terminalName}`);
}
// Optional wait before capture
if (wait) {
await new Promise((resolve) => setTimeout(resolve, wait));
}
return capturePaneContent(paneId, lines, false);
return capturePaneContent(pane.id, lines, false);
}
/**
* Send text to a terminal, optionally press Enter, optionally return output
* Send text to a terminal by name, optionally press Enter, optionally return output
*/
export async function sendText(
terminalId: string,
terminalName: string,
text: string,
pressEnter: boolean = false,
return_output?: { lines?: number; wait?: number }
return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number }
): Promise<string | void> {
// Terminal ID is window ID, get the first pane
const paneId = `${terminalId}.0`;
const session = await findSessionByName(DEFAULT_SESSION);
if (!session) {
throw new Error(`Session '${DEFAULT_SESSION}' not found.`);
}
// Resolve terminal name to window
const window = await findWindowByName(session.id, terminalName);
if (!window) {
const windows = await listWindows(session.id);
const availableNames = windows.map((w) => w.name).join(", ");
throw new Error(
`Terminal '${terminalName}' not found. Available terminals: ${availableNames}`
);
}
// Get the first pane
const panes = await listPanes(window.id);
const pane = panes[0];
if (!pane) {
throw new Error(`No pane found for terminal ${terminalName}`);
}
return tmuxSendText({
paneId,
paneId: pane.id,
text,
pressEnter,
return_output,
@@ -176,11 +219,51 @@ export async function sendText(
}
/**
* Rename a terminal
* Send special keys or key combinations to a terminal by name
* Useful for TUI navigation, control sequences, and interactive applications
*/
export async function sendKeys(
terminalName: string,
keys: string,
repeat: number = 1,
return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number }
): Promise<string | void> {
const session = await findSessionByName(DEFAULT_SESSION);
if (!session) {
throw new Error(`Session '${DEFAULT_SESSION}' not found.`);
}
// Resolve terminal name to window
const window = await findWindowByName(session.id, terminalName);
if (!window) {
const windows = await listWindows(session.id);
const availableNames = windows.map((w) => w.name).join(", ");
throw new Error(
`Terminal '${terminalName}' not found. Available terminals: ${availableNames}`
);
}
// Get the first pane
const panes = await listPanes(window.id);
const pane = panes[0];
if (!pane) {
throw new Error(`No pane found for terminal ${terminalName}`);
}
return tmuxSendKeys({
paneId: pane.id,
keys,
repeat,
return_output,
});
}
/**
* Rename a terminal by name
* Validates that the new name is unique
*/
export async function renameTerminal(
terminalId: string,
terminalName: string,
newName: string
): Promise<void> {
const session = await findSessionByName(DEFAULT_SESSION);
@@ -189,13 +272,29 @@ export async function renameTerminal(
throw new Error(`Session '${DEFAULT_SESSION}' not found.`);
}
// renameWindow handles uniqueness validation internally
await renameWindow(session.id, terminalId, newName);
// renameWindow handles uniqueness validation and name resolution internally
await renameWindow(session.id, terminalName, newName);
}
/**
* Kill (close/destroy) a terminal
* Kill (close/destroy) a terminal by name
*/
export async function killTerminal(terminalId: string): Promise<void> {
await killWindow(terminalId);
export async function killTerminal(terminalName: string): Promise<void> {
const session = await findSessionByName(DEFAULT_SESSION);
if (!session) {
throw new Error(`Session '${DEFAULT_SESSION}' not found.`);
}
// Resolve terminal name to window
const window = await findWindowByName(session.id, terminalName);
if (!window) {
const windows = await listWindows(session.id);
const availableNames = windows.map((w) => w.name).join(", ");
throw new Error(
`Terminal '${terminalName}' not found. Available terminals: ${availableNames}`
);
}
await killWindow(window.id);
}
packages/voice-assistant/src/server/daemon/tmux.ts
+63 -12
View File
@@ -191,9 +191,18 @@ export async function capturePaneContent(
includeColors: boolean = false
): Promise<string> {
const colorFlag = includeColors ? "-e" : "";
return executeTmux(
`capture-pane -p ${colorFlag} -t '${paneId}' -S -${lines} -E -`
// Capture a large range to ensure we have enough content
const captureLines = Math.max(lines, 1000);
const output = await executeTmux(
`capture-pane -p ${colorFlag} -t '${paneId}' -S -${captureLines} -E -`
);
// Trim trailing whitespace, split by lines, take last N lines, rejoin
const trimmed = output.trimEnd();
const allLines = trimmed.split('\n');
const lastLines = allLines.slice(-lines);
return lastLines.join('\n');
}
/**
@@ -800,7 +809,7 @@ export async function sendKeys({
paneId: string;
keys: string;
repeat?: number;
return_output?: { lines?: number; wait?: number };
return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number };
}): Promise<string | void> {
// Repeat the key press the specified number of times
for (let i = 0; i < repeat; i++) {
@@ -810,11 +819,49 @@ export async function sendKeys({
// If return_output is requested, wait and capture pane content
if (return_output) {
await new Promise((resolve) =>
setTimeout(resolve, return_output.wait ?? 500)
);
const lines = return_output.lines || 200;
return capturePaneContent(paneId, lines, false);
const waitForSettled = return_output.waitForSettled ?? true;
const maxWait = return_output.maxWait ?? 120000; // 2 minutes default
if (waitForSettled) {
return waitForPaneActivityToSettle(paneId, maxWait, lines);
} else {
return capturePaneContent(paneId, lines, false);
}
}
}
async function waitForPaneActivityToSettle(
paneId: string,
maxWait: number,
lines: number
): Promise<string> {
const settleTime = 500; // Hardcoded debounce
const pollInterval = 100; // Poll every 100ms
let lastContent = '';
let lastChangeTime = Date.now();
const startTime = Date.now();
while (true) {
const elapsed = Date.now() - startTime;
if (elapsed >= maxWait) {
// Timeout - return what we have
return lastContent;
}
const content = await capturePaneContent(paneId, lines, false);
if (content !== lastContent) {
// Activity detected - reset settle timer
lastContent = content;
lastChangeTime = Date.now();
} else if (Date.now() - lastChangeTime >= settleTime) {
// No changes for settleTime ms - settled!
return content;
}
await new Promise(resolve => setTimeout(resolve, pollInterval));
}
}
@@ -827,7 +874,7 @@ export async function sendText({
paneId: string;
text: string;
pressEnter?: boolean;
return_output?: { lines?: number; wait?: number };
return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number };
}): Promise<string | void> {
// Send each character with -l flag for literal interpretation
for (const char of text) {
@@ -843,10 +890,14 @@ export async function sendText({
// If return_output is requested, wait and capture pane content
if (return_output) {
if (return_output.wait) {
await new Promise((resolve) => setTimeout(resolve, return_output.wait));
}
const lines = return_output.lines || 200;
return capturePaneContent(paneId, lines, false);
const waitForSettled = return_output.waitForSettled ?? true;
const maxWait = return_output.maxWait ?? 120000; // 2 minutes default
if (waitForSettled) {
return waitForPaneActivityToSettle(paneId, maxWait, lines);
} else {
return capturePaneContent(paneId, lines, false);
}
}
}
packages/voice-assistant/src/server/daemon/tool-definitions.ts
+8 -3
View File
@@ -102,16 +102,21 @@ export const terminalTools = [
return_output: {
type: "object",
description:
"Optional. If provided, will wait and return the terminal output after sending the text.",
"Optional. If provided, will wait for terminal activity to settle and return the output.",
properties: {
lines: {
type: "number",
description: "Number of lines to capture. Defaults to 200.",
},
wait: {
waitForSettled: {
type: "boolean",
description:
"Whether to wait for terminal activity to settle before returning output. Polls terminal output and waits 500ms after last change. Defaults to true.",
},
maxWait: {
type: "number",
description:
"Milliseconds to wait before capturing output. Defaults to 500ms.",
"Maximum milliseconds to wait for activity to settle. Defaults to 120000 (2 minutes).",
},
},
},
packages/voice-assistant/src/server/index.ts
+19 -68
View File
@@ -2,11 +2,7 @@ import "dotenv/config";
import express from "express";
import path from "path";
import { fileURLToPath } from "url";
import {
createServer as createHTTPServer,
Server as HttpServer,
RequestListener,
} from "http";
import { createServer as createHTTPServer, Server as HttpServer } from "http";
import { readFile } from "fs/promises";
import { v4 as uuidv4 } from "uuid";
import { createServer as createViteServer } from "vite";
@@ -22,9 +18,8 @@ import {
killTerminal,
} from "./daemon/terminal-manager.js";
import { initializeSTT, transcribeAudio } from "./agent/stt-openai.js";
import { initializeTTS, synthesizeSpeech } from "./agent/tts-openai.js";
import { initializeTTS } from "./agent/tts-openai.js";
import {
createConversation,
processUserMessage,
cleanupConversations,
} from "./agent/orchestrator.js";
@@ -70,20 +65,19 @@ async function createServer(httpServer: HttpServer, config: ServerConfig) {
initialCommand: undefined,
});
testResults.push(
` ✓ Created terminal: ${terminal.id} (${terminal.name})`
` ✓ Created terminal: ${terminal.name}`
);
// 4. Send a command to the terminal
testResults.push('4. Sending command "echo hello world"...');
await sendText(terminal.id, 'echo "hello world"', true, {
await sendText(terminal.name, 'echo "hello world"', true, {
lines: 20,
wait: 500,
});
testResults.push(" ✓ Command sent");
// 5. Capture output
testResults.push("5. Capturing terminal output...");
const output = await captureTerminal(terminal.id, 20);
const output = await captureTerminal(terminal.name, 20);
testResults.push(` ✓ Captured ${output.split("\n").length} lines`);
testResults.push(` Output preview: ${output.substring(0, 100)}...`);
@@ -95,12 +89,12 @@ async function createServer(httpServer: HttpServer, config: ServerConfig) {
(t) => t.name === "test-terminal"
);
if (testTerminal) {
testResults.push(` ✓ Found test-terminal: ${testTerminal.id}`);
testResults.push(` ✓ Found test-terminal`);
}
// 7. Kill the test terminal
testResults.push("7. Killing test terminal...");
await killTerminal(terminal.id);
await killTerminal(terminal.name);
testResults.push(" ✓ Terminal killed");
// 8. Verify it's gone
@@ -231,59 +225,8 @@ async function main() {
);
}
// Create default conversation
const defaultConversationId = createConversation();
console.log(`✓ Default conversation created: ${defaultConversationId}`);
// Helper function to process message and optionally generate TTS
async function processMessageWithOptionalTTS(
message: string,
enableTTS: boolean = false
): Promise<string> {
const response = await processUserMessage({
conversationId: defaultConversationId,
message,
wsServer,
});
// Generate TTS for the response if enabled
if (enableTTS && apiKey) {
try {
console.log("[TTS] Generating speech for assistant response...");
const speechResult = await synthesizeSpeech(response);
// Convert audio buffer to base64
const base64Audio = speechResult.audio.toString("base64");
// Send audio to client via WebSocket
wsServer.broadcast({
type: "audio_output",
payload: {
id: uuidv4(),
audio: base64Audio,
format: speechResult.format,
},
});
console.log(
`[TTS] Sent audio to client via WebSocket (${speechResult.audio.length} bytes)`
);
} catch (error: any) {
console.error("[TTS] Error generating speech:", error);
wsServer.broadcastActivityLog({
id: uuidv4(),
timestamp: new Date(),
type: "error",
content: `TTS error: ${error.message}`,
});
}
}
return response;
}
// Wire orchestrator to WebSocket for text messages
wsServer.setMessageHandler(async (message: string) => {
wsServer.setMessageHandler(async (conversationId: string, message: string, abortSignal: AbortSignal) => {
try {
// Broadcast user's text message as activity log
wsServer.broadcastActivityLog({
@@ -294,9 +237,11 @@ async function main() {
});
await processUserMessage({
conversationId: defaultConversationId,
conversationId,
message,
wsServer,
enableTTS: true,
abortSignal,
});
} catch (error: any) {
console.error("[Orchestrator] Error processing message:", error);
@@ -311,7 +256,7 @@ async function main() {
// Wire audio handler to WebSocket for voice input (STT)
wsServer.setAudioHandler(
async (audio: Buffer, format: string): Promise<string> => {
async (conversationId: string, audio: Buffer, format: string, abortSignal: AbortSignal): Promise<string> => {
try {
// Transcribe audio using OpenAI Whisper
const result = await transcribeAudio(audio, format);
@@ -330,7 +275,13 @@ async function main() {
// Process the transcribed text through the orchestrator WITH TTS enabled
// Since this came from voice input, respond with voice output
await processMessageWithOptionalTTS(result.text, true);
await processUserMessage({
conversationId,
message: result.text,
wsServer,
enableTTS: true,
abortSignal,
});
return result.text;
} catch (error: any) {
packages/voice-assistant/src/server/types.ts
+5 -1
View File
@@ -15,7 +15,7 @@ export interface ActivityLogEntry {
export interface WebSocketMessage {
type: 'activity_log' | 'status' | 'ping' | 'pong' | 'user_message' | 'assistant_chunk'
| 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result';
| 'audio_chunk' | 'audio_output' | 'recording_state' | 'transcription_result' | 'audio_played';
payload: unknown;
}
@@ -40,3 +40,7 @@ export interface AudioOutputPayload {
format: string; // 'mp3'
id: string; // unique ID for queue management
}
export interface AudioPlayedPayload {
id: string; // unique ID of the audio that finished playing
}
packages/voice-assistant/src/server/websocket-server.ts
+86 -9
View File
@@ -4,13 +4,18 @@ import type {
WebSocketMessage,
ActivityLogEntry,
AudioChunkPayload,
AudioPlayedPayload,
} from "./types.js";
import { confirmAudioPlayed } from "./agent/tts-manager.js";
import { createConversation, deleteConversation } from "./agent/orchestrator.js";
export class VoiceAssistantWebSocketServer {
private wss: WebSocketServer;
private clients: Map<WebSocket, string> = new Map(); // Map ws to client ID
private messageHandler?: (message: string) => Promise<void>;
private audioHandler?: (audio: Buffer, format: string) => Promise<string>;
private conversationIds: Map<WebSocket, string> = new Map(); // Map ws to conversation ID
private abortControllers: Map<WebSocket, AbortController> = new Map(); // Map ws to AbortController
private messageHandler?: (conversationId: string, message: string, abortSignal: AbortSignal) => Promise<void>;
private audioHandler?: (conversationId: string, audio: Buffer, format: string, abortSignal: AbortSignal) => Promise<string>;
private audioBuffers: Map<string, { chunks: Buffer[]; format: string }> =
new Map();
private clientIdCounter: number = 0;
@@ -29,8 +34,17 @@ export class VoiceAssistantWebSocketServer {
// Generate unique client ID
const clientId = `client-${++this.clientIdCounter}`;
this.clients.set(ws, clientId);
// Create new conversation for this client
const conversationId = createConversation();
this.conversationIds.set(ws, conversationId);
// Create AbortController for this client
const abortController = new AbortController();
this.abortControllers.set(ws, abortController);
console.log(
`[WS] Client connected: ${clientId} (total: ${this.clients.size})`
`[WS] Client connected: ${clientId} with conversation ${conversationId} (total: ${this.clients.size})`
);
// Send welcome message
@@ -48,20 +62,50 @@ export class VoiceAssistantWebSocketServer {
ws.on("close", () => {
const clientId = this.clients.get(ws);
const conversationId = this.conversationIds.get(ws);
const abortController = this.abortControllers.get(ws);
// Abort any ongoing operations
if (abortController) {
abortController.abort();
this.abortControllers.delete(ws);
console.log(`[WS] Aborted operations for ${clientId}`);
}
if (clientId) {
this.clients.delete(ws);
console.log(
`[WS] Client disconnected: ${clientId} (total: ${this.clients.size})`
);
}
if (conversationId) {
deleteConversation(conversationId);
this.conversationIds.delete(ws);
console.log(`[WS] Conversation ${conversationId} deleted`);
}
});
ws.on("error", (error) => {
console.error("[WS] Client error:", error);
const clientId = this.clients.get(ws);
const conversationId = this.conversationIds.get(ws);
const abortController = this.abortControllers.get(ws);
// Abort any ongoing operations
if (abortController) {
abortController.abort();
this.abortControllers.delete(ws);
}
if (clientId) {
this.clients.delete(ws);
}
if (conversationId) {
deleteConversation(conversationId);
this.conversationIds.delete(ws);
}
});
}
@@ -82,8 +126,18 @@ export class VoiceAssistantWebSocketServer {
case "user_message":
// Handle user message through orchestrator
const payload = message.payload as { message: string };
const conversationId = this.conversationIds.get(ws);
const abortController = this.abortControllers.get(ws);
if (!conversationId) {
console.error("[WS] No conversation found for client");
break;
}
if (!abortController) {
console.error("[WS] No abort controller found for client");
break;
}
if (this.messageHandler) {
await this.messageHandler(payload.message);
await this.messageHandler(conversationId, payload.message, abortController.signal);
} else {
console.warn("[WS] No message handler registered");
}
@@ -94,6 +148,12 @@ export class VoiceAssistantWebSocketServer {
await this.handleAudioChunk(ws, message.payload as AudioChunkPayload);
break;
case "audio_played":
// Handle audio playback confirmation
const audioPlayedPayload = message.payload as AudioPlayedPayload;
confirmAudioPlayed(audioPlayedPayload.id);
break;
default:
console.warn(`[WS] Unknown message type: ${message.type}`);
}
@@ -136,12 +196,12 @@ export class VoiceAssistantWebSocketServer {
});
}
public setMessageHandler(handler: (message: string) => Promise<void>): void {
public setMessageHandler(handler: (conversationId: string, message: string, abortSignal: AbortSignal) => Promise<void>): void {
this.messageHandler = handler;
}
public setAudioHandler(
handler: (audio: Buffer, format: string) => Promise<string>
handler: (conversationId: string, audio: Buffer, format: string, abortSignal: AbortSignal) => Promise<string>
): void {
this.audioHandler = handler;
}
@@ -151,8 +211,19 @@ export class VoiceAssistantWebSocketServer {
payload: AudioChunkPayload
): Promise<void> {
try {
// Use a client-specific key for buffering (in case multiple clients)
const clientId = "default"; // Could be enhanced with per-client tracking
// Use client-specific key for buffering
const clientId = this.clients.get(ws);
if (!clientId) {
console.error("[WS] No client ID found for WebSocket");
return;
}
// Get conversation ID for this client
const conversationId = this.conversationIds.get(ws);
if (!conversationId) {
console.error("[WS] No conversation found for client");
return;
}
// Decode base64 audio data
const audioBuffer = Buffer.from(payload.audio, "base64");
@@ -188,6 +259,12 @@ export class VoiceAssistantWebSocketServer {
this.audioBuffers.delete(clientId);
// Process audio through handler (STT)
const abortController = this.abortControllers.get(ws);
if (!abortController) {
console.error("[WS] No abort controller found for client");
return;
}
if (this.audioHandler) {
this.broadcastActivityLog({
id: Date.now().toString(),
@@ -196,7 +273,7 @@ export class VoiceAssistantWebSocketServer {
content: "Transcribing audio...",
});
const transcript = await this.audioHandler(completeAudio, format);
const transcript = await this.audioHandler(conversationId, completeAudio, format, abortController.signal);
// Send transcription result back to client
this.sendToClient(ws, {
packages/voice-assistant/src/ui/App.css
+311 -35
View File
@@ -1,30 +1,71 @@
.app {
display: flex;
flex-direction: column;
min-height: 100vh;
height: 100svh;
width: 100%;
overflow: hidden;
}
.header {
display: flex;
display: none;
justify-content: space-between;
align-items: center;
padding: 1rem 2rem;
padding: 0.5rem 1rem;
background-color: #1a1a1a;
border-bottom: 1px solid #333;
flex-shrink: 0;
min-height: 0;
overflow: hidden;
max-width: 100%;
min-width: 0;
}
.header h1 {
font-size: 1.5rem;
font-size: 1rem;
font-weight: 600;
margin: 0;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
min-width: 0;
flex: 1;
}
.header-status {
display: flex;
align-items: center;
gap: 0.5rem;
flex-shrink: 0;
min-width: 0;
}
.audio-playing-indicator {
width: 8px;
height: 8px;
background-color: #0f766e;
border-radius: 50%;
animation: pulse-audio 1.5s ease-in-out infinite;
}
@keyframes pulse-audio {
0%, 100% {
opacity: 1;
transform: scale(1);
}
50% {
opacity: 0.5;
transform: scale(1.2);
}
}
.status-indicator {
padding: 0.5rem 1rem;
padding: 0.25rem 0.75rem;
border-radius: 4px;
font-size: 0.75rem;
font-weight: 500;
text-transform: uppercase;
flex-shrink: 0;
white-space: nowrap;
}
.status-indicator.disconnected {
@@ -55,20 +96,24 @@
.main {
display: flex;
flex: 1;
gap: 2rem;
max-height: 100vh;
overflow: hidden;
gap: 0.5rem;
overflow: hidden;
padding: 0.5rem;
min-height: 0;
max-width: 100%;
}
.chat-interface {
flex: 1;
background-color: #1a1a1a;
border-radius: 8px;
padding: 2rem;
padding: 1rem;
display: flex;
flex-direction: column;
gap: 1rem;
height: 100%;
gap: 0.5rem;
min-height: 0;
max-width: 100%;
overflow: hidden;
}
.chat-interface h2 {
@@ -113,23 +158,31 @@ overflow: hidden;
display: flex;
flex-direction: column;
overflow: hidden;
min-height: 0;
max-width: 100%;
}
.log-entries {
flex: 1;
overflow-y: auto;
overflow-x: hidden;
display: flex;
flex-direction: column;
gap: 0.5rem;
gap: 0.25rem;
min-height: 0;
max-width: 100%;
}
.log-entry {
padding: 0.75rem;
padding: 0.5rem;
border-radius: 4px;
font-size: 0.875rem;
line-height: 1.4;
display: flex;
gap: 0.75rem;
gap: 0.5rem;
flex-shrink: 0;
max-width: 100%;
overflow: hidden;
}
.log-time {
@@ -140,6 +193,9 @@ overflow: hidden;
.log-message {
flex: 1;
min-width: 0;
word-wrap: break-word;
overflow-wrap: break-word;
}
.log-entry.system {
@@ -197,6 +253,8 @@ overflow: hidden;
.log-metadata {
margin-top: 0.5rem;
max-width: 100%;
overflow: hidden;
}
.log-metadata summary {
@@ -212,24 +270,40 @@ overflow: hidden;
border-radius: 4px;
overflow-x: auto;
font-size: 0.75rem;
max-width: 100%;
word-wrap: break-word;
overflow-wrap: break-word;
white-space: pre-wrap;
}
.message-form {
display: flex;
align-items: flex-end;
gap: 0.5rem;
padding-top: 1rem;
padding-top: 0.5rem;
border-top: 1px solid #333;
flex-shrink: 0;
min-width: 0;
max-width: 100%;
overflow: hidden;
}
.message-input {
flex: 1;
padding: 0.75rem 1rem;
min-width: 0;
padding: 0.5rem 0.75rem;
background-color: #2a2a2a;
border: 1px solid #444;
border-radius: 6px;
color: #fff;
font-size: 0.875rem;
font-family: inherit;
outline: none;
resize: none;
overflow-y: auto;
max-height: 200px;
line-height: 1.5;
min-height: 26px;
}
.message-input:focus {
@@ -242,27 +316,74 @@ overflow: hidden;
}
.send-button {
padding: 0.75rem 1.5rem;
background-color: #22c55e;
color: #000;
width: 44px;
height: 44px;
min-width: 44px;
min-height: 44px;
padding: 0;
background-color: #3b82f6;
color: #fff;
border: none;
border-radius: 6px;
font-size: 0.875rem;
font-weight: 500;
border-radius: 8px;
cursor: pointer;
transition: background-color 0.2s;
transition: all 0.2s;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
}
.send-button:hover:not(:disabled) {
.send-button.has-text {
background-color: #22c55e;
color: #000;
}
.send-button.has-text:hover:not(:disabled) {
background-color: #16a34a;
}
.send-button:not(.has-text):hover:not(:disabled) {
background-color: #2563eb;
}
.send-button.recording {
background-color: #ef4444;
animation: pulse-red 1.5s ease-in-out infinite;
}
.send-button.recording:hover:not(:disabled) {
background-color: #dc2626;
}
.send-button.processing {
background-color: #f59e0b;
}
.send-button:disabled {
background-color: #444;
color: #999;
cursor: not-allowed;
}
.send-button .recording-indicator {
display: inline-block;
width: 12px;
height: 12px;
background-color: #fff;
border-radius: 50%;
animation: pulse 1s ease-in-out infinite;
}
.send-button .processing-indicator {
display: inline-block;
width: 12px;
height: 12px;
border: 2px solid #fff;
border-top-color: transparent;
border-radius: 50%;
animation: spin 0.8s linear infinite;
}
.test-controls {
padding-top: 0.5rem;
border-top: 1px solid #333;
@@ -274,11 +395,14 @@ overflow: hidden;
.voice-controls {
display: flex;
flex-direction: column;
gap: 0.75rem;
padding: 1rem;
gap: 0.5rem;
padding: 0.75rem;
background-color: #2a2a2a;
border-radius: 6px;
margin-bottom: 1rem;
flex-shrink: 0;
min-width: 0;
max-width: 100%;
overflow: hidden;
}
.voice-button {
@@ -286,15 +410,17 @@ overflow: hidden;
align-items: center;
justify-content: center;
gap: 0.5rem;
padding: 1rem 2rem;
padding: 0.75rem 1.5rem;
background-color: #3b82f6;
color: #fff;
border: none;
border-radius: 8px;
font-size: 1rem;
font-size: 0.875rem;
font-weight: 500;
cursor: pointer;
transition: all 0.2s;
max-width: 100%;
overflow: hidden;
}
.voice-button:hover:not(:disabled) {
@@ -376,13 +502,138 @@ overflow: hidden;
font-size: 0.875rem;
}
.playing-indicator {
padding: 0.5rem;
background-color: #0f766e;
color: #ccfbf1;
border-radius: 4px;
/* Tool Call Card Styles */
.tool-call-card {
background-color: #2a2a2a;
border-radius: 6px;
overflow: hidden;
margin: 0.25rem 0;
border: 1px solid #444;
flex-shrink: 0;
}
.tool-call-card[data-status="executing"] {
border-color: #f59e0b;
}
.tool-call-card[data-status="completed"] {
border-color: #22c55e;
}
.tool-call-header {
width: 100%;
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.75rem;
background: none;
border: none;
color: #e2e8f0;
cursor: pointer;
font-family: inherit;
font-size: 0.875rem;
text-align: center;
text-align: left;
transition: background-color 0.2s;
}
.tool-call-header:hover {
background-color: #333;
}
.tool-call-icon {
display: flex;
align-items: center;
justify-content: center;
color: #999;
flex-shrink: 0;
}
.tool-call-name {
flex: 1;
font-family: monospace;
font-weight: 500;
color: #e2e8f0;
}
.tool-call-status {
display: flex;
align-items: center;
gap: 0.375rem;
padding: 0.25rem 0.5rem;
border-radius: 4px;
font-size: 0.75rem;
flex-shrink: 0;
}
.tool-call-card[data-status="executing"] .tool-call-status {
background-color: rgba(245, 158, 11, 0.2);
color: #fbbf24;
}
.tool-call-card[data-status="completed"] .tool-call-status {
background-color: rgba(34, 197, 94, 0.2);
color: #86efac;
}
.status-icon {
flex-shrink: 0;
}
.status-icon.spinning {
animation: spin 1s linear infinite;
}
.status-text {
font-weight: 500;
text-transform: capitalize;
}
.tool-call-details {
padding: 0 0.75rem 0.75rem 0.75rem;
display: flex;
flex-direction: column;
gap: 0.75rem;
animation: expand 0.2s ease-out;
}
@keyframes expand {
from {
opacity: 0;
max-height: 0;
}
to {
opacity: 1;
max-height: 1000px;
}
}
.tool-call-section {
display: flex;
flex-direction: column;
gap: 0.375rem;
}
.section-header {
font-size: 0.75rem;
font-weight: 600;
color: #999;
text-transform: uppercase;
letter-spacing: 0.025em;
}
.section-content {
padding: 0.5rem;
background-color: #1a1a1a;
border-radius: 4px;
font-family: monospace;
font-size: 0.75rem;
color: #e2e8f0;
overflow-x: auto;
border: 1px solid #444;
margin: 0;
white-space: pre-wrap;
word-wrap: break-word;
overflow-wrap: break-word;
}
@media (max-width: 1024px) {
@@ -390,3 +641,28 @@ overflow: hidden;
flex-direction: column;
}
}
@media (max-width: 768px) {
.main {
padding: 0.25rem;
gap: 0.25rem;
}
.chat-interface {
padding: 0.5rem;
}
.header {
padding: 0.5rem 0.5rem;
gap: 0.5rem;
}
.header h1 {
font-size: 0.875rem;
}
.status-indicator {
padding: 0.25rem 0.5rem;
font-size: 0.7rem;
}
}
packages/voice-assistant/src/ui/App.tsx
+198 -92
View File
@@ -1,16 +1,28 @@
import { useState, useEffect, useRef } from "react";
import { Mic, Send } from "lucide-react";
import { useWebSocket } from "./hooks/useWebSocket";
import { VoiceControls } from "./components/VoiceControls";
import { createAudioPlayer } from "./lib/audio-playback";
import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture";
import { ToolCallCard } from "./components/ToolCallCard";
import "./App.css";
interface LogEntry {
id: string;
timestamp: number;
type: "system" | "info" | "success" | "error" | "user" | "assistant" | "tool";
message: string;
metadata?: Record<string, unknown>;
}
type LogEntry =
| {
type: "system" | "info" | "success" | "error" | "user" | "assistant";
id: string;
timestamp: number;
message: string;
metadata?: Record<string, unknown>;
}
| {
type: "tool_call";
id: string;
timestamp: number;
toolName: string;
args: any;
result?: any;
status: "executing" | "completed";
};
function App() {
const [logs, setLogs] = useState<LogEntry[]>([
@@ -25,8 +37,11 @@ function App() {
const [currentAssistantMessage, setCurrentAssistantMessage] = useState("");
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
const [isRecording, setIsRecording] = useState(false);
const logEndRef = useRef<HTMLDivElement>(null);
const audioPlayerRef = useRef(createAudioPlayer());
const recorderRef = useRef<AudioRecorder | null>(null);
const textareaRef = useRef<HTMLTextAreaElement>(null);
// WebSocket URL - use ws://localhost:3000/ws in dev, or construct from current host in prod
const wsUrl = `${window.location.protocol === "https:" ? "wss" : "ws"}://${
@@ -35,8 +50,20 @@ function App() {
const ws = useWebSocket(wsUrl);
useEffect(() => {
recorderRef.current = createAudioRecorder();
}, []);
useEffect(() => {
if (textareaRef.current) {
textareaRef.current.style.height = 'auto';
const newHeight = Math.max(textareaRef.current.scrollHeight, 44);
textareaRef.current.style.height = `${newHeight}px`;
}
}, [userInput]);
const addLog = (
type: LogEntry["type"],
type: "system" | "info" | "success" | "error" | "user" | "assistant",
message: string,
metadata?: Record<string, unknown>
) => {
@@ -67,22 +94,59 @@ function App() {
// Listen for activity log messages
const unsubActivity = ws.on("activity_log", (payload: unknown) => {
const data = payload as {
id: string;
type: string;
content: string;
metadata?: Record<string, unknown>;
};
// Handle tool calls
if (data.type === "tool_call") {
const { toolCallId, toolName, arguments: args } = data.metadata as {
toolCallId: string;
toolName: string;
arguments: any;
};
setLogs((prev) => [
...prev,
{
type: "tool_call",
id: toolCallId,
timestamp: Date.now(),
toolName,
args,
status: "executing",
},
]);
return;
}
if (data.type === "tool_result") {
const { toolCallId, result } = data.metadata as {
toolCallId: string;
result: any;
};
setLogs((prev) =>
prev.map((log) =>
log.type === "tool_call" && log.id === toolCallId
? { ...log, result, status: "completed" as const }
: log
)
);
return;
}
// Map activity log types to UI log types
let logType: LogEntry["type"] = "info";
let logType: "system" | "info" | "success" | "error" | "user" | "assistant" = "info";
if (data.type === "transcript") logType = "user";
else if (data.type === "assistant") logType = "assistant";
else if (data.type === "tool_call" || data.type === "tool_result")
logType = "tool";
else if (data.type === "error") logType = "error";
addLog(logType, data.content, data.metadata);
// Clear streaming state when complete assistant message arrives
// Clear streaming state when assistant segment is received
if (data.type === "assistant") {
setCurrentAssistantMessage("");
}
@@ -97,7 +161,7 @@ function App() {
// Listen for transcription results
const unsubTranscription = ws.on(
"transcription_result",
(payload: unknown) => {
(_payload: unknown) => {
// Note: Transcription is already broadcast as activity_log with type "transcript"
// No need to log it again here to avoid duplication
setIsProcessingAudio(false);
@@ -109,7 +173,6 @@ function App() {
const data = payload as { audio: string; format: string; id: string };
try {
addLog("info", "Playing assistant audio response...");
setIsPlayingAudio(true);
// Decode base64 audio
@@ -127,8 +190,13 @@ function App() {
// Play audio
await audioPlayerRef.current.play(audioBlob);
// Send confirmation back to server
ws.send({
type: "audio_played",
payload: { id: data.id },
});
setIsPlayingAudio(false);
addLog("success", "Audio playback complete");
} catch (error: any) {
console.error("[App] Audio playback error:", error);
addLog("error", `Audio playback failed: ${error.message}`);
@@ -159,13 +227,7 @@ function App() {
logEndRef.current?.scrollIntoView({ behavior: "smooth" });
}, [logs, currentAssistantMessage]);
const handlePing = () => {
ws.sendPing();
addLog("info", "Sent ping to server");
};
const handleSendMessage = (e: React.FormEvent) => {
e.preventDefault();
const handleSendMessage = () => {
if (!userInput.trim() || !ws.isConnected) return;
// Send message to server
@@ -176,42 +238,74 @@ function App() {
setCurrentAssistantMessage("");
};
const handleAudioRecorded = async (audioBlob: Blob, format: string) => {
if (!ws.isConnected) {
addLog("error", "Cannot send audio - not connected to server");
return;
}
const handleToggleRecording = async () => {
const recorder = recorderRef.current;
if (!recorder || !ws.isConnected) return;
try {
setIsProcessingAudio(true);
addLog("info", "Sending audio to server...");
if (isRecording) {
console.log('[App] Stopping recording...');
const audioBlob = await recorder.stop();
setIsRecording(false);
// Convert blob to base64
const arrayBuffer = await audioBlob.arrayBuffer();
const base64Audio = btoa(
new Uint8Array(arrayBuffer).reduce(
(data, byte) => data + String.fromCharCode(byte),
""
)
);
const format = audioBlob.type || 'audio/webm';
console.log(`[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`);
// Send audio chunk via WebSocket
ws.send({
type: "audio_chunk",
payload: {
audio: base64Audio,
format: format,
isLast: true,
},
});
setIsProcessingAudio(true);
addLog("info", "Sending audio to server...");
console.log(
`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`
);
const arrayBuffer = await audioBlob.arrayBuffer();
const base64Audio = btoa(
new Uint8Array(arrayBuffer).reduce(
(data, byte) => data + String.fromCharCode(byte),
""
)
);
ws.send({
type: "audio_chunk",
payload: {
audio: base64Audio,
format: format,
isLast: true,
},
});
console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`);
} else {
console.log('[App] Starting recording...');
await recorder.start();
setIsRecording(true);
}
} catch (error: any) {
console.error("[App] Error sending audio:", error);
addLog("error", `Failed to send audio: ${error.message}`);
setIsProcessingAudio(false);
console.error('[App] Recording error:', error);
addLog("error", `Failed to record audio: ${error.message}`);
setIsRecording(false);
}
};
const handleButtonClick = async (e: React.MouseEvent<HTMLButtonElement>) => {
e.preventDefault();
if (userInput.trim()) {
handleSendMessage();
} else {
await handleToggleRecording();
}
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
// Detect if device is desktop (not mobile/tablet)
const isDesktop = window.matchMedia('(pointer: fine)').matches;
if (e.key === 'Enter') {
if (isDesktop && !e.shiftKey && userInput.trim()) {
// Desktop: plain Enter sends message
e.preventDefault();
handleSendMessage();
}
// Desktop: Shift+Enter creates new line (default behavior)
// Mobile: Enter creates new line (default behavior)
}
};
@@ -219,12 +313,15 @@ function App() {
<div className="app">
<header className="header">
<h1>Voice Assistant</h1>
<div
className={`status-indicator ${
ws.isConnected ? "connected" : "disconnected"
}`}
>
{ws.isConnected ? "connected" : "disconnected"}
<div className="header-status">
{isPlayingAudio && <div className="audio-playing-indicator" title="Playing audio" />}
<div
className={`status-indicator ${
ws.isConnected ? "connected" : "disconnected"
}`}
>
{ws.isConnected ? "connected" : "disconnected"}
</div>
</div>
</header>
@@ -232,25 +329,29 @@ function App() {
<div className="chat-interface">
<div className="activity-log">
<div className="log-entries">
{logs.map((log) => (
<div key={log.id} className={`log-entry ${log.type}`}>
<span className="log-time">
{new Date(log.timestamp).toLocaleTimeString()}
</span>
<span className="log-message">{log.message}</span>
{log.metadata && (
<details className="log-metadata">
<summary>Details</summary>
<pre>{JSON.stringify(log.metadata, null, 2)}</pre>
</details>
)}
</div>
))}
{logs.map((log) =>
log.type === "tool_call" ? (
<ToolCallCard
key={log.id}
toolName={log.toolName}
args={log.args}
result={log.result}
status={log.status}
/>
) : (
<div key={log.id} className={`log-entry ${log.type}`}>
<span className="log-message">{log.message}</span>
{log.metadata && (
<details className="log-metadata">
<summary>Details</summary>
<pre>{JSON.stringify(log.metadata, null, 2)}</pre>
</details>
)}
</div>
)
)}
{currentAssistantMessage && (
<div className="log-entry assistant streaming">
<span className="log-time">
{new Date().toLocaleTimeString()}
</span>
<span className="log-message">{currentAssistantMessage}</span>
<span className="streaming-indicator">...</span>
</div>
@@ -259,29 +360,34 @@ function App() {
</div>
</div>
<form onSubmit={handleSendMessage} className="message-form">
<input
type="text"
<div className="message-form">
<textarea
ref={textareaRef}
value={userInput}
onChange={(e) => setUserInput(e.target.value)}
placeholder="Type a message to the assistant..."
disabled={!ws.isConnected}
onKeyDown={handleKeyDown}
placeholder="Type a message or tap mic to talk..."
disabled={!ws.isConnected || isRecording}
className="message-input"
rows={1}
/>
<button
type="submit"
disabled={!ws.isConnected || !userInput.trim()}
className="send-button"
type="button"
onClick={handleButtonClick}
disabled={!ws.isConnected || isProcessingAudio}
className={`send-button ${isRecording ? 'recording' : ''} ${isProcessingAudio ? 'processing' : ''} ${userInput.trim() ? 'has-text' : ''}`}
>
Send
{isRecording ? (
<span className="recording-indicator" />
) : isProcessingAudio ? (
<span className="processing-indicator" />
) : userInput.trim() ? (
<Send size={20} />
) : (
<Mic size={20} />
)}
</button>
</form>
<VoiceControls
onAudioRecorded={handleAudioRecorded}
isProcessing={isProcessingAudio}
isPlaying={isPlayingAudio}
/>
</div>
</div>
</main>
</div>
packages/voice-assistant/src/ui/components/ToolCallCard.tsx
+61
View File
@@ -0,0 +1,61 @@
import { useState } from "react";
import { ChevronDown, ChevronRight, CheckCircle, Loader2 } from "lucide-react";
export interface ToolCallCardProps {
toolName: string;
args: any;
result?: any;
status: "executing" | "completed";
}
export function ToolCallCard({
toolName,
args,
result,
status,
}: ToolCallCardProps) {
const [isExpanded, setIsExpanded] = useState(false);
return (
<div className="tool-call-card" data-status={status}>
<button
className="tool-call-header"
onClick={() => setIsExpanded(!isExpanded)}
type="button"
>
<div className="tool-call-icon">
{isExpanded ? (
<ChevronDown size={16} />
) : (
<ChevronRight size={16} />
)}
</div>
<span className="tool-call-name">{toolName}</span>
<div className="tool-call-status">
{status === "executing" ? (
<Loader2 size={14} className="status-icon spinning" />
) : (
<CheckCircle size={14} className="status-icon" />
)}
<span className="status-text">{status}</span>
</div>
</button>
{isExpanded && (
<div className="tool-call-details">
<div className="tool-call-section">
<div className="section-header">Arguments</div>
<pre className="section-content">{JSON.stringify(args, null, 2)}</pre>
</div>
{result !== undefined && (
<div className="tool-call-section">
<div className="section-header">Result</div>
<pre className="section-content">{JSON.stringify(result, null, 2)}</pre>
</div>
)}
</div>
)}
</div>
);
}
packages/voice-assistant/src/ui/components/VoiceControls.tsx
+2 -6
View File
@@ -4,10 +4,9 @@ import { createAudioRecorder, type AudioRecorder } from '../lib/audio-capture';
interface VoiceControlsProps {
onAudioRecorded: (audio: Blob, format: string) => void;
isProcessing: boolean;
isPlaying?: boolean;
}
export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: VoiceControlsProps) {
export function VoiceControls({ onAudioRecorded, isProcessing }: VoiceControlsProps) {
const [isRecording, setIsRecording] = useState(false);
const [error, setError] = useState<string | null>(null);
const [permissionState, setPermissionState] = useState<'prompt' | 'granted' | 'denied'>('prompt');
@@ -70,8 +69,7 @@ export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: Voic
}
}
const canRecord = !isProcessing && !isPlaying;
const buttonDisabled = !canRecord || permissionState === 'denied';
const buttonDisabled = isProcessing || permissionState === 'denied';
return (
<div className="voice-controls">
@@ -106,8 +104,6 @@ export function VoiceControls({ onAudioRecorded, isProcessing, isPlaying }: Voic
Microphone access denied. Please enable it in your browser settings.
</div>
)}
{isPlaying && <div className="playing-indicator">🔊 Playing response...</div>}
</div>
);
}
packages/voice-assistant/src/ui/index.css
+8 -7
View File
@@ -13,6 +13,11 @@
-moz-osx-font-smoothing: grayscale;
}
html {
overflow-x: hidden;
max-width: 100vw;
}
* {
box-sizing: border-box;
margin: 0;
@@ -21,11 +26,7 @@
body {
margin: 0;
min-width: 320px;
min-height: 100vh;
}
#root {
width: 100%;
min-height: 100vh;
padding: 0;
overflow-x: hidden;
overflow-y: hidden;
}
packages/voice-assistant/src/ui/lib/audio-playback.ts
+49 -21
View File
@@ -1,24 +1,32 @@
export interface AudioPlayer {
play(audioData: Blob): Promise<void>;
play(audioData: Blob): Promise<number>;
stop(): void;
isPlaying(): boolean;
clearQueue(): void;
}
interface QueuedAudio {
audioData: Blob;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
export function createAudioPlayer(): AudioPlayer {
let currentAudio: HTMLAudioElement | null = null;
let playing = false;
let queue: Blob[] = [];
let queue: QueuedAudio[] = [];
let isProcessingQueue = false;
async function play(audioData: Blob): Promise<void> {
// Add to queue
queue.push(audioData);
async function play(audioData: Blob): Promise<number> {
return new Promise((resolve, reject) => {
// Add to queue with its promise handlers
queue.push({ audioData, resolve, reject });
// Start processing queue if not already processing
if (!isProcessingQueue) {
processQueue();
}
// Start processing queue if not already processing
if (!isProcessingQueue) {
processQueue();
}
});
}
async function processQueue(): Promise<void> {
@@ -29,14 +37,19 @@ export function createAudioPlayer(): AudioPlayer {
isProcessingQueue = true;
while (queue.length > 0) {
const audioData = queue.shift()!;
await playAudio(audioData);
const item = queue.shift()!;
try {
const duration = await playAudio(item.audioData);
item.resolve(duration);
} catch (error) {
item.reject(error as Error);
}
}
isProcessingQueue = false;
}
async function playAudio(audioData: Blob): Promise<void> {
async function playAudio(audioData: Blob): Promise<number> {
return new Promise((resolve, reject) => {
try {
// Create blob URL
@@ -47,40 +60,45 @@ export function createAudioPlayer(): AudioPlayer {
currentAudio = audio;
playing = true;
console.log(`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`);
console.log(
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
);
audio.onended = () => {
console.log('[AudioPlayer] Playback finished');
const duration = audio.duration;
console.log(
`[AudioPlayer] Playback finished (duration: ${duration}s)`
);
playing = false;
currentAudio = null;
// Clean up blob URL
URL.revokeObjectURL(audioUrl);
resolve();
resolve(duration);
};
audio.onerror = (error) => {
console.error('[AudioPlayer] Playback error:', error);
console.error("[AudioPlayer] Playback error:", error);
playing = false;
currentAudio = null;
// Clean up blob URL
URL.revokeObjectURL(audioUrl);
reject(new Error('Audio playback failed'));
reject(new Error("Audio playback failed"));
};
// Start playback
audio.play().catch((error) => {
console.error('[AudioPlayer] Failed to start playback:', error);
console.error("[AudioPlayer] Failed to start playback:", error);
playing = false;
currentAudio = null;
URL.revokeObjectURL(audioUrl);
reject(error);
});
} catch (error) {
console.error('[AudioPlayer] Error creating audio element:', error);
console.error("[AudioPlayer] Error creating audio element:", error);
playing = false;
currentAudio = null;
reject(error);
@@ -95,7 +113,13 @@ export function createAudioPlayer(): AudioPlayer {
currentAudio = null;
}
playing = false;
queue = [];
// Reject all pending promises in the queue
while (queue.length > 0) {
const item = queue.shift()!;
item.reject(new Error("Playback stopped"));
}
isProcessingQueue = false;
}
@@ -104,7 +128,11 @@ export function createAudioPlayer(): AudioPlayer {
}
function clearQueue(): void {
queue = [];
// Reject all pending promises in the queue
while (queue.length > 0) {
const item = queue.shift()!;
item.reject(new Error("Queue cleared"));
}
}
return {