diff --git a/packages/voice-assistant/src/server/agent/llm-openai.ts b/packages/voice-assistant/src/server/agent/llm-openai.ts index 12e0a5c31..34701ac64 100644 --- a/packages/voice-assistant/src/server/agent/llm-openai.ts +++ b/packages/voice-assistant/src/server/agent/llm-openai.ts @@ -67,15 +67,15 @@ export const terminalTools = { .number() .optional() .describe("Number of lines to capture (default: 200)"), - wait: z + maxWait: z .number() .optional() .describe( - "Milliseconds to wait before capturing output. Useful for slow commands." + "Maximum milliseconds to wait for terminal activity to settle before capturing. Polls every 100ms and waits for 1s of no changes. Useful for commands with delayed output." ), }), - execute: async ({ terminalName, lines, wait }) => { - const output = await captureTerminal(terminalName, lines, wait); + execute: async ({ terminalName, lines, maxWait }) => { + const output = await captureTerminal(terminalName, lines, maxWait); return { output }; }, }), @@ -102,14 +102,22 @@ export const terminalTools = { .number() .optional() .describe("Number of lines to capture (default: 200)"), - wait: z + waitForSettled: z + .boolean() + .optional() + .describe( + "Wait for terminal activity to settle before returning output. Polls terminal and waits 500ms after last change (default: true)" + ), + maxWait: z .number() .optional() - .describe("Milliseconds to wait before capturing output"), + .describe( + "Maximum milliseconds to wait for activity to settle (default: 120000 = 2 minutes)" + ), }) .optional() .describe( - "Capture terminal output after sending text. Specify 'wait' for slow commands." + "Capture terminal output after sending text. By default waits for activity to settle." ), }), execute: async ({ terminalName, text, pressEnter, return_output }) => { @@ -144,14 +152,22 @@ export const terminalTools = { .number() .optional() .describe("Number of lines to capture (default: 200)"), - wait: z + waitForSettled: z + .boolean() + .optional() + .describe( + "Wait for terminal activity to settle before returning output. Polls terminal and waits 500ms after last change (default: true)" + ), + maxWait: z .number() .optional() - .describe("Milliseconds to wait before capturing output"), + .describe( + "Maximum milliseconds to wait for activity to settle (default: 120000 = 2 minutes)" + ), }) .optional() .describe( - "Capture terminal output after sending keys. Specify 'wait' for slow commands." + "Capture terminal output after sending keys. By default waits for activity to settle." ), }), execute: async ({ terminalName, keys, repeat, return_output }) => { @@ -211,8 +227,18 @@ export interface StreamLLMParams { abortSignal?: AbortSignal; onChunk?: (chunk: string) => void | Promise; onTextSegment?: (segment: string) => void; - onToolCall?: (toolCallId: string, toolName: string, args: any) => Promise; + onToolCall?: ( + toolCallId: string, + toolName: string, + args: any + ) => Promise; onToolResult?: (toolCallId: string, toolName: string, result: any) => void; + onToolError?: ( + toolCallId: string, + toolName: string, + error: unknown + ) => void | Promise; + onError?: (error: unknown) => void | Promise; onFinish?: (fullText: string) => void | Promise; } @@ -246,7 +272,11 @@ export async function streamLLM(params: StreamLLMParams): Promise { // Emit tool call event if (params.onToolCall) { - await params.onToolCall(chunk.toolCallId, chunk.toolName, chunk.input); + await params.onToolCall( + chunk.toolCallId, + chunk.toolName, + chunk.input + ); } } else if (chunk.type === "tool-result") { // Emit tool result event @@ -255,6 +285,12 @@ export async function streamLLM(params: StreamLLMParams): Promise { } } }, + onError: async (error) => { + // Emit general stream error + if (params.onError) { + await params.onError(error); + } + }, stopWhen: stepCountIs(10), }); @@ -268,8 +304,15 @@ export async function streamLLM(params: StreamLLMParams): Promise { textBuffer = ""; } - for await (const _part of result.fullStream) { - // console.log("part", _part); + for await (const part of result.fullStream) { + // console.log("part", part); + + // Handle tool-error chunks (not available in onChunk callback) + if (part.type === "tool-error") { + if (params.onToolError) { + await params.onToolError(part.toolCallId, part.toolName, part.error); + } + } } // Flush any remaining text at the end diff --git a/packages/voice-assistant/src/server/daemon/terminal-manager.ts b/packages/voice-assistant/src/server/daemon/terminal-manager.ts index cb33a84ac..45bcfc12b 100644 --- a/packages/voice-assistant/src/server/daemon/terminal-manager.ts +++ b/packages/voice-assistant/src/server/daemon/terminal-manager.ts @@ -13,6 +13,7 @@ import { isWindowNameUnique, getCurrentWorkingDirectory, getCurrentCommand, + waitForPaneActivityToSettle, } from "./tmux.js"; const DEFAULT_SESSION = "voice-dev"; @@ -22,7 +23,6 @@ const DEFAULT_SESSION = "voice-dev"; export interface TerminalInfo { name: string; - active: boolean; workingDirectory: string; currentCommand: string; lastLines?: string; @@ -34,8 +34,8 @@ export interface CreateTerminalParams { initialCommand?: string; } -export interface Terminal extends TerminalInfo { - sessionId: string; +export interface CreateTerminalResult extends TerminalInfo { + commandOutput?: string; } /** @@ -58,7 +58,9 @@ export async function listTerminals(): Promise { const session = await findSessionByName(DEFAULT_SESSION); if (!session) { - throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`); + throw new Error( + `Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.` + ); } const windows = await listWindows(session.id); @@ -69,28 +71,16 @@ export async function listTerminals(): Promise { // Get the first (and only) pane in this window const paneId = `${window.id}.0`; - try { - const workingDirectory = await getCurrentWorkingDirectory(paneId); - const currentCommand = await getCurrentCommand(paneId); - const lastLines = await capturePaneContent(paneId, 5, false); + const workingDirectory = await getCurrentWorkingDirectory(paneId); + const currentCommand = await getCurrentCommand(paneId); + const lastLines = await capturePaneContent(paneId, 5, false); - terminals.push({ - name: window.name, - active: window.active, - workingDirectory, - currentCommand, - lastLines, - }); - } catch (error) { - // If we can't get pane info, still include the terminal with empty values - terminals.push({ - name: window.name, - active: window.active, - workingDirectory: "", - currentCommand: "", - lastLines: "", - }); - } + terminals.push({ + name: window.name, + workingDirectory, + currentCommand, + lastLines, + }); } return terminals; @@ -100,11 +90,15 @@ export async function listTerminals(): Promise { * Create a new terminal (tmux window) with specified name and working directory * Optionally execute an initial command */ -export async function createTerminal(params: CreateTerminalParams): Promise { +export async function createTerminal( + params: CreateTerminalParams +): Promise { const session = await findSessionByName(DEFAULT_SESSION); if (!session) { - throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`); + throw new Error( + `Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.` + ); } // Validate name uniqueness @@ -133,21 +127,21 @@ export async function createTerminal(params: CreateTerminalParams): Promise { const session = await findSessionByName(DEFAULT_SESSION); if (!session) { @@ -171,9 +165,9 @@ export async function captureTerminal( throw new Error(`No pane found for terminal ${terminalName}`); } - // Optional wait before capture - if (wait) { - await new Promise((resolve) => setTimeout(resolve, wait)); + // Wait for activity to settle if maxWait is provided + if (maxWait) { + return waitForPaneActivityToSettle(pane.id, maxWait, lines); } return capturePaneContent(pane.id, lines, false); diff --git a/packages/voice-assistant/src/server/daemon/tmux.ts b/packages/voice-assistant/src/server/daemon/tmux.ts index 9b51e7675..d23495e05 100644 --- a/packages/voice-assistant/src/server/daemon/tmux.ts +++ b/packages/voice-assistant/src/server/daemon/tmux.ts @@ -199,10 +199,10 @@ export async function capturePaneContent( // Trim trailing whitespace, split by lines, take last N lines, rejoin const trimmed = output.trimEnd(); - const allLines = trimmed.split('\n'); + const allLines = trimmed.split("\n"); const lastLines = allLines.slice(-lines); - return lastLines.join('\n'); + return lastLines.join("\n"); } /** @@ -212,7 +212,9 @@ export async function getCurrentWorkingDirectory( paneId: string ): Promise { try { - const tmuxPath = await executeTmux(`display-message -p -t '${paneId}' '#{pane_current_path}'`); + const tmuxPath = await executeTmux( + `display-message -p -t '${paneId}' '#{pane_current_path}'` + ); // If tmux returns a valid path, use it if (tmuxPath && tmuxPath.trim()) { @@ -220,8 +222,12 @@ export async function getCurrentWorkingDirectory( } // Fallback: get the PID and use lsof to find the actual CWD - const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`); - const { stdout } = await exec(`lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-`); + const shellPid = await executeTmux( + `display-message -p -t '${paneId}' '#{pane_pid}'` + ); + const { stdout } = await exec( + `lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-` + ); return stdout.trim() || tmuxPath; } catch (error) { // If all else fails, return empty string @@ -236,15 +242,21 @@ export async function getCurrentWorkingDirectory( export async function getCurrentCommand(paneId: string): Promise { try { // Get the shell PID (the pane's main process) - const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`); + const shellPid = await executeTmux( + `display-message -p -t '${paneId}' '#{pane_pid}'` + ); // First, check if there's a child process using comm= (works for all programs including top) // Use 'ax' flags to see all processes - const { stdout: childPid } = await exec(`ps ax -o pid=,ppid=,comm= | awk '$2 == ${shellPid.trim()} { print $1; exit }'`); + const { stdout: childPid } = await exec( + `ps ax -o pid=,ppid=,comm= | awk '$2 == ${shellPid.trim()} { print $1; exit }'` + ); if (childPid.trim()) { // Found a child process, get its full command with args - const { stdout: fullCmd } = await exec(`ps -p ${childPid.trim()} -o args= | sed 's/\\\\012.*//'`); + const { stdout: fullCmd } = await exec( + `ps -p ${childPid.trim()} -o args= | sed 's/\\\\012.*//'` + ); const command = fullCmd.trim(); if (command) { return command; @@ -256,7 +268,9 @@ export async function getCurrentCommand(paneId: string): Promise { return shellCmd.trim(); } catch (error) { // Fallback to just the command name if ps fails - return executeTmux(`display-message -p -t '${paneId}' '#{pane_current_command}'`); + return executeTmux( + `display-message -p -t '${paneId}' '#{pane_current_command}'` + ); } } @@ -277,11 +291,11 @@ export async function createSession(name: string): Promise { * Expand tilde in path to home directory */ function expandTilde(path: string): string { - if (path.startsWith('~/')) { + if (path.startsWith("~/")) { const homeDir = process.env.HOME || os.homedir(); - return path.replace('~', homeDir); + return path.replace("~", homeDir); } - if (path === '~') { + if (path === "~") { return process.env.HOME || os.homedir(); } return path; @@ -331,17 +345,15 @@ export async function createWindow( // If command is provided, execute it in the new pane if (options?.command && defaultPane) { - await sendText({ + commandOutput = (await sendText({ paneId: defaultPane.id, text: options.command, pressEnter: true, - }); - - // Sleep for 1 second to allow command to execute - await new Promise((resolve) => setTimeout(resolve, 1000)); - - // Capture the pane content - commandOutput = await capturePaneContent(defaultPane.id, 200, false); + return_output: { + waitForSettled: true, + maxWait: 120000, + }, + })) as string; } return { @@ -809,7 +821,11 @@ export async function sendKeys({ paneId: string; keys: string; repeat?: number; - return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number }; + return_output?: { + lines?: number; + waitForSettled?: boolean; + maxWait?: number; + }; }): Promise { // Repeat the key press the specified number of times for (let i = 0; i < repeat; i++) { @@ -831,15 +847,15 @@ export async function sendKeys({ } } -async function waitForPaneActivityToSettle( +export async function waitForPaneActivityToSettle( paneId: string, maxWait: number, lines: number ): Promise { - const settleTime = 500; // Hardcoded debounce - const pollInterval = 100; // Poll every 100ms + const settleTime = 1000; // Hardcoded debounce + const pollInterval = 100; // Poll every 100ms - let lastContent = ''; + let lastContent = ""; let lastChangeTime = Date.now(); const startTime = Date.now(); @@ -861,7 +877,7 @@ async function waitForPaneActivityToSettle( return content; } - await new Promise(resolve => setTimeout(resolve, pollInterval)); + await new Promise((resolve) => setTimeout(resolve, pollInterval)); } } @@ -874,7 +890,11 @@ export async function sendText({ paneId: string; text: string; pressEnter?: boolean; - return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number }; + return_output?: { + lines?: number; + waitForSettled?: boolean; + maxWait?: number; + }; }): Promise { // Send each character with -l flag for literal interpretation for (const char of text) {