feat: enhance capture_terminal with activity settling detection

Extend capture_terminal to wait for terminal activity to settle before
capturing output. The wait parameter is renamed to maxWait and now triggers
intelligent polling that detects when terminal output has stabilized (1s of
no changes) rather than a simple sleep.

Changes:
- Export waitForPaneActivityToSettle from tmux.ts for reuse
- Update captureTerminal to use settling logic when maxWait provided
- Update capture_terminal tool schema with new maxWait parameter
- Add error handling callbacks to StreamLLMParams interface
- Improve send_text/send_keys tool descriptions for clarity

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

packages/voice-assistant/src/server/agent/llm-openai.ts

@@ -67,15 +67,15 @@ export const terminalTools = {
         .number()
         .optional()
         .describe("Number of lines to capture (default: 200)"),
-      wait: z
+      maxWait: z
         .number()
         .optional()
         .describe(
-          "Milliseconds to wait before capturing output. Useful for slow commands."
+          "Maximum milliseconds to wait for terminal activity to settle before capturing. Polls every 100ms and waits for 1s of no changes. Useful for commands with delayed output."
         ),
     }),
-    execute: async ({ terminalName, lines, wait }) => {
+    execute: async ({ terminalName, lines, maxWait }) => {
-      const output = await captureTerminal(terminalName, lines, wait);
+      const output = await captureTerminal(terminalName, lines, maxWait);
       return { output };
     },
   }),
@@ -102,14 +102,22 @@ export const terminalTools = {
             .number()
             .optional()
             .describe("Number of lines to capture (default: 200)"),
-          wait: z
+          waitForSettled: z
+            .boolean()
+            .optional()
+            .describe(
+              "Wait for terminal activity to settle before returning output. Polls terminal and waits 500ms after last change (default: true)"
+            ),
+          maxWait: z
             .number()
             .optional()
-            .describe("Milliseconds to wait before capturing output"),
+            .describe(
+              "Maximum milliseconds to wait for activity to settle (default: 120000 = 2 minutes)"
+            ),
         })
         .optional()
         .describe(
-          "Capture terminal output after sending text. Specify 'wait' for slow commands."
+          "Capture terminal output after sending text. By default waits for activity to settle."
         ),
     }),
     execute: async ({ terminalName, text, pressEnter, return_output }) => {
@@ -144,14 +152,22 @@ export const terminalTools = {
             .number()
             .optional()
             .describe("Number of lines to capture (default: 200)"),
-          wait: z
+          waitForSettled: z
+            .boolean()
+            .optional()
+            .describe(
+              "Wait for terminal activity to settle before returning output. Polls terminal and waits 500ms after last change (default: true)"
+            ),
+          maxWait: z
             .number()
             .optional()
-            .describe("Milliseconds to wait before capturing output"),
+            .describe(
+              "Maximum milliseconds to wait for activity to settle (default: 120000 = 2 minutes)"
+            ),
         })
         .optional()
         .describe(
-          "Capture terminal output after sending keys. Specify 'wait' for slow commands."
+          "Capture terminal output after sending keys. By default waits for activity to settle."
         ),
     }),
     execute: async ({ terminalName, keys, repeat, return_output }) => {
@@ -211,8 +227,18 @@ export interface StreamLLMParams {
   abortSignal?: AbortSignal;
   onChunk?: (chunk: string) => void | Promise<void>;
   onTextSegment?: (segment: string) => void;
-  onToolCall?: (toolCallId: string, toolName: string, args: any) => Promise<void>;
+  onToolCall?: (
+    toolCallId: string,
+    toolName: string,
+    args: any
+  ) => Promise<void>;
   onToolResult?: (toolCallId: string, toolName: string, result: any) => void;
+  onToolError?: (
+    toolCallId: string,
+    toolName: string,
+    error: unknown
+  ) => void | Promise<void>;
+  onError?: (error: unknown) => void | Promise<void>;
   onFinish?: (fullText: string) => void | Promise<void>;
 }
 
@@ -246,7 +272,11 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
 
         // Emit tool call event
         if (params.onToolCall) {
-          await params.onToolCall(chunk.toolCallId, chunk.toolName, chunk.input);
+          await params.onToolCall(
+            chunk.toolCallId,
+            chunk.toolName,
+            chunk.input
+          );
         }
       } else if (chunk.type === "tool-result") {
         // Emit tool result event
@@ -255,6 +285,12 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
         }
       }
     },
+    onError: async (error) => {
+      // Emit general stream error
+      if (params.onError) {
+        await params.onError(error);
+      }
+    },
     stopWhen: stepCountIs(10),
   });
 
@@ -268,8 +304,15 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
     textBuffer = "";
   }
 
-  for await (const _part of result.fullStream) {
+  for await (const part of result.fullStream) {
-    // console.log("part", _part);
+    // console.log("part", part);
+
+    // Handle tool-error chunks (not available in onChunk callback)
+    if (part.type === "tool-error") {
+      if (params.onToolError) {
+        await params.onToolError(part.toolCallId, part.toolName, part.error);
+      }
+    }
   }
 
   // Flush any remaining text at the end

packages/voice-assistant/src/server/daemon/terminal-manager.ts

@@ -13,6 +13,7 @@ import {
   isWindowNameUnique,
   getCurrentWorkingDirectory,
   getCurrentCommand,
+  waitForPaneActivityToSettle,
 } from "./tmux.js";
 
 const DEFAULT_SESSION = "voice-dev";
@@ -22,7 +23,6 @@ const DEFAULT_SESSION = "voice-dev";
 
 export interface TerminalInfo {
   name: string;
-  active: boolean;
   workingDirectory: string;
   currentCommand: string;
   lastLines?: string;
@@ -34,8 +34,8 @@ export interface CreateTerminalParams {
   initialCommand?: string;
 }
 
-export interface Terminal extends TerminalInfo {
+export interface CreateTerminalResult extends TerminalInfo {
-  sessionId: string;
+  commandOutput?: string;
 }
 
 /**
@@ -58,7 +58,9 @@ export async function listTerminals(): Promise<TerminalInfo[]> {
   const session = await findSessionByName(DEFAULT_SESSION);
 
   if (!session) {
-    throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`);
+    throw new Error(
+      `Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`
+    );
   }
 
   const windows = await listWindows(session.id);
@@ -69,28 +71,16 @@ export async function listTerminals(): Promise<TerminalInfo[]> {
     // Get the first (and only) pane in this window
     const paneId = `${window.id}.0`;
 
-    try {
+    const workingDirectory = await getCurrentWorkingDirectory(paneId);
-      const workingDirectory = await getCurrentWorkingDirectory(paneId);
+    const currentCommand = await getCurrentCommand(paneId);
-      const currentCommand = await getCurrentCommand(paneId);
+    const lastLines = await capturePaneContent(paneId, 5, false);
-      const lastLines = await capturePaneContent(paneId, 5, false);
 
-      terminals.push({
+    terminals.push({
-        name: window.name,
+      name: window.name,
-        active: window.active,
+      workingDirectory,
-        workingDirectory,
+      currentCommand,
-        currentCommand,
+      lastLines,
-        lastLines,
+    });
-      });
-    } catch (error) {
-      // If we can't get pane info, still include the terminal with empty values
-      terminals.push({
-        name: window.name,
-        active: window.active,
-        workingDirectory: "",
-        currentCommand: "",
-        lastLines: "",
-      });
-    }
   }
 
   return terminals;
@@ -100,11 +90,15 @@ export async function listTerminals(): Promise<TerminalInfo[]> {
  * Create a new terminal (tmux window) with specified name and working directory
  * Optionally execute an initial command
  */
-export async function createTerminal(params: CreateTerminalParams): Promise<Terminal> {
+export async function createTerminal(
+  params: CreateTerminalParams
+): Promise<CreateTerminalResult> {
   const session = await findSessionByName(DEFAULT_SESSION);
 
   if (!session) {
-    throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`);
+    throw new Error(
+      `Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`
+    );
   }
 
   // Validate name uniqueness
@@ -133,21 +127,21 @@ export async function createTerminal(params: CreateTerminalParams): Promise<Term
 
   return {
     name: windowResult.name,
-    active: windowResult.active,
     workingDirectory,
     currentCommand,
-    sessionId: session.id,
+    commandOutput: windowResult.output,
   };
 }
 
 /**
  * Capture output from a terminal by name
  * Returns the last N lines of terminal content
+ * If maxWait is provided, waits for terminal activity to settle before capturing
  */
 export async function captureTerminal(
   terminalName: string,
   lines: number = 200,
-  wait?: number
+  maxWait?: number
 ): Promise<string> {
   const session = await findSessionByName(DEFAULT_SESSION);
   if (!session) {
@@ -171,9 +165,9 @@ export async function captureTerminal(
     throw new Error(`No pane found for terminal ${terminalName}`);
   }
 
-  // Optional wait before capture
+  // Wait for activity to settle if maxWait is provided
-  if (wait) {
+  if (maxWait) {
-    await new Promise((resolve) => setTimeout(resolve, wait));
+    return waitForPaneActivityToSettle(pane.id, maxWait, lines);
   }
 
   return capturePaneContent(pane.id, lines, false);

packages/voice-assistant/src/server/daemon/tmux.ts

@@ -199,10 +199,10 @@ export async function capturePaneContent(
 
   // Trim trailing whitespace, split by lines, take last N lines, rejoin
   const trimmed = output.trimEnd();
-  const allLines = trimmed.split('\n');
+  const allLines = trimmed.split("\n");
   const lastLines = allLines.slice(-lines);
 
-  return lastLines.join('\n');
+  return lastLines.join("\n");
 }
 
 /**
@@ -212,7 +212,9 @@ export async function getCurrentWorkingDirectory(
   paneId: string
 ): Promise<string> {
   try {
-    const tmuxPath = await executeTmux(`display-message -p -t '${paneId}' '#{pane_current_path}'`);
+    const tmuxPath = await executeTmux(
+      `display-message -p -t '${paneId}' '#{pane_current_path}'`
+    );
 
     // If tmux returns a valid path, use it
     if (tmuxPath && tmuxPath.trim()) {
@@ -220,8 +222,12 @@ export async function getCurrentWorkingDirectory(
     }
 
     // Fallback: get the PID and use lsof to find the actual CWD
-    const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`);
+    const shellPid = await executeTmux(
-    const { stdout } = await exec(`lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-`);
+      `display-message -p -t '${paneId}' '#{pane_pid}'`
+    );
+    const { stdout } = await exec(
+      `lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-`
+    );
     return stdout.trim() || tmuxPath;
   } catch (error) {
     // If all else fails, return empty string
@@ -236,15 +242,21 @@ export async function getCurrentWorkingDirectory(
 export async function getCurrentCommand(paneId: string): Promise<string> {
   try {
     // Get the shell PID (the pane's main process)
-    const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`);
+    const shellPid = await executeTmux(
+      `display-message -p -t '${paneId}' '#{pane_pid}'`
+    );
 
     // First, check if there's a child process using comm= (works for all programs including top)
     // Use 'ax' flags to see all processes
-    const { stdout: childPid } = await exec(`ps ax -o pid=,ppid=,comm= | awk '$2 == ${shellPid.trim()} { print $1; exit }'`);
+    const { stdout: childPid } = await exec(
+      `ps ax -o pid=,ppid=,comm= | awk '$2 == ${shellPid.trim()} { print $1; exit }'`
+    );
 
     if (childPid.trim()) {
       // Found a child process, get its full command with args
-      const { stdout: fullCmd } = await exec(`ps -p ${childPid.trim()} -o args= | sed 's/\\\\012.*//'`);
+      const { stdout: fullCmd } = await exec(
+        `ps -p ${childPid.trim()} -o args= | sed 's/\\\\012.*//'`
+      );
       const command = fullCmd.trim();
       if (command) {
         return command;
@@ -256,7 +268,9 @@ export async function getCurrentCommand(paneId: string): Promise<string> {
     return shellCmd.trim();
   } catch (error) {
     // Fallback to just the command name if ps fails
-    return executeTmux(`display-message -p -t '${paneId}' '#{pane_current_command}'`);
+    return executeTmux(
+      `display-message -p -t '${paneId}' '#{pane_current_command}'`
+    );
   }
 }
 
@@ -277,11 +291,11 @@ export async function createSession(name: string): Promise<TmuxSession | null> {
  * Expand tilde in path to home directory
  */
 function expandTilde(path: string): string {
-  if (path.startsWith('~/')) {
+  if (path.startsWith("~/")) {
     const homeDir = process.env.HOME || os.homedir();
-    return path.replace('~', homeDir);
+    return path.replace("~", homeDir);
   }
-  if (path === '~') {
+  if (path === "~") {
     return process.env.HOME || os.homedir();
   }
   return path;
@@ -331,17 +345,15 @@ export async function createWindow(
 
   // If command is provided, execute it in the new pane
   if (options?.command && defaultPane) {
-    await sendText({
+    commandOutput = (await sendText({
       paneId: defaultPane.id,
       text: options.command,
       pressEnter: true,
-    });
+      return_output: {
-
+        waitForSettled: true,
-    // Sleep for 1 second to allow command to execute
+        maxWait: 120000,
-    await new Promise((resolve) => setTimeout(resolve, 1000));
+      },
-
+    })) as string;
-    // Capture the pane content
-    commandOutput = await capturePaneContent(defaultPane.id, 200, false);
   }
 
   return {
@@ -809,7 +821,11 @@ export async function sendKeys({
   paneId: string;
   keys: string;
   repeat?: number;
-  return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number };
+  return_output?: {
+    lines?: number;
+    waitForSettled?: boolean;
+    maxWait?: number;
+  };
 }): Promise<string | void> {
   // Repeat the key press the specified number of times
   for (let i = 0; i < repeat; i++) {
@@ -831,15 +847,15 @@ export async function sendKeys({
   }
 }
 
-async function waitForPaneActivityToSettle(
+export async function waitForPaneActivityToSettle(
   paneId: string,
   maxWait: number,
   lines: number
 ): Promise<string> {
-  const settleTime = 500;      // Hardcoded debounce
+  const settleTime = 1000; // Hardcoded debounce
-  const pollInterval = 100;     // Poll every 100ms
+  const pollInterval = 100; // Poll every 100ms
 
-  let lastContent = '';
+  let lastContent = "";
   let lastChangeTime = Date.now();
   const startTime = Date.now();
 
@@ -861,7 +877,7 @@ async function waitForPaneActivityToSettle(
       return content;
     }
 
-    await new Promise(resolve => setTimeout(resolve, pollInterval));
+    await new Promise((resolve) => setTimeout(resolve, pollInterval));
   }
 }
 
@@ -874,7 +890,11 @@ export async function sendText({
   paneId: string;
   text: string;
   pressEnter?: boolean;
-  return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number };
+  return_output?: {
+    lines?: number;
+    waitForSettled?: boolean;
+    maxWait?: number;
+  };
 }): Promise<string | void> {
   // Send each character with -l flag for literal interpretation
   for (const char of text) {