jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance capture_terminal with activity settling detection

Browse Source 
Extend capture_terminal to wait for terminal activity to settle before
capturing output. The wait parameter is renamed to maxWait and now triggers
intelligent polling that detects when terminal output has stabilized (1s of
no changes) rather than a simple sleep.

Changes:
- Export waitForPaneActivityToSettle from tmux.ts for reuse
- Update captureTerminal to use settling logic when maxWait provided
- Update capture_terminal tool schema with new maxWait parameter
- Add error handling callbacks to StreamLLMParams interface
- Improve send_text/send_keys tool descriptions for clarity

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Mohamed Boudra
2025-10-20 00:04:35 +02:00
co-authored by Claude
parent 0bfda11c1f
commit 853a73cc05
3 changed files with 131 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/voice-assistant/src/server/agent/llm-openai.ts
+57 -14
View File
@@ -67,15 +67,15 @@ export const terminalTools = {
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
maxWait: z
.number()
.optional()
.describe(
"Milliseconds to wait before capturing output. Useful for slow commands."
"Maximum milliseconds to wait for terminal activity to settle before capturing. Polls every 100ms and waits for 1s of no changes. Useful for commands with delayed output."
),
}),
execute: async ({ terminalName, lines, wait }) => {
const output = await captureTerminal(terminalName, lines, wait);
execute: async ({ terminalName, lines, maxWait }) => {
const output = await captureTerminal(terminalName, lines, maxWait);
return { output };
},
}),
@@ -102,14 +102,22 @@ export const terminalTools = {
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
waitForSettled: z
.boolean()
.optional()
.describe(
"Wait for terminal activity to settle before returning output. Polls terminal and waits 500ms after last change (default: true)"
),
maxWait: z
.number()
.optional()
.describe("Milliseconds to wait before capturing output"),
.describe(
"Maximum milliseconds to wait for activity to settle (default: 120000 = 2 minutes)"
),
})
.optional()
.describe(
"Capture terminal output after sending text. Specify 'wait' for slow commands."
"Capture terminal output after sending text. By default waits for activity to settle."
),
}),
execute: async ({ terminalName, text, pressEnter, return_output }) => {
@@ -144,14 +152,22 @@ export const terminalTools = {
.number()
.optional()
.describe("Number of lines to capture (default: 200)"),
wait: z
waitForSettled: z
.boolean()
.optional()
.describe(
"Wait for terminal activity to settle before returning output. Polls terminal and waits 500ms after last change (default: true)"
),
maxWait: z
.number()
.optional()
.describe("Milliseconds to wait before capturing output"),
.describe(
"Maximum milliseconds to wait for activity to settle (default: 120000 = 2 minutes)"
),
})
.optional()
.describe(
"Capture terminal output after sending keys. Specify 'wait' for slow commands."
"Capture terminal output after sending keys. By default waits for activity to settle."
),
}),
execute: async ({ terminalName, keys, repeat, return_output }) => {
@@ -211,8 +227,18 @@ export interface StreamLLMParams {
abortSignal?: AbortSignal;
onChunk?: (chunk: string) => void | Promise<void>;
onTextSegment?: (segment: string) => void;
onToolCall?: (toolCallId: string, toolName: string, args: any) => Promise<void>;
onToolCall?: (
toolCallId: string,
toolName: string,
args: any
) => Promise<void>;
onToolResult?: (toolCallId: string, toolName: string, result: any) => void;
onToolError?: (
toolCallId: string,
toolName: string,
error: unknown
) => void | Promise<void>;
onError?: (error: unknown) => void | Promise<void>;
onFinish?: (fullText: string) => void | Promise<void>;
}
@@ -246,7 +272,11 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
// Emit tool call event
if (params.onToolCall) {
await params.onToolCall(chunk.toolCallId, chunk.toolName, chunk.input);
await params.onToolCall(
chunk.toolCallId,
chunk.toolName,
chunk.input
);
}
} else if (chunk.type === "tool-result") {
// Emit tool result event
@@ -255,6 +285,12 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
}
}
},
onError: async (error) => {
// Emit general stream error
if (params.onError) {
await params.onError(error);
}
},
stopWhen: stepCountIs(10),
});
@@ -268,8 +304,15 @@ export async function streamLLM(params: StreamLLMParams): Promise<string> {
textBuffer = "";
}
for await (const _part of result.fullStream) {
// console.log("part", _part);
for await (const part of result.fullStream) {
// console.log("part", part);
// Handle tool-error chunks (not available in onChunk callback)
if (part.type === "tool-error") {
if (params.onToolError) {
await params.onToolError(part.toolCallId, part.toolName, part.error);
}
}
}
// Flush any remaining text at the end
packages/voice-assistant/src/server/daemon/terminal-manager.ts
+27 -33
View File
@@ -13,6 +13,7 @@ import {
isWindowNameUnique,
getCurrentWorkingDirectory,
getCurrentCommand,
waitForPaneActivityToSettle,
} from "./tmux.js";
const DEFAULT_SESSION = "voice-dev";
@@ -22,7 +23,6 @@ const DEFAULT_SESSION = "voice-dev";
export interface TerminalInfo {
name: string;
active: boolean;
workingDirectory: string;
currentCommand: string;
lastLines?: string;
@@ -34,8 +34,8 @@ export interface CreateTerminalParams {
initialCommand?: string;
}
export interface Terminal extends TerminalInfo {
sessionId: string;
export interface CreateTerminalResult extends TerminalInfo {
commandOutput?: string;
}
/**
@@ -58,7 +58,9 @@ export async function listTerminals(): Promise<TerminalInfo[]> {
const session = await findSessionByName(DEFAULT_SESSION);
if (!session) {
throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`);
throw new Error(
`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`
);
}
const windows = await listWindows(session.id);
@@ -69,28 +71,16 @@ export async function listTerminals(): Promise<TerminalInfo[]> {
// Get the first (and only) pane in this window
const paneId = `${window.id}.0`;
try {
const workingDirectory = await getCurrentWorkingDirectory(paneId);
const currentCommand = await getCurrentCommand(paneId);
const lastLines = await capturePaneContent(paneId, 5, false);
const workingDirectory = await getCurrentWorkingDirectory(paneId);
const currentCommand = await getCurrentCommand(paneId);
const lastLines = await capturePaneContent(paneId, 5, false);
terminals.push({
name: window.name,
active: window.active,
workingDirectory,
currentCommand,
lastLines,
});
} catch (error) {
// If we can't get pane info, still include the terminal with empty values
terminals.push({
name: window.name,
active: window.active,
workingDirectory: "",
currentCommand: "",
lastLines: "",
});
}
terminals.push({
name: window.name,
workingDirectory,
currentCommand,
lastLines,
});
}
return terminals;
@@ -100,11 +90,15 @@ export async function listTerminals(): Promise<TerminalInfo[]> {
* Create a new terminal (tmux window) with specified name and working directory
* Optionally execute an initial command
*/
export async function createTerminal(params: CreateTerminalParams): Promise<Terminal> {
export async function createTerminal(
params: CreateTerminalParams
): Promise<CreateTerminalResult> {
const session = await findSessionByName(DEFAULT_SESSION);
if (!session) {
throw new Error(`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`);
throw new Error(
`Session '${DEFAULT_SESSION}' not found. Call initializeDefaultSession() first.`
);
}
// Validate name uniqueness
@@ -133,21 +127,21 @@ export async function createTerminal(params: CreateTerminalParams): Promise<Term
return {
name: windowResult.name,
active: windowResult.active,
workingDirectory,
currentCommand,
sessionId: session.id,
commandOutput: windowResult.output,
};
}
/**
* Capture output from a terminal by name
* Returns the last N lines of terminal content
* If maxWait is provided, waits for terminal activity to settle before capturing
*/
export async function captureTerminal(
terminalName: string,
lines: number = 200,
wait?: number
maxWait?: number
): Promise<string> {
const session = await findSessionByName(DEFAULT_SESSION);
if (!session) {
@@ -171,9 +165,9 @@ export async function captureTerminal(
throw new Error(`No pane found for terminal ${terminalName}`);
}
// Optional wait before capture
if (wait) {
await new Promise((resolve) => setTimeout(resolve, wait));
// Wait for activity to settle if maxWait is provided
if (maxWait) {
return waitForPaneActivityToSettle(pane.id, maxWait, lines);
}
return capturePaneContent(pane.id, lines, false);
packages/voice-assistant/src/server/daemon/tmux.ts
+47 -27
View File
@@ -199,10 +199,10 @@ export async function capturePaneContent(
// Trim trailing whitespace, split by lines, take last N lines, rejoin
const trimmed = output.trimEnd();
const allLines = trimmed.split('\n');
const allLines = trimmed.split("\n");
const lastLines = allLines.slice(-lines);
return lastLines.join('\n');
return lastLines.join("\n");
}
/**
@@ -212,7 +212,9 @@ export async function getCurrentWorkingDirectory(
paneId: string
): Promise<string> {
try {
const tmuxPath = await executeTmux(`display-message -p -t '${paneId}' '#{pane_current_path}'`);
const tmuxPath = await executeTmux(
`display-message -p -t '${paneId}' '#{pane_current_path}'`
);
// If tmux returns a valid path, use it
if (tmuxPath && tmuxPath.trim()) {
@@ -220,8 +222,12 @@ export async function getCurrentWorkingDirectory(
}
// Fallback: get the PID and use lsof to find the actual CWD
const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`);
const { stdout } = await exec(`lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-`);
const shellPid = await executeTmux(
`display-message -p -t '${paneId}' '#{pane_pid}'`
);
const { stdout } = await exec(
`lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-`
);
return stdout.trim() || tmuxPath;
} catch (error) {
// If all else fails, return empty string
@@ -236,15 +242,21 @@ export async function getCurrentWorkingDirectory(
export async function getCurrentCommand(paneId: string): Promise<string> {
try {
// Get the shell PID (the pane's main process)
const shellPid = await executeTmux(`display-message -p -t '${paneId}' '#{pane_pid}'`);
const shellPid = await executeTmux(
`display-message -p -t '${paneId}' '#{pane_pid}'`
);
// First, check if there's a child process using comm= (works for all programs including top)
// Use 'ax' flags to see all processes
const { stdout: childPid } = await exec(`ps ax -o pid=,ppid=,comm= | awk '$2 == ${shellPid.trim()} { print $1; exit }'`);
const { stdout: childPid } = await exec(
`ps ax -o pid=,ppid=,comm= | awk '$2 == ${shellPid.trim()} { print $1; exit }'`
);
if (childPid.trim()) {
// Found a child process, get its full command with args
const { stdout: fullCmd } = await exec(`ps -p ${childPid.trim()} -o args= | sed 's/\\\\012.*//'`);
const { stdout: fullCmd } = await exec(
`ps -p ${childPid.trim()} -o args= | sed 's/\\\\012.*//'`
);
const command = fullCmd.trim();
if (command) {
return command;
@@ -256,7 +268,9 @@ export async function getCurrentCommand(paneId: string): Promise<string> {
return shellCmd.trim();
} catch (error) {
// Fallback to just the command name if ps fails
return executeTmux(`display-message -p -t '${paneId}' '#{pane_current_command}'`);
return executeTmux(
`display-message -p -t '${paneId}' '#{pane_current_command}'`
);
}
}
@@ -277,11 +291,11 @@ export async function createSession(name: string): Promise<TmuxSession | null> {
* Expand tilde in path to home directory
*/
function expandTilde(path: string): string {
if (path.startsWith('~/')) {
if (path.startsWith("~/")) {
const homeDir = process.env.HOME || os.homedir();
return path.replace('~', homeDir);
return path.replace("~", homeDir);
}
if (path === '~') {
if (path === "~") {
return process.env.HOME || os.homedir();
}
return path;
@@ -331,17 +345,15 @@ export async function createWindow(
// If command is provided, execute it in the new pane
if (options?.command && defaultPane) {
await sendText({
commandOutput = (await sendText({
paneId: defaultPane.id,
text: options.command,
pressEnter: true,
});
// Sleep for 1 second to allow command to execute
await new Promise((resolve) => setTimeout(resolve, 1000));
// Capture the pane content
commandOutput = await capturePaneContent(defaultPane.id, 200, false);
return_output: {
waitForSettled: true,
maxWait: 120000,
},
})) as string;
}
return {
@@ -809,7 +821,11 @@ export async function sendKeys({
paneId: string;
keys: string;
repeat?: number;
return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number };
return_output?: {
lines?: number;
waitForSettled?: boolean;
maxWait?: number;
};
}): Promise<string | void> {
// Repeat the key press the specified number of times
for (let i = 0; i < repeat; i++) {
@@ -831,15 +847,15 @@ export async function sendKeys({
}
}
async function waitForPaneActivityToSettle(
export async function waitForPaneActivityToSettle(
paneId: string,
maxWait: number,
lines: number
): Promise<string> {
const settleTime = 500; // Hardcoded debounce
const pollInterval = 100; // Poll every 100ms
const settleTime = 1000; // Hardcoded debounce
const pollInterval = 100; // Poll every 100ms
let lastContent = '';
let lastContent = "";
let lastChangeTime = Date.now();
const startTime = Date.now();
@@ -861,7 +877,7 @@ async function waitForPaneActivityToSettle(
return content;
}
await new Promise(resolve => setTimeout(resolve, pollInterval));
await new Promise((resolve) => setTimeout(resolve, pollInterval));
}
}
@@ -874,7 +890,11 @@ export async function sendText({
paneId: string;
text: string;
pressEnter?: boolean;
return_output?: { lines?: number; waitForSettled?: boolean; maxWait?: number };
return_output?: {
lines?: number;
waitForSettled?: boolean;
maxWait?: number;
};
}): Promise<string | void> {
// Send each character with -l flag for literal interpretation
for (const char of text) {