From cb2e4aacfe340df85c46ff6a84ce6db43f473e21 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 12 Feb 2026 18:04:39 +0700 Subject: [PATCH] feat(cli): add --output-schema flag for structured agent output --- packages/cli/src/cli.ts | 1 + packages/cli/src/commands/agent/index.ts | 1 + packages/cli/src/commands/agent/run.ts | 238 +++++++++++++++++++- packages/cli/src/commands/onboard.ts | 1 + packages/cli/src/output/with-output.ts | 12 + packages/cli/tests/05-agent-run.test.ts | 52 ++++- packages/server/src/client/daemon-client.ts | 4 + packages/server/src/server/session.ts | 26 ++- packages/server/src/shared/messages.ts | 2 + packages/website/src/routes/docs/cli.tsx | 28 ++- 10 files changed, 344 insertions(+), 21 deletions(-) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 1794118bb..44a0e5264 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -67,6 +67,7 @@ export function createCli(): Command { .option('--cwd ', 'Working directory (default: current)') .option('--label ', 'Add label(s) to the agent (can be used multiple times)', collectMultiple, []) .option('--ui', 'Mark as UI agent (equivalent to --label ui=true)') + .option('--output-schema ', 'Output JSON matching the provided schema file path or inline JSON schema') .option('--json', 'Output in JSON format') .option('--host ', 'Daemon host:port (default: localhost:6767)') .action(withOutput(runRunCommand)) diff --git a/packages/cli/src/commands/agent/index.ts b/packages/cli/src/commands/agent/index.ts index a24555d1d..4004e2911 100644 --- a/packages/cli/src/commands/agent/index.ts +++ b/packages/cli/src/commands/agent/index.ts @@ -43,6 +43,7 @@ export function createAgentCommand(): Command { .option('--cwd ', 'Working directory (default: current)') .option('--label ', 'Add label(s) to the agent (can be used multiple times)', collectMultiple, []) .option('--ui', 'Mark as UI agent (equivalent to --label ui=true)') + .option('--output-schema ', 'Output JSON matching the provided schema file path or inline JSON schema') .option('--json', 'Output in JSON format') .option('--host ', 'Daemon host:port (default: localhost:6767)') .action(withOutput(runRunCommand)) diff --git a/packages/cli/src/commands/agent/run.ts b/packages/cli/src/commands/agent/run.ts index 01c22739f..27472c328 100644 --- a/packages/cli/src/commands/agent/run.ts +++ b/packages/cli/src/commands/agent/run.ts @@ -9,7 +9,7 @@ import { lookup } from 'mime-types' /** Result type for agent run command */ export interface AgentRunResult { agentId: string - status: 'created' | 'running' + status: 'created' | 'running' | 'completed' | 'timeout' | 'permission' | 'error' provider: string cwd: string title: string | null @@ -39,24 +39,178 @@ export interface AgentRunOptions extends CommandOptions { cwd?: string label?: string[] ui?: boolean + outputSchema?: string } -function toRunResult(agent: AgentSnapshotPayload): AgentRunResult { +function toRunResult( + agent: AgentSnapshotPayload, + statusOverride?: AgentRunResult['status'] +): AgentRunResult { return { agentId: agent.id, - status: agent.status === 'running' ? 'running' : 'created', + status: statusOverride ?? (agent.status === 'running' ? 'running' : 'created'), provider: agent.provider, cwd: agent.cwd, title: agent.title, } } +function loadOutputSchema(value: string): Record { + const trimmed = value.trim() + if (!trimmed) { + const error: CommandError = { + code: 'INVALID_OUTPUT_SCHEMA', + message: '--output-schema cannot be empty', + details: 'Provide a JSON schema file path or inline JSON object', + } + throw error + } + + let source = trimmed + if (!trimmed.startsWith('{')) { + try { + source = readFileSync(resolve(trimmed), 'utf8') + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + const error: CommandError = { + code: 'INVALID_OUTPUT_SCHEMA', + message: `Failed to read output schema file: ${trimmed}`, + details: message, + } + throw error + } + } + + let parsed: unknown + try { + parsed = JSON.parse(source) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + const error: CommandError = { + code: 'INVALID_OUTPUT_SCHEMA', + message: 'Failed to parse output schema JSON', + details: message, + } + throw error + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + const error: CommandError = { + code: 'INVALID_OUTPUT_SCHEMA', + message: 'Output schema must be a JSON object', + } + throw error + } + + return parsed as Record +} + +function extractFirstJsonObject(text: string): string | null { + const source = text.trim() + if (!source) { + return null + } + + const startIndexes: number[] = [] + for (let i = 0; i < source.length; i += 1) { + if (source[i] === '{') { + startIndexes.push(i) + } + } + + for (const start of startIndexes) { + let depth = 0 + let inString = false + let escaped = false + + for (let i = start; i < source.length; i += 1) { + const ch = source[i]! + + if (inString) { + if (escaped) { + escaped = false + continue + } + if (ch === '\\') { + escaped = true + continue + } + if (ch === '"') { + inString = false + } + continue + } + + if (ch === '"') { + inString = true + continue + } + + if (ch === '{') { + depth += 1 + continue + } + if (ch === '}') { + depth -= 1 + if (depth === 0) { + const candidate = source.slice(start, i + 1).trim() + try { + JSON.parse(candidate) + return candidate + } catch { + // Keep scanning. + } + } + } + } + } + + return null +} + +function parseStructuredOutput(lastMessage: string): Record { + const trimmed = lastMessage.trim() + const fenced = trimmed.match(/```(?:json)?\s*\n([\s\S]*?)\n```/) + const jsonText = fenced?.[1]?.trim() ?? extractFirstJsonObject(trimmed) ?? trimmed + + let parsed: unknown + try { + parsed = JSON.parse(jsonText) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: 'Agent response is not valid JSON', + details: message, + } + throw error + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: 'Agent response JSON must be an object', + } + throw error + } + + return parsed as Record +} + +function structuredRunSchema(output: Record): OutputSchema { + return { + ...agentRunSchema, + serialize: () => output, + } +} + export async function runRunCommand( prompt: string, options: AgentRunOptions, _command: Command ): Promise> { const host = getDaemonHost({ host: options.host as string | undefined }) + const outputSchema = options.outputSchema ? loadOutputSchema(options.outputSchema) : undefined // Validate prompt is provided if (!prompt || prompt.trim().length === 0) { @@ -78,6 +232,16 @@ export async function runRunCommand( throw error } + // --output-schema always runs in attached/wait mode + if (outputSchema && options.detach) { + const error: CommandError = { + code: 'INVALID_OPTIONS', + message: '--output-schema cannot be used with --detach', + details: 'Structured output requires waiting for the agent to finish', + } + throw error + } + let client try { client = await connectToDaemon({ host: options.host as string | undefined }) @@ -162,12 +326,75 @@ export async function runRunCommand( modeId: options.mode, model: options.model, initialPrompt: prompt, + outputSchema, images, git, worktreeName: options.worktree, labels: Object.keys(labels).length > 0 ? labels : undefined, }) + if (outputSchema) { + const state = await client.waitForFinish(agent.id, 10 * 60 * 1000) + + if (state.status === 'timeout') { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: 'Timed out waiting for structured output', + } + throw error + } + + if (state.status === 'permission') { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: 'Agent is waiting for permission before producing structured output', + } + throw error + } + + if (state.status === 'error') { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: state.error ?? 'Agent failed before producing structured output', + } + throw error + } + + const lastMessage = state.lastMessage?.trim() + if (!lastMessage) { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: 'Agent finished without a structured output message', + } + throw error + } + + const output = parseStructuredOutput(lastMessage) + await client.close() + + return { + type: 'single', + data: toRunResult(agent, 'completed'), + schema: structuredRunSchema(output), + } + } + + // Default run behavior is foreground: wait for completion unless --detach is set. + if (!options.detach) { + const state = await client.waitForFinish(agent.id, 10 * 60 * 1000) + await client.close() + + const finalAgent = state.final ?? agent + const status: AgentRunResult['status'] = + state.status === 'idle' ? 'completed' : state.status + + return { + type: 'single', + data: toRunResult(finalAgent, status), + schema: agentRunSchema, + } + } + await client.close() return { @@ -177,6 +404,11 @@ export async function runRunCommand( } } catch (err) { await client.close().catch(() => {}) + + if (err && typeof err === 'object' && 'code' in err) { + throw err + } + const message = err instanceof Error ? err.message : String(err) const error: CommandError = { code: 'AGENT_CREATE_FAILED', diff --git a/packages/cli/src/commands/onboard.ts b/packages/cli/src/commands/onboard.ts index 970cf6235..2b5100c69 100644 --- a/packages/cli/src/commands/onboard.ts +++ b/packages/cli/src/commands/onboard.ts @@ -483,6 +483,7 @@ function printNextSteps(pairingUrl: string | null, paseoHome: string, richUi: bo '2. Web app: https://app.paseo.sh', '3. Desktop app: https://github.com/getpaseo/paseo/releases/latest', '4. Docs: https://paseo.sh/docs', + '5. Example: paseo run --output-schema schema.json "extract fields"', ] const quickReferenceLines = [ '1. paseo --help', diff --git a/packages/cli/src/output/with-output.ts b/packages/cli/src/output/with-output.ts index 4028d69de..a9b5e7700 100644 --- a/packages/cli/src/output/with-output.ts +++ b/packages/cli/src/output/with-output.ts @@ -31,6 +31,18 @@ function normalizeFormat(raw: unknown): OutputOptions['format'] { /** Extract output options from command options */ function extractOutputOptions(options: CommandOptions): OutputOptions { + const hasStructuredOutputSchema = + typeof options.outputSchema === 'string' && options.outputSchema.trim().length > 0 + + if (hasStructuredOutputSchema) { + return { + format: 'json', + quiet: false, + noHeaders: options.headers === false, + noColor: options.color === false, + } + } + return { format: options.json ? 'json' : normalizeFormat(options.format ?? defaultOutputOptions.format), quiet: options.quiet ?? defaultOutputOptions.quiet, diff --git a/packages/cli/tests/05-agent-run.test.ts b/packages/cli/tests/05-agent-run.test.ts index 22c409bf1..b9688072b 100644 --- a/packages/cli/tests/05-agent-run.test.ts +++ b/packages/cli/tests/05-agent-run.test.ts @@ -22,7 +22,7 @@ import assert from 'node:assert' import { $ } from 'zx' -import { mkdtemp, rm } from 'fs/promises' +import { mkdtemp, rm, writeFile } from 'fs/promises' import { tmpdir } from 'os' import { join } from 'path' @@ -33,6 +33,18 @@ console.log('=== Run Command Tests ===\n') // Get random port that's definitely not in use (never 6767) const port = 10000 + Math.floor(Math.random() * 50000) const paseoHome = await mkdtemp(join(tmpdir(), 'paseo-test-home-')) +const schemaPath = join(paseoHome, 'run-output-schema.json') +await writeFile( + schemaPath, + JSON.stringify({ + type: 'object', + properties: { + status: { type: 'string' }, + }, + required: ['status'], + additionalProperties: false, + }) +) try { // Test 1: run --help shows options @@ -46,6 +58,7 @@ try { assert(result.stdout.includes('--provider'), 'help should mention --provider option') assert(result.stdout.includes('--mode'), 'help should mention --mode option') assert(result.stdout.includes('--cwd'), 'help should mention --cwd option') + assert(result.stdout.includes('--output-schema'), 'help should mention --output-schema option') assert(result.stdout.includes('--host'), 'help should mention --host option') assert(result.stdout.includes(''), 'help should mention prompt argument') console.log('✓ run --help shows options\n') @@ -138,9 +151,34 @@ try { console.log('✓ run --cwd flag is accepted\n') } - // Test 9: -q (quiet) flag is accepted with run + // Test 9: run --output-schema flag is accepted { - console.log('Test 9: -q (quiet) flag is accepted with run') + console.log('Test 9: run --output-schema flag is accepted') + const result = + await $`PASEO_HOST=localhost:${port} PASEO_HOME=${paseoHome} npx paseo run --output-schema ${schemaPath} "test prompt"`.nothrow() + const output = result.stdout + result.stderr + assert(!output.includes('unknown option'), 'should accept --output-schema flag') + assert(!output.includes('error: option'), 'should not have option parsing error') + console.log('✓ run --output-schema flag is accepted\n') + } + + // Test 10: run --output-schema cannot be used with --detach + { + console.log('Test 10: run --output-schema cannot be used with --detach') + const result = + await $`PASEO_HOST=localhost:${port} PASEO_HOME=${paseoHome} npx paseo run -d --output-schema ${schemaPath} "test prompt"`.nothrow() + assert.notStrictEqual(result.exitCode, 0, 'should fail with --detach and --output-schema') + const output = result.stdout + result.stderr + assert( + output.includes('--output-schema cannot be used with --detach'), + 'error should explain detach incompatibility' + ) + console.log('✓ run --output-schema cannot be used with --detach\n') + } + + // Test 11: -q (quiet) flag is accepted with run + { + console.log('Test 11: -q (quiet) flag is accepted with run') const result = await $`PASEO_HOST=localhost:${port} PASEO_HOME=${paseoHome} npx paseo -q run -d "test prompt"`.nothrow() const output = result.stdout + result.stderr @@ -149,9 +187,9 @@ try { console.log('✓ -q (quiet) flag is accepted with run\n') } - // Test 10: Combined flags work together + // Test 12: Combined flags work together { - console.log('Test 10: Combined flags work together') + console.log('Test 12: Combined flags work together') const result = await $`PASEO_HOST=localhost:${port} PASEO_HOME=${paseoHome} npx paseo -q run -d --name "test-fixer" --provider claude --mode bypass --cwd /tmp "Fix the tests"`.nothrow() const output = result.stdout + result.stderr @@ -160,9 +198,9 @@ try { console.log('✓ Combined flags work together\n') } - // Test 11: paseo --help shows run command + // Test 13: paseo --help shows run command { - console.log('Test 11: paseo --help shows run command') + console.log('Test 13: paseo --help shows run command') const result = await $`npx paseo --help`.nothrow() assert.strictEqual(result.exitCode, 0, 'paseo --help should exit 0') assert(result.stdout.includes('run'), 'help should mention run command') diff --git a/packages/server/src/client/daemon-client.ts b/packages/server/src/client/daemon-client.ts index 79efe75b7..4fedb5630 100644 --- a/packages/server/src/client/daemon-client.ts +++ b/packages/server/src/client/daemon-client.ts @@ -169,6 +169,7 @@ export type CreateAgentRequestOptions = { provider?: AgentProvider; cwd?: string; initialPrompt?: string; + outputSchema?: Record; images?: CreateAgentRequestMessage["images"]; git?: GitSetupOptions; worktreeName?: string; @@ -224,6 +225,7 @@ export type WaitForFinishResult = { status: "idle" | "error" | "permission" | "timeout"; final: AgentSnapshotPayload | null; error: string | null; + lastMessage: string | null; }; type Waiter = { @@ -984,6 +986,7 @@ export class DaemonClient { requestId, config, ...(options.initialPrompt ? { initialPrompt: options.initialPrompt } : {}), + ...(options.outputSchema ? { outputSchema: options.outputSchema } : {}), ...(options.images && options.images.length > 0 ? { images: options.images } : {}), @@ -2260,6 +2263,7 @@ export class DaemonClient { status: payload.status, final: payload.final, error: payload.error, + lastMessage: payload.lastMessage, }; } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index d13b51398..64046c1bf 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -63,6 +63,7 @@ import type { AgentPermissionResponse, AgentPromptContentBlock, AgentPromptInput, + AgentRunOptions, McpServerConfig, AgentSessionConfig, AgentStreamEvent, @@ -683,7 +684,8 @@ export class Session { */ private startAgentStream( agentId: string, - prompt: AgentPromptInput + prompt: AgentPromptInput, + runOptions?: AgentRunOptions ): { ok: true } | { ok: false; error: string } { this.sessionLogger.info( { agentId }, @@ -692,7 +694,7 @@ export class Session { let iterator: AsyncGenerator; try { - iterator = this.agentManager.streamAgent(agentId, prompt); + iterator = this.agentManager.streamAgent(agentId, prompt, runOptions); } catch (error) { this.handleAgentRunError(agentId, error, "Failed to start agent run"); const message = @@ -2036,7 +2038,8 @@ export class Session { agentId: string, text: string, messageId?: string, - images?: Array<{ data: string; mimeType: string }> + images?: Array<{ data: string; mimeType: string }>, + runOptions?: AgentRunOptions ): Promise { this.sessionLogger.info( { agentId, textPreview: text.substring(0, 50), imageCount: images?.length ?? 0 }, @@ -2076,7 +2079,7 @@ export class Session { ); } - this.startAgentStream(agentId, prompt); + this.startAgentStream(agentId, prompt, runOptions); } /** @@ -2124,7 +2127,7 @@ export class Session { private async handleCreateAgentRequest( msg: Extract ): Promise { - const { config, worktreeName, requestId, initialPrompt, git, images, labels } = msg; + const { config, worktreeName, requestId, initialPrompt, outputSchema, git, images, labels } = msg; this.sessionLogger.info( { cwd: config.cwd, provider: config.provider, worktreeName }, `Creating agent in ${config.cwd} (${config.provider})${ @@ -2163,7 +2166,8 @@ export class Session { snapshot.id, trimmedPrompt, uuidv4(), - images + images, + outputSchema ? { outputSchema } : undefined ); } catch (promptError) { this.sessionLogger.error( @@ -4832,7 +4836,7 @@ export class Session { if (!resolved.ok) { this.emit({ type: "wait_for_finish_response", - payload: { requestId, status: "error", final: null, error: resolved.error }, + payload: { requestId, status: "error", final: null, error: resolved.error, lastMessage: null }, }); return; } @@ -4849,6 +4853,7 @@ export class Session { status: "error", final: null, error: `Agent not found: ${agentId}`, + lastMessage: null, }, }); return; @@ -4862,7 +4867,7 @@ export class Session { : "idle"; this.emit({ type: "wait_for_finish_response", - payload: { requestId, status, final, error: null }, + payload: { requestId, status, final, error: null, lastMessage: null }, }); return; } @@ -4892,7 +4897,7 @@ export class Session { this.emit({ type: "wait_for_finish_response", - payload: { requestId, status, final, error: null }, + payload: { requestId, status, final, error: null, lastMessage: result.lastMessage }, }); } catch (error) { const isAbort = @@ -4910,6 +4915,7 @@ export class Session { status: "error", final, error: message, + lastMessage: null, }, }); return; @@ -4921,7 +4927,7 @@ export class Session { } this.emit({ type: "wait_for_finish_response", - payload: { requestId, status: "timeout", final, error: null }, + payload: { requestId, status: "timeout", final, error: null, lastMessage: null }, }); } finally { clearTimeout(timeoutHandle); diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index 869ba67ca..bfa2f456e 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -548,6 +548,7 @@ export const CreateAgentRequestMessageSchema = z.object({ config: AgentSessionConfigSchema, worktreeName: z.string().optional(), initialPrompt: z.string().optional(), + outputSchema: z.record(z.unknown()).optional(), images: z.array(z.object({ data: z.string(), // base64 encoded image mimeType: z.string(), // e.g., "image/jpeg", "image/png" @@ -1360,6 +1361,7 @@ export const WaitForFinishResponseMessageSchema = z.object({ status: z.enum(["idle", "error", "permission", "timeout"]), final: AgentSnapshotPayloadSchema.nullable(), error: z.string().nullable(), + lastMessage: z.string().nullable(), }), }); diff --git a/packages/website/src/routes/docs/cli.tsx b/packages/website/src/routes/docs/cli.tsx index 0a561e5c1..f57bc9f5c 100644 --- a/packages/website/src/routes/docs/cli.tsx +++ b/packages/website/src/routes/docs/cli.tsx @@ -64,12 +64,23 @@ paseo stop # Stop an agent`} 
{`paseo run "implement user authentication"
 paseo run --provider codex "refactor the API layer"
 paseo run --detach "run the full test suite"  # background
-paseo run --worktree feature-x "implement feature X"`}
+paseo run --worktree feature-x "implement feature X" +paseo run --output-schema schema.json "extract release notes" +paseo run --output-schema '{"type":"object","properties":{"summary":{"type":"string"}},"required":["summary"]}' "summarize release notes"`}

The --worktree flag creates the agent in an isolated git worktree, useful for parallel feature development.

+

+ Use --output-schema to return only matching JSON output. + You can pass a schema file path or an inline JSON schema object. + This mode cannot be used with --detach. +

+

+ By default, paseo run waits for completion. Use{' '} + --detach to run in the background. +

{/* Listing agents */} @@ -187,6 +198,21 @@ paseo daemon stop # Stop the daemon`} paseo run --detach "implement the API" --name api-agent paseo wait api-agent paseo logs api-agent --tail 5`} + +

+ Simple implement + verify loop: +

+ + 
{`# Requires jq
+while true; do
+  paseo run --provider codex "make the tests pass" >/dev/null
+
+  verdict=$(paseo run --provider claude --output-schema '{"type":"object","properties":{"criteria_met":{"type":"boolean"}},"required":["criteria_met"],"additionalProperties":false}' "ensure tests all pass")
+  if echo "$verdict" | jq -e '.criteria_met == true' >/dev/null; then
+    echo "criteria met"
+    break
+  fi
+done`}

This pattern enables hierarchical task decomposition — a lead agent can break down work,