diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b75a4ba9..9f1d08fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [0.1.9] - 2026-02-17 +### Improved +- Unified structured-output generation through a single shared schema-validation and retry pipeline. +- Reused provider availability checks for structured generation fallback selection. +- Added structured generation waterfall ordering for internal metadata and git text generation: Claude Haiku, then Codex, then OpenCode. + +### Fixed +- Fixed CLI `run --output-schema` to use the shared structured-output path instead of ad-hoc JSON parsing. +- Fixed `run --output-schema` failures where providers returned empty `lastMessage` by recovering from timeline assistant output. +- Fixed internal commit message, pull request text, and agent metadata generation to follow one consistent structured pipeline. + ## [0.1.8] - 2026-02-17 ### Added - Added a cross-platform confirm dialog flow for daemon restarts. diff --git a/packages/cli/src/commands/agent/run.ts b/packages/cli/src/commands/agent/run.ts index 27472c328..77496b6f5 100644 --- a/packages/cli/src/commands/agent/run.ts +++ b/packages/cli/src/commands/agent/run.ts @@ -1,5 +1,9 @@ import type { Command } from 'commander' -import type { AgentSnapshotPayload } from '@getpaseo/server' +import { + getStructuredAgentResponse, + StructuredAgentResponseError, + type AgentSnapshotPayload, +} from '@getpaseo/server' import { connectToDaemon, getDaemonHost } from '../../utils/client.js' import type { CommandOptions, SingleResult, OutputSchema, CommandError } from '../../output/index.js' import { readFileSync } from 'node:fs' @@ -105,98 +109,55 @@ function loadOutputSchema(value: string): Record { return parsed as Record } -function extractFirstJsonObject(text: string): string | null { - const source = text.trim() - if (!source) { - return null +class StructuredRunStatusError extends Error { + readonly kind: 'timeout' | 'permission' | 'error' | 'empty' + + constructor(kind: 'timeout' | 'permission' | 'error' | 'empty', message: string) { + super(message) + this.name = 'StructuredRunStatusError' + this.kind = kind + } +} + +type ConnectedDaemonClient = Awaited> + +export interface StructuredResponseTimelineClient { + fetchAgentTimeline: ConnectedDaemonClient['fetchAgentTimeline'] +} + +export async function resolveStructuredResponseMessage(options: { + client: StructuredResponseTimelineClient + agentId: string + lastMessage: string | null +}): Promise { + const direct = options.lastMessage?.trim() + if (direct) { + return direct } - const startIndexes: number[] = [] - for (let i = 0; i < source.length; i += 1) { - if (source[i] === '{') { - startIndexes.push(i) - } - } - - for (const start of startIndexes) { - let depth = 0 - let inString = false - let escaped = false - - for (let i = start; i < source.length; i += 1) { - const ch = source[i]! - - if (inString) { - if (escaped) { - escaped = false - continue - } - if (ch === '\\') { - escaped = true - continue - } - if (ch === '"') { - inString = false - } + try { + const timeline = await options.client.fetchAgentTimeline(options.agentId, { + direction: 'tail', + projection: 'projected', + limit: 200, + }) + for (let index = timeline.entries.length - 1; index >= 0; index -= 1) { + const entry = timeline.entries[index] + if (!entry || entry.item.type !== 'assistant_message') { continue } - - if (ch === '"') { - inString = true - continue - } - - if (ch === '{') { - depth += 1 - continue - } - if (ch === '}') { - depth -= 1 - if (depth === 0) { - const candidate = source.slice(start, i + 1).trim() - try { - JSON.parse(candidate) - return candidate - } catch { - // Keep scanning. - } - } + const text = entry.item.text.trim() + if (text.length > 0) { + return text } } + } catch { + // Leave empty; caller will surface a consistent structured-output failure message. } return null } -function parseStructuredOutput(lastMessage: string): Record { - const trimmed = lastMessage.trim() - const fenced = trimmed.match(/```(?:json)?\s*\n([\s\S]*?)\n```/) - const jsonText = fenced?.[1]?.trim() ?? extractFirstJsonObject(trimmed) ?? trimmed - - let parsed: unknown - try { - parsed = JSON.parse(jsonText) - } catch (err) { - const message = err instanceof Error ? err.message : String(err) - const error: CommandError = { - code: 'OUTPUT_SCHEMA_FAILED', - message: 'Agent response is not valid JSON', - details: message, - } - throw error - } - - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - const error: CommandError = { - code: 'OUTPUT_SCHEMA_FAILED', - message: 'Agent response JSON must be an object', - } - throw error - } - - return parsed as Record -} - function structuredRunSchema(output: Record): OutputSchema { return { ...agentRunSchema, @@ -318,6 +279,107 @@ export async function runRunCommand( labels['ui'] = 'true' } + if (outputSchema) { + let structuredAgent: AgentSnapshotPayload | null = null + + const callStructuredTurn = async (structuredPrompt: string): Promise => { + if (!structuredAgent) { + structuredAgent = await client.createAgent({ + provider: (options.provider as 'claude' | 'codex' | 'opencode') ?? 'claude', + cwd, + title: options.name, + modeId: options.mode, + model: options.model, + initialPrompt: structuredPrompt, + images, + git, + worktreeName: options.worktree, + labels: Object.keys(labels).length > 0 ? labels : undefined, + }) + } else { + await client.sendMessage(structuredAgent.id, structuredPrompt) + } + + const state = await client.waitForFinish(structuredAgent.id, 10 * 60 * 1000) + if (state.status === 'timeout') { + throw new StructuredRunStatusError('timeout', 'Timed out waiting for structured output') + } + if (state.status === 'permission') { + throw new StructuredRunStatusError( + 'permission', + 'Agent is waiting for permission before producing structured output' + ) + } + if (state.status === 'error') { + throw new StructuredRunStatusError( + 'error', + state.error ?? 'Agent failed before producing structured output' + ) + } + + const lastMessage = await resolveStructuredResponseMessage({ + client, + agentId: structuredAgent.id, + lastMessage: state.lastMessage, + }) + if (!lastMessage) { + throw new StructuredRunStatusError( + 'empty', + 'Agent finished without a structured output message' + ) + } + + return lastMessage + } + + let output: Record + try { + output = await getStructuredAgentResponse>({ + caller: callStructuredTurn, + prompt, + schema: outputSchema, + schemaName: 'RunOutput', + maxRetries: 2, + }) + } catch (err) { + if (err instanceof StructuredRunStatusError) { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: err.message, + } + throw error + } + if (err instanceof StructuredAgentResponseError) { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: 'Agent response did not match the required output schema', + details: + err.validationErrors.length > 0 + ? err.validationErrors.join('\n') + : err.lastResponse || 'No response', + } + throw error + } + throw err + } + + if (!structuredAgent) { + const error: CommandError = { + code: 'OUTPUT_SCHEMA_FAILED', + message: 'Agent finished without a structured output message', + } + throw error + } + + await client.close() + + return { + type: 'single', + data: toRunResult(structuredAgent, 'completed'), + schema: structuredRunSchema(output), + } + } + // Create the agent const agent = await client.createAgent({ provider: (options.provider as 'claude' | 'codex' | 'opencode') ?? 'claude', @@ -326,59 +388,12 @@ export async function runRunCommand( modeId: options.mode, model: options.model, initialPrompt: prompt, - outputSchema, images, git, worktreeName: options.worktree, labels: Object.keys(labels).length > 0 ? labels : undefined, }) - if (outputSchema) { - const state = await client.waitForFinish(agent.id, 10 * 60 * 1000) - - if (state.status === 'timeout') { - const error: CommandError = { - code: 'OUTPUT_SCHEMA_FAILED', - message: 'Timed out waiting for structured output', - } - throw error - } - - if (state.status === 'permission') { - const error: CommandError = { - code: 'OUTPUT_SCHEMA_FAILED', - message: 'Agent is waiting for permission before producing structured output', - } - throw error - } - - if (state.status === 'error') { - const error: CommandError = { - code: 'OUTPUT_SCHEMA_FAILED', - message: state.error ?? 'Agent failed before producing structured output', - } - throw error - } - - const lastMessage = state.lastMessage?.trim() - if (!lastMessage) { - const error: CommandError = { - code: 'OUTPUT_SCHEMA_FAILED', - message: 'Agent finished without a structured output message', - } - throw error - } - - const output = parseStructuredOutput(lastMessage) - await client.close() - - return { - type: 'single', - data: toRunResult(agent, 'completed'), - schema: structuredRunSchema(output), - } - } - // Default run behavior is foreground: wait for completion unless --detach is set. if (!options.detach) { const state = await client.waitForFinish(agent.id, 10 * 60 * 1000) diff --git a/packages/cli/tests/21-run-output-schema-helper.test.ts b/packages/cli/tests/21-run-output-schema-helper.test.ts new file mode 100644 index 000000000..edbcf3a6b --- /dev/null +++ b/packages/cli/tests/21-run-output-schema-helper.test.ts @@ -0,0 +1,97 @@ +#!/usr/bin/env npx tsx + +import assert from 'node:assert' +import { + resolveStructuredResponseMessage, + type StructuredResponseTimelineClient, +} from '../src/commands/agent/run.ts' + +type TimelineEntry = { + item: { + type: string + text?: string + } +} + +function createClient(options: { + entries?: TimelineEntry[] + throwOnFetch?: boolean + onFetch?: () => void +}): StructuredResponseTimelineClient { + return { + fetchAgentTimeline: async () => { + options.onFetch?.() + if (options.throwOnFetch) { + throw new Error('timeline unavailable') + } + return { + entries: options.entries ?? [], + } as Awaited> + }, + } +} + +console.log('=== Run Output Schema Helper Tests ===\n') + +// Test 1: Direct lastMessage is returned without timeline fetch. +{ + let fetchCount = 0 + const client = createClient({ onFetch: () => fetchCount += 1 }) + const result = await resolveStructuredResponseMessage({ + client, + agentId: 'agent-1', + lastMessage: ' {"summary":"ok"} ', + }) + assert.strictEqual(result, '{"summary":"ok"}') + assert.strictEqual(fetchCount, 0, 'should not fetch timeline when lastMessage exists') + console.log('✓ returns direct lastMessage when present') +} + +// Test 2: Falls back to latest assistant message from timeline. +{ + const client = createClient({ + entries: [ + { item: { type: 'user_message', text: 'prompt' } }, + { item: { type: 'assistant_message', text: '' } }, + { item: { type: 'assistant_message', text: ' {"summary":"from timeline"} ' } }, + ], + }) + const result = await resolveStructuredResponseMessage({ + client, + agentId: 'agent-2', + lastMessage: null, + }) + assert.strictEqual(result, '{"summary":"from timeline"}') + console.log('✓ falls back to latest assistant timeline entry') +} + +// Test 3: Returns null when timeline has no assistant messages. +{ + const client = createClient({ + entries: [ + { item: { type: 'user_message', text: 'prompt' } }, + { item: { type: 'reasoning', text: 'thinking' } }, + ], + }) + const result = await resolveStructuredResponseMessage({ + client, + agentId: 'agent-3', + lastMessage: null, + }) + assert.strictEqual(result, null) + console.log('✓ returns null when no assistant messages exist') +} + +// Test 4: Returns null if timeline fetch fails. +{ + const client = createClient({ throwOnFetch: true }) + const result = await resolveStructuredResponseMessage({ + client, + agentId: 'agent-4', + lastMessage: null, + }) + assert.strictEqual(result, null) + console.log('✓ returns null when timeline fetch throws') +} + +console.log('\n=== All helper tests passed ===') diff --git a/packages/cli/tests/e2e/run-output-schema.test.ts b/packages/cli/tests/e2e/run-output-schema.test.ts new file mode 100644 index 000000000..d5023828b --- /dev/null +++ b/packages/cli/tests/e2e/run-output-schema.test.ts @@ -0,0 +1,139 @@ +#!/usr/bin/env npx tsx + +import assert from 'node:assert' +import { createE2ETestContext, type TestDaemonContext } from '../helpers/test-daemon.ts' + +interface E2EContext extends TestDaemonContext { + paseo: (args: string[], opts?: { timeout?: number; cwd?: string }) => Promise<{ + exitCode: number + stdout: string + stderr: string + }> +} + +const schema = JSON.stringify({ + type: 'object', + properties: { + summary: { type: 'string' }, + }, + required: ['summary'], + additionalProperties: false, +}) + +const impossibleSchema = JSON.stringify({ + type: 'object', + properties: { + summary: { type: 'string' }, + }, + required: ['summary'], + allOf: [ + { properties: { summary: { type: 'string' } } }, + { properties: { summary: { type: 'number' } } }, + ], +}) + +let ctx: E2EContext + +async function setup(): Promise { + ctx = await createE2ETestContext({ timeout: 180000 }) +} + +async function cleanup(): Promise { + if (ctx) { + await ctx.stop() + } +} + +async function runProviderCase(input: { + provider: 'claude' | 'codex' | 'opencode' + mode: string + model: string +}): Promise { + const result = await ctx.paseo( + [ + 'run', + '--provider', + input.provider, + '--mode', + input.mode, + '--model', + input.model, + '--output-schema', + schema, + `Return valid JSON with a short summary for provider ${input.provider}.`, + ], + { timeout: 180000 } + ) + + assert.strictEqual( + result.exitCode, + 0, + `expected ${input.provider} run to succeed, got ${result.exitCode}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}` + ) + + const parsed = JSON.parse(result.stdout.trim()) as { summary?: unknown } + assert.strictEqual( + typeof parsed.summary, + 'string', + `expected ${input.provider} output to contain string summary, got: ${result.stdout}` + ) + assert(parsed.summary && parsed.summary.length > 0, 'summary must not be empty') +} + +async function test_all_providers_return_structured_output(): Promise { + await runProviderCase({ + provider: 'claude', + mode: 'bypassPermissions', + model: 'haiku', + }) + await runProviderCase({ + provider: 'codex', + mode: 'full-access', + model: 'gpt-5.3-codex', + }) + await runProviderCase({ + provider: 'opencode', + mode: 'default', + model: 'opencode/kimi-k2.5-free', + }) +} + +async function test_schema_validation_is_enforced(): Promise { + const result = await ctx.paseo( + [ + 'run', + '--provider', + 'claude', + '--mode', + 'bypassPermissions', + '--model', + 'haiku', + '--output-schema', + impossibleSchema, + 'Return exactly {"summary":"ok"} and nothing else.', + ], + { timeout: 180000 } + ) + + assert.notStrictEqual(result.exitCode, 0, 'expected impossible schema to fail') + const output = `${result.stdout}\n${result.stderr}` + assert( + output.includes('OUTPUT_SCHEMA_FAILED'), + `expected OUTPUT_SCHEMA_FAILED for impossible schema\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}` + ) +} + +async function main(): Promise { + try { + await setup() + await test_all_providers_return_structured_output() + await test_schema_validation_is_enforced() + } catch (error) { + console.error(error) + process.exitCode = 1 + } finally { + await cleanup() + } +} + +main() diff --git a/packages/server/src/server/agent/agent-metadata-generator.ts b/packages/server/src/server/agent/agent-metadata-generator.ts index cc6cfdf6c..bbee72f45 100644 --- a/packages/server/src/server/agent/agent-metadata-generator.ts +++ b/packages/server/src/server/agent/agent-metadata-generator.ts @@ -4,8 +4,10 @@ import type { Logger } from "pino"; import type { AgentManager } from "./agent-manager.js"; import { + DEFAULT_STRUCTURED_GENERATION_PROVIDERS, + StructuredAgentFallbackError, StructuredAgentResponseError, - generateStructuredAgentResponse, + generateStructuredAgentResponseWithFallback, } from "./agent-response-loop.js"; import { validateBranchSlug } from "../../utils/worktree.js"; import { @@ -14,12 +16,8 @@ import { type CheckoutStatusResult, } from "../../utils/checkout-git.js"; -const AUTO_GEN_PROVIDER = "codex" as const; -const AUTO_GEN_MODEL = "gpt-5.1-codex-mini"; -const AUTO_GEN_REASONING_EFFORT = "low"; - export type AgentMetadataGeneratorDeps = { - generateStructuredAgentResponse?: typeof generateStructuredAgentResponse; + generateStructuredAgentResponseWithFallback?: typeof generateStructuredAgentResponseWithFallback; getCheckoutStatus?: typeof getCheckoutStatus; renameCurrentBranch?: typeof renameCurrentBranch; }; @@ -148,7 +146,9 @@ export async function generateAndApplyAgentMetadata( return; } - const generator = options.deps?.generateStructuredAgentResponse ?? generateStructuredAgentResponse; + const generator = + options.deps?.generateStructuredAgentResponseWithFallback ?? + generateStructuredAgentResponseWithFallback; const getCheckoutStatusImpl = options.deps?.getCheckoutStatus ?? getCheckoutStatus; const renameCurrentBranchImpl = options.deps?.renameCurrentBranch ?? renameCurrentBranch; @@ -157,21 +157,22 @@ export async function generateAndApplyAgentMetadata( try { result = await generator({ manager: options.agentManager, - agentConfig: { - provider: AUTO_GEN_PROVIDER, - model: AUTO_GEN_MODEL, - thinkingOptionId: AUTO_GEN_REASONING_EFFORT, - cwd: options.cwd, - title: "Agent metadata generator", - internal: true, - }, + cwd: options.cwd, prompt: buildPrompt(needs), schema, schemaName: "AgentMetadata", maxRetries: 2, + providers: DEFAULT_STRUCTURED_GENERATION_PROVIDERS, + agentConfigOverrides: { + title: "Agent metadata generator", + internal: true, + }, }); } catch (error) { - if (error instanceof StructuredAgentResponseError) { + if ( + error instanceof StructuredAgentResponseError || + error instanceof StructuredAgentFallbackError + ) { options.logger.warn( { err: error, agentId: options.agentId }, "Structured metadata generation failed" diff --git a/packages/server/src/server/agent/agent-response-loop.test.ts b/packages/server/src/server/agent/agent-response-loop.test.ts index 8759ebc62..d0359e9b8 100644 --- a/packages/server/src/server/agent/agent-response-loop.test.ts +++ b/packages/server/src/server/agent/agent-response-loop.test.ts @@ -2,9 +2,12 @@ import { describe, it, expect } from "vitest"; import { z } from "zod"; import { getStructuredAgentResponse, + generateStructuredAgentResponseWithFallback, + StructuredAgentFallbackError, StructuredAgentResponseError, type AgentCaller, } from "./agent-response-loop.js"; +import type { AgentManager } from "./agent-manager.js"; function createScriptedCaller(responses: string[]) { const prompts: string[] = []; @@ -142,3 +145,134 @@ describe("getStructuredAgentResponse", () => { expect(result).toEqual({ value: 42 }); }); }); + +describe("generateStructuredAgentResponseWithFallback", () => { + const schema = z.object({ summary: z.string() }); + + function createManager(availability: Array<{ provider: string; available: boolean; error: string | null }>) { + return { + listProviderAvailability: async () => availability, + } as unknown as AgentManager; + } + + it("uses the first available provider in the waterfall", async () => { + const calls: Array<{ provider: string; model?: string }> = []; + const manager = createManager([ + { provider: "claude", available: true, error: null }, + { provider: "codex", available: true, error: null }, + { provider: "opencode", available: true, error: null }, + ]); + + const result = await generateStructuredAgentResponseWithFallback({ + manager, + cwd: "/tmp/project", + prompt: "Return JSON", + schema, + providers: [ + { provider: "claude", model: "haiku" }, + { provider: "codex", model: "gpt-5.1-codex-mini" }, + ], + runner: async (options) => { + calls.push({ + provider: options.agentConfig.provider, + model: options.agentConfig.model ?? undefined, + }); + return { summary: "ok" }; + }, + }); + + expect(result).toEqual({ summary: "ok" }); + expect(calls).toEqual([{ provider: "claude", model: "haiku" }]); + }); + + it("skips unavailable providers and uses the next available one", async () => { + const calls: Array<{ provider: string; model?: string }> = []; + const manager = createManager([ + { provider: "claude", available: false, error: "missing auth" }, + { provider: "codex", available: true, error: null }, + { provider: "opencode", available: true, error: null }, + ]); + + const result = await generateStructuredAgentResponseWithFallback({ + manager, + cwd: "/tmp/project", + prompt: "Return JSON", + schema, + providers: [ + { provider: "claude", model: "haiku" }, + { provider: "codex", model: "gpt-5.1-codex-mini" }, + ], + runner: async (options) => { + calls.push({ + provider: options.agentConfig.provider, + model: options.agentConfig.model ?? undefined, + }); + return { summary: "ok" }; + }, + }); + + expect(result).toEqual({ summary: "ok" }); + expect(calls).toEqual([{ provider: "codex", model: "gpt-5.1-codex-mini" }]); + }); + + it("falls back when an available provider fails", async () => { + const calls: Array<{ provider: string; model?: string }> = []; + const manager = createManager([ + { provider: "claude", available: true, error: null }, + { provider: "codex", available: true, error: null }, + { provider: "opencode", available: true, error: null }, + ]); + + const result = await generateStructuredAgentResponseWithFallback({ + manager, + cwd: "/tmp/project", + prompt: "Return JSON", + schema, + providers: [ + { provider: "claude", model: "haiku" }, + { provider: "codex", model: "gpt-5.1-codex-mini" }, + ], + runner: async (options) => { + calls.push({ + provider: options.agentConfig.provider, + model: options.agentConfig.model ?? undefined, + }); + if (options.agentConfig.provider === "claude") { + throw new Error("claude failed"); + } + return { summary: "ok" }; + }, + }); + + expect(result).toEqual({ summary: "ok" }); + expect(calls).toEqual([ + { provider: "claude", model: "haiku" }, + { provider: "codex", model: "gpt-5.1-codex-mini" }, + ]); + }); + + it("throws a fallback error when all providers are unavailable or fail", async () => { + const manager = createManager([ + { provider: "claude", available: false, error: "missing auth" }, + { provider: "codex", available: true, error: null }, + { provider: "opencode", available: false, error: "not installed" }, + ]); + + await expect( + generateStructuredAgentResponseWithFallback({ + manager, + cwd: "/tmp/project", + prompt: "Return JSON", + schema, + providers: [ + { provider: "claude", model: "haiku" }, + { provider: "codex", model: "gpt-5.1-codex-mini" }, + { provider: "opencode", model: "opencode/kimi-k2.5-free" }, + ], + runner: async () => { + throw new Error("failed"); + }, + }) + ).rejects.toBeInstanceOf(StructuredAgentFallbackError); + }); +}); diff --git a/packages/server/src/server/agent/agent-response-loop.ts b/packages/server/src/server/agent/agent-response-loop.ts index 02344ee89..4daf453e1 100644 --- a/packages/server/src/server/agent/agent-response-loop.ts +++ b/packages/server/src/server/agent/agent-response-loop.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import Ajv, { type ErrorObject, type Options as AjvOptions } from "ajv"; -import type { AgentSessionConfig } from "./agent-sdk-types.js"; +import type { AgentProvider, AgentSessionConfig } from "./agent-sdk-types.js"; import type { AgentManager } from "./agent-manager.js"; import { getAgentProviderDefinition } from "./provider-manifest.js"; @@ -21,6 +21,43 @@ export class StructuredAgentResponseError extends Error { } } +export type StructuredGenerationProvider = { + provider: AgentProvider; + model?: string; + thinkingOptionId?: string; +}; + +export type StructuredGenerationAttempt = { + provider: AgentProvider; + model: string | null; + available: boolean; + error: string | null; +}; + +export class StructuredAgentFallbackError extends Error { + readonly attempts: StructuredGenerationAttempt[]; + + constructor(attempts: StructuredGenerationAttempt[]) { + const summary = attempts + .map((attempt) => { + const modelSuffix = attempt.model ? ` (${attempt.model})` : ""; + if (!attempt.available) { + return `${attempt.provider}${modelSuffix}: unavailable${attempt.error ? ` (${attempt.error})` : ""}`; + } + return `${attempt.provider}${modelSuffix}: failed${attempt.error ? ` (${attempt.error})` : ""}`; + }) + .join("; "); + + super( + summary.length > 0 + ? `Structured generation failed for all providers: ${summary}` + : "Structured generation failed for all providers" + ); + this.name = "StructuredAgentFallbackError"; + this.attempts = attempts; + } +} + export interface StructuredAgentResponseOptions { caller: AgentCaller; prompt: string; @@ -39,6 +76,27 @@ export interface StructuredAgentGenerationOptions { schemaName?: string; } +export interface StructuredAgentGenerationWithFallbackOptions { + manager: AgentManager; + cwd: string; + prompt: string; + schema: z.ZodType | JsonSchema; + providers: readonly StructuredGenerationProvider[]; + agentConfigOverrides?: Omit< + AgentSessionConfig, + "provider" | "cwd" | "model" | "thinkingOptionId" + >; + maxRetries?: number; + schemaName?: string; + runner?: (options: StructuredAgentGenerationOptions) => Promise; +} + +export const DEFAULT_STRUCTURED_GENERATION_PROVIDERS: readonly StructuredGenerationProvider[] = [ + { provider: "claude", model: "haiku" }, + { provider: "codex", model: "gpt-5.1-codex-mini" }, + { provider: "opencode", model: "opencode/kimi-k2.5-free" }, +] as const; + interface SchemaValidator { jsonSchema: JsonSchema; validate: (value: unknown) => { ok: true; value: T } | { ok: false; errors: string[] }; @@ -293,3 +351,82 @@ export async function generateStructuredAgentResponse( } } } + +function errorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + return String(error); +} + +export async function generateStructuredAgentResponseWithFallback( + options: StructuredAgentGenerationWithFallbackOptions +): Promise { + const { + manager, + cwd, + prompt, + schema, + providers, + agentConfigOverrides, + maxRetries, + schemaName, + runner, + } = options; + + if (providers.length === 0) { + throw new StructuredAgentFallbackError([]); + } + + const runStructured = + runner ?? + ((input: StructuredAgentGenerationOptions) => + generateStructuredAgentResponse(input)); + const availability = await manager.listProviderAvailability(); + const availabilityByProvider = new Map( + availability.map((entry) => [entry.provider, entry]) + ); + const attempts: StructuredGenerationAttempt[] = []; + + for (const candidate of providers) { + const availabilityEntry = availabilityByProvider.get(candidate.provider); + if (availabilityEntry && !availabilityEntry.available) { + attempts.push({ + provider: candidate.provider, + model: candidate.model ?? null, + available: false, + error: availabilityEntry.error ?? null, + }); + continue; + } + + try { + const result = await runStructured({ + manager, + prompt, + schema, + maxRetries, + schemaName, + agentConfig: { + ...agentConfigOverrides, + provider: candidate.provider, + cwd, + ...(candidate.model ? { model: candidate.model } : {}), + ...(candidate.thinkingOptionId + ? { thinkingOptionId: candidate.thinkingOptionId } + : {}), + }, + }); + return result; + } catch (error) { + attempts.push({ + provider: candidate.provider, + model: candidate.model ?? null, + available: true, + error: errorMessage(error), + }); + } + } + + throw new StructuredAgentFallbackError(attempts); +} diff --git a/packages/server/src/server/exports.ts b/packages/server/src/server/exports.ts index 5095dc93e..dee16d888 100644 --- a/packages/server/src/server/exports.ts +++ b/packages/server/src/server/exports.ts @@ -34,6 +34,20 @@ export type { // Agent activity curator for CLI logs export { curateAgentActivity } from "./agent/activity-curator.js"; +export { + getStructuredAgentResponse, + StructuredAgentResponseError, + StructuredAgentFallbackError, + DEFAULT_STRUCTURED_GENERATION_PROVIDERS, + generateStructuredAgentResponseWithFallback, + type AgentCaller, + type JsonSchema, + type StructuredGenerationAttempt, + type StructuredGenerationProvider, + type StructuredAgentGenerationOptions, + type StructuredAgentGenerationWithFallbackOptions, + type StructuredAgentResponseOptions, +} from "./agent/agent-response-loop.js"; // WebSocket message types for CLI streaming export type { diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 54a4a61fd..a97704b10 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -81,8 +81,10 @@ import { } from "./agent/timeline-append.js"; import { projectTimelineRows, type TimelineProjectionMode } from "./agent/timeline-projection.js"; import { + DEFAULT_STRUCTURED_GENERATION_PROVIDERS, + StructuredAgentFallbackError, StructuredAgentResponseError, - generateStructuredAgentResponse, + generateStructuredAgentResponseWithFallback, } from "./agent/agent-response-loop.js"; import type { AgentPermissionResponse, @@ -166,12 +168,6 @@ const TERMINAL_STREAM_WINDOW_BYTES = 256 * 1024; const TERMINAL_STREAM_MAX_PENDING_BYTES = 2 * 1024 * 1024; const TERMINAL_STREAM_MAX_PENDING_CHUNKS = 2048; -/** - * Default model used for auto-generating commit messages and PR descriptions. - * Uses Claude Haiku for speed and cost efficiency. - */ -const AUTO_GEN_MODEL = "haiku"; - function deriveRemoteProjectKey(remoteUrl: string | null): string | null { if (!remoteUrl) { return null; @@ -3042,23 +3038,25 @@ export class Session { patch.length > 0 ? patch : "(No diff available)", ].join("\n"); try { - const result = await generateStructuredAgentResponse({ + const result = await generateStructuredAgentResponseWithFallback({ manager: this.agentManager, - agentConfig: { - provider: "claude", - model: AUTO_GEN_MODEL, - cwd, - title: "Commit generator", - internal: true, - }, + cwd, prompt, schema, schemaName: "CommitMessage", maxRetries: 2, + providers: DEFAULT_STRUCTURED_GENERATION_PROVIDERS, + agentConfigOverrides: { + title: "Commit generator", + internal: true, + }, }); return result.message; } catch (error) { - if (error instanceof StructuredAgentResponseError) { + if ( + error instanceof StructuredAgentResponseError || + error instanceof StructuredAgentFallbackError + ) { return "Update files"; } throw error; @@ -3107,22 +3105,24 @@ export class Session { patch.length > 0 ? patch : "(No diff available)", ].join("\n"); try { - return await generateStructuredAgentResponse({ + return await generateStructuredAgentResponseWithFallback({ manager: this.agentManager, - agentConfig: { - provider: "claude", - model: AUTO_GEN_MODEL, - cwd, - title: "PR generator", - internal: true, - }, + cwd, prompt, schema, schemaName: "PullRequest", maxRetries: 2, + providers: DEFAULT_STRUCTURED_GENERATION_PROVIDERS, + agentConfigOverrides: { + title: "PR generator", + internal: true, + }, }); } catch (error) { - if (error instanceof StructuredAgentResponseError) { + if ( + error instanceof StructuredAgentResponseError || + error instanceof StructuredAgentFallbackError + ) { return { title: "Update changes", body: "Automated PR generated by Paseo.",