refactor: unify structured generation pipeline

2026-02-17 14:52:58 +07:00
parent abc146b186
commit ce5d9d6dc8
9 changed files with 719 additions and 171 deletions
@@ -0,0 +1,97 @@
+#!/usr/bin/env npx tsx
+
+import assert from 'node:assert'
+import {
+  resolveStructuredResponseMessage,
+  type StructuredResponseTimelineClient,
+} from '../src/commands/agent/run.ts'
+
+type TimelineEntry = {
+  item: {
+    type: string
+    text?: string
+  }
+}
+
+function createClient(options: {
+  entries?: TimelineEntry[]
+  throwOnFetch?: boolean
+  onFetch?: () => void
+}): StructuredResponseTimelineClient {
+  return {
+    fetchAgentTimeline: async () => {
+      options.onFetch?.()
+      if (options.throwOnFetch) {
+        throw new Error('timeline unavailable')
+      }
+      return {
+        entries: options.entries ?? [],
+      } as Awaited<ReturnType<StructuredResponseTimelineClient['fetchAgentTimeline']>>
+    },
+  }
+}
+
+console.log('=== Run Output Schema Helper Tests ===\n')
+
+// Test 1: Direct lastMessage is returned without timeline fetch.
+{
+  let fetchCount = 0
+  const client = createClient({ onFetch: () => fetchCount += 1 })
+  const result = await resolveStructuredResponseMessage({
+    client,
+    agentId: 'agent-1',
+    lastMessage: '  {"summary":"ok"}  ',
+  })
+  assert.strictEqual(result, '{"summary":"ok"}')
+  assert.strictEqual(fetchCount, 0, 'should not fetch timeline when lastMessage exists')
+  console.log('✓ returns direct lastMessage when present')
+}
+
+// Test 2: Falls back to latest assistant message from timeline.
+{
+  const client = createClient({
+    entries: [
+      { item: { type: 'user_message', text: 'prompt' } },
+      { item: { type: 'assistant_message', text: '' } },
+      { item: { type: 'assistant_message', text: ' {"summary":"from timeline"} ' } },
+    ],
+  })
+  const result = await resolveStructuredResponseMessage({
+    client,
+    agentId: 'agent-2',
+    lastMessage: null,
+  })
+  assert.strictEqual(result, '{"summary":"from timeline"}')
+  console.log('✓ falls back to latest assistant timeline entry')
+}
+
+// Test 3: Returns null when timeline has no assistant messages.
+{
+  const client = createClient({
+    entries: [
+      { item: { type: 'user_message', text: 'prompt' } },
+      { item: { type: 'reasoning', text: 'thinking' } },
+    ],
+  })
+  const result = await resolveStructuredResponseMessage({
+    client,
+    agentId: 'agent-3',
+    lastMessage: null,
+  })
+  assert.strictEqual(result, null)
+  console.log('✓ returns null when no assistant messages exist')
+}
+
+// Test 4: Returns null if timeline fetch fails.
+{
+  const client = createClient({ throwOnFetch: true })
+  const result = await resolveStructuredResponseMessage({
+    client,
+    agentId: 'agent-4',
+    lastMessage: null,
+  })
+  assert.strictEqual(result, null)
+  console.log('✓ returns null when timeline fetch throws')
+}
+
+console.log('\n=== All helper tests passed ===')
@@ -0,0 +1,139 @@
+#!/usr/bin/env npx tsx
+
+import assert from 'node:assert'
+import { createE2ETestContext, type TestDaemonContext } from '../helpers/test-daemon.ts'
+
+interface E2EContext extends TestDaemonContext {
+  paseo: (args: string[], opts?: { timeout?: number; cwd?: string }) => Promise<{
+    exitCode: number
+    stdout: string
+    stderr: string
+  }>
+}
+
+const schema = JSON.stringify({
+  type: 'object',
+  properties: {
+    summary: { type: 'string' },
+  },
+  required: ['summary'],
+  additionalProperties: false,
+})
+
+const impossibleSchema = JSON.stringify({
+  type: 'object',
+  properties: {
+    summary: { type: 'string' },
+  },
+  required: ['summary'],
+  allOf: [
+    { properties: { summary: { type: 'string' } } },
+    { properties: { summary: { type: 'number' } } },
+  ],
+})
+
+let ctx: E2EContext
+
+async function setup(): Promise<void> {
+  ctx = await createE2ETestContext({ timeout: 180000 })
+}
+
+async function cleanup(): Promise<void> {
+  if (ctx) {
+    await ctx.stop()
+  }
+}
+
+async function runProviderCase(input: {
+  provider: 'claude' | 'codex' | 'opencode'
+  mode: string
+  model: string
+}): Promise<void> {
+  const result = await ctx.paseo(
+    [
+      'run',
+      '--provider',
+      input.provider,
+      '--mode',
+      input.mode,
+      '--model',
+      input.model,
+      '--output-schema',
+      schema,
+      `Return valid JSON with a short summary for provider ${input.provider}.`,
+    ],
+    { timeout: 180000 }
+  )
+
+  assert.strictEqual(
+    result.exitCode,
+    0,
+    `expected ${input.provider} run to succeed, got ${result.exitCode}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`
+  )
+
+  const parsed = JSON.parse(result.stdout.trim()) as { summary?: unknown }
+  assert.strictEqual(
+    typeof parsed.summary,
+    'string',
+    `expected ${input.provider} output to contain string summary, got: ${result.stdout}`
+  )
+  assert(parsed.summary && parsed.summary.length > 0, 'summary must not be empty')
+}
+
+async function test_all_providers_return_structured_output(): Promise<void> {
+  await runProviderCase({
+    provider: 'claude',
+    mode: 'bypassPermissions',
+    model: 'haiku',
+  })
+  await runProviderCase({
+    provider: 'codex',
+    mode: 'full-access',
+    model: 'gpt-5.3-codex',
+  })
+  await runProviderCase({
+    provider: 'opencode',
+    mode: 'default',
+    model: 'opencode/kimi-k2.5-free',
+  })
+}
+
+async function test_schema_validation_is_enforced(): Promise<void> {
+  const result = await ctx.paseo(
+    [
+      'run',
+      '--provider',
+      'claude',
+      '--mode',
+      'bypassPermissions',
+      '--model',
+      'haiku',
+      '--output-schema',
+      impossibleSchema,
+      'Return exactly {"summary":"ok"} and nothing else.',
+    ],
+    { timeout: 180000 }
+  )
+
+  assert.notStrictEqual(result.exitCode, 0, 'expected impossible schema to fail')
+  const output = `${result.stdout}\n${result.stderr}`
+  assert(
+    output.includes('OUTPUT_SCHEMA_FAILED'),
+    `expected OUTPUT_SCHEMA_FAILED for impossible schema\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`
+  )
+}
+
+async function main(): Promise<void> {
+  try {
+    await setup()
+    await test_all_providers_return_structured_output()
+    await test_schema_validation_is_enforced()
+  } catch (error) {
+    console.error(error)
+    process.exitCode = 1
+  } finally {
+    await cleanup()
+  }
+}
+
+main()