test: harden daemon lifecycle and e2e reliability across app/cli/server
This commit is contained in:
Mohamed Boudra
@@ -4,7 +4,8 @@
|
||||
* Phase 15: Provider Command Tests
|
||||
*
|
||||
* Tests provider commands for listing providers and models.
|
||||
* Provider data is static and doesn't require a running daemon.
|
||||
* Provider ls data is static, while provider models are fetched via daemon integration.
|
||||
* This test uses an isolated daemon to avoid coupling to a user's long-running daemon.
|
||||
*
|
||||
* Tests:
|
||||
* - provider --help shows subcommands
|
||||
@@ -19,129 +20,209 @@
|
||||
*/
|
||||
|
||||
import assert from 'node:assert'
|
||||
import { $ } from 'zx'
|
||||
|
||||
$.verbose = false
|
||||
import { createE2ETestContext } from './helpers/test-daemon.ts'
|
||||
|
||||
console.log('=== Provider Commands ===\n')
|
||||
|
||||
// Test 1: provider --help shows subcommands
|
||||
{
|
||||
console.log('Test 1: provider --help shows subcommands')
|
||||
const result = await $`npx paseo provider --help`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'provider --help should exit 0')
|
||||
assert(result.stdout.includes('ls'), 'help should mention ls')
|
||||
assert(result.stdout.includes('models'), 'help should mention models')
|
||||
console.log('✓ provider --help shows subcommands\n')
|
||||
type ProviderModel = {
|
||||
model: string
|
||||
id: string
|
||||
description?: string
|
||||
}
|
||||
|
||||
// Test 2: provider ls lists all providers
|
||||
{
|
||||
console.log('Test 2: provider ls lists all providers')
|
||||
const result = await $`npx paseo provider ls`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'provider ls should exit 0')
|
||||
assert(result.stdout.includes('claude'), 'output should include claude')
|
||||
assert(result.stdout.includes('codex'), 'output should include codex')
|
||||
assert(result.stdout.includes('opencode'), 'output should include opencode')
|
||||
assert(result.stdout.includes('available'), 'output should show available status')
|
||||
console.log('✓ provider ls lists all providers\n')
|
||||
let claudeModelIdsFromJson: string[] = []
|
||||
let claudeModelsFromJson: ProviderModel[] = []
|
||||
|
||||
const ctx = await createE2ETestContext({ timeout: 120000 })
|
||||
|
||||
async function runProviderModelsJson(
|
||||
provider: 'claude' | 'codex' | 'opencode'
|
||||
): Promise<ProviderModel[]> {
|
||||
const transientNeedles = ['transport closed', 'timed out', 'timeout', 'socket', 'econn']
|
||||
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
const result = await ctx.paseo(['provider', 'models', provider, '--json'])
|
||||
if (result.exitCode === 0) {
|
||||
return JSON.parse(result.stdout.trim()) as ProviderModel[]
|
||||
}
|
||||
|
||||
const combined = `${result.stdout}\n${result.stderr}`
|
||||
const normalized = combined.toLowerCase()
|
||||
const isTransient = transientNeedles.some((needle) => normalized.includes(needle))
|
||||
|
||||
if (!isTransient || attempt === 3) {
|
||||
assert.fail(`provider models ${provider} should exit 0\n${combined}`)
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 250 * attempt))
|
||||
}
|
||||
|
||||
assert.fail(`provider models ${provider} exhausted retries`)
|
||||
}
|
||||
|
||||
// Test 3: provider ls --json outputs valid JSON
|
||||
{
|
||||
console.log('Test 3: provider ls --json outputs valid JSON')
|
||||
const result = await $`npx paseo provider ls --json`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'should exit 0')
|
||||
const data = JSON.parse(result.stdout.trim())
|
||||
assert(Array.isArray(data), 'output should be an array')
|
||||
assert.strictEqual(data.length, 3, 'should have 3 providers')
|
||||
assert(data.some((p: { provider: string }) => p.provider === 'claude'), 'should include claude')
|
||||
assert(data.some((p: { provider: string }) => p.provider === 'codex'), 'should include codex')
|
||||
assert(data.some((p: { provider: string }) => p.provider === 'opencode'), 'should include opencode')
|
||||
console.log('✓ provider ls --json outputs valid JSON\n')
|
||||
}
|
||||
try {
|
||||
// Test 1: provider --help shows subcommands
|
||||
{
|
||||
console.log('Test 1: provider --help shows subcommands')
|
||||
const result = await ctx.paseo(['provider', '--help'])
|
||||
assert.strictEqual(result.exitCode, 0, 'provider --help should exit 0')
|
||||
assert(result.stdout.includes('ls'), 'help should mention ls')
|
||||
assert(result.stdout.includes('models'), 'help should mention models')
|
||||
console.log('✓ provider --help shows subcommands\n')
|
||||
}
|
||||
|
||||
// Test 4: provider ls --quiet outputs provider names only
|
||||
{
|
||||
console.log('Test 4: provider ls --quiet outputs provider names only')
|
||||
const result = await $`npx paseo provider ls --quiet`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'should exit 0')
|
||||
const lines = result.stdout.trim().split('\n')
|
||||
assert.strictEqual(lines.length, 3, 'should have 3 lines')
|
||||
assert(lines.includes('claude'), 'should include claude')
|
||||
assert(lines.includes('codex'), 'should include codex')
|
||||
assert(lines.includes('opencode'), 'should include opencode')
|
||||
console.log('✓ provider ls --quiet outputs provider names only\n')
|
||||
}
|
||||
// Test 2: provider ls lists all providers
|
||||
{
|
||||
console.log('Test 2: provider ls lists all providers')
|
||||
const result = await ctx.paseo(['provider', 'ls'])
|
||||
assert.strictEqual(result.exitCode, 0, 'provider ls should exit 0')
|
||||
assert(result.stdout.includes('claude'), 'output should include claude')
|
||||
assert(result.stdout.includes('codex'), 'output should include codex')
|
||||
assert(result.stdout.includes('opencode'), 'output should include opencode')
|
||||
assert(result.stdout.includes('available'), 'output should show available status')
|
||||
console.log('✓ provider ls lists all providers\n')
|
||||
}
|
||||
|
||||
// Test 5: provider models claude lists claude models
|
||||
{
|
||||
console.log('Test 5: provider models claude lists claude models')
|
||||
const result = await $`npx paseo provider models claude`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'provider models claude should exit 0')
|
||||
assert(result.stdout.includes('claude-sonnet-4-20250514'), 'output should include claude-sonnet-4')
|
||||
assert(result.stdout.includes('claude-opus-4-20250514'), 'output should include claude-opus-4')
|
||||
assert(result.stdout.includes('claude-3-5-haiku-20241022'), 'output should include claude-haiku')
|
||||
console.log('✓ provider models claude lists claude models\n')
|
||||
}
|
||||
// Test 3: provider ls --json outputs valid JSON
|
||||
{
|
||||
console.log('Test 3: provider ls --json outputs valid JSON')
|
||||
const result = await ctx.paseo(['provider', 'ls', '--json'])
|
||||
assert.strictEqual(result.exitCode, 0, 'should exit 0')
|
||||
const data = JSON.parse(result.stdout.trim())
|
||||
assert(Array.isArray(data), 'output should be an array')
|
||||
assert.strictEqual(data.length, 3, 'should have 3 providers')
|
||||
assert(data.some((p: { provider: string }) => p.provider === 'claude'), 'should include claude')
|
||||
assert(data.some((p: { provider: string }) => p.provider === 'codex'), 'should include codex')
|
||||
assert(data.some((p: { provider: string }) => p.provider === 'opencode'), 'should include opencode')
|
||||
console.log('✓ provider ls --json outputs valid JSON\n')
|
||||
}
|
||||
|
||||
// Test 6: provider models codex lists codex models
|
||||
{
|
||||
console.log('Test 6: provider models codex lists codex models')
|
||||
const result = await $`npx paseo provider models codex`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'provider models codex should exit 0')
|
||||
assert(result.stdout.includes('o3-mini'), 'output should include o3-mini')
|
||||
assert(result.stdout.includes('o4-mini'), 'output should include o4-mini')
|
||||
console.log('✓ provider models codex lists codex models\n')
|
||||
}
|
||||
// Test 4: provider ls --quiet outputs provider names only
|
||||
{
|
||||
console.log('Test 4: provider ls --quiet outputs provider names only')
|
||||
const result = await ctx.paseo(['provider', 'ls', '--quiet'])
|
||||
assert.strictEqual(result.exitCode, 0, 'should exit 0')
|
||||
const lines = result.stdout.trim().split('\n')
|
||||
assert.strictEqual(lines.length, 3, 'should have 3 lines')
|
||||
assert(lines.includes('claude'), 'should include claude')
|
||||
assert(lines.includes('codex'), 'should include codex')
|
||||
assert(lines.includes('opencode'), 'should include opencode')
|
||||
console.log('✓ provider ls --quiet outputs provider names only\n')
|
||||
}
|
||||
|
||||
// Test 7: provider models opencode lists opencode models
|
||||
{
|
||||
console.log('Test 7: provider models opencode lists opencode models')
|
||||
const result = await $`npx paseo provider models opencode`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'provider models opencode should exit 0')
|
||||
// opencode supports both claude and codex models
|
||||
assert(result.stdout.includes('claude-sonnet-4-20250514'), 'output should include claude models')
|
||||
assert(result.stdout.includes('o3-mini'), 'output should include codex models')
|
||||
console.log('✓ provider models opencode lists opencode models\n')
|
||||
}
|
||||
// Test 5: provider models claude lists canonical model aliases
|
||||
{
|
||||
console.log('Test 5: provider models claude lists canonical model aliases')
|
||||
const data = await runProviderModelsJson('claude')
|
||||
assert.strictEqual(data.length, 3, 'should have exactly 3 claude models')
|
||||
const byId = new Map(data.map((model) => [model.id, model]))
|
||||
const ids = [...byId.keys()].sort()
|
||||
assert.deepStrictEqual(ids, ['default', 'haiku', 'opus'], 'claude IDs should be default/opus/haiku')
|
||||
assert.strictEqual(byId.get('default')?.model, 'Sonnet 4.5', 'default claude alias should map to Sonnet 4.5')
|
||||
assert.strictEqual(byId.get('opus')?.model, 'Opus 4.6', 'opus claude alias should map to Opus 4.6')
|
||||
assert.strictEqual(byId.get('haiku')?.model, 'Haiku 4.5', 'haiku claude alias should map to Haiku 4.5')
|
||||
assert(
|
||||
(byId.get('default')?.description ?? '').includes('Best for everyday tasks'),
|
||||
'default claude description should mention everyday tasks'
|
||||
)
|
||||
assert(
|
||||
(byId.get('opus')?.description ?? '').includes('Most capable'),
|
||||
'opus claude description should mention most capable'
|
||||
)
|
||||
assert(
|
||||
(byId.get('haiku')?.description ?? '').includes('Fastest'),
|
||||
'haiku claude description should mention fastest'
|
||||
)
|
||||
console.log('✓ provider models claude lists canonical model aliases\n')
|
||||
}
|
||||
|
||||
// Test 8: provider models unknown fails with error
|
||||
{
|
||||
console.log('Test 8: provider models unknown fails with error')
|
||||
const result = await $`npx paseo provider models unknown`.nothrow()
|
||||
assert.notStrictEqual(result.exitCode, 0, 'should fail for unknown provider')
|
||||
const output = result.stdout + result.stderr
|
||||
assert(
|
||||
output.toLowerCase().includes('unknown') || output.toLowerCase().includes('provider'),
|
||||
'error should mention unknown provider'
|
||||
)
|
||||
console.log('✓ provider models unknown fails with error\n')
|
||||
}
|
||||
// Test 6: provider models codex includes concrete codex model IDs
|
||||
{
|
||||
console.log('Test 6: provider models codex includes concrete codex model IDs')
|
||||
const data = await runProviderModelsJson('codex')
|
||||
assert(data.length >= 6, 'codex model list should include current codex lineup')
|
||||
const ids = data.map((m) => m.id)
|
||||
assert.strictEqual(new Set(ids).size, ids.length, 'codex model IDs should be unique')
|
||||
assert(ids.includes('gpt-5.3-codex'), 'codex output should include gpt-5.3-codex')
|
||||
assert(ids.includes('gpt-5.3-codex-spark'), 'codex output should include gpt-5.3-codex-spark')
|
||||
assert(ids.includes('gpt-5.1-codex-max'), 'codex output should include gpt-5.1-codex-max')
|
||||
assert(ids.includes('gpt-5.1-codex-mini'), 'codex output should include gpt-5.1-codex-mini')
|
||||
console.log('✓ provider models codex includes concrete codex model IDs\n')
|
||||
}
|
||||
|
||||
// Test 9: provider models --json outputs valid JSON
|
||||
{
|
||||
console.log('Test 9: provider models --json outputs valid JSON')
|
||||
const result = await $`npx paseo provider models claude --json`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'should exit 0')
|
||||
const data = JSON.parse(result.stdout.trim())
|
||||
assert(Array.isArray(data), 'output should be an array')
|
||||
assert.strictEqual(data.length, 3, 'should have 3 models for claude')
|
||||
assert(data.every((m: { model: string; id: string }) => m.model && m.id), 'each model should have name and id')
|
||||
console.log('✓ provider models --json outputs valid JSON\n')
|
||||
}
|
||||
// Test 7: provider models opencode returns namespaced model IDs
|
||||
{
|
||||
console.log('Test 7: provider models opencode returns namespaced model IDs')
|
||||
const data = await runProviderModelsJson('opencode')
|
||||
assert(data.length >= 3, 'opencode model list should not be empty')
|
||||
const ids = data.map((m) => m.id)
|
||||
assert(data.every((m) => m.id.includes('/')), 'opencode model IDs should be provider-namespaced')
|
||||
assert(ids.includes('opencode/gpt-5-nano'), 'opencode output should include opencode/gpt-5-nano')
|
||||
assert(ids.includes('openai/o3-mini'), 'opencode output should include openai/o3-mini')
|
||||
assert(
|
||||
ids.includes('openai/gpt-5.3-codex-spark'),
|
||||
'opencode output should include openai/gpt-5.3-codex-spark'
|
||||
)
|
||||
console.log('✓ provider models opencode returns namespaced model IDs\n')
|
||||
}
|
||||
|
||||
// Test 10: provider models --quiet outputs model IDs only
|
||||
{
|
||||
console.log('Test 10: provider models --quiet outputs model IDs only')
|
||||
const result = await $`npx paseo provider models claude --quiet`.nothrow()
|
||||
assert.strictEqual(result.exitCode, 0, 'should exit 0')
|
||||
const lines = result.stdout.trim().split('\n')
|
||||
assert.strictEqual(lines.length, 3, 'should have 3 lines')
|
||||
assert(lines.includes('claude-sonnet-4-20250514'), 'should include claude-sonnet-4')
|
||||
assert(lines.includes('claude-opus-4-20250514'), 'should include claude-opus-4')
|
||||
assert(lines.includes('claude-3-5-haiku-20241022'), 'should include claude-haiku')
|
||||
console.log('✓ provider models --quiet outputs model IDs only\n')
|
||||
// Test 8: provider models unknown fails with error
|
||||
{
|
||||
console.log('Test 8: provider models unknown fails with error')
|
||||
const result = await ctx.paseo(['provider', 'models', 'unknown'])
|
||||
assert.notStrictEqual(result.exitCode, 0, 'should fail for unknown provider')
|
||||
const output = result.stdout + result.stderr
|
||||
assert(
|
||||
output.toLowerCase().includes('unknown') || output.toLowerCase().includes('provider'),
|
||||
'error should mention unknown provider'
|
||||
)
|
||||
console.log('✓ provider models unknown fails with error\n')
|
||||
}
|
||||
|
||||
// Test 9: provider models --json outputs valid JSON
|
||||
{
|
||||
console.log('Test 9: provider models --json outputs valid JSON')
|
||||
const data = await runProviderModelsJson('claude')
|
||||
assert(Array.isArray(data), 'output should be an array')
|
||||
assert.strictEqual(data.length, 3, 'should have exactly 3 models for claude')
|
||||
assert(data.every((m) => m.model && m.id), 'each model should have name and id')
|
||||
const byId = new Map(data.map((model) => [model.id, model]))
|
||||
const ids = [...byId.keys()].sort()
|
||||
assert.deepStrictEqual(ids, ['default', 'haiku', 'opus'], 'claude JSON IDs should be deterministic')
|
||||
assert.strictEqual(byId.get('default')?.model, 'Sonnet 4.5', 'default claude alias should map to Sonnet 4.5')
|
||||
assert.strictEqual(byId.get('opus')?.model, 'Opus 4.6', 'opus claude alias should map to Opus 4.6')
|
||||
assert.strictEqual(byId.get('haiku')?.model, 'Haiku 4.5', 'haiku claude alias should map to Haiku 4.5')
|
||||
claudeModelIdsFromJson = data.map((m) => m.id)
|
||||
claudeModelsFromJson = data
|
||||
console.log('✓ provider models --json outputs valid JSON\n')
|
||||
}
|
||||
|
||||
// Test 10: provider models --quiet outputs model IDs only
|
||||
{
|
||||
console.log('Test 10: provider models --quiet outputs model IDs only')
|
||||
assert(claudeModelIdsFromJson.length > 0, 'claude model IDs should be captured from --json output')
|
||||
const result = await ctx.paseo(['provider', 'models', 'claude', '--quiet'])
|
||||
assert.strictEqual(result.exitCode, 0, 'should exit 0')
|
||||
const lines = result.stdout.trim().split('\n').filter(Boolean)
|
||||
assert.strictEqual(lines.length, 3, 'should have 3 lines')
|
||||
assert.deepStrictEqual(
|
||||
[...lines].sort(),
|
||||
[...claudeModelIdsFromJson].sort(),
|
||||
'--quiet should print the same model IDs returned by --json'
|
||||
)
|
||||
assert.deepStrictEqual(
|
||||
[...lines].sort(),
|
||||
['default', 'haiku', 'opus'],
|
||||
'--quiet should print canonical claude IDs'
|
||||
)
|
||||
assert(
|
||||
claudeModelsFromJson.some((m) => m.id === 'default'),
|
||||
'captured --json output should still include default claude model id'
|
||||
)
|
||||
console.log('✓ provider models --quiet outputs model IDs only\n')
|
||||
}
|
||||
} finally {
|
||||
await ctx.stop()
|
||||
}
|
||||
|
||||
console.log('=== All provider tests passed ===')
|
||||
|
||||
Reference in New Issue
Block a user