Switch to double quotes and reformat codebase with Biome
This commit is contained in:
Mohamed Boudra
@@ -22,238 +22,250 @@
|
||||
* - Clean up resources after test completes
|
||||
*/
|
||||
|
||||
import assert from 'node:assert'
|
||||
import { createE2ETestContext, type TestDaemonContext } from '../helpers/test-daemon.ts'
|
||||
import assert from "node:assert";
|
||||
import { createE2ETestContext, type TestDaemonContext } from "../helpers/test-daemon.ts";
|
||||
|
||||
interface E2EContext extends TestDaemonContext {
|
||||
paseo: (args: string[], opts?: { timeout?: number; cwd?: string }) => Promise<{
|
||||
exitCode: number
|
||||
stdout: string
|
||||
stderr: string
|
||||
}>
|
||||
paseo: (
|
||||
args: string[],
|
||||
opts?: { timeout?: number; cwd?: string },
|
||||
) => Promise<{
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
let ctx: E2EContext
|
||||
let ctx: E2EContext;
|
||||
|
||||
async function setup(): Promise<void> {
|
||||
console.log('Setting up E2E test context...')
|
||||
console.log('Starting test daemon on random port (this may take a few seconds)...')
|
||||
console.log("Setting up E2E test context...");
|
||||
console.log("Starting test daemon on random port (this may take a few seconds)...");
|
||||
|
||||
try {
|
||||
ctx = await createE2ETestContext({ timeout: 45000 })
|
||||
console.log(`Test daemon started on port ${ctx.port}`)
|
||||
console.log(`PASEO_HOME: ${ctx.paseoHome}`)
|
||||
console.log(`Work directory: ${ctx.workDir}`)
|
||||
ctx = await createE2ETestContext({ timeout: 45000 });
|
||||
console.log(`Test daemon started on port ${ctx.port}`);
|
||||
console.log(`PASEO_HOME: ${ctx.paseoHome}`);
|
||||
console.log(`Work directory: ${ctx.workDir}`);
|
||||
} catch (err) {
|
||||
console.error('Failed to start test daemon:', err)
|
||||
throw err
|
||||
console.error("Failed to start test daemon:", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanup(): Promise<void> {
|
||||
console.log('\nCleaning up...')
|
||||
console.log("\nCleaning up...");
|
||||
if (ctx) {
|
||||
await ctx.stop()
|
||||
console.log('Test daemon stopped and temp directories removed')
|
||||
await ctx.stop();
|
||||
console.log("Test daemon stopped and temp directories removed");
|
||||
}
|
||||
}
|
||||
|
||||
async function test_agent_ls_empty(): Promise<void> {
|
||||
console.log('\n--- Test: agent ls with empty list ---')
|
||||
console.log("\n--- Test: agent ls with empty list ---");
|
||||
|
||||
const result = await ctx.paseo(['ls', '--json'])
|
||||
const result = await ctx.paseo(["ls", "--json"]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent ls should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent ls should succeed");
|
||||
|
||||
const agents = JSON.parse(result.stdout.trim())
|
||||
assert(Array.isArray(agents), 'Output should be JSON array')
|
||||
assert.strictEqual(agents.length, 0, 'Should have no agents initially')
|
||||
const agents = JSON.parse(result.stdout.trim());
|
||||
assert(Array.isArray(agents), "Output should be JSON array");
|
||||
assert.strictEqual(agents.length, 0, "Should have no agents initially");
|
||||
|
||||
console.log('PASS: agent ls returns empty list')
|
||||
console.log("PASS: agent ls returns empty list");
|
||||
}
|
||||
|
||||
async function test_agent_run_detached(): Promise<string> {
|
||||
console.log('\n--- Test: agent run detached ---')
|
||||
console.log("\n--- Test: agent run detached ---");
|
||||
|
||||
// Use quiet mode to get just the agent ID
|
||||
// CRITICAL: Use haiku model for fast, cheap tests
|
||||
// CRITICAL: Use bypassPermissions mode so agent doesn't wait for permission approvals
|
||||
const result = await ctx.paseo(
|
||||
[
|
||||
'-q',
|
||||
'run',
|
||||
'-d',
|
||||
'--provider',
|
||||
'claude',
|
||||
'--model',
|
||||
'claude-3-5-haiku-20241022',
|
||||
'--mode',
|
||||
'bypassPermissions',
|
||||
'--name',
|
||||
'E2E Test Agent',
|
||||
'Say hello world',
|
||||
"-q",
|
||||
"run",
|
||||
"-d",
|
||||
"--provider",
|
||||
"claude",
|
||||
"--model",
|
||||
"claude-3-5-haiku-20241022",
|
||||
"--mode",
|
||||
"bypassPermissions",
|
||||
"--name",
|
||||
"E2E Test Agent",
|
||||
"Say hello world",
|
||||
],
|
||||
{ timeout: 60000 }
|
||||
)
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent run should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent run should succeed");
|
||||
|
||||
const agentId = result.stdout.trim()
|
||||
assert(agentId.length > 0, 'Should return agent ID')
|
||||
assert(agentId.match(/^[a-z0-9-]+$/), `Agent ID should be alphanumeric: ${agentId}`)
|
||||
const agentId = result.stdout.trim();
|
||||
assert(agentId.length > 0, "Should return agent ID");
|
||||
assert(agentId.match(/^[a-z0-9-]+$/), `Agent ID should be alphanumeric: ${agentId}`);
|
||||
|
||||
console.log(`PASS: agent created with ID: ${agentId}`)
|
||||
return agentId
|
||||
console.log(`PASS: agent created with ID: ${agentId}`);
|
||||
return agentId;
|
||||
}
|
||||
|
||||
async function test_agent_ls_shows_agent(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: agent ls shows created agent ---')
|
||||
console.log("\n--- Test: agent ls shows created agent ---");
|
||||
|
||||
const result = await ctx.paseo(['ls', '--json'])
|
||||
const result = await ctx.paseo(["ls", "--json"]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent ls should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent ls should succeed");
|
||||
|
||||
const agents = JSON.parse(result.stdout.trim()) as Array<{ id: string; title?: string; status?: string }>
|
||||
assert(Array.isArray(agents), 'Output should be JSON array')
|
||||
assert(agents.length >= 1, 'Should have at least one agent')
|
||||
const agents = JSON.parse(result.stdout.trim()) as Array<{
|
||||
id: string;
|
||||
title?: string;
|
||||
status?: string;
|
||||
}>;
|
||||
assert(Array.isArray(agents), "Output should be JSON array");
|
||||
assert(agents.length >= 1, "Should have at least one agent");
|
||||
|
||||
// Find our agent by ID prefix
|
||||
const ourAgent = agents.find((a) => agentId.startsWith(a.id) || a.id.startsWith(agentId.slice(0, 7)))
|
||||
assert(ourAgent, `Our agent ${agentId} should be in the list`)
|
||||
const ourAgent = agents.find(
|
||||
(a) => agentId.startsWith(a.id) || a.id.startsWith(agentId.slice(0, 7)),
|
||||
);
|
||||
assert(ourAgent, `Our agent ${agentId} should be in the list`);
|
||||
|
||||
console.log('Agent found:', ourAgent)
|
||||
console.log('PASS: agent ls shows created agent')
|
||||
console.log("Agent found:", ourAgent);
|
||||
console.log("PASS: agent ls shows created agent");
|
||||
}
|
||||
|
||||
async function test_agent_wait(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: agent wait ---')
|
||||
console.log("\n--- Test: agent wait ---");
|
||||
|
||||
// Wait for agent to become idle (with generous timeout for haiku model)
|
||||
const result = await ctx.paseo(['wait', '--timeout', '120s', agentId], { timeout: 130000 })
|
||||
const result = await ctx.paseo(["wait", "--timeout", "120s", agentId], { timeout: 130000 });
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent wait should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent wait should succeed");
|
||||
|
||||
console.log('PASS: agent wait completed successfully')
|
||||
console.log("PASS: agent wait completed successfully");
|
||||
}
|
||||
|
||||
async function test_agent_inspect(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: agent inspect ---')
|
||||
console.log("\n--- Test: agent inspect ---");
|
||||
|
||||
const result = await ctx.paseo(['inspect', agentId])
|
||||
const result = await ctx.paseo(["inspect", agentId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent inspect should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent inspect should succeed");
|
||||
|
||||
// Check that output contains expected fields (table format)
|
||||
const output = result.stdout
|
||||
assert(output.includes('Id'), 'Output should contain Id field')
|
||||
assert(output.includes('Provider'), 'Output should contain Provider field')
|
||||
assert(output.includes('Status'), 'Output should contain Status field')
|
||||
assert(output.includes('claude'), 'Output should mention claude provider')
|
||||
const output = result.stdout;
|
||||
assert(output.includes("Id"), "Output should contain Id field");
|
||||
assert(output.includes("Provider"), "Output should contain Provider field");
|
||||
assert(output.includes("Status"), "Output should contain Status field");
|
||||
assert(output.includes("claude"), "Output should mention claude provider");
|
||||
|
||||
console.log('PASS: agent inspect shows expected fields')
|
||||
console.log("PASS: agent inspect shows expected fields");
|
||||
}
|
||||
|
||||
async function test_agent_logs(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: agent logs ---')
|
||||
console.log("\n--- Test: agent logs ---");
|
||||
|
||||
const result = await ctx.paseo(['logs', '--tail', '20', agentId])
|
||||
const result = await ctx.paseo(["logs", "--tail", "20", agentId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout length:', result.stdout.length)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout length:", result.stdout.length);
|
||||
// Don't print full logs as they can be verbose
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent logs should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent logs should succeed");
|
||||
// Logs should have some content (agent was asked to say hello)
|
||||
assert(result.stdout.length > 0, 'Logs should have some content')
|
||||
assert(result.stdout.length > 0, "Logs should have some content");
|
||||
|
||||
console.log('PASS: agent logs returns content')
|
||||
console.log("PASS: agent logs returns content");
|
||||
}
|
||||
|
||||
async function test_agent_stop(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: agent stop ---')
|
||||
console.log("\n--- Test: agent stop ---");
|
||||
|
||||
const result = await ctx.paseo(['stop', agentId])
|
||||
const result = await ctx.paseo(["stop", agentId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent stop should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent stop should succeed");
|
||||
|
||||
const inspectResult = await ctx.paseo(['inspect', agentId])
|
||||
assert.strictEqual(inspectResult.exitCode, 0, 'agent should remain inspectable after stop')
|
||||
assert(!inspectResult.stdout.toLowerCase().includes('running'), 'Agent should not be running after stop')
|
||||
const inspectResult = await ctx.paseo(["inspect", agentId]);
|
||||
assert.strictEqual(inspectResult.exitCode, 0, "agent should remain inspectable after stop");
|
||||
assert(
|
||||
!inspectResult.stdout.toLowerCase().includes("running"),
|
||||
"Agent should not be running after stop",
|
||||
);
|
||||
|
||||
console.log('PASS: agent stop interrupted without deleting the agent')
|
||||
console.log("PASS: agent stop interrupted without deleting the agent");
|
||||
}
|
||||
|
||||
async function test_agent_delete(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: agent delete ---')
|
||||
console.log("\n--- Test: agent delete ---");
|
||||
|
||||
const result = await ctx.paseo(['delete', agentId])
|
||||
const result = await ctx.paseo(["delete", agentId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent delete should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent delete should succeed");
|
||||
|
||||
const inspectResult = await ctx.paseo(['inspect', agentId])
|
||||
assert.notStrictEqual(inspectResult.exitCode, 0, 'deleted agent should not be inspectable')
|
||||
const inspectResult = await ctx.paseo(["inspect", agentId]);
|
||||
assert.notStrictEqual(inspectResult.exitCode, 0, "deleted agent should not be inspectable");
|
||||
|
||||
console.log('PASS: agent delete removed the agent')
|
||||
console.log("PASS: agent delete removed the agent");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log('=== E2E Test: Agent Lifecycle ===\n')
|
||||
console.log('This test creates REAL agents with REAL daemons.')
|
||||
console.log('It may take some time as it waits for agent responses.\n')
|
||||
console.log("=== E2E Test: Agent Lifecycle ===\n");
|
||||
console.log("This test creates REAL agents with REAL daemons.");
|
||||
console.log("It may take some time as it waits for agent responses.\n");
|
||||
|
||||
try {
|
||||
await setup()
|
||||
await setup();
|
||||
|
||||
// Run tests in sequence
|
||||
await test_agent_ls_empty()
|
||||
await test_agent_ls_empty();
|
||||
|
||||
const agentId = await test_agent_run_detached()
|
||||
const agentId = await test_agent_run_detached();
|
||||
|
||||
await test_agent_ls_shows_agent(agentId)
|
||||
await test_agent_ls_shows_agent(agentId);
|
||||
|
||||
await test_agent_wait(agentId)
|
||||
await test_agent_wait(agentId);
|
||||
|
||||
await test_agent_inspect(agentId)
|
||||
await test_agent_inspect(agentId);
|
||||
|
||||
await test_agent_logs(agentId)
|
||||
await test_agent_logs(agentId);
|
||||
|
||||
await test_agent_stop(agentId)
|
||||
await test_agent_delete(agentId)
|
||||
await test_agent_stop(agentId);
|
||||
await test_agent_delete(agentId);
|
||||
|
||||
console.log('\n=== All E2E tests passed! ===')
|
||||
console.log("\n=== All E2E tests passed! ===");
|
||||
} catch (err) {
|
||||
console.error('\n=== E2E test FAILED ===')
|
||||
console.error(err)
|
||||
process.exitCode = 1
|
||||
console.error("\n=== E2E test FAILED ===");
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await cleanup()
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
main();
|
||||
|
||||
@@ -21,185 +21,189 @@
|
||||
* - Clean up resources after test completes
|
||||
*/
|
||||
|
||||
import assert from 'node:assert'
|
||||
import { createE2ETestContext, type TestDaemonContext } from '../helpers/test-daemon.ts'
|
||||
import assert from "node:assert";
|
||||
import { createE2ETestContext, type TestDaemonContext } from "../helpers/test-daemon.ts";
|
||||
|
||||
interface E2EContext extends TestDaemonContext {
|
||||
paseo: (args: string[], opts?: { timeout?: number; cwd?: string }) => Promise<{
|
||||
exitCode: number
|
||||
stdout: string
|
||||
stderr: string
|
||||
}>
|
||||
paseo: (
|
||||
args: string[],
|
||||
opts?: { timeout?: number; cwd?: string },
|
||||
) => Promise<{
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
let ctx: E2EContext
|
||||
let ctx: E2EContext;
|
||||
|
||||
async function setup(): Promise<void> {
|
||||
console.log('Setting up E2E test context...')
|
||||
console.log('Starting test daemon on random port (this may take a few seconds)...')
|
||||
console.log("Setting up E2E test context...");
|
||||
console.log("Starting test daemon on random port (this may take a few seconds)...");
|
||||
|
||||
try {
|
||||
ctx = await createE2ETestContext({ timeout: 45000 })
|
||||
console.log(`Test daemon started on port ${ctx.port}`)
|
||||
console.log(`PASEO_HOME: ${ctx.paseoHome}`)
|
||||
console.log(`Work directory: ${ctx.workDir}`)
|
||||
ctx = await createE2ETestContext({ timeout: 45000 });
|
||||
console.log(`Test daemon started on port ${ctx.port}`);
|
||||
console.log(`PASEO_HOME: ${ctx.paseoHome}`);
|
||||
console.log(`Work directory: ${ctx.workDir}`);
|
||||
} catch (err) {
|
||||
console.error('Failed to start test daemon:', err)
|
||||
throw err
|
||||
console.error("Failed to start test daemon:", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanup(): Promise<void> {
|
||||
console.log('\nCleaning up...')
|
||||
console.log("\nCleaning up...");
|
||||
if (ctx) {
|
||||
await ctx.stop()
|
||||
console.log('Test daemon stopped and temp directories removed')
|
||||
await ctx.stop();
|
||||
console.log("Test daemon stopped and temp directories removed");
|
||||
}
|
||||
}
|
||||
|
||||
async function test_create_agent(): Promise<string> {
|
||||
console.log('\n--- Test: Create agent in detached mode ---')
|
||||
console.log("\n--- Test: Create agent in detached mode ---");
|
||||
|
||||
// CRITICAL: Use haiku model for fast, cheap tests
|
||||
// CRITICAL: Use bypassPermissions mode so agent doesn't wait for permission approvals
|
||||
const result = await ctx.paseo(
|
||||
[
|
||||
'-q',
|
||||
'run',
|
||||
'-d',
|
||||
'--provider',
|
||||
'claude',
|
||||
'--model',
|
||||
'claude-3-5-haiku-20241022',
|
||||
'--mode',
|
||||
'bypassPermissions',
|
||||
'--name',
|
||||
'Send Test Agent',
|
||||
"-q",
|
||||
"run",
|
||||
"-d",
|
||||
"--provider",
|
||||
"claude",
|
||||
"--model",
|
||||
"claude-3-5-haiku-20241022",
|
||||
"--mode",
|
||||
"bypassPermissions",
|
||||
"--name",
|
||||
"Send Test Agent",
|
||||
'Say "initial task complete"',
|
||||
],
|
||||
{ timeout: 60000 }
|
||||
)
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent run should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent run should succeed");
|
||||
|
||||
const agentId = result.stdout.trim()
|
||||
assert(agentId.length > 0, 'Should return agent ID')
|
||||
assert(agentId.match(/^[a-z0-9-]+$/), `Agent ID should be alphanumeric: ${agentId}`)
|
||||
const agentId = result.stdout.trim();
|
||||
assert(agentId.length > 0, "Should return agent ID");
|
||||
assert(agentId.match(/^[a-z0-9-]+$/), `Agent ID should be alphanumeric: ${agentId}`);
|
||||
|
||||
console.log(`PASS: agent created with ID: ${agentId}`)
|
||||
return agentId
|
||||
console.log(`PASS: agent created with ID: ${agentId}`);
|
||||
return agentId;
|
||||
}
|
||||
|
||||
async function test_wait_for_initial_task(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: Wait for initial task to complete ---')
|
||||
console.log("\n--- Test: Wait for initial task to complete ---");
|
||||
|
||||
const result = await ctx.paseo(['wait', '--timeout', '120s', agentId], { timeout: 130000 })
|
||||
const result = await ctx.paseo(["wait", "--timeout", "120s", agentId], { timeout: 130000 });
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent wait should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent wait should succeed");
|
||||
|
||||
console.log('PASS: Initial task completed')
|
||||
console.log("PASS: Initial task completed");
|
||||
}
|
||||
|
||||
async function test_agent_send(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: Send follow-up message ---')
|
||||
console.log("\n--- Test: Send follow-up message ---");
|
||||
|
||||
// Send a follow-up message to the agent
|
||||
const result = await ctx.paseo(
|
||||
['send', agentId, 'Now say "follow-up task complete"'],
|
||||
{ timeout: 180000 }
|
||||
)
|
||||
const result = await ctx.paseo(["send", agentId, 'Now say "follow-up task complete"'], {
|
||||
timeout: 180000,
|
||||
});
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent send should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent send should succeed");
|
||||
|
||||
// Verify output contains expected status
|
||||
assert(
|
||||
result.stdout.includes('completed') || result.stdout.includes('sent'),
|
||||
'Should indicate message was sent or completed'
|
||||
)
|
||||
result.stdout.includes("completed") || result.stdout.includes("sent"),
|
||||
"Should indicate message was sent or completed",
|
||||
);
|
||||
|
||||
console.log('PASS: Follow-up message sent successfully')
|
||||
console.log("PASS: Follow-up message sent successfully");
|
||||
}
|
||||
|
||||
async function test_verify_logs(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: Verify agent processed both messages ---')
|
||||
console.log("\n--- Test: Verify agent processed both messages ---");
|
||||
|
||||
const result = await ctx.paseo(['logs', '--tail', '50', agentId])
|
||||
const result = await ctx.paseo(["logs", "--tail", "50", agentId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout length:', result.stdout.length)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout length:", result.stdout.length);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent logs should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent logs should succeed");
|
||||
|
||||
// Logs should have content from both tasks
|
||||
assert(result.stdout.length > 0, 'Logs should have content')
|
||||
assert(result.stdout.length > 0, "Logs should have content");
|
||||
|
||||
// The agent was asked to say specific things, check if either appears in logs
|
||||
// This is a loose check since exact log format may vary
|
||||
const logsLower = result.stdout.toLowerCase()
|
||||
const hasInitialTask = logsLower.includes('initial') || logsLower.includes('hello') || logsLower.includes('task')
|
||||
const hasFollowUp = logsLower.includes('follow') || logsLower.includes('complete') || logsLower.includes('task')
|
||||
const logsLower = result.stdout.toLowerCase();
|
||||
const hasInitialTask =
|
||||
logsLower.includes("initial") || logsLower.includes("hello") || logsLower.includes("task");
|
||||
const hasFollowUp =
|
||||
logsLower.includes("follow") || logsLower.includes("complete") || logsLower.includes("task");
|
||||
|
||||
// At minimum, there should be log entries (we can't guarantee exact content)
|
||||
assert(result.stdout.split('\n').length > 3, 'Should have multiple log entries from both tasks')
|
||||
assert(result.stdout.split("\n").length > 3, "Should have multiple log entries from both tasks");
|
||||
|
||||
console.log('PASS: Agent logs show activity from both tasks')
|
||||
console.log("PASS: Agent logs show activity from both tasks");
|
||||
}
|
||||
|
||||
async function test_agent_delete(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: Delete agent ---')
|
||||
console.log("\n--- Test: Delete agent ---");
|
||||
|
||||
const result = await ctx.paseo(['delete', agentId])
|
||||
const result = await ctx.paseo(["delete", agentId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent delete should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent delete should succeed");
|
||||
|
||||
console.log('PASS: Agent deleted successfully')
|
||||
console.log("PASS: Agent deleted successfully");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log('=== E2E Test: Agent Send Command ===\n')
|
||||
console.log('This test verifies the `send` command with REAL agents.')
|
||||
console.log('It may take some time as it waits for agent responses.\n')
|
||||
console.log("=== E2E Test: Agent Send Command ===\n");
|
||||
console.log("This test verifies the `send` command with REAL agents.");
|
||||
console.log("It may take some time as it waits for agent responses.\n");
|
||||
|
||||
try {
|
||||
await setup()
|
||||
await setup();
|
||||
|
||||
// Create agent and wait for initial task
|
||||
const agentId = await test_create_agent()
|
||||
await test_wait_for_initial_task(agentId)
|
||||
const agentId = await test_create_agent();
|
||||
await test_wait_for_initial_task(agentId);
|
||||
|
||||
// Send follow-up message (this waits for completion by default)
|
||||
await test_agent_send(agentId)
|
||||
await test_agent_send(agentId);
|
||||
|
||||
// Verify both messages were processed
|
||||
await test_verify_logs(agentId)
|
||||
await test_verify_logs(agentId);
|
||||
|
||||
// Cleanup agent
|
||||
await test_agent_delete(agentId)
|
||||
await test_agent_delete(agentId);
|
||||
|
||||
console.log('\n=== All agent-send E2E tests passed! ===')
|
||||
console.log("\n=== All agent-send E2E tests passed! ===");
|
||||
} catch (err) {
|
||||
console.error('\n=== E2E test FAILED ===')
|
||||
console.error(err)
|
||||
process.exitCode = 1
|
||||
console.error("\n=== E2E test FAILED ===");
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await cleanup()
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
main();
|
||||
|
||||
@@ -21,106 +21,111 @@
|
||||
* - Clean up resources after test completes
|
||||
*/
|
||||
|
||||
import assert from 'node:assert'
|
||||
import { createE2ETestContext, type TestDaemonContext } from '../helpers/test-daemon.ts'
|
||||
import assert from "node:assert";
|
||||
import { createE2ETestContext, type TestDaemonContext } from "../helpers/test-daemon.ts";
|
||||
|
||||
interface E2EContext extends TestDaemonContext {
|
||||
paseo: (args: string[], opts?: { timeout?: number; cwd?: string }) => Promise<{
|
||||
exitCode: number
|
||||
stdout: string
|
||||
stderr: string
|
||||
}>
|
||||
paseo: (
|
||||
args: string[],
|
||||
opts?: { timeout?: number; cwd?: string },
|
||||
) => Promise<{
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
let ctx: E2EContext
|
||||
let ctx: E2EContext;
|
||||
|
||||
async function setup(): Promise<void> {
|
||||
console.log('Setting up E2E test context...')
|
||||
console.log('Starting test daemon on random port (this may take a few seconds)...')
|
||||
console.log("Setting up E2E test context...");
|
||||
console.log("Starting test daemon on random port (this may take a few seconds)...");
|
||||
|
||||
try {
|
||||
ctx = await createE2ETestContext({ timeout: 45000 })
|
||||
console.log(`Test daemon started on port ${ctx.port}`)
|
||||
console.log(`PASEO_HOME: ${ctx.paseoHome}`)
|
||||
console.log(`Work directory: ${ctx.workDir}`)
|
||||
ctx = await createE2ETestContext({ timeout: 45000 });
|
||||
console.log(`Test daemon started on port ${ctx.port}`);
|
||||
console.log(`PASEO_HOME: ${ctx.paseoHome}`);
|
||||
console.log(`Work directory: ${ctx.workDir}`);
|
||||
} catch (err) {
|
||||
console.error('Failed to start test daemon:', err)
|
||||
throw err
|
||||
console.error("Failed to start test daemon:", err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanup(): Promise<void> {
|
||||
console.log('\nCleaning up...')
|
||||
console.log("\nCleaning up...");
|
||||
if (ctx) {
|
||||
await ctx.stop()
|
||||
console.log('Test daemon stopped and temp directories removed')
|
||||
await ctx.stop();
|
||||
console.log("Test daemon stopped and temp directories removed");
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function test_create_agent_with_permissions(): Promise<string> {
|
||||
console.log('\n--- Test: Create agent in default mode (will request permissions) ---')
|
||||
console.log("\n--- Test: Create agent in default mode (will request permissions) ---");
|
||||
|
||||
// Create agent WITHOUT bypassPermissions - it will need to request permissions
|
||||
// Use a task that is very likely to trigger a tool use (and thus permission request)
|
||||
const result = await ctx.paseo(
|
||||
[
|
||||
'-q',
|
||||
'agent',
|
||||
'run',
|
||||
'-d',
|
||||
'--provider',
|
||||
'claude',
|
||||
'--model',
|
||||
'claude-3-5-haiku-20241022',
|
||||
'--name',
|
||||
'Permission Test Agent',
|
||||
"-q",
|
||||
"agent",
|
||||
"run",
|
||||
"-d",
|
||||
"--provider",
|
||||
"claude",
|
||||
"--model",
|
||||
"claude-3-5-haiku-20241022",
|
||||
"--name",
|
||||
"Permission Test Agent",
|
||||
// This prompt should trigger file read or bash command, requiring permission
|
||||
'List the files in the current directory using ls',
|
||||
"List the files in the current directory using ls",
|
||||
],
|
||||
{ timeout: 60000 }
|
||||
)
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent run should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent run should succeed");
|
||||
|
||||
const agentId = result.stdout.trim()
|
||||
assert(agentId.length > 0, 'Should return agent ID')
|
||||
assert(agentId.match(/^[a-z0-9-]+$/), `Agent ID should be alphanumeric: ${agentId}`)
|
||||
const agentId = result.stdout.trim();
|
||||
assert(agentId.length > 0, "Should return agent ID");
|
||||
assert(agentId.match(/^[a-z0-9-]+$/), `Agent ID should be alphanumeric: ${agentId}`);
|
||||
|
||||
console.log(`PASS: agent created with ID: ${agentId}`)
|
||||
return agentId
|
||||
console.log(`PASS: agent created with ID: ${agentId}`);
|
||||
return agentId;
|
||||
}
|
||||
|
||||
async function test_wait_for_permission_request(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: Wait for agent to request permission ---')
|
||||
console.log("\n--- Test: Wait for agent to request permission ---");
|
||||
|
||||
// Poll for permission requests with timeout
|
||||
const maxWait = 60000 // 60 seconds max
|
||||
const pollInterval = 1000 // 1 second
|
||||
const startTime = Date.now()
|
||||
const maxWait = 60000; // 60 seconds max
|
||||
const pollInterval = 1000; // 1 second
|
||||
const startTime = Date.now();
|
||||
|
||||
while (Date.now() - startTime < maxWait) {
|
||||
const result = await ctx.paseo(['permit', 'ls', '--json'])
|
||||
const result = await ctx.paseo(["permit", "ls", "--json"]);
|
||||
|
||||
if (result.exitCode === 0) {
|
||||
try {
|
||||
const permissions = JSON.parse(result.stdout.trim())
|
||||
const permissions = JSON.parse(result.stdout.trim());
|
||||
if (Array.isArray(permissions) && permissions.length > 0) {
|
||||
// Check if any permission is for our agent
|
||||
const ourPermission = permissions.find(
|
||||
(p: { agentId?: string }) => p.agentId?.startsWith(agentId.slice(0, 7)) || agentId.startsWith(p.agentId?.slice(0, 7) || '')
|
||||
)
|
||||
(p: { agentId?: string }) =>
|
||||
p.agentId?.startsWith(agentId.slice(0, 7)) ||
|
||||
agentId.startsWith(p.agentId?.slice(0, 7) || ""),
|
||||
);
|
||||
if (ourPermission) {
|
||||
console.log('Permission request detected:', ourPermission)
|
||||
console.log('PASS: Agent requested permission')
|
||||
return
|
||||
console.log("Permission request detected:", ourPermission);
|
||||
console.log("PASS: Agent requested permission");
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -128,158 +133,166 @@ async function test_wait_for_permission_request(agentId: string): Promise<void>
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(pollInterval)
|
||||
await sleep(pollInterval);
|
||||
}
|
||||
|
||||
// If we get here, check agent status - it might have already completed
|
||||
const statusResult = await ctx.paseo(['inspect', agentId])
|
||||
console.log('Agent status:', statusResult.stdout)
|
||||
const statusResult = await ctx.paseo(["inspect", agentId]);
|
||||
console.log("Agent status:", statusResult.stdout);
|
||||
|
||||
// It's possible the agent completed without needing permissions (e.g., haiku might not use tools)
|
||||
// In that case, we should skip the permission tests
|
||||
if (statusResult.stdout.includes('idle') || statusResult.stdout.includes('completed')) {
|
||||
console.log('SKIP: Agent completed without requiring permissions (model chose not to use tools)')
|
||||
throw new Error('SKIP_PERMISSION_TEST')
|
||||
if (statusResult.stdout.includes("idle") || statusResult.stdout.includes("completed")) {
|
||||
console.log(
|
||||
"SKIP: Agent completed without requiring permissions (model chose not to use tools)",
|
||||
);
|
||||
throw new Error("SKIP_PERMISSION_TEST");
|
||||
}
|
||||
|
||||
throw new Error('Timeout waiting for agent to request permission')
|
||||
throw new Error("Timeout waiting for agent to request permission");
|
||||
}
|
||||
|
||||
async function test_permit_ls(): Promise<{ agentShortId: string; requestId: string }> {
|
||||
console.log('\n--- Test: List pending permissions with permit ls ---')
|
||||
console.log("\n--- Test: List pending permissions with permit ls ---");
|
||||
|
||||
const result = await ctx.paseo(['permit', 'ls', '--json'])
|
||||
const result = await ctx.paseo(["permit", "ls", "--json"]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'permit ls should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "permit ls should succeed");
|
||||
|
||||
const permissions = JSON.parse(result.stdout.trim())
|
||||
assert(Array.isArray(permissions), 'Output should be JSON array')
|
||||
assert(permissions.length > 0, 'Should have at least one pending permission')
|
||||
const permissions = JSON.parse(result.stdout.trim());
|
||||
assert(Array.isArray(permissions), "Output should be JSON array");
|
||||
assert(permissions.length > 0, "Should have at least one pending permission");
|
||||
|
||||
const permission = permissions[0]
|
||||
assert(permission.id, 'Permission should have id')
|
||||
assert(permission.agentShortId, 'Permission should have agentShortId')
|
||||
assert(permission.name, 'Permission should have tool name')
|
||||
const permission = permissions[0];
|
||||
assert(permission.id, "Permission should have id");
|
||||
assert(permission.agentShortId, "Permission should have agentShortId");
|
||||
assert(permission.name, "Permission should have tool name");
|
||||
|
||||
console.log(`PASS: Found ${permissions.length} pending permission(s)`)
|
||||
console.log(` Request ID: ${permission.id}`)
|
||||
console.log(` Agent: ${permission.agentShortId}`)
|
||||
console.log(` Tool: ${permission.name}`)
|
||||
console.log(`PASS: Found ${permissions.length} pending permission(s)`);
|
||||
console.log(` Request ID: ${permission.id}`);
|
||||
console.log(` Agent: ${permission.agentShortId}`);
|
||||
console.log(` Tool: ${permission.name}`);
|
||||
|
||||
return {
|
||||
agentShortId: permission.agentShortId,
|
||||
requestId: permission.id,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function test_permit_allow(agentShortId: string, requestId: string): Promise<void> {
|
||||
console.log('\n--- Test: Allow permission with permit allow ---')
|
||||
console.log("\n--- Test: Allow permission with permit allow ---");
|
||||
|
||||
const result = await ctx.paseo(['permit', 'allow', agentShortId, requestId])
|
||||
const result = await ctx.paseo(["permit", "allow", agentShortId, requestId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'permit allow should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "permit allow should succeed");
|
||||
|
||||
// Check that output shows the permission was allowed
|
||||
const output = result.stdout.toLowerCase()
|
||||
assert(output.includes('allowed') || output.includes(requestId.slice(0, 8).toLowerCase()), 'Should confirm permission was allowed')
|
||||
const output = result.stdout.toLowerCase();
|
||||
assert(
|
||||
output.includes("allowed") || output.includes(requestId.slice(0, 8).toLowerCase()),
|
||||
"Should confirm permission was allowed",
|
||||
);
|
||||
|
||||
console.log('PASS: Permission allowed successfully')
|
||||
console.log("PASS: Permission allowed successfully");
|
||||
}
|
||||
|
||||
async function test_agent_continues(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: Verify agent continues after permission granted ---')
|
||||
console.log("\n--- Test: Verify agent continues after permission granted ---");
|
||||
|
||||
// Wait for agent to become idle (should continue after permission)
|
||||
const result = await ctx.paseo(['wait', '--timeout', '120s', agentId], { timeout: 130000 })
|
||||
const result = await ctx.paseo(["wait", "--timeout", "120s", agentId], { timeout: 130000 });
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent wait should succeed after permission granted')
|
||||
assert.strictEqual(result.exitCode, 0, "agent wait should succeed after permission granted");
|
||||
|
||||
// Verify agent status
|
||||
const inspectResult = await ctx.paseo(['inspect', agentId])
|
||||
console.log('Final agent status:', inspectResult.stdout)
|
||||
const inspectResult = await ctx.paseo(["inspect", agentId]);
|
||||
console.log("Final agent status:", inspectResult.stdout);
|
||||
|
||||
assert(inspectResult.stdout.includes('idle') || inspectResult.stdout.includes('completed'), 'Agent should be idle after completing')
|
||||
assert(
|
||||
inspectResult.stdout.includes("idle") || inspectResult.stdout.includes("completed"),
|
||||
"Agent should be idle after completing",
|
||||
);
|
||||
|
||||
console.log('PASS: Agent completed after permission was granted')
|
||||
console.log("PASS: Agent completed after permission was granted");
|
||||
}
|
||||
|
||||
async function test_agent_delete(agentId: string): Promise<void> {
|
||||
console.log('\n--- Test: Delete agent ---')
|
||||
console.log("\n--- Test: Delete agent ---");
|
||||
|
||||
const result = await ctx.paseo(['delete', agentId])
|
||||
const result = await ctx.paseo(["delete", agentId]);
|
||||
|
||||
console.log('Exit code:', result.exitCode)
|
||||
console.log('Stdout:', result.stdout)
|
||||
if (result.stderr) console.log('Stderr:', result.stderr)
|
||||
console.log("Exit code:", result.exitCode);
|
||||
console.log("Stdout:", result.stdout);
|
||||
if (result.stderr) console.log("Stderr:", result.stderr);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'agent delete should succeed')
|
||||
assert.strictEqual(result.exitCode, 0, "agent delete should succeed");
|
||||
|
||||
console.log('PASS: Agent deleted successfully')
|
||||
console.log("PASS: Agent deleted successfully");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log('=== E2E Test: Permissions Workflow ===\n')
|
||||
console.log('This test verifies the permission request/allow workflow.')
|
||||
console.log('It creates an agent that will request tool permissions.\n')
|
||||
console.log("=== E2E Test: Permissions Workflow ===\n");
|
||||
console.log("This test verifies the permission request/allow workflow.");
|
||||
console.log("It creates an agent that will request tool permissions.\n");
|
||||
|
||||
let agentId: string | undefined
|
||||
let agentId: string | undefined;
|
||||
|
||||
try {
|
||||
await setup()
|
||||
await setup();
|
||||
|
||||
// Create agent that will need permissions
|
||||
agentId = await test_create_agent_with_permissions()
|
||||
agentId = await test_create_agent_with_permissions();
|
||||
|
||||
try {
|
||||
// Wait for permission request
|
||||
await test_wait_for_permission_request(agentId)
|
||||
await test_wait_for_permission_request(agentId);
|
||||
|
||||
// List and verify permissions
|
||||
const { agentShortId, requestId } = await test_permit_ls()
|
||||
const { agentShortId, requestId } = await test_permit_ls();
|
||||
|
||||
// Allow the permission
|
||||
await test_permit_allow(agentShortId, requestId)
|
||||
await test_permit_allow(agentShortId, requestId);
|
||||
|
||||
// Verify agent continues and completes
|
||||
await test_agent_continues(agentId)
|
||||
await test_agent_continues(agentId);
|
||||
|
||||
console.log('\n=== All permissions E2E tests passed! ===')
|
||||
console.log("\n=== All permissions E2E tests passed! ===");
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message === 'SKIP_PERMISSION_TEST') {
|
||||
console.log('\n=== Permissions test skipped (agent completed without tool use) ===')
|
||||
console.log('This can happen when the model chooses not to use tools.')
|
||||
if (err instanceof Error && err.message === "SKIP_PERMISSION_TEST") {
|
||||
console.log("\n=== Permissions test skipped (agent completed without tool use) ===");
|
||||
console.log("This can happen when the model chooses not to use tools.");
|
||||
// Still considered a pass - the CLI works, just the model behavior varies
|
||||
} else {
|
||||
throw err
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('\n=== E2E test FAILED ===')
|
||||
console.error(err)
|
||||
process.exitCode = 1
|
||||
console.error("\n=== E2E test FAILED ===");
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
// Always try to delete the agent if it was created
|
||||
if (agentId) {
|
||||
try {
|
||||
await test_agent_delete(agentId)
|
||||
await test_agent_delete(agentId);
|
||||
} catch {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
await cleanup()
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
main();
|
||||
|
||||
@@ -1,141 +1,144 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
import assert from 'node:assert'
|
||||
import { createE2ETestContext, type TestDaemonContext } from '../helpers/test-daemon.ts'
|
||||
import assert from "node:assert";
|
||||
import { createE2ETestContext, type TestDaemonContext } from "../helpers/test-daemon.ts";
|
||||
|
||||
interface E2EContext extends TestDaemonContext {
|
||||
paseo: (args: string[], opts?: { timeout?: number; cwd?: string }) => Promise<{
|
||||
exitCode: number
|
||||
stdout: string
|
||||
stderr: string
|
||||
}>
|
||||
paseo: (
|
||||
args: string[],
|
||||
opts?: { timeout?: number; cwd?: string },
|
||||
) => Promise<{
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
const schema = JSON.stringify({
|
||||
type: 'object',
|
||||
type: "object",
|
||||
properties: {
|
||||
summary: { type: 'string' },
|
||||
summary: { type: "string" },
|
||||
},
|
||||
required: ['summary'],
|
||||
required: ["summary"],
|
||||
additionalProperties: false,
|
||||
})
|
||||
});
|
||||
|
||||
const impossibleSchema = JSON.stringify({
|
||||
type: 'object',
|
||||
type: "object",
|
||||
properties: {
|
||||
summary: { type: 'string' },
|
||||
summary: { type: "string" },
|
||||
},
|
||||
required: ['summary'],
|
||||
required: ["summary"],
|
||||
allOf: [
|
||||
{ properties: { summary: { type: 'string' } } },
|
||||
{ properties: { summary: { type: 'number' } } },
|
||||
{ properties: { summary: { type: "string" } } },
|
||||
{ properties: { summary: { type: "number" } } },
|
||||
],
|
||||
})
|
||||
});
|
||||
|
||||
const OPEN_CODE_STRUCTURED_MODEL = 'opencode/gpt-5-nano'
|
||||
const OPEN_CODE_STRUCTURED_MODEL = "opencode/gpt-5-nano";
|
||||
|
||||
let ctx: E2EContext
|
||||
let ctx: E2EContext;
|
||||
|
||||
async function setup(): Promise<void> {
|
||||
ctx = await createE2ETestContext({ timeout: 180000 })
|
||||
ctx = await createE2ETestContext({ timeout: 180000 });
|
||||
}
|
||||
|
||||
async function cleanup(): Promise<void> {
|
||||
if (ctx) {
|
||||
await ctx.stop()
|
||||
await ctx.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function runProviderCase(input: {
|
||||
provider: 'claude' | 'codex' | 'opencode'
|
||||
mode: string
|
||||
model: string
|
||||
provider: "claude" | "codex" | "opencode";
|
||||
mode: string;
|
||||
model: string;
|
||||
}): Promise<void> {
|
||||
const result = await ctx.paseo(
|
||||
[
|
||||
'run',
|
||||
'--provider',
|
||||
"run",
|
||||
"--provider",
|
||||
input.provider,
|
||||
'--mode',
|
||||
"--mode",
|
||||
input.mode,
|
||||
'--model',
|
||||
"--model",
|
||||
input.model,
|
||||
'--output-schema',
|
||||
"--output-schema",
|
||||
schema,
|
||||
`Return valid JSON with a short summary for provider ${input.provider}.`,
|
||||
],
|
||||
{ timeout: 180000 }
|
||||
)
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
|
||||
assert.strictEqual(
|
||||
result.exitCode,
|
||||
0,
|
||||
`expected ${input.provider} run to succeed, got ${result.exitCode}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`
|
||||
)
|
||||
`expected ${input.provider} run to succeed, got ${result.exitCode}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
|
||||
);
|
||||
|
||||
const parsed = JSON.parse(result.stdout.trim()) as { summary?: unknown }
|
||||
const parsed = JSON.parse(result.stdout.trim()) as { summary?: unknown };
|
||||
assert.strictEqual(
|
||||
typeof parsed.summary,
|
||||
'string',
|
||||
`expected ${input.provider} output to contain string summary, got: ${result.stdout}`
|
||||
)
|
||||
assert(parsed.summary && parsed.summary.length > 0, 'summary must not be empty')
|
||||
"string",
|
||||
`expected ${input.provider} output to contain string summary, got: ${result.stdout}`,
|
||||
);
|
||||
assert(parsed.summary && parsed.summary.length > 0, "summary must not be empty");
|
||||
}
|
||||
|
||||
async function test_all_providers_return_structured_output(): Promise<void> {
|
||||
await runProviderCase({
|
||||
provider: 'claude',
|
||||
mode: 'bypassPermissions',
|
||||
model: 'haiku',
|
||||
})
|
||||
provider: "claude",
|
||||
mode: "bypassPermissions",
|
||||
model: "haiku",
|
||||
});
|
||||
await runProviderCase({
|
||||
provider: 'codex',
|
||||
mode: 'full-access',
|
||||
model: 'gpt-5.3-codex',
|
||||
})
|
||||
provider: "codex",
|
||||
mode: "full-access",
|
||||
model: "gpt-5.3-codex",
|
||||
});
|
||||
await runProviderCase({
|
||||
provider: 'opencode',
|
||||
mode: 'default',
|
||||
provider: "opencode",
|
||||
mode: "default",
|
||||
model: OPEN_CODE_STRUCTURED_MODEL,
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
async function test_schema_validation_is_enforced(): Promise<void> {
|
||||
const result = await ctx.paseo(
|
||||
[
|
||||
'run',
|
||||
'--provider',
|
||||
'claude',
|
||||
'--mode',
|
||||
'bypassPermissions',
|
||||
'--model',
|
||||
'haiku',
|
||||
'--output-schema',
|
||||
"run",
|
||||
"--provider",
|
||||
"claude",
|
||||
"--mode",
|
||||
"bypassPermissions",
|
||||
"--model",
|
||||
"haiku",
|
||||
"--output-schema",
|
||||
impossibleSchema,
|
||||
'Return exactly {"summary":"ok"} and nothing else.',
|
||||
],
|
||||
{ timeout: 180000 }
|
||||
)
|
||||
{ timeout: 180000 },
|
||||
);
|
||||
|
||||
assert.notStrictEqual(result.exitCode, 0, 'expected impossible schema to fail')
|
||||
const output = `${result.stdout}\n${result.stderr}`
|
||||
assert.notStrictEqual(result.exitCode, 0, "expected impossible schema to fail");
|
||||
const output = `${result.stdout}\n${result.stderr}`;
|
||||
assert(
|
||||
output.includes('OUTPUT_SCHEMA_FAILED'),
|
||||
`expected OUTPUT_SCHEMA_FAILED for impossible schema\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`
|
||||
)
|
||||
output.includes("OUTPUT_SCHEMA_FAILED"),
|
||||
`expected OUTPUT_SCHEMA_FAILED for impossible schema\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
try {
|
||||
await setup()
|
||||
await test_all_providers_return_structured_output()
|
||||
await test_schema_validation_is_enforced()
|
||||
await setup();
|
||||
await test_all_providers_return_structured_output();
|
||||
await test_schema_validation_is_enforced();
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await cleanup()
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
main();
|
||||
|
||||
Reference in New Issue
Block a user