Route real provider tests through OpenRouter

This commit is contained in:
Mohamed Boudra
2026-05-31 13:57:17 +07:00
parent 8cf80c2486
commit d1f37b5d72
45 changed files with 636 additions and 405 deletions
@@ -7,13 +7,19 @@ import { createOpencodeClient } from "@opencode-ai/sdk/v2/client";
import { AgentManager } from "./agent-manager.js"; import { AgentManager } from "./agent-manager.js";
import { AgentStorage } from "./agent-storage.js"; import { AgentStorage } from "./agent-storage.js";
import { createAllClients, shutdownProviders } from "./provider-registry.js"; import { shutdownProviders } from "./provider-registry.js";
import { generateAndApplyAgentMetadata } from "./agent-metadata-generator.js"; import { generateAndApplyAgentMetadata } from "./agent-metadata-generator.js";
import { isProviderAvailable } from "../daemon-e2e/agent-configs.js";
import { OpenCodeServerManager } from "./providers/opencode/server-manager.js"; import { OpenCodeServerManager } from "./providers/opencode/server-manager.js";
import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "../daemon-e2e/real-provider-test-config.js";
const CODEX_TEST_MODEL = "gpt-5.4-mini"; const CODEX_TEST_MODEL = getRealProviderConfig("codex").model;
const CODEX_TEST_THINKING_OPTION_ID = "low"; const CODEX_TEST_THINKING_OPTION_ID = getRealProviderConfig("codex").thinkingOptionId;
const CLAUDE_TEST_MODEL = getRealProviderConfig("claude").model;
const OPENCODE_TEST_MODEL = getRealProviderConfig("opencode").model;
function collectFilesRecursively(root: string, filter: (name: string) => boolean): Set<string> { function collectFilesRecursively(root: string, filter: (name: string) => boolean): Set<string> {
const results = new Set<string>(); const results = new Set<string>();
@@ -75,9 +81,9 @@ describe("agent metadata generation (real agents)", () => {
beforeAll(async () => { beforeAll(async () => {
[codexAvailable, claudeAvailable, opencodeAvailable] = await Promise.all([ [codexAvailable, claudeAvailable, opencodeAvailable] = await Promise.all([
isProviderAvailable("codex"), canRunRealProvider("codex"),
isProviderAvailable("claude"), canRunRealProvider("claude"),
isProviderAvailable("opencode"), canRunRealProvider("opencode"),
]); ]);
}); });
@@ -86,7 +92,7 @@ describe("agent metadata generation (real agents)", () => {
paseoHome = tmpCwd("metadata-paseo-home-"); paseoHome = tmpCwd("metadata-paseo-home-");
storage = new AgentStorage(path.join(paseoHome, "agents"), logger); storage = new AgentStorage(path.join(paseoHome, "agents"), logger);
manager = new AgentManager({ manager = new AgentManager({
clients: createAllClients(logger), clients: createRealProviderClients(["codex", "claude", "opencode"], logger),
registry: storage, registry: storage,
logger, logger,
}); });
@@ -106,7 +112,9 @@ describe("agent metadata generation (real agents)", () => {
{ {
provider: "codex", provider: "codex",
model: CODEX_TEST_MODEL, model: CODEX_TEST_MODEL,
thinkingOptionId: CODEX_TEST_THINKING_OPTION_ID, ...(CODEX_TEST_THINKING_OPTION_ID
? { thinkingOptionId: CODEX_TEST_THINKING_OPTION_ID }
: {}),
modeId: "auto", modeId: "auto",
cwd: cwd, cwd: cwd,
title: "Main Agent", title: "Main Agent",
@@ -144,7 +152,7 @@ describe("agent metadata generation (real agents)", () => {
const agent = await manager.createAgent( const agent = await manager.createAgent(
{ {
provider: "claude", provider: "claude",
model: "haiku", model: CLAUDE_TEST_MODEL,
thinkingOptionId: "on", thinkingOptionId: "on",
cwd: cwd, cwd: cwd,
title: "Main Claude Agent", title: "Main Claude Agent",
@@ -182,7 +190,7 @@ describe("agent metadata generation (real agents)", () => {
const agent = await manager.createAgent( const agent = await manager.createAgent(
{ {
provider: "opencode", provider: "opencode",
model: "opencode/gpt-5-nano", model: OPENCODE_TEST_MODEL,
modeId: "build", modeId: "build",
cwd: cwd, cwd: cwd,
title: "Main OpenCode Agent", title: "Main OpenCode Agent",
@@ -12,14 +12,17 @@ import { generateStructuredAgentResponse } from "./agent-response-loop.js";
import { AgentManager } from "./agent-manager.js"; import { AgentManager } from "./agent-manager.js";
import { AgentStorage } from "./agent-storage.js"; import { AgentStorage } from "./agent-storage.js";
import { createAgentMcpServer } from "./mcp-server.js"; import { createAgentMcpServer } from "./mcp-server.js";
import { createAllClients, shutdownProviders } from "./provider-registry.js"; import { shutdownProviders } from "./provider-registry.js";
import { isProviderAvailable } from "../daemon-e2e/agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "../daemon-e2e/real-provider-test-config.js";
import pino from "pino"; import pino from "pino";
const CODEX_TEST_MODEL = "gpt-5.4-mini"; const CODEX_TEST_MODEL = getRealProviderConfig("codex").model;
const CODEX_TEST_THINKING_OPTION_ID = "low"; const CODEX_TEST_THINKING_OPTION_ID = getRealProviderConfig("codex").thinkingOptionId;
const CLAUDE_TEST_MODEL = getRealProviderConfig("claude").model;
const hasOpenAICredentials = !!process.env.OPENAI_API_KEY;
interface AgentMcpServerHandle { interface AgentMcpServerHandle {
url: string; url: string;
@@ -154,8 +157,8 @@ describe("getStructuredAgentResponse (e2e)", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
beforeAll(async () => { beforeAll(async () => {
canRunCodex = !process.env.CI && hasOpenAICredentials; canRunCodex = await canRunRealProvider("codex");
canRunClaude = await isProviderAvailable("claude"); canRunClaude = await canRunRealProvider("claude");
if (!canRunCodex && !canRunClaude) { if (!canRunCodex && !canRunClaude) {
return; return;
} }
@@ -169,7 +172,7 @@ describe("getStructuredAgentResponse (e2e)", () => {
beforeEach(async () => { beforeEach(async () => {
cwd = mkdtempSync(path.join(tmpdir(), "agent-response-loop-")); cwd = mkdtempSync(path.join(tmpdir(), "agent-response-loop-"));
manager = new AgentManager({ manager = new AgentManager({
clients: createAllClients(logger), clients: createRealProviderClients(["codex", "claude"], logger),
logger, logger,
}); });
}); });
@@ -193,7 +196,9 @@ describe("getStructuredAgentResponse (e2e)", () => {
agentConfig: { agentConfig: {
provider: "codex", provider: "codex",
model: CODEX_TEST_MODEL, model: CODEX_TEST_MODEL,
thinkingOptionId: CODEX_TEST_THINKING_OPTION_ID, ...(CODEX_TEST_THINKING_OPTION_ID
? { thinkingOptionId: CODEX_TEST_THINKING_OPTION_ID }
: {}),
cwd, cwd,
title: "Structured Response Test", title: "Structured Response Test",
}, },
@@ -222,7 +227,7 @@ describe("getStructuredAgentResponse (e2e)", () => {
manager, manager,
agentConfig: { agentConfig: {
provider: "claude", provider: "claude",
model: "haiku", model: CLAUDE_TEST_MODEL,
thinkingOptionId: "on", thinkingOptionId: "on",
cwd, cwd,
title: "Claude Haiku Structured Test", title: "Claude Haiku Structured Test",
@@ -1,15 +1,17 @@
import { beforeAll, beforeEach, describe, expect, test } from "vitest"; import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import pino from "pino"; import pino from "pino";
import { isProviderAvailable } from "../../../daemon-e2e/agent-configs.js"; import {
import { ClaudeAgentClient } from "./agent.js"; canRunRealProvider,
createRealProviderClient,
} from "../../../daemon-e2e/real-provider-test-config.js";
// Real-Claude contract coverage: validates slash command shape from a live Claude CLI session. // Real-Claude contract coverage: validates slash command shape from a live Claude CLI session.
describe("claude agent commands contract (real)", () => { describe("claude agent commands contract (real)", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -19,9 +21,7 @@ describe("claude agent commands contract (real)", () => {
}); });
test("lists slash commands with the expected contract", async () => { test("lists slash commands with the expected contract", async () => {
const client = new ClaudeAgentClient({ const client = createRealProviderClient("claude", pino({ level: "silent" }));
logger: pino({ level: "silent" }),
});
const session = await client.createSession({ const session = await client.createSession({
provider: "claude", provider: "claude",
cwd: process.cwd(), cwd: process.cwd(),
@@ -19,15 +19,18 @@ import {
type AgentSession, type AgentSession,
type AgentStreamEvent, type AgentStreamEvent,
} from "../../agent-sdk-types.js"; } from "../../agent-sdk-types.js";
import { isProviderAvailable } from "../../../daemon-e2e/agent-configs.js"; import {
import { ClaudeAgentClient } from "./agent.js"; canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
} from "../../../daemon-e2e/real-provider-test-config.js";
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Helpers // Helpers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const client = new ClaudeAgentClient({ logger }); const client = createRealProviderClient("claude", logger);
function tmpCwd(prefix: string): string { function tmpCwd(prefix: string): string {
return mkdtempSync(path.join(tmpdir(), prefix)); return mkdtempSync(path.join(tmpdir(), prefix));
@@ -62,11 +65,10 @@ async function createSession(params?: {
}): Promise<{ cwd: string; session: AgentSession }> { }): Promise<{ cwd: string; session: AgentSession }> {
const cwd = tmpCwd(params?.cwdPrefix ?? "event-stream-integration-"); const cwd = tmpCwd(params?.cwdPrefix ?? "event-stream-integration-");
const session = await client.createSession({ const session = await client.createSession({
provider: "claude", ...getRealProviderConfig("claude"),
cwd, cwd,
title: "event-stream integration", title: "event-stream integration",
modeId: "acceptEdits", modeId: "acceptEdits",
model: "haiku",
}); });
return { cwd, session }; return { cwd, session };
} }
@@ -202,7 +204,7 @@ function assertInvariants(events: AgentStreamEvent[], foregroundTurnIds: string[
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -2,8 +2,10 @@ import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import pino from "pino"; import pino from "pino";
import type { AgentStreamEvent, AgentSession } from "../../agent-sdk-types.js"; import type { AgentStreamEvent, AgentSession } from "../../agent-sdk-types.js";
import { isProviderAvailable } from "../../../daemon-e2e/agent-configs.js"; import {
import { ClaudeAgentClient } from "./agent.js"; canRunRealProvider,
createRealProviderClient,
} from "../../../daemon-e2e/real-provider-test-config.js";
import { streamSession } from "../test-utils/session-stream-adapter.js"; import { streamSession } from "../test-utils/session-stream-adapter.js";
function isTerminalEvent(event: AgentStreamEvent): boolean { function isTerminalEvent(event: AgentStreamEvent): boolean {
@@ -29,7 +31,7 @@ describe("Claude max effort availability (real)", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -39,9 +41,7 @@ describe("Claude max effort availability (real)", () => {
}); });
test("surfaces the Claude stderr diagnostic when bypassPermissions + max effort is unavailable", async () => { test("surfaces the Claude stderr diagnostic when bypassPermissions + max effort is unavailable", async () => {
const client = new ClaudeAgentClient({ const client = createRealProviderClient("claude", pino({ level: "silent" }));
logger: pino({ level: "silent" }),
});
const session = await client.createSession({ const session = await client.createSession({
provider: "claude", provider: "claude",
cwd: process.cwd(), cwd: process.cwd(),
@@ -10,15 +10,19 @@ import type {
AgentStreamEvent, AgentStreamEvent,
ToolCallTimelineItem, ToolCallTimelineItem,
} from "../../agent-sdk-types.js"; } from "../../agent-sdk-types.js";
import { isProviderAvailable } from "../../../daemon-e2e/agent-configs.js"; import {
canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
getRealProviderRuntimeSettings,
} from "../../../daemon-e2e/real-provider-test-config.js";
import { findExecutable } from "../../../../utils/executable.js"; import { findExecutable } from "../../../../utils/executable.js";
import { withTimeout } from "../../../../utils/promise-timeout.js"; import { withTimeout } from "../../../../utils/promise-timeout.js";
import { ClaudeAgentClient } from "./agent.js";
import { claudeQuery } from "./query.js"; import { claudeQuery } from "./query.js";
import { streamSession } from "../test-utils/session-stream-adapter.js"; import { streamSession } from "../test-utils/session-stream-adapter.js";
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const client = new ClaudeAgentClient({ logger }); const client = createRealProviderClient("claude", logger);
function tmpCwd(prefix: string): string { function tmpCwd(prefix: string): string {
return mkdtempSync(path.join(tmpdir(), prefix)); return mkdtempSync(path.join(tmpdir(), prefix));
@@ -150,11 +154,10 @@ async function createSession(params?: {
}): Promise<{ cwd: string; session: AgentSession }> { }): Promise<{ cwd: string; session: AgentSession }> {
const cwd = tmpCwd(params?.cwdPrefix ?? "claude-agent-integration-"); const cwd = tmpCwd(params?.cwdPrefix ?? "claude-agent-integration-");
const session = await client.createSession({ const session = await client.createSession({
provider: "claude", ...getRealProviderConfig("claude"),
cwd, cwd,
title: params?.title ?? "ClaudeAgentSession integration", title: params?.title ?? "ClaudeAgentSession integration",
modeId: params?.modeId ?? "acceptEdits", modeId: params?.modeId ?? "acceptEdits",
model: "haiku",
}); });
return { cwd, session }; return { cwd, session };
} }
@@ -175,7 +178,7 @@ describe("ClaudeAgentSession integration", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -233,16 +236,19 @@ describe("ClaudeAgentSession integration", () => {
test("supportedModels returns the current abstract Claude SDK model shape", async () => { test("supportedModels returns the current abstract Claude SDK model shape", async () => {
const claudeBinary = await findExecutable("claude"); const claudeBinary = await findExecutable("claude");
if (!claudeBinary) throw new Error("claude binary required for this integration test"); if (!claudeBinary) throw new Error("claude binary required for this integration test");
const query = claudeQuery({ const query = claudeQuery(
prompt: createEmptyPrompt(), {
options: { prompt: createEmptyPrompt(),
cwd: process.cwd(), options: {
permissionMode: "plan", cwd: process.cwd(),
includePartialMessages: false, permissionMode: "plan",
settingSources: ["user", "project"], includePartialMessages: false,
pathToClaudeCodeExecutable: claudeBinary, settingSources: ["user", "project"],
pathToClaudeCodeExecutable: claudeBinary,
},
}, },
}); { runtimeSettings: getRealProviderRuntimeSettings("claude") },
);
try { try {
const models = await query.supportedModels(); const models = await query.supportedModels();
@@ -6,7 +6,10 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import { beforeAll, beforeEach, describe, expect, test } from "vitest"; import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import type { SDKMessage, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; import type { SDKMessage, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk";
import { isProviderAvailable } from "../../../daemon-e2e/agent-configs.js"; import {
canRunRealProvider,
getRealProviderRuntimeSettings,
} from "../../../daemon-e2e/real-provider-test-config.js";
import { findExecutable } from "../../../../utils/executable.js"; import { findExecutable } from "../../../../utils/executable.js";
import { claudeQuery } from "./query.js"; import { claudeQuery } from "./query.js";
@@ -89,7 +92,7 @@ describe("Claude SDK direct behavior", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -104,20 +107,23 @@ describe("Claude SDK direct behavior", () => {
const claudeBinary = await findExecutable("claude"); const claudeBinary = await findExecutable("claude");
// Use same options as the Claude provider // Use same options as the Claude provider
const q = claudeQuery({ const q = claudeQuery(
prompt: input, {
options: { prompt: input,
cwd, options: {
includePartialMessages: true, cwd,
permissionMode: "bypassPermissions", includePartialMessages: true,
...(claudeBinary ? { pathToClaudeCodeExecutable: claudeBinary } : {}), permissionMode: "bypassPermissions",
systemPrompt: { ...(claudeBinary ? { pathToClaudeCodeExecutable: claudeBinary } : {}),
type: "preset", systemPrompt: {
preset: "claude_code", type: "preset",
preset: "claude_code",
},
settingSources: ["user", "project"],
}, },
settingSources: ["user", "project"],
}, },
}); { runtimeSettings: getRealProviderRuntimeSettings("claude") },
);
try { try {
// Send first message // Send first message
@@ -8,12 +8,8 @@ import { once } from "node:events";
import { CodexAppServerAgentClient } from "./codex-app-server-agent.js"; import { CodexAppServerAgentClient } from "./codex-app-server-agent.js";
import { createTestLogger } from "../../../test-utils/test-logger.js"; import { createTestLogger } from "../../../test-utils/test-logger.js";
import { agentConfigs } from "../../daemon-e2e/agent-configs.js";
import type { AgentStreamEvent } from "../agent-sdk-types.js"; import type { AgentStreamEvent } from "../agent-sdk-types.js";
const CODEX_TEST_MODEL = agentConfigs.codex.model;
const CODEX_TEST_THINKING_OPTION_ID = agentConfigs.codex.thinkingOptionId;
function isCodexInstalled(): boolean { function isCodexInstalled(): boolean {
try { try {
const out = execFileSync("which", ["codex"], { encoding: "utf8" }).trim(); const out = execFileSync("which", ["codex"], { encoding: "utf8" }).trim();
@@ -217,35 +213,7 @@ function waitForEvent<TEvent extends AgentStreamEvent>(params: {
}); });
} }
describe("Codex app-server provider (e2e)", () => { describe("Codex app-server provider (local e2e)", () => {
test.runIf(isCodexInstalled())(
"lists models and runs a simple prompt",
async () => {
const client = new CodexAppServerAgentClient(createTestLogger());
const cwd = mkdtempSync(path.join(os.tmpdir(), "codex-app-server-e2e-"));
const models = await client.listModels({ cwd, force: false });
expect(models.some((m) => m.id.includes("gpt-5.1-codex"))).toBe(true);
const session = await client.createSession({
provider: "codex",
cwd,
modeId: "auto",
model: CODEX_TEST_MODEL,
thinkingOptionId: CODEX_TEST_THINKING_OPTION_ID,
});
try {
expect(session.features?.some((feature) => feature.id === "plan_mode")).toBe(true);
const result = await session.run("Say hello in one sentence.");
expect(result.finalText.length).toBeGreaterThan(0);
} finally {
await session.close();
rmSync(cwd, { recursive: true, force: true });
}
},
30000,
);
test.runIf(isCodexInstalled())( test.runIf(isCodexInstalled())(
"surfaces request_user_input from the app-server as question permissions and timeline tool calls", "surfaces request_user_input from the app-server as question permissions and timeline tool calls",
async () => { async () => {
@@ -0,0 +1,47 @@
import { mkdtempSync, rmSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import { createTestLogger } from "../../../test-utils/test-logger.js";
import {
canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
} from "../../daemon-e2e/real-provider-test-config.js";
describe("Codex app-server provider (real)", () => {
let canRun = false;
beforeAll(async () => {
canRun = await canRunRealProvider("codex");
});
beforeEach((context) => {
if (!canRun) {
context.skip();
}
});
test("lists models and runs a simple prompt", async () => {
const client = createRealProviderClient("codex", createTestLogger());
const cwd = mkdtempSync(path.join(os.tmpdir(), "codex-app-server-e2e-"));
const models = await client.listModels({ cwd, force: false });
expect(models.length).toBeGreaterThan(0);
const session = await client.createSession({
...getRealProviderConfig("codex"),
cwd,
modeId: "auto",
});
try {
expect(session.features?.some((feature) => feature.id === "plan_mode")).toBe(true);
const result = await session.run("Say hello in one sentence.");
expect(result.finalText.length).toBeGreaterThan(0);
} finally {
await session.close();
rmSync(cwd, { recursive: true, force: true });
}
}, 30_000);
});
@@ -3,9 +3,12 @@ import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import { CodexAppServerAgentClient } from "./codex-app-server-agent.js";
import { createTestLogger } from "../../../test-utils/test-logger.js"; import { createTestLogger } from "../../../test-utils/test-logger.js";
import { isProviderAvailable } from "../../daemon-e2e/agent-configs.js"; import {
canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
} from "../../daemon-e2e/real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "codex-plan-mode-real-")); return mkdtempSync(path.join(tmpdir(), "codex-plan-mode-real-"));
@@ -15,7 +18,7 @@ describe("Codex app-server provider (real) plan mode", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("codex"); canRun = await canRunRealProvider("codex");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -26,14 +29,13 @@ describe("Codex app-server provider (real) plan mode", () => {
test("maps gpt-5.4 markdown plans to a plan tool call instead of todo items", async () => { test("maps gpt-5.4 markdown plans to a plan tool call instead of todo items", async () => {
const cwd = tmpCwd(); const cwd = tmpCwd();
const client = new CodexAppServerAgentClient(createTestLogger()); const client = createRealProviderClient("codex", createTestLogger());
try { try {
const session = await client.createSession({ const session = await client.createSession({
provider: "codex", ...getRealProviderConfig("codex"),
cwd, cwd,
modeId: "auto", modeId: "auto",
model: "gpt-5.4",
thinkingOptionId: "medium", thinkingOptionId: "medium",
}); });
@@ -1,18 +1,21 @@
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
import pino from "pino"; import pino from "pino";
import { isCommandAvailable } from "../../../utils/executable.js";
import { OpenCodeAgentClient } from "./opencode-agent.js";
import { OpenCodeServerManager } from "./opencode/server-manager.js"; import { OpenCodeServerManager } from "./opencode/server-manager.js";
import {
canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
} from "../../daemon-e2e/real-provider-test-config.js";
const BIG_PICKLE_MODEL = "opencode/big-pickle"; const OPENCODE_REAL_TEST_MODEL = getRealProviderConfig("opencode").model;
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
describe("opencode agent commands contract (real)", () => { describe("opencode agent commands contract (real)", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isCommandAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -26,13 +29,11 @@ describe("opencode agent commands contract (real)", () => {
}); });
test("lists slash commands with the expected contract", async () => { test("lists slash commands with the expected contract", async () => {
expect(await isCommandAvailable("opencode")).toBe(true); const client = createRealProviderClient("opencode", logger);
const client = new OpenCodeAgentClient(logger);
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd: process.cwd(), cwd: process.cwd(),
model: BIG_PICKLE_MODEL, model: OPENCODE_REAL_TEST_MODEL,
modeId: "plan", modeId: "plan",
}); });
@@ -57,13 +58,11 @@ describe("opencode agent commands contract (real)", () => {
}, 60_000); }, 60_000);
test("executes a slash command without arguments", async () => { test("executes a slash command without arguments", async () => {
expect(await isCommandAvailable("opencode")).toBe(true); const client = createRealProviderClient("opencode", logger);
const client = new OpenCodeAgentClient(logger);
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd: process.cwd(), cwd: process.cwd(),
model: BIG_PICKLE_MODEL, model: OPENCODE_REAL_TEST_MODEL,
modeId: "plan", modeId: "plan",
}); });
@@ -6,21 +6,24 @@ import path from "node:path";
import { beforeAll, beforeEach, describe, expect, test } from "vitest"; import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import pino from "pino"; import pino from "pino";
import { isCommandAvailable } from "../../../utils/executable.js";
import { createDaemonTestContext, type DaemonTestContext } from "../../test-utils/index.js"; import { createDaemonTestContext, type DaemonTestContext } from "../../test-utils/index.js";
import { OpenCodeAgentClient } from "./opencode-agent.js";
import { OpenCodeServerManager } from "./opencode/server-manager.js"; import { OpenCodeServerManager } from "./opencode/server-manager.js";
import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "../../daemon-e2e/real-provider-test-config.js";
const COMMAND_NAME = "paseo-issue-903-big-pickle"; const COMMAND_NAME = "paseo-issue-903-big-pickle";
const COMMAND_FILE_NAME = `${COMMAND_NAME}.md`; const COMMAND_FILE_NAME = `${COMMAND_NAME}.md`;
const BIG_PICKLE_MODEL = "opencode/big-pickle"; const OPENCODE_REAL_TEST_MODEL = getRealProviderConfig("opencode").model;
const EXPECTED_RESPONSE = "PASEO_ISSUE_903_BIG_PICKLE_OK"; const EXPECTED_RESPONSE = "PASEO_ISSUE_903_BIG_PICKLE_OK";
describe("opencode custom command Big Pickle E2E (real)", () => { describe("opencode custom command Big Pickle E2E (real)", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isCommandAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -59,21 +62,19 @@ describe("opencode custom command Big Pickle E2E (real)", () => {
ctx = await createDaemonTestContext({ ctx = await createDaemonTestContext({
logger, logger,
agentClients: { agentClients: createRealProviderClients(["opencode"], logger),
opencode: new OpenCodeAgentClient(logger),
},
}); });
const agent = await ctx.client.createAgent({ const agent = await ctx.client.createAgent({
provider: "opencode", provider: "opencode",
cwd: projectDir, cwd: projectDir,
model: BIG_PICKLE_MODEL, model: OPENCODE_REAL_TEST_MODEL,
modeId: "plan", modeId: "plan",
title: "OpenCode issue 903 Big Pickle custom command", title: "OpenCode issue 903 Big Pickle custom command",
}); });
expect(agent.provider).toBe("opencode"); expect(agent.provider).toBe("opencode");
expect(agent.model).toBe(BIG_PICKLE_MODEL); expect(agent.model).toBe(OPENCODE_REAL_TEST_MODEL);
expect(agent.status).toBe("idle"); expect(agent.status).toBe("idle");
const commands = await ctx.client.listCommands(agent.id); const commands = await ctx.client.listCommands(agent.id);
@@ -2,8 +2,10 @@ import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import pino from "pino"; import pino from "pino";
import type { AgentStreamEvent } from "../agent-sdk-types.js"; import type { AgentStreamEvent } from "../agent-sdk-types.js";
import { isCommandAvailable } from "../../../utils/executable.js"; import {
import { OpenCodeAgentClient } from "./opencode-agent.js"; canRunRealProvider,
createRealProviderClient,
} from "../../daemon-e2e/real-provider-test-config.js";
import { streamSession } from "./test-utils/session-stream-adapter.js"; import { streamSession } from "./test-utils/session-stream-adapter.js";
function isTerminalEvent(event: AgentStreamEvent): boolean { function isTerminalEvent(event: AgentStreamEvent): boolean {
@@ -24,7 +26,7 @@ describe("opencode agent error handling (real)", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isCommandAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -34,7 +36,7 @@ describe("opencode agent error handling (real)", () => {
}); });
test("surfaces error for the exact opencode/<bad-model> path used by paseo run", async () => { test("surfaces error for the exact opencode/<bad-model> path used by paseo run", async () => {
const client = new OpenCodeAgentClient(pino({ level: "silent" })); const client = createRealProviderClient("opencode", pino({ level: "silent" }));
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd: process.cwd(), cwd: process.cwd(),
@@ -59,7 +61,7 @@ describe("opencode agent error handling (real)", () => {
}, 45_000); }, 45_000);
test("surfaces error for unknown provider model (fast path)", async () => { test("surfaces error for unknown provider model (fast path)", async () => {
const client = new OpenCodeAgentClient(pino({ level: "silent" })); const client = createRealProviderClient("opencode", pino({ level: "silent" }));
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd: process.cwd(), cwd: process.cwd(),
@@ -84,7 +86,7 @@ describe("opencode agent error handling (real)", () => {
}, 30_000); }, 30_000);
test("sequential sessions: second session works after first errors", async () => { test("sequential sessions: second session works after first errors", async () => {
const client = new OpenCodeAgentClient(pino({ level: "silent" })); const client = createRealProviderClient("opencode", pino({ level: "silent" }));
// Session 1: bogus model, will error quickly // Session 1: bogus model, will error quickly
const s1 = await client.createSession({ const s1 = await client.createSession({
@@ -121,7 +123,7 @@ describe("opencode agent error handling (real)", () => {
test("surfaces error for known provider with nonexistent model (retry path)", async () => { test("surfaces error for known provider with nonexistent model (retry path)", async () => {
// When the provider is recognized (anthropic) but the model doesn't exist, // When the provider is recognized (anthropic) but the model doesn't exist,
// OpenCode retries before surfacing the error. This must not hang. // OpenCode retries before surfacing the error. This must not hang.
const client = new OpenCodeAgentClient(pino({ level: "silent" })); const client = createRealProviderClient("opencode", pino({ level: "silent" }));
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd: process.cwd(), cwd: process.cwd(),
@@ -4,19 +4,22 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "./opencode-agent.js";
import { OpenCodeServerManager } from "./opencode/server-manager.js"; import { OpenCodeServerManager } from "./opencode/server-manager.js";
import { isProviderAvailable } from "../../daemon-e2e/agent-configs.js";
import type { AgentStreamEvent } from "../agent-sdk-types.js"; import type { AgentStreamEvent } from "../agent-sdk-types.js";
import {
canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
} from "../../daemon-e2e/real-provider-test-config.js";
const BIG_PICKLE_MODEL = "opencode/big-pickle"; const OPENCODE_REAL_TEST_MODEL = getRealProviderConfig("opencode").model;
describe("OpenCode assistant message", () => { describe("OpenCode assistant message", () => {
let canRun = false; let canRun = false;
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -29,15 +32,15 @@ describe("OpenCode assistant message", () => {
await OpenCodeServerManager.getInstance(logger).shutdown(); await OpenCodeServerManager.getInstance(logger).shutdown();
}); });
test("assistant_message appears in live stream with opencode/big-pickle", async () => { test("assistant_message appears in live stream", async () => {
const cwd = mkdtempSync(path.join(tmpdir(), "opencode-msg-")); const cwd = mkdtempSync(path.join(tmpdir(), "opencode-msg-"));
const client = new OpenCodeAgentClient(logger); const client = createRealProviderClient("opencode", logger);
try { try {
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd, cwd,
model: BIG_PICKLE_MODEL, model: OPENCODE_REAL_TEST_MODEL,
modeId: "build", modeId: "build",
}); });
@@ -53,13 +56,13 @@ describe("OpenCode assistant message", () => {
test("streamHistory returns assistant_message after a completed turn", async () => { test("streamHistory returns assistant_message after a completed turn", async () => {
const cwd = mkdtempSync(path.join(tmpdir(), "opencode-history-")); const cwd = mkdtempSync(path.join(tmpdir(), "opencode-history-"));
const client = new OpenCodeAgentClient(logger); const client = createRealProviderClient("opencode", logger);
try { try {
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd, cwd,
model: BIG_PICKLE_MODEL, model: OPENCODE_REAL_TEST_MODEL,
modeId: "build", modeId: "build",
}); });
@@ -4,19 +4,22 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "./opencode-agent.js";
import { OpenCodeServerManager } from "./opencode/server-manager.js"; import { OpenCodeServerManager } from "./opencode/server-manager.js";
import { isProviderAvailable } from "../../daemon-e2e/agent-configs.js";
import type { AgentStreamEvent } from "../agent-sdk-types.js"; import type { AgentStreamEvent } from "../agent-sdk-types.js";
import {
canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
} from "../../daemon-e2e/real-provider-test-config.js";
const BIG_PICKLE_MODEL = "opencode/big-pickle"; const OPENCODE_REAL_TEST_MODEL = getRealProviderConfig("opencode").model;
describe("OpenCode reasoning dedup", () => { describe("OpenCode reasoning dedup", () => {
let canRun = false; let canRun = false;
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -31,13 +34,13 @@ describe("OpenCode reasoning dedup", () => {
test("reasoning content is not duplicated as assistant_message", async () => { test("reasoning content is not duplicated as assistant_message", async () => {
const cwd = mkdtempSync(path.join(tmpdir(), "opencode-reasoning-dedup-")); const cwd = mkdtempSync(path.join(tmpdir(), "opencode-reasoning-dedup-"));
const client = new OpenCodeAgentClient(logger); const client = createRealProviderClient("opencode", logger);
try { try {
const session = await client.createSession({ const session = await client.createSession({
provider: "opencode", provider: "opencode",
cwd, cwd,
model: BIG_PICKLE_MODEL, model: OPENCODE_REAL_TEST_MODEL,
modeId: "build", modeId: "build",
}); });
@@ -2,12 +2,9 @@
* Shared agent configurations for e2e tests. * Shared agent configurations for e2e tests.
* Enables running the same tests against Claude, Codex, and OpenCode providers. * Enables running the same tests against Claude, Codex, and OpenCode providers.
*/ */
import { existsSync } from "node:fs"; import { dirname, resolve } from "node:path";
import { join, dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url"; import { fileURLToPath } from "node:url";
import { homedir } from "node:os";
import dotenv from "dotenv"; import dotenv from "dotenv";
import { isCommandAvailable } from "../../utils/executable.js";
const serverRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); const serverRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
dotenv.config({ path: resolve(serverRoot, ".env.test"), override: true }); dotenv.config({ path: resolve(serverRoot, ".env.test"), override: true });
@@ -63,7 +60,6 @@ export const agentConfigs = {
} as const satisfies Record<string, AgentTestConfig>; } as const satisfies Record<string, AgentTestConfig>;
export type AgentProvider = keyof typeof agentConfigs; export type AgentProvider = keyof typeof agentConfigs;
const providerAvailabilityCache = new Map<AgentProvider, Promise<boolean>>();
/** /**
* Get test config for creating an agent with full permissions (no prompts). * Get test config for creating an agent with full permissions (no prompts).
@@ -93,56 +89,7 @@ export function getAskModeConfig(provider: AgentProvider) {
}; };
} }
/**
* Whether a real provider can run in this environment.
* Checks binary availability AND credentials (env vars, OAuth tokens, auth files).
*
* Credentials are typically loaded from .env.test via the vitest setup file.
* This MUST be a function (not a const) so process.env is read at call time,
* after dotenv has injected the test credentials.
*/
export function isProviderAvailable(provider: AgentProvider): Promise<boolean> {
const cached = providerAvailabilityCache.get(provider);
if (cached) {
return cached;
}
const availability = (async (): Promise<boolean> => {
switch (provider) {
case "claude":
const hasClaudeEnvCredentials =
Boolean(process.env.CLAUDE_CODE_OAUTH_TOKEN) || Boolean(process.env.ANTHROPIC_API_KEY);
return (await isCommandAvailable("claude")) && (!process.env.CI || hasClaudeEnvCredentials);
case "codex":
return (
(await isCommandAvailable("codex")) &&
(existsSync(join(homedir(), ".codex", "auth.json")) ||
Boolean(process.env.OPENAI_API_KEY))
);
case "copilot":
return await isCommandAvailable("copilot");
case "opencode":
return await isCommandAvailable("opencode");
case "pi":
return (
(await isCommandAvailable(
process.env.PI_COMMAND ?? process.env.PI_ACP_PI_COMMAND ?? "pi",
)) &&
(Boolean(process.env.OPENAI_API_KEY) ||
Boolean(process.env.ANTHROPIC_API_KEY) ||
Boolean(process.env.OPENROUTER_API_KEY) ||
existsSync(join(homedir(), ".pi", "agent", "auth.json")))
);
default:
return false;
}
})();
providerAvailabilityCache.set(provider, availability);
return availability;
}
/** /**
* Helper to run a test for each provider. * Helper to run a test for each provider.
*/ */
export const allProviders: AgentProvider[] = ["claude", "codex", "copilot", "opencode", "pi"]; export const allProviders: AgentProvider[] = ["claude", "codex", "opencode", "pi"];
@@ -4,10 +4,13 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-claude-autonomous-simple-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-claude-autonomous-simple-"));
@@ -21,7 +24,7 @@ describe("daemon E2E (real claude) - autonomous wake simple", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -34,7 +37,7 @@ describe("daemon E2E (real claude) - autonomous wake simple", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -48,7 +51,7 @@ describe("daemon E2E (real claude) - autonomous wake simple", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-autonomous-simple-real", title: "claude-autonomous-simple-real",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
const autonomousWakeToken = `AUTONOMOUS_SIMPLE_${Date.now().toString(36)}`; const autonomousWakeToken = `AUTONOMOUS_SIMPLE_${Date.now().toString(36)}`;
@@ -4,10 +4,13 @@ import path from "node:path";
import pino from "pino"; import pino from "pino";
import { beforeAll, beforeEach, describe, expect, test } from "vitest"; import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import { WebSocket } from "ws"; import { WebSocket } from "ws";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-claude-autonomous-wake-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-claude-autonomous-wake-"));
@@ -427,7 +430,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -440,7 +443,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -454,7 +457,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-autonomous-abc-a", title: "claude-autonomous-abc-a",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
const autonomousWakeToken = `AUTONOMOUS_WAKE_${Date.now().toString(36)}`; const autonomousWakeToken = `AUTONOMOUS_WAKE_${Date.now().toString(36)}`;
@@ -513,7 +516,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -527,7 +530,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-autonomous-abc-b", title: "claude-autonomous-abc-b",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT); await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT);
@@ -586,7 +589,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -600,7 +603,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-autonomous-abc-c", title: "claude-autonomous-abc-c",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT); await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT);
@@ -652,7 +655,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -666,7 +669,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-autonomous-wake-real", title: "claude-autonomous-wake-real",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await client.sendMessage( await client.sendMessage(
@@ -710,7 +713,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -724,7 +727,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-autonomous-followup-real", title: "claude-autonomous-followup-real",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await client.sendMessage( await client.sendMessage(
@@ -755,7 +758,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const wsUrl = `ws://127.0.0.1:${daemon.port}/ws`; const wsUrl = `ws://127.0.0.1:${daemon.port}/ws`;
@@ -770,7 +773,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-hang-repro-real", title: "claude-hang-repro-real",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
for (let cycle = 0; cycle < 20; cycle += 1) { for (let cycle = 0; cycle < 20; cycle += 1) {
@@ -858,7 +861,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -872,7 +875,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-background-repro-real", title: "claude-background-repro-real",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT); await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT);
@@ -930,7 +933,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -944,7 +947,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-autonomous-race-stress-real", title: "claude-autonomous-race-stress-real",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
for (let cycle = 0; cycle < 20; cycle += 1) { for (let cycle = 0; cycle < 20; cycle += 1) {
@@ -1016,7 +1019,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -1030,7 +1033,7 @@ describe("daemon E2E (real claude) - autonomous wake from background task", () =
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-transcript-parity-race", title: "claude-transcript-parity-race",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT); await client.sendMessage(agent.id, BACKGROUND_TASK_SLEEP_5_PROMPT);
@@ -1,13 +1,16 @@
import { readdir, writeFile } from "node:fs/promises"; import { readdir, writeFile } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js";
import type { AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentTimelineItem } from "../agent/agent-sdk-types.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
import { import {
closeRewindSession, closeRewindSession,
fetchTimelineItems, fetchTimelineItems,
@@ -114,7 +117,7 @@ async function launchClaudeRewindSession(
const agent = await harness.client.createAgent({ const agent = await harness.client.createAgent({
cwd, cwd,
title, title,
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
return { agentId: agent.id, cwd }; return { agentId: agent.id, cwd };
@@ -212,12 +215,17 @@ function assertTimeline(items: AgentTimelineItem[], expectedKept: ClaudeTurnSpec
} }
describe("daemon E2E (real claude) - rewind", () => { describe("daemon E2E (real claude) - rewind", () => {
let canRun = false;
let harness: ClaudeRewindHarness; let harness: ClaudeRewindHarness;
beforeAll(async () => { beforeAll(async () => {
canRun = await canRunRealProvider("claude");
if (!canRun) {
return;
}
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ const client = new DaemonClient({
@@ -235,9 +243,16 @@ describe("daemon E2E (real claude) - rewind", () => {
await harness?.daemon.close().catch(() => undefined); await harness?.daemon.close().catch(() => undefined);
}); });
beforeEach((context) => {
if (!canRun) {
context.skip();
}
});
test.each(MATRIX)( test.each(MATRIX)(
"$name", "$name",
async (scenario) => { async (scenario) => {
if (!harness) throw new Error("Claude rewind harness was not initialized");
const session = await launchClaudeRewindSession( const session = await launchClaudeRewindSession(
harness, harness,
`claude-rewind-${scenario.mode}-${scenario.turnCount}-${scenario.rewindTurn}`, `claude-rewind-${scenario.mode}-${scenario.turnCount}-${scenario.rewindTurn}`,
@@ -5,10 +5,13 @@ import path from "node:path";
import pino from "pino"; import pino from "pino";
import type { AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentTimelineItem } from "../agent/agent-sdk-types.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-claude-thinking-memory-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-claude-thinking-memory-"));
@@ -48,7 +51,7 @@ describe("daemon E2E (real claude) - thinking effort memory", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -61,7 +64,7 @@ describe("daemon E2E (real claude) - thinking effort memory", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -81,7 +84,7 @@ describe("daemon E2E (real claude) - thinking effort memory", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "claude-thinking-memory-real", title: "claude-thinking-memory-real",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
model: model.id, model: model.id,
}); });
@@ -4,11 +4,14 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { CodexAppServerAgentClient } from "../agent/providers/codex-app-server-agent.js";
import { resolveBinaryVersion } from "../agent/providers/diagnostic-utils.js"; import { resolveBinaryVersion } from "../agent/providers/diagnostic-utils.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-codex-goal-")); return mkdtempSync(path.join(tmpdir(), "daemon-codex-goal-"));
@@ -89,7 +92,7 @@ describe("daemon E2E (real codex) - /goal command", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
if (!(await isProviderAvailable("codex"))) { if (!(await canRunRealProvider("codex"))) {
return; return;
} }
const versionOutput = await resolveBinaryVersion("codex"); const versionOutput = await resolveBinaryVersion("codex");
@@ -106,7 +109,7 @@ describe("daemon E2E (real codex) - /goal command", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { codex: new CodexAppServerAgentClient(logger) }, agentClients: createRealProviderClients(["codex"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -118,7 +121,7 @@ describe("daemon E2E (real codex) - /goal command", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "codex-goal-set", title: "codex-goal-set",
...getFullAccessConfig("codex"), ...getRealProviderConfig("codex"),
}); });
const sentinel = "TX9Q4Z"; const sentinel = "TX9Q4Z";
@@ -160,7 +163,7 @@ describe("daemon E2E (real codex) - /goal command", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { codex: new CodexAppServerAgentClient(logger) }, agentClients: createRealProviderClients(["codex"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -172,7 +175,7 @@ describe("daemon E2E (real codex) - /goal command", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "codex-goal-mid-turn", title: "codex-goal-mid-turn",
...getFullAccessConfig("codex"), ...getRealProviderConfig("codex"),
}); });
// Set an initial goal so pause has something to act on. // Set an initial goal so pause has something to act on.
@@ -237,7 +240,7 @@ describe("daemon E2E (real codex) - /goal command", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { codex: new CodexAppServerAgentClient(logger) }, agentClients: createRealProviderClients(["codex"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -249,7 +252,7 @@ describe("daemon E2E (real codex) - /goal command", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "codex-goal-clear", title: "codex-goal-clear",
...getFullAccessConfig("codex"), ...getRealProviderConfig("codex"),
}); });
await client.sendMessage(agent.id, "/goal something to be cleared"); await client.sendMessage(agent.id, "/goal something to be cleared");
@@ -1,13 +1,16 @@
import { readdir, writeFile } from "node:fs/promises"; import { readdir, writeFile } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
import { CodexAppServerAgentClient } from "../agent/providers/codex-app-server-agent.js";
import type { AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentTimelineItem } from "../agent/agent-sdk-types.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
import { import {
closeRewindSession, closeRewindSession,
fetchTimelineItems, fetchTimelineItems,
@@ -84,7 +87,7 @@ async function launchCodexRewindSession(
const agent = await harness.client.createAgent({ const agent = await harness.client.createAgent({
cwd, cwd,
title, title,
...getFullAccessConfig("codex"), ...getRealProviderConfig("codex"),
}); });
return { agentId: agent.id, cwd, scratchPath }; return { agentId: agent.id, cwd, scratchPath };
@@ -142,12 +145,17 @@ async function askCodexToEditFile(
} }
describe("daemon E2E (real codex) - rewind", () => { describe("daemon E2E (real codex) - rewind", () => {
let canRun = false;
let harness: CodexRewindHarness; let harness: CodexRewindHarness;
beforeAll(async () => { beforeAll(async () => {
canRun = await canRunRealProvider("codex");
if (!canRun) {
return;
}
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { codex: new CodexAppServerAgentClient(logger) }, agentClients: createRealProviderClients(["codex"], logger),
logger, logger,
}); });
const client = new DaemonClient({ const client = new DaemonClient({
@@ -165,7 +173,14 @@ describe("daemon E2E (real codex) - rewind", () => {
await harness?.daemon.close().catch(() => undefined); await harness?.daemon.close().catch(() => undefined);
}); });
beforeEach((context) => {
if (!canRun) {
context.skip();
}
});
test("rewinds a real Codex conversation after a single edit turn while leaving the file edit on disk", async () => { test("rewinds a real Codex conversation after a single edit turn while leaving the file edit on disk", async () => {
if (!harness) throw new Error("Codex rewind harness was not initialized");
const session = await launchCodexRewindSession(harness, "codex-rewind-single-edit-real"); const session = await launchCodexRewindSession(harness, "codex-rewind-single-edit-real");
try { try {
@@ -6,8 +6,11 @@ import pino from "pino";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js"; import {
import { isProviderAvailable } from "./agent-configs.js"; canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-claude-model-init-")); return mkdtempSync(path.join(tmpdir(), "daemon-claude-model-init-"));
@@ -17,7 +20,7 @@ describe("daemon E2E (real claude) - model resolution on init", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -30,7 +33,7 @@ describe("daemon E2E (real claude) - model resolution on init", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -45,9 +48,8 @@ describe("daemon E2E (real claude) - model resolution on init", () => {
const catalogModelIds = new Set(modelsResult.models.map((m) => m.id)); const catalogModelIds = new Set(modelsResult.models.map((m) => m.id));
const agent = await client.createAgent({ const agent = await client.createAgent({
provider: "claude", ...getRealProviderConfig("claude"),
cwd, cwd,
model: "haiku",
title: "model-init-test", title: "model-init-test",
}); });
@@ -6,8 +6,7 @@ import pino from "pino";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js"; import { canRunRealProvider, createRealProviderClients } from "./real-provider-test-config.js";
import { isProviderAvailable } from "./agent-configs.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-claude-runtime-model-reconcile-")); return mkdtempSync(path.join(tmpdir(), "daemon-claude-runtime-model-reconcile-"));
@@ -17,7 +16,7 @@ describe("daemon E2E (real claude) - runtime model reconciliation", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -30,7 +29,7 @@ describe("daemon E2E (real claude) - runtime model reconciliation", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -4,10 +4,9 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { isProviderAvailable } from "./agent-configs.js"; import { canRunRealProvider, createRealProviderClients } from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-custom-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-custom-"));
@@ -19,7 +18,7 @@ async function createHarness(): Promise<{
}> { }> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -32,7 +31,7 @@ describe("daemon E2E (real opencode) - custom agent discovery", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -5,12 +5,11 @@ import path from "node:path";
import pino from "pino"; import pino from "pino";
import { beforeAll, beforeEach, describe, expect, test } from "vitest"; import { beforeAll, beforeEach, describe, expect, test } from "vitest";
import type { PersistedAgentDescriptor } from "../agent/agent-sdk-types.js"; import type { AgentClient, PersistedAgentDescriptor } from "../agent/agent-sdk-types.js";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { OpenCodeServerManager } from "../agent/providers/opencode/server-manager.js"; import { OpenCodeServerManager } from "../agent/providers/opencode/server-manager.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { isProviderAvailable } from "./agent-configs.js"; import { canRunRealProvider, createRealProviderClient } from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
const dir = mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-draft-features-")); const dir = mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-draft-features-"));
@@ -18,7 +17,7 @@ function tmpCwd(): string {
} }
async function withConnectedOpenCodeDaemon( async function withConnectedOpenCodeDaemon(
provider: OpenCodeAgentClient, provider: AgentClient,
run: (context: { client: DaemonClient }) => Promise<void>, run: (context: { client: DaemonClient }) => Promise<void>,
): Promise<void> { ): Promise<void> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
@@ -41,7 +40,7 @@ async function withConnectedOpenCodeDaemon(
} }
async function deletePersistedSessions( async function deletePersistedSessions(
provider: OpenCodeAgentClient, provider: AgentClient,
sessions: ReadonlyArray<PersistedAgentDescriptor>, sessions: ReadonlyArray<PersistedAgentDescriptor>,
): Promise<void> { ): Promise<void> {
await Promise.all( await Promise.all(
@@ -56,7 +55,7 @@ describe("daemon E2E (real opencode) - draft feature discovery", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -67,12 +66,12 @@ describe("daemon E2E (real opencode) - draft feature discovery", () => {
test("listing draft features does not leave an OpenCode provider session behind", async () => { test("listing draft features does not leave an OpenCode provider session behind", async () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const provider = new OpenCodeAgentClient(logger); const provider = createRealProviderClient("opencode", logger);
const cwd = tmpCwd(); const cwd = tmpCwd();
let after: PersistedAgentDescriptor[] = []; let after: PersistedAgentDescriptor[] = [];
try { try {
expect(await provider.listPersistedAgents({ cwd })).toEqual([]); expect(await provider.listPersistedAgents?.({ cwd })).toEqual([]);
await withConnectedOpenCodeDaemon(provider, async ({ client }) => { await withConnectedOpenCodeDaemon(provider, async ({ client }) => {
const response = await client.listProviderFeatures({ const response = await client.listProviderFeatures({
@@ -85,7 +84,7 @@ describe("daemon E2E (real opencode) - draft feature discovery", () => {
expect(response.features ?? []).toEqual([]); expect(response.features ?? []).toEqual([]);
}); });
after = await provider.listPersistedAgents({ cwd }); after = (await provider.listPersistedAgents?.({ cwd })) ?? [];
} finally { } finally {
await deletePersistedSessions(provider, after); await deletePersistedSessions(provider, after);
rmSync(cwd, { recursive: true, force: true }); rmSync(cwd, { recursive: true, force: true });
@@ -97,12 +96,12 @@ describe("daemon E2E (real opencode) - draft feature discovery", () => {
test("listing draft commands does not leave an OpenCode provider session behind", async () => { test("listing draft commands does not leave an OpenCode provider session behind", async () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const provider = new OpenCodeAgentClient(logger); const provider = createRealProviderClient("opencode", logger);
const cwd = tmpCwd(); const cwd = tmpCwd();
let after: PersistedAgentDescriptor[] = []; let after: PersistedAgentDescriptor[] = [];
try { try {
expect(await provider.listPersistedAgents({ cwd })).toEqual([]); expect(await provider.listPersistedAgents?.({ cwd })).toEqual([]);
await withConnectedOpenCodeDaemon(provider, async ({ client }) => { await withConnectedOpenCodeDaemon(provider, async ({ client }) => {
const response = await client.listCommands("draft-opencode-agent", { const response = await client.listCommands("draft-opencode-agent", {
@@ -116,7 +115,7 @@ describe("daemon E2E (real opencode) - draft feature discovery", () => {
expect(response.commands.length).toBeGreaterThan(0); expect(response.commands.length).toBeGreaterThan(0);
}); });
after = await provider.listPersistedAgents({ cwd }); after = (await provider.listPersistedAgents?.({ cwd })) ?? [];
} finally { } finally {
await deletePersistedSessions(provider, after); await deletePersistedSessions(provider, after);
rmSync(cwd, { recursive: true, force: true }); rmSync(cwd, { recursive: true, force: true });
@@ -6,13 +6,16 @@ import path from "node:path";
import pino from "pino"; import pino from "pino";
import type { AgentPersistenceHandle, AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentPersistenceHandle, AgentTimelineItem } from "../agent/agent-sdk-types.js";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
import type { FetchRecentProviderSessionEntry } from "../../client/daemon-client.js"; import type { FetchRecentProviderSessionEntry } from "../../client/daemon-client.js";
const OPENCODE_REAL_TEST_MODEL = "opencode/big-pickle"; const OPENCODE_REAL_TEST_MODEL = getRealProviderConfig("opencode").model;
const OPENCODE_REAL_TEST_TIMEOUT_MS = 180_000; const OPENCODE_REAL_TEST_TIMEOUT_MS = 180_000;
function tmpCwd(): string { function tmpCwd(): string {
@@ -25,7 +28,7 @@ async function withConnectedOpenCodeDaemon(
): Promise<void> { ): Promise<void> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -112,7 +115,7 @@ describe("daemon E2E (real opencode) - persisted import resume", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -4,12 +4,15 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
const OPENCODE_REAL_TEST_MODEL = "opencode/big-pickle"; const OPENCODE_REAL_TEST_MODEL = getRealProviderConfig("opencode").model;
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-init-prompt-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-init-prompt-"));
@@ -21,7 +24,7 @@ async function createHarness(): Promise<{
}> { }> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -34,7 +37,7 @@ describe("daemon E2E (real opencode) - initial prompt wait", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -4,10 +4,9 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { isProviderAvailable } from "./agent-configs.js"; import { canRunRealProvider, createRealProviderClients } from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-invalid-mode-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-invalid-mode-"));
@@ -19,7 +18,7 @@ async function createHarness(): Promise<{
}> { }> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -32,7 +31,7 @@ describe("daemon E2E (real opencode) - invalid mode handling", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -4,10 +4,9 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { isProviderAvailable } from "./agent-configs.js"; import { canRunRealProvider, createRealProviderClients } from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-invalid-model-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-invalid-model-"));
@@ -19,7 +18,7 @@ async function createHarness(): Promise<{
}> { }> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -32,7 +31,7 @@ describe("daemon E2E (real opencode) - invalid model handling", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -4,10 +4,9 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { isProviderAvailable } from "./agent-configs.js"; import { canRunRealProvider, createRealProviderClients } from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-opencode-"));
@@ -19,7 +18,7 @@ async function createHarness(): Promise<{
}> { }> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -32,7 +31,7 @@ describe("daemon E2E (real opencode) - plan mode and clarifying questions", () =
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -2,14 +2,17 @@ import { execFileSync } from "node:child_process";
import { readdir, writeFile } from "node:fs/promises"; import { readdir, writeFile } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import type { AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentTimelineItem } from "../agent/agent-sdk-types.js";
import type { AgentLifecycleStatus } from "@getpaseo/protocol/agent-lifecycle"; import type { AgentLifecycleStatus } from "@getpaseo/protocol/agent-lifecycle";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
import { import {
closeRewindSession, closeRewindSession,
fetchTimelineItems, fetchTimelineItems,
@@ -56,7 +59,7 @@ async function launchOpenCodeRewindSession(
const agent = await harness.client.createAgent({ const agent = await harness.client.createAgent({
cwd, cwd,
title, title,
...getFullAccessConfig("opencode"), ...getRealProviderConfig("opencode"),
}); });
return { agentId: agent.id, cwd, scratchPath }; return { agentId: agent.id, cwd, scratchPath };
@@ -212,12 +215,17 @@ async function rewindOpenCode(
} }
describe("daemon E2E (real opencode) - rewind", () => { describe("daemon E2E (real opencode) - rewind", () => {
let canRun = false;
let harness: OpenCodeRewindHarness; let harness: OpenCodeRewindHarness;
beforeAll(async () => { beforeAll(async () => {
canRun = await canRunRealProvider("opencode");
if (!canRun) {
return;
}
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ const client = new DaemonClient({
@@ -235,7 +243,14 @@ describe("daemon E2E (real opencode) - rewind", () => {
await harness?.daemon.close().catch(() => undefined); await harness?.daemon.close().catch(() => undefined);
}); });
beforeEach((context) => {
if (!canRun) {
context.skip();
}
});
test("rewinds conversation and files after a single real OpenCode edit turn", async () => { test("rewinds conversation and files after a single real OpenCode edit turn", async () => {
if (!harness) throw new Error("OpenCode rewind harness was not initialized");
const session = await launchOpenCodeRewindSession(harness, "opencode-rewind-single-edit-real"); const session = await launchOpenCodeRewindSession(harness, "opencode-rewind-single-edit-real");
try { try {
@@ -4,11 +4,10 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient, type WaitForFinishResult } from "../test-utils/daemon-client.js"; import { DaemonClient, type WaitForFinishResult } from "../test-utils/daemon-client.js";
import { createMessageCollector } from "../test-utils/message-collector.js"; import { createMessageCollector } from "../test-utils/message-collector.js";
import { isProviderAvailable } from "./agent-configs.js"; import { canRunRealProvider, createRealProviderClients } from "./real-provider-test-config.js";
import type { AgentPermissionRequest } from "../agent/agent-sdk-types.js"; import type { AgentPermissionRequest } from "../agent/agent-sdk-types.js";
import type { SessionOutboundMessage } from "../messages.js"; import type { SessionOutboundMessage } from "../messages.js";
@@ -252,7 +251,7 @@ async function createHarness(): Promise<{
}> { }> {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { opencode: new OpenCodeAgentClient(logger) }, agentClients: createRealProviderClients(["opencode"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -265,7 +264,7 @@ describe("daemon E2E (real opencode) - send while working and interrupt", () =>
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("opencode"); canRun = await canRunRealProvider("opencode");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -1,11 +1,11 @@
import { readdir } from "node:fs/promises"; import { readdir } from "node:fs/promises";
import pino from "pino"; import pino from "pino";
import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { afterAll, beforeAll, beforeEach, describe, expect, test } from "vitest";
import { PiRpcAgentClient } from "../agent/providers/pi/agent.js";
import type { AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentTimelineItem } from "../agent/agent-sdk-types.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { canRunRealProvider, createRealProviderClients } from "./real-provider-test-config.js";
import { import {
closeRewindSession, closeRewindSession,
fetchTimelineItems, fetchTimelineItems,
@@ -103,12 +103,17 @@ async function askPi(
} }
describe("daemon E2E (real pi) - rewind", () => { describe("daemon E2E (real pi) - rewind", () => {
let canRun = false;
let harness: PiRewindHarness; let harness: PiRewindHarness;
beforeAll(async () => { beforeAll(async () => {
canRun = await canRunRealProvider("pi");
if (!canRun) {
return;
}
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { pi: new PiRpcAgentClient({ logger }) }, agentClients: createRealProviderClients(["pi"], logger),
logger, logger,
}); });
const client = new DaemonClient({ const client = new DaemonClient({
@@ -126,7 +131,14 @@ describe("daemon E2E (real pi) - rewind", () => {
await harness?.daemon.close().catch(() => undefined); await harness?.daemon.close().catch(() => undefined);
}); });
beforeEach((context) => {
if (!canRun) {
context.skip();
}
});
test("rewinds a real Pi conversation after a single turn and keeps the session usable", async () => { test("rewinds a real Pi conversation after a single turn and keeps the session usable", async () => {
if (!harness) throw new Error("Pi rewind harness was not initialized");
const session = await launchPiRewindSession(harness, "pi-rewind-single-conversation-real"); const session = await launchPiRewindSession(harness, "pi-rewind-single-conversation-real");
try { try {
@@ -6,19 +6,24 @@ import { beforeAll, beforeEach, expect, test } from "vitest";
import pino from "pino"; import pino from "pino";
import type { import type {
AgentClient,
AgentPersistenceHandle, AgentPersistenceHandle,
AgentStreamEvent, AgentStreamEvent,
AgentTimelineItem, AgentTimelineItem,
} from "../agent/agent-sdk-types.js"; } from "../agent/agent-sdk-types.js";
import { PiRpcAgentClient } from "../agent/providers/pi/agent.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClient,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
process.env.PASEO_SUPERVISED = "0"; process.env.PASEO_SUPERVISED = "0";
const PI_TEST_TIMEOUT_MS = 240_000; const PI_TEST_TIMEOUT_MS = 240_000;
const PI_REAL_TEST_MODEL = "openrouter/google/gemini-2.5-flash-lite"; const PI_REAL_TEST_MODEL = getRealProviderConfig("pi").model;
type ToolCallItem = Extract<AgentTimelineItem, { type: "tool_call" }>; type ToolCallItem = Extract<AgentTimelineItem, { type: "tool_call" }>;
@@ -26,14 +31,14 @@ function tmpCwd(prefix = "daemon-real-pi-"): string {
return mkdtempSync(path.join(tmpdir(), prefix)); return mkdtempSync(path.join(tmpdir(), prefix));
} }
function createPiClient(): PiRpcAgentClient { function createPiClient(): AgentClient {
return new PiRpcAgentClient({ logger: pino({ level: "silent" }) }); return createRealProviderClient("pi", pino({ level: "silent" }));
} }
function createPiToolDaemon() { function createPiToolDaemon() {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
return createTestPaseoDaemon({ return createTestPaseoDaemon({
agentClients: { pi: new PiRpcAgentClient({ logger }) }, agentClients: createRealProviderClients(["pi"], logger),
logger, logger,
}); });
} }
@@ -87,7 +92,7 @@ async function withConnectedPiDaemon(
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("pi"); canRun = await canRunRealProvider("pi");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -0,0 +1,170 @@
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import type { Logger } from "pino";
import type { AgentClient, AgentProvider, AgentSessionConfig } from "../agent/agent-sdk-types.js";
import type { ProviderRuntimeSettings } from "../agent/provider-launch-config.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js";
import { CodexAppServerAgentClient } from "../agent/providers/codex-app-server-agent.js";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { PiRpcAgentClient } from "../agent/providers/pi/agent.js";
import { isCommandAvailable } from "../../utils/executable.js";
export const realProviders = ["claude", "codex", "opencode", "pi"] as const;
export type RealProvider = (typeof realProviders)[number];
export type RealProviderConfig = Pick<
AgentSessionConfig,
"provider" | "model" | "modeId" | "thinkingOptionId"
>;
const OPENROUTER_BASE_URL = "https://openrouter.ai/api";
const OPENROUTER_OPENAI_BASE_URL = "https://openrouter.ai/api";
const CLAUDE_REAL_TEST_MODEL = "haiku";
const CODEX_REAL_TEST_MODEL = "~openai/gpt-latest";
const OPENCODE_REAL_TEST_MODEL = "openrouter/google/gemini-2.5-flash-lite";
const PI_REAL_TEST_MODEL = "openrouter/google/gemini-2.5-flash-lite";
const availabilityCache = new Map<RealProvider, Promise<boolean>>();
export function getRealProviderConfig(provider: RealProvider): RealProviderConfig {
switch (provider) {
case "claude":
return {
provider,
model: CLAUDE_REAL_TEST_MODEL,
modeId: "bypassPermissions",
};
case "codex":
return {
provider,
model: CODEX_REAL_TEST_MODEL,
thinkingOptionId: "low",
modeId: "full-access",
};
case "opencode":
return {
provider,
model: OPENCODE_REAL_TEST_MODEL,
modeId: "build",
};
case "pi":
return {
provider,
model: PI_REAL_TEST_MODEL,
thinkingOptionId: "medium",
};
}
}
export function getRealProviderRuntimeSettings(provider: RealProvider): ProviderRuntimeSettings {
const apiKey = getOpenRouterApiKey();
switch (provider) {
case "claude":
return {
env: {
OPENROUTER_API_KEY: apiKey,
ANTHROPIC_BASE_URL: OPENROUTER_BASE_URL,
ANTHROPIC_AUTH_TOKEN: apiKey,
ANTHROPIC_API_KEY: "",
ANTHROPIC_DEFAULT_HAIKU_MODEL: "~anthropic/claude-haiku-latest",
ANTHROPIC_DEFAULT_SONNET_MODEL: "~anthropic/claude-sonnet-latest",
ANTHROPIC_DEFAULT_OPUS_MODEL: "~anthropic/claude-opus-latest",
CLAUDE_CODE_SUBAGENT_MODEL: "~anthropic/claude-haiku-latest",
},
};
case "codex":
return {
env: {
OPENROUTER_API_KEY: apiKey,
OPENAI_API_KEY: apiKey,
OPENAI_BASE_URL: OPENROUTER_OPENAI_BASE_URL,
},
};
case "opencode": {
const root = mkdtempSync(path.join(tmpdir(), "paseo-real-opencode-"));
return {
env: {
OPENROUTER_API_KEY: apiKey,
OPENCODE_DISABLE_AUTO_UPDATE: "1",
XDG_CONFIG_HOME: path.join(root, "config"),
XDG_DATA_HOME: path.join(root, "data"),
XDG_CACHE_HOME: path.join(root, "cache"),
},
};
}
case "pi":
return {
env: {
OPENROUTER_API_KEY: apiKey,
},
};
}
}
export function createRealProviderClient(provider: RealProvider, logger: Logger): AgentClient {
const runtimeSettings = getRealProviderRuntimeSettings(provider);
switch (provider) {
case "claude":
return new ClaudeAgentClient({ logger, runtimeSettings });
case "codex":
return new CodexAppServerAgentClient(logger, runtimeSettings, {
customProvider: {
id: "codex-openrouter",
label: "Codex OpenRouter",
extends: "codex",
},
});
case "opencode":
return new OpenCodeAgentClient(logger, runtimeSettings);
case "pi":
return new PiRpcAgentClient({ logger, runtimeSettings });
}
}
export function createRealProviderClients(
providers: readonly RealProvider[],
logger: Logger,
): Partial<Record<AgentProvider, AgentClient>> {
return Object.fromEntries(
providers.map((provider) => [provider, createRealProviderClient(provider, logger)]),
);
}
export function canRunRealProvider(provider: RealProvider): Promise<boolean> {
const cached = availabilityCache.get(provider);
if (cached) {
return cached;
}
const availability = (async () => {
if (!getOpenRouterApiKeyOrNull()) {
return false;
}
return await isCommandAvailable(getProviderBinary(provider));
})();
availabilityCache.set(provider, availability);
return availability;
}
function getOpenRouterApiKey(): string {
const apiKey = getOpenRouterApiKeyOrNull();
if (!apiKey) {
throw new Error("OPENROUTER_API_KEY is required for real provider tests");
}
return apiKey;
}
function getOpenRouterApiKeyOrNull(): string | null {
const value = process.env.OPENROUTER_API_KEY?.trim();
return value && value.length > 0 ? value : null;
}
function getProviderBinary(provider: RealProvider): string {
if (provider === "pi") {
return process.env.PI_COMMAND ?? process.env.PI_ACP_PI_COMMAND ?? "pi";
}
return provider;
}
@@ -6,8 +6,11 @@ import pino from "pino";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js"; import {
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-rewind-dedupe-real-claude-")); return mkdtempSync(path.join(tmpdir(), "daemon-rewind-dedupe-real-claude-"));
@@ -17,7 +20,7 @@ describe("daemon E2E (real claude) - rewind user message dedupe", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -30,7 +33,7 @@ describe("daemon E2E (real claude) - rewind user message dedupe", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -44,7 +47,7 @@ describe("daemon E2E (real claude) - rewind user message dedupe", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "rewind-user-message-dedupe-real-claude", title: "rewind-user-message-dedupe-real-claude",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await client.sendMessage(agent.id, "Reply with exactly: READY"); await client.sendMessage(agent.id, "Reply with exactly: READY");
@@ -6,8 +6,11 @@ import pino from "pino";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js"; import {
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
import { createMessageCollector } from "../test-utils/message-collector.js"; import { createMessageCollector } from "../test-utils/message-collector.js";
import type { AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentTimelineItem } from "../agent/agent-sdk-types.js";
import type { SessionOutboundMessage } from "../messages.js"; import type { SessionOutboundMessage } from "../messages.js";
@@ -171,7 +174,7 @@ describe("daemon E2E (real claude) - send message during tool call", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -184,7 +187,7 @@ describe("daemon E2E (real claude) - send message during tool call", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
@@ -197,7 +200,7 @@ describe("daemon E2E (real claude) - send message during tool call", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "tool-interrupt-repro", title: "tool-interrupt-repro",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
const collector = createMessageCollector(client); const collector = createMessageCollector(client);
@@ -4,13 +4,16 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import { CodexAppServerAgentClient } from "../agent/providers/codex-app-server-agent.js";
import type { AgentTimelineItem } from "../agent/agent-sdk-types.js"; import type { AgentTimelineItem } from "../agent/agent-sdk-types.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createMessageCollector } from "../test-utils/message-collector.js"; import { createMessageCollector } from "../test-utils/message-collector.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import type { SessionOutboundMessage } from "../messages.js"; import type { SessionOutboundMessage } from "../messages.js";
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
function tmpCwd(): string { function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-real-codex-tool-interrupt-")); return mkdtempSync(path.join(tmpdir(), "daemon-real-codex-tool-interrupt-"));
@@ -143,7 +146,7 @@ describe("daemon E2E (real codex) - send message during tool call", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("codex"); canRun = await canRunRealProvider("codex");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -156,7 +159,7 @@ describe("daemon E2E (real codex) - send message during tool call", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { codex: new CodexAppServerAgentClient(logger) }, agentClients: createRealProviderClients(["codex"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -169,7 +172,7 @@ describe("daemon E2E (real codex) - send message during tool call", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "codex-tool-interrupt-repro", title: "codex-tool-interrupt-repro",
...getFullAccessConfig("codex"), ...getRealProviderConfig("codex"),
}); });
collector = createMessageCollector(client); collector = createMessageCollector(client);
@@ -232,7 +235,7 @@ describe("daemon E2E (real codex) - send message during tool call", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { codex: new CodexAppServerAgentClient(logger) }, agentClients: createRealProviderClients(["codex"], logger),
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -245,7 +248,7 @@ describe("daemon E2E (real codex) - send message during tool call", () => {
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: "codex-quick-follow-up-repro", title: "codex-quick-follow-up-repro",
...getFullAccessConfig("codex"), ...getRealProviderConfig("codex"),
}); });
collector = createMessageCollector(client); collector = createMessageCollector(client);
@@ -6,8 +6,11 @@ import pino from "pino";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js"; import {
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
import { applyAgentInputProcessingTransition } from "./send-while-running-stuck-test-utils.js"; import { applyAgentInputProcessingTransition } from "./send-while-running-stuck-test-utils.js";
function tmpCwd(): string { function tmpCwd(): string {
@@ -18,7 +21,7 @@ describe("daemon E2E (real claude) - send while running recovery", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("claude"); canRun = await canRunRealProvider("claude");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -31,7 +34,7 @@ describe("daemon E2E (real claude) - send while running recovery", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { claude: new ClaudeAgentClient({ logger }) }, agentClients: createRealProviderClients(["claude"], logger),
logger, logger,
}); });
@@ -47,7 +50,7 @@ describe("daemon E2E (real claude) - send while running recovery", () => {
const agent = await primary.createAgent({ const agent = await primary.createAgent({
cwd, cwd,
title: "stuck-repro-real-claude", title: "stuck-repro-real-claude",
...getFullAccessConfig("claude"), ...getRealProviderConfig("claude"),
}); });
await primary.sendMessage( await primary.sendMessage(
@@ -6,8 +6,11 @@ import pino from "pino";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { CodexAppServerAgentClient } from "../agent/providers/codex-app-server-agent.js"; import {
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; canRunRealProvider,
createRealProviderClients,
getRealProviderConfig,
} from "./real-provider-test-config.js";
import { applyAgentInputProcessingTransition } from "./send-while-running-stuck-test-utils.js"; import { applyAgentInputProcessingTransition } from "./send-while-running-stuck-test-utils.js";
function tmpCwd(): string { function tmpCwd(): string {
@@ -18,7 +21,7 @@ describe("daemon E2E (real codex) - send while running recovery", () => {
let canRun = false; let canRun = false;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable("codex"); canRun = await canRunRealProvider("codex");
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -31,7 +34,7 @@ describe("daemon E2E (real codex) - send while running recovery", () => {
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { codex: new CodexAppServerAgentClient(logger) }, agentClients: createRealProviderClients(["codex"], logger),
logger, logger,
}); });
@@ -47,7 +50,7 @@ describe("daemon E2E (real codex) - send while running recovery", () => {
const agent = await primary.createAgent({ const agent = await primary.createAgent({
cwd, cwd,
title: "stuck-repro-real-codex", title: "stuck-repro-real-codex",
...getFullAccessConfig("codex"), ...getRealProviderConfig("codex"),
}); });
await primary.sendMessage(agent.id, "Run: sleep 5"); await primary.sendMessage(agent.id, "Run: sleep 5");
@@ -4,18 +4,15 @@ import { tmpdir } from "node:os";
import path from "node:path"; import path from "node:path";
import pino from "pino"; import pino from "pino";
import type { AgentClient } from "../agent/agent-sdk-types.js";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js";
import { CodexAppServerAgentClient } from "../agent/providers/codex-app-server-agent.js";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { import {
allProviders, canRunRealProvider,
getFullAccessConfig, createRealProviderClients,
isProviderAvailable, getRealProviderConfig,
type AgentProvider, realProviders,
} from "./agent-configs.js"; type RealProvider,
} from "./real-provider-test-config.js";
type UiAction = type UiAction =
| { | {
@@ -109,7 +106,7 @@ function longRunningPrompt(sleepSeconds: number, token: string): string {
].join(" "); ].join(" ");
} }
function buildNormalUsageScenario(provider: AgentProvider, seed: number): UiScenario { function buildNormalUsageScenario(provider: RealProvider, seed: number): UiScenario {
const token1 = `UI_${provider}_${seed}_A`; const token1 = `UI_${provider}_${seed}_A`;
const token2 = `UI_${provider}_${seed}_B`; const token2 = `UI_${provider}_${seed}_B`;
return { return {
@@ -146,7 +143,7 @@ function buildNormalUsageScenario(provider: AgentProvider, seed: number): UiScen
} }
function buildOverlapScenario( function buildOverlapScenario(
provider: AgentProvider, provider: RealProvider,
seed: number, seed: number,
firstPick: "first" | "last", firstPick: "first" | "last",
): UiScenario { ): UiScenario {
@@ -444,21 +441,11 @@ async function runUiScenario(params: {
} }
} }
function createRealAgentClient(provider: AgentProvider, logger: pino.Logger): AgentClient { describe.each(realProviders)("daemon E2E (real %s) - UI action stress", (provider) => {
if (provider === "claude") {
return new ClaudeAgentClient({ logger });
}
if (provider === "codex") {
return new CodexAppServerAgentClient(logger);
}
return new OpenCodeAgentClient(logger);
}
describe.each(allProviders)("daemon E2E (real %s) - UI action stress", (provider) => {
let shouldRun = false; let shouldRun = false;
beforeAll(async () => { beforeAll(async () => {
shouldRun = await isProviderAvailable(provider); shouldRun = await canRunRealProvider(provider);
}); });
beforeEach((context) => { beforeEach((context) => {
@@ -471,9 +458,7 @@ describe.each(allProviders)("daemon E2E (real %s) - UI action stress", (provider
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { agentClients: createRealProviderClients([provider], logger),
[provider]: createRealAgentClient(provider, logger),
} as Partial<Record<AgentProvider, AgentClient>>,
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -486,7 +471,7 @@ describe.each(allProviders)("daemon E2E (real %s) - UI action stress", (provider
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: `uist-n-${provider}`, title: `uist-n-${provider}`,
...getFullAccessConfig(provider), ...getRealProviderConfig(provider),
}); });
await runUiScenario({ await runUiScenario({
@@ -507,9 +492,7 @@ describe.each(allProviders)("daemon E2E (real %s) - UI action stress", (provider
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
const cwd = tmpCwd(); const cwd = tmpCwd();
const daemon = await createTestPaseoDaemon({ const daemon = await createTestPaseoDaemon({
agentClients: { agentClients: createRealProviderClients([provider], logger),
[provider]: createRealAgentClient(provider, logger),
} as Partial<Record<AgentProvider, AgentClient>>,
logger, logger,
}); });
const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` }); const client = new DaemonClient({ url: `ws://127.0.0.1:${daemon.port}/ws` });
@@ -528,7 +511,7 @@ describe.each(allProviders)("daemon E2E (real %s) - UI action stress", (provider
const agent = await client.createAgent({ const agent = await client.createAgent({
cwd, cwd,
title: `uist-o-${provider}`, title: `uist-o-${provider}`,
...getFullAccessConfig(provider), ...getRealProviderConfig(provider),
}); });
await runUiScenario({ await runUiScenario({
@@ -4,65 +4,54 @@ import path from "node:path";
import pino from "pino"; import pino from "pino";
import { afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest"; import { afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest";
import { ClaudeAgentClient } from "../agent/providers/claude/agent.js";
import { CodexAppServerAgentClient } from "../agent/providers/codex-app-server-agent.js";
import { OpenCodeAgentClient } from "../agent/providers/opencode-agent.js";
import { PiRpcAgentClient } from "../agent/providers/pi/agent.js";
import type { import type {
AgentClient,
AgentProvider, AgentProvider,
AgentSessionConfig, AgentSessionConfig,
AgentStreamEvent, AgentStreamEvent,
} from "../agent/agent-sdk-types.js"; } from "../agent/agent-sdk-types.js";
import { DaemonClient } from "../test-utils/daemon-client.js"; import { DaemonClient } from "../test-utils/daemon-client.js";
import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js"; import { createTestPaseoDaemon, type TestPaseoDaemon } from "../test-utils/paseo-daemon.js";
import { getFullAccessConfig, isProviderAvailable } from "./agent-configs.js"; import {
canRunRealProvider,
createRealProviderClient,
getRealProviderConfig,
type RealProvider,
} from "./real-provider-test-config.js";
import { fetchTimelineItems } from "./test-utils/rewind-helpers.js"; import { fetchTimelineItems } from "./test-utils/rewind-helpers.js";
type ContractProvider = Extract<AgentProvider, "claude" | "codex" | "opencode" | "pi">; type ContractProvider = Extract<AgentProvider, RealProvider>;
interface ProviderContractCase { interface ProviderContractCase {
provider: ContractProvider; provider: ContractProvider;
title: string; title: string;
timeoutMs: number; timeoutMs: number;
createClient: (logger: pino.Logger) => AgentClient;
createConfig: () => AgentSessionConfig; createConfig: () => AgentSessionConfig;
} }
const PI_REAL_TEST_MODEL = "openrouter/google/gemini-2.5-flash-lite";
const CONTRACT_CASES: ProviderContractCase[] = [ const CONTRACT_CASES: ProviderContractCase[] = [
{ {
provider: "claude", provider: "claude",
title: "Claude", title: "Claude",
timeoutMs: 180_000, timeoutMs: 180_000,
createClient: (logger) => new ClaudeAgentClient({ logger }), createConfig: () => getRealProviderConfig("claude"),
createConfig: () => getFullAccessConfig("claude"),
}, },
{ {
provider: "codex", provider: "codex",
title: "Codex", title: "Codex",
timeoutMs: 180_000, timeoutMs: 180_000,
createClient: (logger) => new CodexAppServerAgentClient(logger), createConfig: () => getRealProviderConfig("codex"),
createConfig: () => getFullAccessConfig("codex"),
}, },
{ {
provider: "opencode", provider: "opencode",
title: "OpenCode", title: "OpenCode",
timeoutMs: 180_000, timeoutMs: 180_000,
createClient: (logger) => new OpenCodeAgentClient(logger), createConfig: () => getRealProviderConfig("opencode"),
createConfig: () => getFullAccessConfig("opencode"),
}, },
{ {
provider: "pi", provider: "pi",
title: "Pi", title: "Pi",
timeoutMs: 240_000, timeoutMs: 240_000,
createClient: (logger) => new PiRpcAgentClient({ logger }), createConfig: () => getRealProviderConfig("pi"),
createConfig: () => ({
provider: "pi",
model: PI_REAL_TEST_MODEL,
thinkingOptionId: "medium",
}),
}, },
]; ];
@@ -94,7 +83,7 @@ describe.each(CONTRACT_CASES)("daemon E2E (real $provider) - user_message contra
let cwd: string | null = null; let cwd: string | null = null;
beforeAll(async () => { beforeAll(async () => {
canRun = await isProviderAvailable(entry.provider); canRun = await canRunRealProvider(entry.provider);
}); });
beforeEach(async (context) => { beforeEach(async (context) => {
@@ -106,7 +95,7 @@ describe.each(CONTRACT_CASES)("daemon E2E (real $provider) - user_message contra
cwd = tmpCwd(entry.provider); cwd = tmpCwd(entry.provider);
const logger = pino({ level: "silent" }); const logger = pino({ level: "silent" });
daemon = await createTestPaseoDaemon({ daemon = await createTestPaseoDaemon({
agentClients: { [entry.provider]: entry.createClient(logger) }, agentClients: { [entry.provider]: createRealProviderClient(entry.provider, logger) },
logger, logger,
}); });
client = new DaemonClient({ client = new DaemonClient({