diff --git a/packages/server/src/server/agent/providers/opencode-agent.test.ts b/packages/server/src/server/agent/providers/opencode-agent.test.ts index 7d924d4fb..6708274fc 100644 --- a/packages/server/src/server/agent/providers/opencode-agent.test.ts +++ b/packages/server/src/server/agent/providers/opencode-agent.test.ts @@ -710,7 +710,7 @@ describe("OpenCode adapter startTurn error handling", () => { ); }); - test("fails a turn when OpenCode retry status does not recover", async () => { + test("keeps a turn active while OpenCode is retrying", async () => { vi.useFakeTimers(); const retryStream: AsyncIterable = { [Symbol.asyncIterator]: () => { @@ -746,6 +746,8 @@ describe("OpenCode adapter startTurn error handling", () => { event: vi.fn().mockResolvedValue({ stream: retryStream }), }, session: { + abort: vi.fn().mockResolvedValue({ error: null }), + update: vi.fn().mockResolvedValue({ error: null }), promptAsync: vi.fn().mockResolvedValue({ data: {}, error: undefined }), }, } as never; @@ -761,15 +763,24 @@ describe("OpenCode adapter startTurn error handling", () => { const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); - await session.startTurn("hello"); - await vi.advanceTimersByTimeAsync(10_000); + try { + await session.startTurn("hello"); + await vi.advanceTimersByTimeAsync(10_000); - const failed = events.find((event) => event.type === "turn_failed"); - expect(failed).toMatchObject({ - type: "turn_failed", - error: expect.stringContaining("model does not exist"), - }); - vi.useRealTimers(); + expect(events).toContainEqual({ + type: "timeline", + provider: "opencode", + item: { + type: "error", + message: "Provider retry (attempt 1): model does not exist", + }, + turnId: "opencode-turn-0", + }); + expect(events.some((event) => event.type === "turn_failed")).toBe(false); + await session.close(); + } finally { + vi.useRealTimers(); + } }); test("deletes provider session on close when persistence is disabled", async () => { diff --git a/packages/server/src/server/agent/providers/opencode-agent.ts b/packages/server/src/server/agent/providers/opencode-agent.ts index 937e70f72..e5d1cc488 100644 --- a/packages/server/src/server/agent/providers/opencode-agent.ts +++ b/packages/server/src/server/agent/providers/opencode-agent.ts @@ -76,7 +76,6 @@ const OPENCODE_BUILD_MODE_ID = "build"; const OPENCODE_FULL_ACCESS_MODE_ID = "full-access"; const OPENCODE_STORAGE_SESSION_LIMIT = 200; const OPENCODE_PENDING_ABORT_START_TIMEOUT_MS = 10_000; -const OPENCODE_RETRY_STATUS_FAILURE_MS = 10_000; const DEFAULT_MODES: AgentMode[] = [ { @@ -2229,7 +2228,6 @@ class OpenCodeAgentSession implements AgentSession { private releaseServer: (() => void) | null; private readonly persistSession: boolean; private deletedFromProvider = false; - private retryFailureTimer: ReturnType | null = null; constructor( config: OpenCodeAgentConfig, client: OpencodeClient, @@ -2372,7 +2370,6 @@ class OpenCodeAgentSession implements AgentSession { this.subAgentsByCallId.clear(); this.subAgentCallIdByChildSessionId.clear(); this.pendingChildToolPartsBySessionId.clear(); - this.clearRetryFailureTimer(); const turnAbortController = new AbortController(); this.abortController = turnAbortController; await this.ensureMcpServersConfigured(); @@ -2700,8 +2697,6 @@ class OpenCodeAgentSession implements AgentSession { }); return false; } - - this.armRetryFailureTimerForStatus(event, turnId); const translated = await this.translateEvent(event); this.traceOpenCode("provider.opencode.parsed_event", { turnId, @@ -2753,7 +2748,6 @@ class OpenCodeAgentSession implements AgentSession { } else { this.runningToolCalls.clear(); } - this.clearRetryFailureTimer(); this.activeForegroundTurnId = null; // Abort the SSE connection so the SDK tears down the underlying fetch. this.abortController?.abort(); @@ -2769,44 +2763,6 @@ class OpenCodeAgentSession implements AgentSession { this.runningToolCalls.delete(item.callId); } - private armRetryFailureTimerForStatus(event: OpenCodeEvent, turnId: string): void { - if (this.retryFailureTimer || event.type !== "session.status") { - return; - } - if (event.properties.sessionID !== this.sessionId || event.properties.status.type !== "retry") { - return; - } - - const retry = event.properties.status; - const message = typeof retry.message === "string" ? retry.message.trim() : ""; - const error = message - ? `OpenCode provider retry did not recover: ${message}` - : "OpenCode provider retry did not recover"; - - this.retryFailureTimer = setTimeout(() => { - this.retryFailureTimer = null; - if (this.activeForegroundTurnId !== turnId) { - return; - } - this.finishForegroundTurn( - { - type: "turn_failed", - provider: "opencode", - error, - }, - turnId, - ); - }, OPENCODE_RETRY_STATUS_FAILURE_MS); - } - - private clearRetryFailureTimer(): void { - if (!this.retryFailureTimer) { - return; - } - clearTimeout(this.retryFailureTimer); - this.retryFailureTimer = null; - } - private synthesizeInterruptedToolCalls(turnId: string): void { for (const item of this.runningToolCalls.values()) { const error = { message: "Tool execution aborted" }; diff --git a/packages/server/src/server/agent/providers/opencode-retry-timeout.real.e2e.test.ts b/packages/server/src/server/agent/providers/opencode-retry-timeout.real.e2e.test.ts new file mode 100644 index 000000000..bf3bcc875 --- /dev/null +++ b/packages/server/src/server/agent/providers/opencode-retry-timeout.real.e2e.test.ts @@ -0,0 +1,156 @@ +import http from "node:http"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { expect, test } from "vitest"; + +import type { AgentStreamEvent } from "../agent-sdk-types.js"; +import { OpenCodeAgentClient } from "./opencode-agent.js"; +import { OpenCodeServerManager } from "./opencode/server-manager.js"; +import { createTestLogger } from "../../../test-utils/test-logger.js"; + +function listen(server: http.Server): Promise { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("server did not bind to a TCP port")); + return; + } + resolve(address.port); + }); + }); +} + +test("does not fail an active OpenCode provider retry before the advertised retry delay elapses", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "paseo-opencode-retry-")); + const projectDir = path.join(root, "project"); + const xdgConfigHome = path.join(root, "config"); + const xdgDataHome = path.join(root, "data"); + const xdgCacheHome = path.join(root, "cache"); + await mkdir(projectDir, { recursive: true }); + await mkdir(xdgConfigHome, { recursive: true }); + await mkdir(xdgDataHome, { recursive: true }); + await mkdir(xdgCacheHome, { recursive: true }); + + let completionRequests = 0; + let turnStartedAt = 0; + const requestLog: Array<{ elapsedMs: number; request: number }> = []; + const providerServer = http.createServer((req, res) => { + if (req.method === "GET" && req.url === "/v1/models") { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ object: "list", data: [{ id: "flaky-model", object: "model" }] })); + return; + } + if (req.method === "POST" && req.url === "/v1/chat/completions") { + completionRequests += 1; + requestLog.push({ + elapsedMs: turnStartedAt === 0 ? -1 : Date.now() - turnStartedAt, + request: completionRequests, + }); + req.resume(); + res.writeHead(503, { + "content-type": "application/json", + "retry-after-ms": "15000", + }); + res.end( + JSON.stringify({ + error: { + message: `engine overloaded request ${completionRequests}`, + type: "server_error", + code: "server_error", + }, + }), + ); + return; + } + res.writeHead(404, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "not found" })); + }); + + const port = await listen(providerServer); + await writeFile( + path.join(projectDir, "opencode.json"), + JSON.stringify( + { + $schema: "https://opencode.ai/config.json", + model: "flaky/flaky-model", + enabled_providers: ["flaky"], + provider: { + flaky: { + npm: "@ai-sdk/openai-compatible", + name: "Flaky local provider", + options: { + baseURL: `http://127.0.0.1:${port}/v1`, + apiKey: "test-key", + timeout: false, + }, + models: { + "flaky-model": { + name: "Flaky Model", + limit: { + context: 128000, + output: 8192, + }, + }, + }, + }, + }, + }, + null, + 2, + ), + ); + + const logger = createTestLogger(); + const runtimeSettings = { + env: { + XDG_CONFIG_HOME: xdgConfigHome, + XDG_DATA_HOME: xdgDataHome, + XDG_CACHE_HOME: xdgCacheHome, + OPENCODE_DISABLE_AUTO_UPDATE: "1", + }, + }; + const client = new OpenCodeAgentClient(logger, runtimeSettings, path.join(root, "paseo-storage")); + const events: AgentStreamEvent[] = []; + const eventLog: Array<{ elapsedMs: number; event: AgentStreamEvent }> = []; + let session: Awaited> | undefined; + + try { + session = await client.createSession({ + provider: "opencode", + cwd: projectDir, + model: "flaky/flaky-model", + }); + session.subscribe((event) => { + events.push(event); + eventLog.push({ elapsedMs: Date.now() - turnStartedAt, event }); + }); + + turnStartedAt = Date.now(); + await session.startTurn("Say hello."); + await new Promise((resolve) => setTimeout(resolve, 12_000)); + + const retryEvents = events.filter( + (event) => event.type === "timeline" && event.item.type === "error", + ); + expect(retryEvents.length).toBeGreaterThan(0); + + const failedEvents = events.filter((event) => event.type === "turn_failed"); + expect( + failedEvents, + JSON.stringify( + { completionRequests, requestLog, eventLog, failedEvents, retryEvents }, + null, + 2, + ), + ).toEqual([]); + } finally { + await session?.close(); + await OpenCodeServerManager.getInstance(logger, runtimeSettings).shutdown(); + providerServer.close(); + await rm(root, { recursive: true, force: true }); + } +}, 45_000);