fix(app): back off and offer retry when timeline sync is refused
A host can refuse a timeline read for as long as it likes. Codex does this whenever another writer holds the thread: every catch-up fails with "already has an active writer" until the other session lets go. The app retried that on a fixed 1s timer with no ceiling, so an idle window sat there issuing a request and logging a warning every second, indefinitely. The refusal is normal lifecycle; the traffic was the bug. Catch-up and subscription retries now double from 1s to a 30s ceiling, resetting on success, reconnect, delivery-mode change, and visibility change so recovery stays immediate once the condition clears. Backing off alone would leave the user waiting up to 30s with no way in, so both failure surfaces get a retry they can press. Background retries stay silent; a retry the user asked for is a fallible user action and owns its pending state, so "retrying" is a status the sync model publishes rather than a React boolean. The blocking "Failed to load agent" screen was the worse half: its load effect self-gates on the error state, so a refused first resume was terminal until the user navigated away. Its retry resets the state to idle, which re-arms the effect.
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,87 @@
|
||||
import { expect, test } from "../support/fixtures";
|
||||
import { installDaemonWebSocketGate } from "../support/helpers/daemon-websocket-gate";
|
||||
import {
|
||||
expectTimelinePromptVisible,
|
||||
openAgentTimeline,
|
||||
seedLongMockAgentTimeline,
|
||||
} from "../support/helpers/timeline-pagination";
|
||||
import {
|
||||
disconnectViewedTimeline,
|
||||
restoreViewedTimelineWithHeldResponse,
|
||||
} from "../support/helpers/timeline-resume";
|
||||
|
||||
test.describe("Agent timeline sync retry", () => {
|
||||
test("a refused catch-up offers a retry that reports pending, failure, and recovery", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
test.setTimeout(120_000);
|
||||
const gate = await installDaemonWebSocketGate(page);
|
||||
const agent = await seedLongMockAgentTimeline({ turns: 2 });
|
||||
try {
|
||||
await openAgentTimeline(page, agent);
|
||||
await expectTimelinePromptVisible(page, agent.newestPrompt);
|
||||
|
||||
gate.failTimelineResponses(agent.agentId);
|
||||
await disconnectViewedTimeline(page, gate);
|
||||
await restoreViewedTimelineWithHeldResponse(page, gate);
|
||||
|
||||
const callout = page.getByTestId("agent-timeline-sync-error");
|
||||
const retry = page.getByTestId("agent-timeline-sync-retry");
|
||||
await expect(callout).toBeVisible({ timeout: 30_000 });
|
||||
await expect(retry).toBeEnabled();
|
||||
await page.screenshot({ path: testInfo.outputPath("1-sync-error.png"), fullPage: true });
|
||||
|
||||
gate.holdTimelineResponses(agent.agentId);
|
||||
await retry.click();
|
||||
await gate.waitForHeldTimelineResponse();
|
||||
await expect(retry).toBeDisabled();
|
||||
await expect(retry).toHaveText("Retrying…");
|
||||
await page.screenshot({ path: testInfo.outputPath("2-retry-pending.png"), fullPage: true });
|
||||
|
||||
gate.releaseHeldTimelineResponses();
|
||||
await expect(retry).toBeEnabled({ timeout: 30_000 });
|
||||
await expect(callout).toBeVisible();
|
||||
await page.screenshot({ path: testInfo.outputPath("3-retry-failed.png"), fullPage: true });
|
||||
|
||||
gate.allowTimelineResponses();
|
||||
await retry.click();
|
||||
await expect(callout).toBeHidden({ timeout: 30_000 });
|
||||
await expectTimelinePromptVisible(page, agent.newestPrompt);
|
||||
await page.screenshot({ path: testInfo.outputPath("4-recovered.png"), fullPage: true });
|
||||
} finally {
|
||||
gate.restore();
|
||||
await agent.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("a refused first load offers a retry instead of a dead panel", async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
test.setTimeout(120_000);
|
||||
const gate = await installDaemonWebSocketGate(page);
|
||||
const agent = await seedLongMockAgentTimeline({ turns: 2 });
|
||||
try {
|
||||
gate.failTimelineResponses(agent.agentId);
|
||||
await openAgentTimeline(page, agent);
|
||||
|
||||
const loadError = page.getByTestId("agent-load-error");
|
||||
const retry = page.getByTestId("agent-load-error-retry");
|
||||
await expect(loadError).toBeVisible({ timeout: 60_000 });
|
||||
await expect(loadError).toContainText("already has an active writer");
|
||||
await page.screenshot({ path: testInfo.outputPath("5-load-error.png"), fullPage: true });
|
||||
|
||||
await retry.click();
|
||||
await expect(loadError).toBeVisible({ timeout: 30_000 });
|
||||
await page.screenshot({ path: testInfo.outputPath("6-load-retry-failed.png") });
|
||||
|
||||
gate.allowTimelineResponses();
|
||||
await retry.click();
|
||||
await expect(loadError).toBeHidden({ timeout: 30_000 });
|
||||
await expectTimelinePromptVisible(page, agent.newestPrompt);
|
||||
await page.screenshot({ path: testInfo.outputPath("7-load-recovered.png"), fullPage: true });
|
||||
} finally {
|
||||
gate.restore();
|
||||
await agent.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -133,6 +133,23 @@ function forceTimelineReset(message: string | Buffer, enabled: boolean): string
|
||||
return JSON.stringify(envelope);
|
||||
}
|
||||
|
||||
// The daemon refusal this reproduces: a Codex thread that already has an active writer.
|
||||
const TIMELINE_WRITER_CONFLICT_ERROR =
|
||||
"Failed to resume Codex thread playwright-thread: thread playwright-thread already has an active writer";
|
||||
|
||||
function failTimelineResponse(message: string | Buffer, agentId: string | null): string | Buffer {
|
||||
if (!agentId || typeof message !== "string") return message;
|
||||
const envelope = JSON.parse(message) as {
|
||||
message?: { payload?: Record<string, unknown> };
|
||||
payload?: Record<string, unknown>;
|
||||
};
|
||||
const payload = envelope.message?.payload ?? envelope.payload;
|
||||
if (!payload || payload.agentId !== agentId) return message;
|
||||
payload.error = TIMELINE_WRITER_CONFLICT_ERROR;
|
||||
payload.entries = [];
|
||||
return JSON.stringify(envelope);
|
||||
}
|
||||
|
||||
function rewriteShellToolCommand(
|
||||
message: string | Buffer,
|
||||
command: string | null,
|
||||
@@ -284,6 +301,10 @@ export async function installDaemonWebSocketGate(page: Page) {
|
||||
let stripAssistantMessageIds = false;
|
||||
let stripCanonicalSubmittedPromptsFeature = false;
|
||||
let shellToolCommandOverride: string | null = null;
|
||||
let failingTimelineAgentId: string | null = null;
|
||||
let holdingTimelineAgentId: string | null = null;
|
||||
const heldTimelineResponses: Array<() => void> = [];
|
||||
const heldTimelineResponseWaiters = new Set<() => void>();
|
||||
let heldClientRequestType: string | null = null;
|
||||
let heldClientRequest: { server: WebSocketRoute; message: string | Buffer } | null = null;
|
||||
let resolveHeldClientRequest: (() => void) | null = null;
|
||||
@@ -487,11 +508,25 @@ export async function installDaemonWebSocketGate(page: Page) {
|
||||
stripCanonicalSubmittedPromptsFeature,
|
||||
serverMessage?.type,
|
||||
);
|
||||
const isTimelineResponse = serverMessage?.type === "fetch_agent_timeline_response";
|
||||
if (isTimelineResponse) {
|
||||
outboundMessage = failTimelineResponse(outboundMessage, failingTimelineAgentId);
|
||||
}
|
||||
const shouldForceTimelineReset =
|
||||
forceTimelineEpochReset && serverMessage?.type === "fetch_agent_timeline_response";
|
||||
outboundMessage = forceTimelineReset(outboundMessage, shouldForceTimelineReset);
|
||||
if (shouldForceTimelineReset) forceTimelineEpochReset = false;
|
||||
recordServerMessage(serverMessage);
|
||||
if (isTimelineResponse && holdingTimelineAgentId) {
|
||||
const payload = (serverMessage as { payload?: { agentId?: unknown } } | null)?.payload;
|
||||
if (payload?.agentId === holdingTimelineAgentId) {
|
||||
const forward = outboundMessage;
|
||||
heldTimelineResponses.push(() => ws.send(forward));
|
||||
for (const resolve of heldTimelineResponseWaiters) resolve();
|
||||
heldTimelineResponseWaiters.clear();
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (holdServerMessage({ browser: ws, message: outboundMessage, parsed: serverMessage }))
|
||||
return;
|
||||
if (holdReadyFileUpdate(ws, outboundMessage, fileMessage)) return;
|
||||
@@ -560,6 +595,24 @@ export async function installDaemonWebSocketGate(page: Page) {
|
||||
restore(): void {
|
||||
acceptingConnections = true;
|
||||
},
|
||||
failTimelineResponses(agentId: string): void {
|
||||
failingTimelineAgentId = agentId;
|
||||
},
|
||||
allowTimelineResponses(): void {
|
||||
failingTimelineAgentId = null;
|
||||
},
|
||||
holdTimelineResponses(agentId: string): void {
|
||||
holdingTimelineAgentId = agentId;
|
||||
},
|
||||
async waitForHeldTimelineResponse(): Promise<void> {
|
||||
while (heldTimelineResponses.length === 0) {
|
||||
await new Promise<void>((resolve) => heldTimelineResponseWaiters.add(resolve));
|
||||
}
|
||||
},
|
||||
releaseHeldTimelineResponses(): void {
|
||||
holdingTimelineAgentId = null;
|
||||
for (const forward of heldTimelineResponses.splice(0)) forward();
|
||||
},
|
||||
restoreFresh(): void {
|
||||
reconnectWithFreshClient = true;
|
||||
acceptingConnections = true;
|
||||
|
||||
Reference in New Issue
Block a user