fix(app): back off and offer retry when timeline sync is refused

A host can refuse a timeline read for as long as it likes. Codex does this whenever another writer holds the thread: every catch-up fails with "already has an active writer" until the other session lets go. The app retried that on a fixed 1s timer with no ceiling, so an idle window sat there issuing a request and logging a warning every second, indefinitely. The refusal is normal lifecycle; the traffic was the bug. Catch-up and subscription retries now double from 1s to a 30s ceiling, resetting on success, reconnect, delivery-mode change, and visibility change so recovery stays immediate once the condition clears. Backing off alone would leave the user waiting up to 30s with no way in, so both failure surfaces get a retry they can press. Background retries stay silent; a retry the user asked for is a fallible user action and owns its pending state, so "retrying" is a status the sync model publishes rather than a React boolean. The blocking "Failed to load agent" screen was the worse half: its load effect self-gates on the error state, so a refused first resume was terminal until the user navigated away. Its retry resets the state to idle, which re-arms the effect.
2026-08-18 14:36:27 +02:00
parent 05ee7a1d31
commit 0f2e58d4bb
17 changed files with 461 additions and 51 deletions
@@ -79,10 +79,11 @@ class TimelineWorld {
    schedule: (task, delayMs) => {
      const scheduled = { task, delayMs };
      this.scheduled.push(scheduled);
-      const waiter = this.retryWaiters.shift();
-      if (waiter && delayMs === 1_000) {
+      const waiterIndex = this.retryWaiters.findIndex((waiter) => waiter.delayMs === delayMs);
+      if (waiterIndex >= 0) {
+        const [waiter] = this.retryWaiters.splice(waiterIndex, 1);
        this.scheduled.splice(this.scheduled.indexOf(scheduled), 1);
-        waiter(task);
+        waiter.resolve(task);
      }
      return () => {
        const index = this.scheduled.indexOf(scheduled);
@@ -100,7 +101,10 @@ class TimelineWorld {
  }> = [];
  private readonly errorWaiters: Array<(message: string) => void> = [];
  private readonly scheduled: Array<{ task: () => void; delayMs: number }> = [];
-  private readonly retryWaiters: Array<(retry: () => void) => void> = [];
+  private readonly retryWaiters: Array<{
+    delayMs: number;
+    resolve(retry: () => void): void;
+  }> = [];

  private fetchTimeline(
    agentId: string,
@@ -147,10 +151,10 @@ class TimelineWorld {
    return new Promise((resolve) => this.errorWaiters.push(resolve));
  }

-  nextRetry(): Promise<() => void> {
-    const index = this.scheduled.findIndex((entry) => entry.delayMs === 1_000);
+  nextRetry(delayMs = 1_000): Promise<() => void> {
+    const index = this.scheduled.findIndex((entry) => entry.delayMs === delayMs);
    if (index >= 0) return Promise.resolve(this.scheduled.splice(index, 1)[0].task);
-    return new Promise((resolve) => this.retryWaiters.push(resolve));
+    return new Promise((resolve) => this.retryWaiters.push({ delayMs, resolve }));
  }

  elapse(elapsedMs: number): void {
@@ -411,6 +415,74 @@ test("a failed catch-up reports once and retries through the explicit retry poli
  world.expectNoPendingMembership();
 });

+test("a failed catch-up retries with exponential backoff", async () => {
+  const world = new TimelineWorld();
+  world.sync.setConnected(true);
+  world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
+  const membership = await world.nextMembership();
+  membership.succeed();
+
+  const first = await world.nextFetch("agent-a");
+  first.fail("timeline unavailable");
+  const [firstError, retryAfterFirstFailure] = await Promise.all([
+    world.nextError(),
+    world.nextRetry(),
+  ]);
+  expect(firstError).toBe("timeline unavailable");
+  retryAfterFirstFailure();
+
+  const second = await world.nextFetch("agent-a");
+  second.fail("timeline unavailable");
+  await world.nextError();
+
+  const retryAfterSecondFailure = await world.nextRetry(2_000);
+  expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error");
+
+  retryAfterSecondFailure();
+  const third = await world.nextFetch("agent-a");
+  third.respond({ hasNewer: false });
+  await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready"));
+});
+
+test("manual retries can immediately re-attempt a failed catch-up", async () => {
+  const world = new TimelineWorld();
+  world.sync.setConnected(true);
+  world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
+  const membership = await world.nextMembership();
+  membership.succeed();
+
+  const failed = await world.nextFetch("agent-a");
+  failed.fail("timeline unavailable");
+  await world.nextRetry();
+  expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error");
+
+  world.sync.retryVisibleAgentTimeline("agent-a");
+  expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("retrying");
+
+  const retry = await world.nextFetch("agent-a");
+  retry.respond({ hasNewer: false });
+  await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready"));
+});
+
+test("a manual retry that fails returns to the error state", async () => {
+  const world = new TimelineWorld();
+  world.sync.setConnected(true);
+  world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
+  const membership = await world.nextMembership();
+  membership.succeed();
+
+  const failed = await world.nextFetch("agent-a");
+  failed.fail("timeline unavailable");
+  await world.nextRetry();
+
+  world.sync.retryVisibleAgentTimeline("agent-a");
+  expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("retrying");
+
+  const retry = await world.nextFetch("agent-a");
+  retry.fail("timeline still unavailable");
+  await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error"));
+});
+
 test("gap recovery supersedes completed catch-up and pages through the current tail", async () => {
  const world = new TimelineWorld();
  world.sync.setConnected(true);
@@ -486,6 +558,34 @@ test("membership failure autonomously retries without another visibility declara
  });
 });

+test("membership failures retry with exponential backoff", async () => {
+  const world = new TimelineWorld();
+  world.sync.setConnected(true);
+  world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
+
+  const first = await world.nextMembership();
+  first.fail("subscription unavailable");
+  const [firstError, retryAfterFirstFailure] = await Promise.all([
+    world.nextError(),
+    world.nextRetry(),
+  ]);
+  expect(firstError).toBe("subscription unavailable");
+
+  retryAfterFirstFailure();
+  const second = await world.nextMembership();
+  second.fail("subscription unavailable again");
+
+  const retryAfterSecondFailure = await world.nextRetry(2_000);
+  expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error");
+
+  retryAfterSecondFailure();
+  const retry = await world.nextMembership();
+  retry.succeed();
+  const catchUp = await world.nextFetch("agent-a");
+  catchUp.respond({ hasNewer: false });
+  await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready"));
+});
+
 test("backgrounding preserves the hot membership without catch-up on return", async () => {
  const world = new TimelineWorld();
  world.sync.setConnected(true);
@@ -23,12 +23,13 @@ interface ViewedTimelineSyncPorts {
 }

 export type TimelineDeliveryMode = "legacy" | "selective";
-export type ViewedTimelineStatus = "ready" | "pending" | "error";
+export type ViewedTimelineStatus = "ready" | "pending" | "error" | "retrying";

 export interface ViewedTimelineUiBridge {
  replaceVisibleAgentIds(sourceId: string, agentIds: string[]): void;
  subscribe(listener: () => void): () => void;
  getAgentTimelineStatus(agentId: string): ViewedTimelineStatus;
+  retryVisibleAgentTimeline(agentId: string): void;
 }

 export interface ViewedTimelineSync extends ViewedTimelineUiBridge {
@@ -40,6 +41,7 @@ export interface ViewedTimelineSync extends ViewedTimelineUiBridge {
 }

 const RETRY_DELAY_MS = 1_000;
+const MAX_RETRY_DELAY_MS = 30_000;
 const VIEWED_TIMELINE_HOT_AGENT_LIMIT = 5;

 type CatchUpStatus = "running" | "complete" | "error";
@@ -49,8 +51,16 @@ interface CatchUpState {
  status: CatchUpStatus;
  request?: ProjectedTimelineForwardFetchPlan;
  cancelRetry?: () => void;
+  retryDelayMs?: number;
 }

+const getNextRetryDelayMs = (previousDelayMs: number | undefined): number => {
+  if (previousDelayMs == null) {
+    return RETRY_DELAY_MS;
+  }
+  return Math.min(previousDelayMs * 2, MAX_RETRY_DELAY_MS);
+};
+
 function isSameCatchUpRequest(
  left: ProjectedTimelineForwardFetchPlan | undefined,
  right: ProjectedTimelineForwardFetchPlan | undefined,
@@ -102,6 +112,9 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
  const pendingCatchUps = new Map<string, ProjectedTimelineForwardFetchPlan>();
  const visibilityCatchUpPending = new Set<string>();
  const visibilityCatchUpErrors = new Set<string>();
+  // User-initiated retries only. Background retries stay silent; a retry the user asked for
+  // owes them a pending state until it settles.
+  const manualRetries = new Set<string>();
  const listeners = new Set<() => void>();
  let active = true;
  let connected = false;
@@ -113,6 +126,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
  let reconciling = false;
  let reconcileRequested = false;
  let membershipNeedsRetry = false;
+  let membershipRetryDelayMs: number | undefined;
  let cancelMembershipRetry: (() => void) | null = null;
  let recentlyViewedAgentIds: string[] = [];

@@ -152,12 +166,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
  const setVisibilityCatchUpReady = (agentId: string) => {
    const wasPending = visibilityCatchUpPending.delete(agentId);
    const hadError = visibilityCatchUpErrors.delete(agentId);
-    if (wasPending || hadError) notifyListeners();
+    const wasRetrying = manualRetries.delete(agentId);
+    if (wasPending || hadError || wasRetrying) notifyListeners();
  };

  const setVisibilityCatchUpError = (agentIds: string[]) => {
    let changed = false;
    for (const agentId of agentIds) {
+      if (manualRetries.delete(agentId)) changed = true;
      if (!visibilityCatchUpPending.delete(agentId)) continue;
      visibilityCatchUpErrors.add(agentId);
      changed = true;
@@ -210,12 +226,19 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
      setVisibilityCatchUpReady(agentId);
    } catch (error) {
      if (catchUps.get(agentId)?.generation === generation) {
+        const nextRetryDelayMs = getNextRetryDelayMs(catchUps.get(agentId)?.retryDelayMs);
        const cancelRetry = ports.schedule(() => {
          const current = catchUps.get(agentId);
          if (current?.generation !== generation || current.status !== "error") return;
          startCatchUp(agentId);
-        }, RETRY_DELAY_MS);
-        catchUps.set(agentId, { generation, status: "error", cancelRetry });
+        }, nextRetryDelayMs);
+        catchUps.set(agentId, {
+          generation,
+          status: "error",
+          request,
+          cancelRetry,
+          retryDelayMs: nextRetryDelayMs,
+        });
        setVisibilityCatchUpError([agentId]);
        ports.reportError(error);
      }
@@ -247,7 +270,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
    current?.cancelRetry?.();
    const generation = (catchUpGenerations.get(agentId) ?? 0) + 1;
    catchUpGenerations.set(agentId, generation);
-    catchUps.set(agentId, { generation, status: "running", request: nextRequest });
+    const retryDelayMs =
+      supersede || current?.status !== "error" ? undefined : current.retryDelayMs;
+    catchUps.set(agentId, {
+      generation,
+      status: "running",
+      request: nextRequest,
+      retryDelayMs,
+    });
    pendingCatchUps.delete(agentId);
    void fetchUntilCurrent(
      agentId,
@@ -279,6 +309,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
      membershipNeedsRetry = true;
      setVisibilityCatchUpError(requested);
      cancelMembershipRetry?.();
+      const nextRetryDelayMs = getNextRetryDelayMs(membershipRetryDelayMs);
      cancelMembershipRetry = ports.schedule(() => {
        cancelMembershipRetry = null;
        if (
@@ -290,12 +321,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
          return;
        }
        void reconcileMembership();
-      }, RETRY_DELAY_MS);
+      }, nextRetryDelayMs);
+      membershipRetryDelayMs = nextRetryDelayMs;
      ports.reportError(error);
      return;
    }
    cancelMembershipRetry?.();
    cancelMembershipRetry = null;
+    membershipRetryDelayMs = undefined;
    if (disposed || !connected || deliveryMode !== "selective") return;
    acknowledged = requested;
    if (generation !== membershipGeneration) {
@@ -338,6 +371,25 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
    }
  };

+  const retryVisibleAgentTimeline = (agentId: string) => {
+    if (!isDesired(agentId) || manualRetries.has(agentId)) return;
+    const catchUp = catchUps.get(agentId);
+    const membershipRetryable = deliveryMode === "selective" && membershipNeedsRetry && connected;
+    if (catchUp?.status !== "error" && !membershipRetryable) return;
+    manualRetries.add(agentId);
+    notifyListeners();
+    if (catchUp?.status === "error") {
+      catchUp.cancelRetry?.();
+      startCatchUp(agentId, { request: catchUp.request, supersede: true });
+      return;
+    }
+    cancelMembershipRetry?.();
+    cancelMembershipRetry = null;
+    membershipRetryDelayMs = undefined;
+    membershipNeedsRetry = false;
+    void reconcileMembership();
+  };
+
  const commitDesiredMembership = (
    nextDesired: string[],
    options: { resetCatchUpStatus?: boolean } = {},
@@ -350,6 +402,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
          statusChanged = true;
        }
        if (visibilityCatchUpErrors.delete(agentId)) statusChanged = true;
+        if (manualRetries.delete(agentId)) statusChanged = true;
      }
    }
    if (sameAgentIds(nextDesired, desired)) {
@@ -364,12 +417,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
        cancelCatchUp(agentId);
        visibilityCatchUpPending.delete(agentId);
        visibilityCatchUpErrors.delete(agentId);
+        manualRetries.delete(agentId);
      }
    }
    for (const agentId of nextDesired) {
      if (!desired.includes(agentId)) {
        visibilityCatchUpPending.add(agentId);
        visibilityCatchUpErrors.delete(agentId);
+        manualRetries.delete(agentId);
      }
    }
    cancelMembershipRetry?.();
@@ -403,6 +458,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
      return () => listeners.delete(listener);
    },
    getAgentTimelineStatus(agentId) {
+      if (manualRetries.has(agentId)) return "retrying";
      if (visibilityCatchUpErrors.has(agentId)) return "error";
      if (!isDesired(agentId) || visibilityCatchUpPending.has(agentId)) return "pending";
      return "ready";
@@ -427,6 +483,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
        commitDesiredMembership(visible, { resetCatchUpStatus: true });
        cancelMembershipRetry?.();
        cancelMembershipRetry = null;
+        membershipRetryDelayMs = undefined;
        acknowledged = [];
        membershipGeneration += 1;
        for (const agentId of desired) cancelCatchUp(agentId);
@@ -445,6 +502,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
      deliveryMode = nextMode;
      cancelMembershipRetry?.();
      cancelMembershipRetry = null;
+      membershipRetryDelayMs = undefined;
      membershipNeedsRetry = false;
      membershipGeneration += 1;
      for (const agentId of desired) cancelCatchUp(agentId);
@@ -453,6 +511,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
      desired = visible;
      visibilityCatchUpPending.clear();
      visibilityCatchUpErrors.clear();
+      manualRetries.clear();
      for (const agentId of desired) visibilityCatchUpPending.add(agentId);
      acknowledged = deliveryMode === "legacy" && connected ? desired : [];
      notifyListeners();
@@ -470,6 +529,8 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
      disposed = true;
      cancelMembershipRetry?.();
      cancelMembershipRetry = null;
+      membershipNeedsRetry = false;
+      membershipRetryDelayMs = undefined;
      sources.clear();
      membershipGeneration += 1;
      for (const agentId of desired) cancelCatchUp(agentId);
@@ -478,8 +539,10 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
      recentlyViewedAgentIds = [];
      visibilityCatchUpPending.clear();
      visibilityCatchUpErrors.clear();
+      manualRetries.clear();
      notifyListeners();
      listeners.clear();
    },
+    retryVisibleAgentTimeline,
  };
 }