From 0f2e58d4bbfe6a31a4a1ab88d4cd9b28e505f0e7 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 18 Aug 2026 14:03:17 +0200 Subject: [PATCH] fix(app): back off and offer retry when timeline sync is refused A host can refuse a timeline read for as long as it likes. Codex does this whenever another writer holds the thread: every catch-up fails with "already has an active writer" until the other session lets go. The app retried that on a fixed 1s timer with no ceiling, so an idle window sat there issuing a request and logging a warning every second, indefinitely. The refusal is normal lifecycle; the traffic was the bug. Catch-up and subscription retries now double from 1s to a 30s ceiling, resetting on success, reconnect, delivery-mode change, and visibility change so recovery stays immediate once the condition clears. Backing off alone would leave the user waiting up to 30s with no way in, so both failure surfaces get a retry they can press. Background retries stay silent; a retry the user asked for is a fallible user action and owns its pending state, so "retrying" is a status the sync model publishes rather than a React boolean. The blocking "Failed to load agent" screen was the worse half: its load effect self-gates on the error state, so a refused first resume was terminal until the user navigated away. Its retry resets the state to idle, which re-arms the effect. --- docs/timeline-sync.md | 10 ++ .../browser/agent-timeline-sync-retry.spec.ts | 87 +++++++++++++ .../support/helpers/daemon-websocket-gate.ts | 53 ++++++++ .../use-agent-screen-state-machine.test.ts | 16 +++ .../hooks/use-agent-screen-state-machine.ts | 11 +- packages/app/src/i18n/resources/ar.ts | 3 +- packages/app/src/i18n/resources/en.ts | 3 +- packages/app/src/i18n/resources/es.ts | 3 +- packages/app/src/i18n/resources/fr.ts | 3 +- packages/app/src/i18n/resources/ja.ts | 3 +- packages/app/src/i18n/resources/ko.ts | 3 +- packages/app/src/i18n/resources/pt-BR.ts | 3 +- packages/app/src/i18n/resources/ru.ts | 3 +- packages/app/src/i18n/resources/zh-CN.ts | 3 +- packages/app/src/panels/agent-panel.tsx | 119 ++++++++++++++---- .../src/timeline/viewed-timeline-sync.test.ts | 114 +++++++++++++++-- .../app/src/timeline/viewed-timeline-sync.ts | 75 ++++++++++- 17 files changed, 461 insertions(+), 51 deletions(-) create mode 100644 packages/app/e2e/browser/agent-timeline-sync-retry.spec.ts diff --git a/docs/timeline-sync.md b/docs/timeline-sync.md index c055cc7ef..b8780c3c6 100644 --- a/docs/timeline-sync.md +++ b/docs/timeline-sync.md @@ -42,6 +42,16 @@ Initialization timeouts guard lack of catch-up progress, not the full multi-page Opening or resuming an agent fetches one bounded latest tail page. Older history remains user-driven by scrolling upward. +A failed catch-up or subscription reconcile retries on its own, doubling from 1s to a 30s ceiling. +A fixed 1s retry turned a persistent daemon-side refusal — a Codex thread that already has an active +writer, say — into a request and a log line every second on an idle app. The delay resets on success, +reconnect, delivery-mode change, and visibility change, so recovery is still immediate once the +condition clears. + +Background retries are silent. The retry the user presses in the sync-error callout is a fallible +user action and owns its pending state: `retrying` is a status the sync model publishes, not a React +boolean, so the button reports in-flight and the callout returns to `error` when the attempt fails. + Reaching the history-start threshold loads one older page and preserves the visible content anchor. Cursor progress does not trigger another page. The user must leave and return to the threshold unless the anchored page still leaves the viewport at history start, as with short or compacted content; in diff --git a/packages/app/e2e/browser/agent-timeline-sync-retry.spec.ts b/packages/app/e2e/browser/agent-timeline-sync-retry.spec.ts new file mode 100644 index 000000000..2f3fcab53 --- /dev/null +++ b/packages/app/e2e/browser/agent-timeline-sync-retry.spec.ts @@ -0,0 +1,87 @@ +import { expect, test } from "../support/fixtures"; +import { installDaemonWebSocketGate } from "../support/helpers/daemon-websocket-gate"; +import { + expectTimelinePromptVisible, + openAgentTimeline, + seedLongMockAgentTimeline, +} from "../support/helpers/timeline-pagination"; +import { + disconnectViewedTimeline, + restoreViewedTimelineWithHeldResponse, +} from "../support/helpers/timeline-resume"; + +test.describe("Agent timeline sync retry", () => { + test("a refused catch-up offers a retry that reports pending, failure, and recovery", async ({ + page, + }, testInfo) => { + test.setTimeout(120_000); + const gate = await installDaemonWebSocketGate(page); + const agent = await seedLongMockAgentTimeline({ turns: 2 }); + try { + await openAgentTimeline(page, agent); + await expectTimelinePromptVisible(page, agent.newestPrompt); + + gate.failTimelineResponses(agent.agentId); + await disconnectViewedTimeline(page, gate); + await restoreViewedTimelineWithHeldResponse(page, gate); + + const callout = page.getByTestId("agent-timeline-sync-error"); + const retry = page.getByTestId("agent-timeline-sync-retry"); + await expect(callout).toBeVisible({ timeout: 30_000 }); + await expect(retry).toBeEnabled(); + await page.screenshot({ path: testInfo.outputPath("1-sync-error.png"), fullPage: true }); + + gate.holdTimelineResponses(agent.agentId); + await retry.click(); + await gate.waitForHeldTimelineResponse(); + await expect(retry).toBeDisabled(); + await expect(retry).toHaveText("Retrying…"); + await page.screenshot({ path: testInfo.outputPath("2-retry-pending.png"), fullPage: true }); + + gate.releaseHeldTimelineResponses(); + await expect(retry).toBeEnabled({ timeout: 30_000 }); + await expect(callout).toBeVisible(); + await page.screenshot({ path: testInfo.outputPath("3-retry-failed.png"), fullPage: true }); + + gate.allowTimelineResponses(); + await retry.click(); + await expect(callout).toBeHidden({ timeout: 30_000 }); + await expectTimelinePromptVisible(page, agent.newestPrompt); + await page.screenshot({ path: testInfo.outputPath("4-recovered.png"), fullPage: true }); + } finally { + gate.restore(); + await agent.cleanup(); + } + }); + + test("a refused first load offers a retry instead of a dead panel", async ({ + page, + }, testInfo) => { + test.setTimeout(120_000); + const gate = await installDaemonWebSocketGate(page); + const agent = await seedLongMockAgentTimeline({ turns: 2 }); + try { + gate.failTimelineResponses(agent.agentId); + await openAgentTimeline(page, agent); + + const loadError = page.getByTestId("agent-load-error"); + const retry = page.getByTestId("agent-load-error-retry"); + await expect(loadError).toBeVisible({ timeout: 60_000 }); + await expect(loadError).toContainText("already has an active writer"); + await page.screenshot({ path: testInfo.outputPath("5-load-error.png"), fullPage: true }); + + await retry.click(); + await expect(loadError).toBeVisible({ timeout: 30_000 }); + await page.screenshot({ path: testInfo.outputPath("6-load-retry-failed.png") }); + + gate.allowTimelineResponses(); + await retry.click(); + await expect(loadError).toBeHidden({ timeout: 30_000 }); + await expectTimelinePromptVisible(page, agent.newestPrompt); + await page.screenshot({ path: testInfo.outputPath("7-load-recovered.png"), fullPage: true }); + } finally { + gate.restore(); + await agent.cleanup(); + } + }); +}); diff --git a/packages/app/e2e/support/helpers/daemon-websocket-gate.ts b/packages/app/e2e/support/helpers/daemon-websocket-gate.ts index 869c8f1e4..83edf569b 100644 --- a/packages/app/e2e/support/helpers/daemon-websocket-gate.ts +++ b/packages/app/e2e/support/helpers/daemon-websocket-gate.ts @@ -133,6 +133,23 @@ function forceTimelineReset(message: string | Buffer, enabled: boolean): string return JSON.stringify(envelope); } +// The daemon refusal this reproduces: a Codex thread that already has an active writer. +const TIMELINE_WRITER_CONFLICT_ERROR = + "Failed to resume Codex thread playwright-thread: thread playwright-thread already has an active writer"; + +function failTimelineResponse(message: string | Buffer, agentId: string | null): string | Buffer { + if (!agentId || typeof message !== "string") return message; + const envelope = JSON.parse(message) as { + message?: { payload?: Record }; + payload?: Record; + }; + const payload = envelope.message?.payload ?? envelope.payload; + if (!payload || payload.agentId !== agentId) return message; + payload.error = TIMELINE_WRITER_CONFLICT_ERROR; + payload.entries = []; + return JSON.stringify(envelope); +} + function rewriteShellToolCommand( message: string | Buffer, command: string | null, @@ -284,6 +301,10 @@ export async function installDaemonWebSocketGate(page: Page) { let stripAssistantMessageIds = false; let stripCanonicalSubmittedPromptsFeature = false; let shellToolCommandOverride: string | null = null; + let failingTimelineAgentId: string | null = null; + let holdingTimelineAgentId: string | null = null; + const heldTimelineResponses: Array<() => void> = []; + const heldTimelineResponseWaiters = new Set<() => void>(); let heldClientRequestType: string | null = null; let heldClientRequest: { server: WebSocketRoute; message: string | Buffer } | null = null; let resolveHeldClientRequest: (() => void) | null = null; @@ -487,11 +508,25 @@ export async function installDaemonWebSocketGate(page: Page) { stripCanonicalSubmittedPromptsFeature, serverMessage?.type, ); + const isTimelineResponse = serverMessage?.type === "fetch_agent_timeline_response"; + if (isTimelineResponse) { + outboundMessage = failTimelineResponse(outboundMessage, failingTimelineAgentId); + } const shouldForceTimelineReset = forceTimelineEpochReset && serverMessage?.type === "fetch_agent_timeline_response"; outboundMessage = forceTimelineReset(outboundMessage, shouldForceTimelineReset); if (shouldForceTimelineReset) forceTimelineEpochReset = false; recordServerMessage(serverMessage); + if (isTimelineResponse && holdingTimelineAgentId) { + const payload = (serverMessage as { payload?: { agentId?: unknown } } | null)?.payload; + if (payload?.agentId === holdingTimelineAgentId) { + const forward = outboundMessage; + heldTimelineResponses.push(() => ws.send(forward)); + for (const resolve of heldTimelineResponseWaiters) resolve(); + heldTimelineResponseWaiters.clear(); + return; + } + } if (holdServerMessage({ browser: ws, message: outboundMessage, parsed: serverMessage })) return; if (holdReadyFileUpdate(ws, outboundMessage, fileMessage)) return; @@ -560,6 +595,24 @@ export async function installDaemonWebSocketGate(page: Page) { restore(): void { acceptingConnections = true; }, + failTimelineResponses(agentId: string): void { + failingTimelineAgentId = agentId; + }, + allowTimelineResponses(): void { + failingTimelineAgentId = null; + }, + holdTimelineResponses(agentId: string): void { + holdingTimelineAgentId = agentId; + }, + async waitForHeldTimelineResponse(): Promise { + while (heldTimelineResponses.length === 0) { + await new Promise((resolve) => heldTimelineResponseWaiters.add(resolve)); + } + }, + releaseHeldTimelineResponses(): void { + holdingTimelineAgentId = null; + for (const forward of heldTimelineResponses.splice(0)) forward(); + }, restoreFresh(): void { reconnectWithFreshClient = true; acceptingConnections = true; diff --git a/packages/app/src/hooks/use-agent-screen-state-machine.test.ts b/packages/app/src/hooks/use-agent-screen-state-machine.test.ts index 88cce1e35..d76f60362 100644 --- a/packages/app/src/hooks/use-agent-screen-state-machine.test.ts +++ b/packages/app/src/hooks/use-agent-screen-state-machine.test.ts @@ -317,6 +317,22 @@ describe("deriveAgentScreenViewState", () => { expect(ready.agent.id).toBe("agent-1"); }); + it("marks the sync error as retrying while a user-requested retry is in flight", () => { + const memory = createBaseMemory({ + hasRenderedReady: true, + lastReadyAgent: createAgent("agent-1"), + }); + const input: AgentScreenMachineInput = { + ...createBaseInput(), + visibilityCatchUpStatus: "retrying", + }; + + const result = deriveAgentScreenViewState({ input, memory }); + const ready = expectReadyState(result.state); + + expect(ready.sync).toEqual({ status: "sync_error", isRetrying: true }); + }); + it("returns blocking error before first paint when refresh fails", () => { const memory = createBaseMemory(); const input: AgentScreenMachineInput = { diff --git a/packages/app/src/hooks/use-agent-screen-state-machine.ts b/packages/app/src/hooks/use-agent-screen-state-machine.ts index 1b8dca725..09a4c907c 100644 --- a/packages/app/src/hooks/use-agent-screen-state-machine.ts +++ b/packages/app/src/hooks/use-agent-screen-state-machine.ts @@ -4,6 +4,7 @@ import type { AgentFeature, AgentProvider, } from "@getpaseo/protocol/agent-types"; +import type { ViewedTimelineStatus } from "@/timeline/viewed-timeline-sync"; export interface AgentScreenAgent { serverId: string; @@ -47,7 +48,7 @@ export interface AgentScreenMachineInput { isArchivingCurrentAgent: boolean; isHistorySyncing: boolean; needsAuthoritativeSync: boolean; - visibilityCatchUpStatus: "ready" | "pending" | "error"; + visibilityCatchUpStatus: ViewedTimelineStatus; continuity: AgentScreenContinuity; hasHydratedHistoryBefore: boolean; } @@ -82,7 +83,7 @@ export type AgentScreenReadySyncState = status: "catching_up"; ui: "overlay" | "silent"; } - | { status: "sync_error" }; + | { status: "sync_error"; isRetrying: boolean }; export type AgentScreenViewState = | { @@ -173,10 +174,10 @@ function resolveAgentScreenSync(args: { return { status: "reconnecting" }; } if (input.missingAgentState.kind === "error") { - return { status: "sync_error" }; + return { status: "sync_error", isRetrying: input.visibilityCatchUpStatus === "retrying" }; } - if (input.visibilityCatchUpStatus === "error") { - return { status: "sync_error" }; + if (input.visibilityCatchUpStatus === "error" || input.visibilityCatchUpStatus === "retrying") { + return { status: "sync_error", isRetrying: input.visibilityCatchUpStatus === "retrying" }; } if ( input.visibilityCatchUpStatus === "pending" || diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index be3badda9..a6c92e800 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -215,7 +215,8 @@ export const ar: TranslationResources = { notFound: "لم يتم العثور على Agent", failedToLoad: "فشل تحميل الوكيل", reconnecting: "جارٍ إعادة الاتصال", - timelineSyncFailed: "تعذر تحديث سجل الوكيل. جارٍ إعادة المحاولة…", + timelineSyncFailed: "تعذر تحديث سجل الوكيل.", + timelineSyncRetrying: "جارٍ إعادة المحاولة…", archivingTitle: "وكيل الارشيف...", archivingSubtitle: "الرجاء الانتظار بينما نقوم بأرشفة هذا الوكيل.", }, diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 8e300872b..686b0e304 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -212,7 +212,8 @@ export const en = { notFound: "Agent not found", failedToLoad: "Failed to load agent", reconnecting: "Reconnecting", - timelineSyncFailed: "Couldn't refresh agent history. Retrying…", + timelineSyncFailed: "Couldn't refresh agent history.", + timelineSyncRetrying: "Retrying…", archivingTitle: "Archiving agent...", archivingSubtitle: "Please wait while we archive this agent.", }, diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 046ffbed0..6f01b6abf 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -215,7 +215,8 @@ export const es: TranslationResources = { notFound: "Agentno encontrado", failedToLoad: "No se pudo cargar el agente", reconnecting: "Reconectando", - timelineSyncFailed: "No se pudo actualizar el historial del agente. Reintentando…", + timelineSyncFailed: "No se pudo actualizar el historial del agente.", + timelineSyncRetrying: "Reintentando…", archivingTitle: "Agente de archivo...", archivingSubtitle: "Espere mientras archivamos este agente.", }, diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 637cd88f7..6ab9e49f3 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -217,7 +217,8 @@ export const fr: TranslationResources = { notFound: "Agentintrouvable", failedToLoad: "Échec du chargement de l'agent", reconnecting: "Reconnexion", - timelineSyncFailed: "Impossible d’actualiser l’historique de l’agent. Nouvelle tentative…", + timelineSyncFailed: "Impossible d’actualiser l’historique de l’agent.", + timelineSyncRetrying: "Nouvelle tentative…", archivingTitle: "Agent d'archivage...", archivingSubtitle: "Veuillez patienter pendant que nous archivons cet agent.", }, diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index cd8cc0a50..08f2f9575 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -215,7 +215,8 @@ export const ja: TranslationResources = { notFound: "エージェントが見つかりません", failedToLoad: "エージェントの読み込みに失敗しました", reconnecting: "再接続中", - timelineSyncFailed: "エージェントの履歴を更新できませんでした。再試行しています…", + timelineSyncFailed: "エージェントの履歴を更新できませんでした。", + timelineSyncRetrying: "再試行しています…", archivingTitle: "エージェントをアーカイブ中...", archivingSubtitle: "このエージェントをアーカイブするまでお待ちください。", }, diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index c13c0ed3c..7305ef42d 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -215,7 +215,8 @@ export const ko: TranslationResources = { notFound: "에이전트를 찾을 수 없습니다", failedToLoad: "에이전트를 불러오지 못했습니다", reconnecting: "다시 연결하는 중", - timelineSyncFailed: "에이전트 기록을 새로고침할 수 없습니다. 재시도 중…", + timelineSyncFailed: "에이전트 기록을 새로고침할 수 없습니다.", + timelineSyncRetrying: "재시도 중…", archivingTitle: "에이전트 보관 중...", archivingSubtitle: "이 에이전트를 보관하는 동안 잠시 기다려 주세요.", }, diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 26340cc50..eae8e3d6c 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -215,7 +215,8 @@ export const ptBR: TranslationResources = { notFound: "Agente não encontrado", failedToLoad: "Falha ao carregar agente", reconnecting: "Reconectando", - timelineSyncFailed: "Não foi possível atualizar o histórico do agente. Tentando novamente…", + timelineSyncFailed: "Não foi possível atualizar o histórico do agente.", + timelineSyncRetrying: "Tentando novamente…", archivingTitle: "Arquivando agente...", archivingSubtitle: "Aguarde enquanto arquivamos este agente.", }, diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index d54d9ae9b..12e543b98 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -215,7 +215,8 @@ export const ru: TranslationResources = { notFound: "Agent не найден", failedToLoad: "Не удалось загрузить агент", reconnecting: "Повторное подключение", - timelineSyncFailed: "Не удалось обновить историю агента. Повторная попытка…", + timelineSyncFailed: "Не удалось обновить историю агента.", + timelineSyncRetrying: "Повторная попытка…", archivingTitle: "Архивный агент...", archivingSubtitle: "Пожалуйста, подождите, пока мы архивируем этого агента.", }, diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 30a88d4d6..86412a29f 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -215,7 +215,8 @@ export const zhCN: TranslationResources = { notFound: "未找到 Agent", failedToLoad: "加载 Agent 失败", reconnecting: "正在重连", - timelineSyncFailed: "无法刷新代理历史记录。正在重试…", + timelineSyncFailed: "无法刷新代理历史记录。", + timelineSyncRetrying: "正在重试…", archivingTitle: "正在归档 Agent...", archivingSubtitle: "请稍候，我们正在归档这个 Agent。", }, diff --git a/packages/app/src/panels/agent-panel.tsx b/packages/app/src/panels/agent-panel.tsx index 4f70c2b68..3f01e5786 100644 --- a/packages/app/src/panels/agent-panel.tsx +++ b/packages/app/src/panels/agent-panel.tsx @@ -1,3 +1,4 @@ +import { Button } from "@/components/ui/button"; import { LoadingSpinner } from "@/components/ui/loading-spinner"; import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import type { TFunction } from "i18next"; @@ -23,7 +24,6 @@ import { AgentStreamView, type AgentStreamViewHandle } from "@/agent-stream/view import { ArchivedAgentCallout } from "@/components/archived-agent-callout"; import { FileDropZone } from "@/components/file-drop/file-drop-zone"; import { useRetainedPanelActive } from "@/components/retained-panel"; -import { SidebarCallout } from "@/components/sidebar-callout"; import { Composer } from "@/composer"; import { resolveComposerTrackTailClearance } from "@/composer/pill-styles"; import { getActiveMessageSubmissions } from "@/composer/submission/model"; @@ -37,7 +37,11 @@ import { } from "@/components/toast-host"; import type { WorkspaceComposerAttachment } from "@/attachments/types"; import { useWorkspaceAttachmentScopeKey } from "@/attachments/workspace-attachments-store"; -import { COMPACT_FORM_FACTOR_WIDTH, useIsCompactFormFactor } from "@/constants/layout"; +import { + COMPACT_FORM_FACTOR_WIDTH, + MAX_CONTENT_WIDTH, + useIsCompactFormFactor, +} from "@/constants/layout"; import { isWeb } from "@/constants/platform"; import { useAgentAttentionClear } from "@/hooks/use-agent-attention-clear"; import { useAgentInitialization } from "@/hooks/use-agent-initialization"; @@ -197,12 +201,30 @@ function buildChatAgentFromState( }; } +function AgentLoadErrorView({ message, onRetry }: { message: string; onRetry: () => void }) { + const { t } = useTranslation(); + return ( + + + {t("agentPanel.states.failedToLoad")} + {message} + + + + + + ); +} + function renderChatAgentNonReadyView(args: { viewState: AgentScreenViewState; effectiveAgent: AgentScreenAgent | null; t: TFunction; + onRetryLoad: () => void; }): React.ReactElement | null { - const { viewState, effectiveAgent, t } = args; + const { viewState, effectiveAgent, t, onRetryLoad } = args; if (viewState.tag === "not_found") { return ( @@ -213,14 +235,7 @@ function renderChatAgentNonReadyView(args: { ); } if (viewState.tag === "error") { - return ( - - - {t("agentPanel.states.failedToLoad")} - {viewState.message} - - - ); + return ; } if (viewState.tag === "boot" || !effectiveAgent) { return ( @@ -603,6 +618,7 @@ function AgentPanelBody({ ); const [lookupState, setLookupState] = useState({ tag: "idle" }); const lookupAttemptTokenRef = useRef(0); + const retryAgentLookup = useCallback(() => setLookupState({ tag: "idle" }), []); const workspaceKey = buildWorkspaceTabPersistenceKey({ serverId, workspaceId }); const resolvePendingAgent = useWorkspaceLayoutStore((state) => state.resolvePendingAgent); @@ -695,14 +711,7 @@ function AgentPanelBody({ } if (lookupState.tag === "error") { - return ( - - - {t("agentPanel.states.failedToLoad")} - {lookupState.message} - - - ); + return ; } const agent: AgentScreenAgent | null = @@ -1029,6 +1038,13 @@ function ChatAgentContent({ streamViewRef.current?.scrollToBottom("message-sent"); }, [agentId]); + const retryAgentLoad = useCallback(() => setMissingAgentState({ kind: "idle" }), []); + + const retryTimelineSync = useCallback(() => { + if (!agentId || !viewedTimelineSync) return; + viewedTimelineSync.retryVisibleAgentTimeline(agentId); + }, [agentId, viewedTimelineSync]); + useEffect(() => { initAttemptTokenRef.current += 1; setMissingAgentState({ kind: "idle" }); @@ -1127,6 +1143,7 @@ function ChatAgentContent({ viewState, effectiveAgent, t, + onRetryLoad: retryAgentLoad, }); if (nonReadyView) return nonReadyView; invariant(agentId, "agent id is defined when agent content is ready"); @@ -1138,6 +1155,10 @@ function ChatAgentContent({ viewState.sync.status === "catching_up" && viewState.sync.ui === "overlay"; const showHistorySyncError = viewState.tag === "ready" && viewState.sync.status === "sync_error"; + const isRetryingHistorySync = + viewState.tag === "ready" && + viewState.sync.status === "sync_error" && + viewState.sync.isRetrying; return ( void; showHistorySyncOverlay: boolean; showHistorySyncError: boolean; + isRetryingHistorySync: boolean; + retryTimelineSync: () => void; cwd: string; onAttentionInputFocus: () => void; onAttentionPromptSend: () => void; @@ -1333,11 +1360,26 @@ const ChatAgentReadyContent = memo(function ChatAgentReadyContent({ {contentContainer} {showHistorySyncError ? ( - + + + + + {t("agentPanel.states.timelineSyncFailed")} + + + + + ) : null} {composerSection} @@ -1745,6 +1787,32 @@ const styles = StyleSheet.create((theme) => ({ overflow: "hidden", ...(isWeb ? { userSelect: "none" as const } : {}), }, + timelineSyncCalloutRail: { + width: "100%", + alignItems: "center", + paddingHorizontal: theme.spacing[4], + paddingTop: theme.spacing[2], + }, + timelineSyncCalloutContent: { + width: "100%", + maxWidth: MAX_CONTENT_WIDTH, + }, + timelineSyncCallout: { + flexDirection: "row", + alignItems: "center", + justifyContent: "center", + gap: theme.spacing[3], + backgroundColor: theme.colors.surface1, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.statusDanger, + borderRadius: theme.borderRadius["2xl"], + paddingVertical: theme.spacing[2], + paddingHorizontal: theme.spacing[4], + }, + timelineSyncCalloutText: { + color: theme.colors.foregroundMuted, + fontSize: theme.fontSize.base, + }, historySyncOverlay: { position: "absolute", top: 0, @@ -1807,6 +1875,9 @@ const styles = StyleSheet.create((theme) => ({ color: theme.colors.foregroundMuted, textAlign: "center", }, + errorAction: { + marginTop: theme.spacing[4], + }, statusText: { marginTop: theme.spacing[2], textAlign: "center", diff --git a/packages/app/src/timeline/viewed-timeline-sync.test.ts b/packages/app/src/timeline/viewed-timeline-sync.test.ts index a114ea2af..f158178e1 100644 --- a/packages/app/src/timeline/viewed-timeline-sync.test.ts +++ b/packages/app/src/timeline/viewed-timeline-sync.test.ts @@ -79,10 +79,11 @@ class TimelineWorld { schedule: (task, delayMs) => { const scheduled = { task, delayMs }; this.scheduled.push(scheduled); - const waiter = this.retryWaiters.shift(); - if (waiter && delayMs === 1_000) { + const waiterIndex = this.retryWaiters.findIndex((waiter) => waiter.delayMs === delayMs); + if (waiterIndex >= 0) { + const [waiter] = this.retryWaiters.splice(waiterIndex, 1); this.scheduled.splice(this.scheduled.indexOf(scheduled), 1); - waiter(task); + waiter.resolve(task); } return () => { const index = this.scheduled.indexOf(scheduled); @@ -100,7 +101,10 @@ class TimelineWorld { }> = []; private readonly errorWaiters: Array<(message: string) => void> = []; private readonly scheduled: Array<{ task: () => void; delayMs: number }> = []; - private readonly retryWaiters: Array<(retry: () => void) => void> = []; + private readonly retryWaiters: Array<{ + delayMs: number; + resolve(retry: () => void): void; + }> = []; private fetchTimeline( agentId: string, @@ -147,10 +151,10 @@ class TimelineWorld { return new Promise((resolve) => this.errorWaiters.push(resolve)); } - nextRetry(): Promise<() => void> { - const index = this.scheduled.findIndex((entry) => entry.delayMs === 1_000); + nextRetry(delayMs = 1_000): Promise<() => void> { + const index = this.scheduled.findIndex((entry) => entry.delayMs === delayMs); if (index >= 0) return Promise.resolve(this.scheduled.splice(index, 1)[0].task); - return new Promise((resolve) => this.retryWaiters.push(resolve)); + return new Promise((resolve) => this.retryWaiters.push({ delayMs, resolve })); } elapse(elapsedMs: number): void { @@ -411,6 +415,74 @@ test("a failed catch-up reports once and retries through the explicit retry poli world.expectNoPendingMembership(); }); +test("a failed catch-up retries with exponential backoff", async () => { + const world = new TimelineWorld(); + world.sync.setConnected(true); + world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]); + const membership = await world.nextMembership(); + membership.succeed(); + + const first = await world.nextFetch("agent-a"); + first.fail("timeline unavailable"); + const [firstError, retryAfterFirstFailure] = await Promise.all([ + world.nextError(), + world.nextRetry(), + ]); + expect(firstError).toBe("timeline unavailable"); + retryAfterFirstFailure(); + + const second = await world.nextFetch("agent-a"); + second.fail("timeline unavailable"); + await world.nextError(); + + const retryAfterSecondFailure = await world.nextRetry(2_000); + expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error"); + + retryAfterSecondFailure(); + const third = await world.nextFetch("agent-a"); + third.respond({ hasNewer: false }); + await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready")); +}); + +test("manual retries can immediately re-attempt a failed catch-up", async () => { + const world = new TimelineWorld(); + world.sync.setConnected(true); + world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]); + const membership = await world.nextMembership(); + membership.succeed(); + + const failed = await world.nextFetch("agent-a"); + failed.fail("timeline unavailable"); + await world.nextRetry(); + expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error"); + + world.sync.retryVisibleAgentTimeline("agent-a"); + expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("retrying"); + + const retry = await world.nextFetch("agent-a"); + retry.respond({ hasNewer: false }); + await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready")); +}); + +test("a manual retry that fails returns to the error state", async () => { + const world = new TimelineWorld(); + world.sync.setConnected(true); + world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]); + const membership = await world.nextMembership(); + membership.succeed(); + + const failed = await world.nextFetch("agent-a"); + failed.fail("timeline unavailable"); + await world.nextRetry(); + + world.sync.retryVisibleAgentTimeline("agent-a"); + expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("retrying"); + + const retry = await world.nextFetch("agent-a"); + retry.fail("timeline still unavailable"); + await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error")); +}); + test("gap recovery supersedes completed catch-up and pages through the current tail", async () => { const world = new TimelineWorld(); world.sync.setConnected(true); @@ -486,6 +558,34 @@ test("membership failure autonomously retries without another visibility declara }); }); +test("membership failures retry with exponential backoff", async () => { + const world = new TimelineWorld(); + world.sync.setConnected(true); + world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]); + + const first = await world.nextMembership(); + first.fail("subscription unavailable"); + const [firstError, retryAfterFirstFailure] = await Promise.all([ + world.nextError(), + world.nextRetry(), + ]); + expect(firstError).toBe("subscription unavailable"); + + retryAfterFirstFailure(); + const second = await world.nextMembership(); + second.fail("subscription unavailable again"); + + const retryAfterSecondFailure = await world.nextRetry(2_000); + expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error"); + + retryAfterSecondFailure(); + const retry = await world.nextMembership(); + retry.succeed(); + const catchUp = await world.nextFetch("agent-a"); + catchUp.respond({ hasNewer: false }); + await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready")); +}); + test("backgrounding preserves the hot membership without catch-up on return", async () => { const world = new TimelineWorld(); world.sync.setConnected(true); diff --git a/packages/app/src/timeline/viewed-timeline-sync.ts b/packages/app/src/timeline/viewed-timeline-sync.ts index 4feed346b..110ce0ace 100644 --- a/packages/app/src/timeline/viewed-timeline-sync.ts +++ b/packages/app/src/timeline/viewed-timeline-sync.ts @@ -23,12 +23,13 @@ interface ViewedTimelineSyncPorts { } export type TimelineDeliveryMode = "legacy" | "selective"; -export type ViewedTimelineStatus = "ready" | "pending" | "error"; +export type ViewedTimelineStatus = "ready" | "pending" | "error" | "retrying"; export interface ViewedTimelineUiBridge { replaceVisibleAgentIds(sourceId: string, agentIds: string[]): void; subscribe(listener: () => void): () => void; getAgentTimelineStatus(agentId: string): ViewedTimelineStatus; + retryVisibleAgentTimeline(agentId: string): void; } export interface ViewedTimelineSync extends ViewedTimelineUiBridge { @@ -40,6 +41,7 @@ export interface ViewedTimelineSync extends ViewedTimelineUiBridge { } const RETRY_DELAY_MS = 1_000; +const MAX_RETRY_DELAY_MS = 30_000; const VIEWED_TIMELINE_HOT_AGENT_LIMIT = 5; type CatchUpStatus = "running" | "complete" | "error"; @@ -49,8 +51,16 @@ interface CatchUpState { status: CatchUpStatus; request?: ProjectedTimelineForwardFetchPlan; cancelRetry?: () => void; + retryDelayMs?: number; } +const getNextRetryDelayMs = (previousDelayMs: number | undefined): number => { + if (previousDelayMs == null) { + return RETRY_DELAY_MS; + } + return Math.min(previousDelayMs * 2, MAX_RETRY_DELAY_MS); +}; + function isSameCatchUpRequest( left: ProjectedTimelineForwardFetchPlan | undefined, right: ProjectedTimelineForwardFetchPlan | undefined, @@ -102,6 +112,9 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed const pendingCatchUps = new Map(); const visibilityCatchUpPending = new Set(); const visibilityCatchUpErrors = new Set(); + // User-initiated retries only. Background retries stay silent; a retry the user asked for + // owes them a pending state until it settles. + const manualRetries = new Set(); const listeners = new Set<() => void>(); let active = true; let connected = false; @@ -113,6 +126,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed let reconciling = false; let reconcileRequested = false; let membershipNeedsRetry = false; + let membershipRetryDelayMs: number | undefined; let cancelMembershipRetry: (() => void) | null = null; let recentlyViewedAgentIds: string[] = []; @@ -152,12 +166,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed const setVisibilityCatchUpReady = (agentId: string) => { const wasPending = visibilityCatchUpPending.delete(agentId); const hadError = visibilityCatchUpErrors.delete(agentId); - if (wasPending || hadError) notifyListeners(); + const wasRetrying = manualRetries.delete(agentId); + if (wasPending || hadError || wasRetrying) notifyListeners(); }; const setVisibilityCatchUpError = (agentIds: string[]) => { let changed = false; for (const agentId of agentIds) { + if (manualRetries.delete(agentId)) changed = true; if (!visibilityCatchUpPending.delete(agentId)) continue; visibilityCatchUpErrors.add(agentId); changed = true; @@ -210,12 +226,19 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed setVisibilityCatchUpReady(agentId); } catch (error) { if (catchUps.get(agentId)?.generation === generation) { + const nextRetryDelayMs = getNextRetryDelayMs(catchUps.get(agentId)?.retryDelayMs); const cancelRetry = ports.schedule(() => { const current = catchUps.get(agentId); if (current?.generation !== generation || current.status !== "error") return; startCatchUp(agentId); - }, RETRY_DELAY_MS); - catchUps.set(agentId, { generation, status: "error", cancelRetry }); + }, nextRetryDelayMs); + catchUps.set(agentId, { + generation, + status: "error", + request, + cancelRetry, + retryDelayMs: nextRetryDelayMs, + }); setVisibilityCatchUpError([agentId]); ports.reportError(error); } @@ -247,7 +270,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed current?.cancelRetry?.(); const generation = (catchUpGenerations.get(agentId) ?? 0) + 1; catchUpGenerations.set(agentId, generation); - catchUps.set(agentId, { generation, status: "running", request: nextRequest }); + const retryDelayMs = + supersede || current?.status !== "error" ? undefined : current.retryDelayMs; + catchUps.set(agentId, { + generation, + status: "running", + request: nextRequest, + retryDelayMs, + }); pendingCatchUps.delete(agentId); void fetchUntilCurrent( agentId, @@ -279,6 +309,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed membershipNeedsRetry = true; setVisibilityCatchUpError(requested); cancelMembershipRetry?.(); + const nextRetryDelayMs = getNextRetryDelayMs(membershipRetryDelayMs); cancelMembershipRetry = ports.schedule(() => { cancelMembershipRetry = null; if ( @@ -290,12 +321,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed return; } void reconcileMembership(); - }, RETRY_DELAY_MS); + }, nextRetryDelayMs); + membershipRetryDelayMs = nextRetryDelayMs; ports.reportError(error); return; } cancelMembershipRetry?.(); cancelMembershipRetry = null; + membershipRetryDelayMs = undefined; if (disposed || !connected || deliveryMode !== "selective") return; acknowledged = requested; if (generation !== membershipGeneration) { @@ -338,6 +371,25 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed } }; + const retryVisibleAgentTimeline = (agentId: string) => { + if (!isDesired(agentId) || manualRetries.has(agentId)) return; + const catchUp = catchUps.get(agentId); + const membershipRetryable = deliveryMode === "selective" && membershipNeedsRetry && connected; + if (catchUp?.status !== "error" && !membershipRetryable) return; + manualRetries.add(agentId); + notifyListeners(); + if (catchUp?.status === "error") { + catchUp.cancelRetry?.(); + startCatchUp(agentId, { request: catchUp.request, supersede: true }); + return; + } + cancelMembershipRetry?.(); + cancelMembershipRetry = null; + membershipRetryDelayMs = undefined; + membershipNeedsRetry = false; + void reconcileMembership(); + }; + const commitDesiredMembership = ( nextDesired: string[], options: { resetCatchUpStatus?: boolean } = {}, @@ -350,6 +402,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed statusChanged = true; } if (visibilityCatchUpErrors.delete(agentId)) statusChanged = true; + if (manualRetries.delete(agentId)) statusChanged = true; } } if (sameAgentIds(nextDesired, desired)) { @@ -364,12 +417,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed cancelCatchUp(agentId); visibilityCatchUpPending.delete(agentId); visibilityCatchUpErrors.delete(agentId); + manualRetries.delete(agentId); } } for (const agentId of nextDesired) { if (!desired.includes(agentId)) { visibilityCatchUpPending.add(agentId); visibilityCatchUpErrors.delete(agentId); + manualRetries.delete(agentId); } } cancelMembershipRetry?.(); @@ -403,6 +458,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed return () => listeners.delete(listener); }, getAgentTimelineStatus(agentId) { + if (manualRetries.has(agentId)) return "retrying"; if (visibilityCatchUpErrors.has(agentId)) return "error"; if (!isDesired(agentId) || visibilityCatchUpPending.has(agentId)) return "pending"; return "ready"; @@ -427,6 +483,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed commitDesiredMembership(visible, { resetCatchUpStatus: true }); cancelMembershipRetry?.(); cancelMembershipRetry = null; + membershipRetryDelayMs = undefined; acknowledged = []; membershipGeneration += 1; for (const agentId of desired) cancelCatchUp(agentId); @@ -445,6 +502,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed deliveryMode = nextMode; cancelMembershipRetry?.(); cancelMembershipRetry = null; + membershipRetryDelayMs = undefined; membershipNeedsRetry = false; membershipGeneration += 1; for (const agentId of desired) cancelCatchUp(agentId); @@ -453,6 +511,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed desired = visible; visibilityCatchUpPending.clear(); visibilityCatchUpErrors.clear(); + manualRetries.clear(); for (const agentId of desired) visibilityCatchUpPending.add(agentId); acknowledged = deliveryMode === "legacy" && connected ? desired : []; notifyListeners(); @@ -470,6 +529,8 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed disposed = true; cancelMembershipRetry?.(); cancelMembershipRetry = null; + membershipNeedsRetry = false; + membershipRetryDelayMs = undefined; sources.clear(); membershipGeneration += 1; for (const agentId of desired) cancelCatchUp(agentId); @@ -478,8 +539,10 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed recentlyViewedAgentIds = []; visibilityCatchUpPending.clear(); visibilityCatchUpErrors.clear(); + manualRetries.clear(); notifyListeners(); listeners.clear(); }, + retryVisibleAgentTimeline, }; }