jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(app): back off and offer retry when timeline sync is refused

Browse Source 
A host can refuse a timeline read for as long as it likes. Codex does
this whenever another writer holds the thread: every catch-up fails with
"already has an active writer" until the other session lets go. The app
retried that on a fixed 1s timer with no ceiling, so an idle window sat
there issuing a request and logging a warning every second, indefinitely.
The refusal is normal lifecycle; the traffic was the bug.

Catch-up and subscription retries now double from 1s to a 30s ceiling,
resetting on success, reconnect, delivery-mode change, and visibility
change so recovery stays immediate once the condition clears.

Backing off alone would leave the user waiting up to 30s with no way in,
so both failure surfaces get a retry they can press. Background retries
stay silent; a retry the user asked for is a fallible user action and
owns its pending state, so "retrying" is a status the sync model
publishes rather than a React boolean.

The blocking "Failed to load agent" screen was the worse half: its load
effect self-gates on the error state, so a refused first resume was
terminal until the user navigated away. Its retry resets the state to
idle, which re-arms the effect.
This commit is contained in:
Mohamed Boudra
2026-08-18 14:36:27 +02:00
parent 05ee7a1d31
commit 0f2e58d4bb
17 changed files with 461 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/e2e/browser/agent-timeline-sync-retry.spec.ts
+87
View File
@@ -0,0 +1,87 @@
import { expect, test } from "../support/fixtures";
import { installDaemonWebSocketGate } from "../support/helpers/daemon-websocket-gate";
import {
expectTimelinePromptVisible,
openAgentTimeline,
seedLongMockAgentTimeline,
} from "../support/helpers/timeline-pagination";
import {
disconnectViewedTimeline,
restoreViewedTimelineWithHeldResponse,
} from "../support/helpers/timeline-resume";
test.describe("Agent timeline sync retry", () => {
test("a refused catch-up offers a retry that reports pending, failure, and recovery", async ({
page,
}, testInfo) => {
test.setTimeout(120_000);
const gate = await installDaemonWebSocketGate(page);
const agent = await seedLongMockAgentTimeline({ turns: 2 });
try {
await openAgentTimeline(page, agent);
await expectTimelinePromptVisible(page, agent.newestPrompt);
gate.failTimelineResponses(agent.agentId);
await disconnectViewedTimeline(page, gate);
await restoreViewedTimelineWithHeldResponse(page, gate);
const callout = page.getByTestId("agent-timeline-sync-error");
const retry = page.getByTestId("agent-timeline-sync-retry");
await expect(callout).toBeVisible({ timeout: 30_000 });
await expect(retry).toBeEnabled();
await page.screenshot({ path: testInfo.outputPath("1-sync-error.png"), fullPage: true });
gate.holdTimelineResponses(agent.agentId);
await retry.click();
await gate.waitForHeldTimelineResponse();
await expect(retry).toBeDisabled();
await expect(retry).toHaveText("Retrying…");
await page.screenshot({ path: testInfo.outputPath("2-retry-pending.png"), fullPage: true });
gate.releaseHeldTimelineResponses();
await expect(retry).toBeEnabled({ timeout: 30_000 });
await expect(callout).toBeVisible();
await page.screenshot({ path: testInfo.outputPath("3-retry-failed.png"), fullPage: true });
gate.allowTimelineResponses();
await retry.click();
await expect(callout).toBeHidden({ timeout: 30_000 });
await expectTimelinePromptVisible(page, agent.newestPrompt);
await page.screenshot({ path: testInfo.outputPath("4-recovered.png"), fullPage: true });
} finally {
gate.restore();
await agent.cleanup();
}
});
test("a refused first load offers a retry instead of a dead panel", async ({
page,
}, testInfo) => {
test.setTimeout(120_000);
const gate = await installDaemonWebSocketGate(page);
const agent = await seedLongMockAgentTimeline({ turns: 2 });
try {
gate.failTimelineResponses(agent.agentId);
await openAgentTimeline(page, agent);
const loadError = page.getByTestId("agent-load-error");
const retry = page.getByTestId("agent-load-error-retry");
await expect(loadError).toBeVisible({ timeout: 60_000 });
await expect(loadError).toContainText("already has an active writer");
await page.screenshot({ path: testInfo.outputPath("5-load-error.png"), fullPage: true });
await retry.click();
await expect(loadError).toBeVisible({ timeout: 30_000 });
await page.screenshot({ path: testInfo.outputPath("6-load-retry-failed.png") });
gate.allowTimelineResponses();
await retry.click();
await expect(loadError).toBeHidden({ timeout: 30_000 });
await expectTimelinePromptVisible(page, agent.newestPrompt);
await page.screenshot({ path: testInfo.outputPath("7-load-recovered.png"), fullPage: true });
} finally {
gate.restore();
await agent.cleanup();
}
});
});
packages/app/e2e/support/helpers/daemon-websocket-gate.ts
+53
View File
@@ -133,6 +133,23 @@ function forceTimelineReset(message: string | Buffer, enabled: boolean): string
return JSON.stringify(envelope);
}
// The daemon refusal this reproduces: a Codex thread that already has an active writer.
const TIMELINE_WRITER_CONFLICT_ERROR =
"Failed to resume Codex thread playwright-thread: thread playwright-thread already has an active writer";
function failTimelineResponse(message: string | Buffer, agentId: string | null): string | Buffer {
if (!agentId || typeof message !== "string") return message;
const envelope = JSON.parse(message) as {
message?: { payload?: Record<string, unknown> };
payload?: Record<string, unknown>;
};
const payload = envelope.message?.payload ?? envelope.payload;
if (!payload || payload.agentId !== agentId) return message;
payload.error = TIMELINE_WRITER_CONFLICT_ERROR;
payload.entries = [];
return JSON.stringify(envelope);
}
function rewriteShellToolCommand(
message: string | Buffer,
command: string | null,
@@ -284,6 +301,10 @@ export async function installDaemonWebSocketGate(page: Page) {
let stripAssistantMessageIds = false;
let stripCanonicalSubmittedPromptsFeature = false;
let shellToolCommandOverride: string | null = null;
let failingTimelineAgentId: string | null = null;
let holdingTimelineAgentId: string | null = null;
const heldTimelineResponses: Array<() => void> = [];
const heldTimelineResponseWaiters = new Set<() => void>();
let heldClientRequestType: string | null = null;
let heldClientRequest: { server: WebSocketRoute; message: string | Buffer } | null = null;
let resolveHeldClientRequest: (() => void) | null = null;
@@ -487,11 +508,25 @@ export async function installDaemonWebSocketGate(page: Page) {
stripCanonicalSubmittedPromptsFeature,
serverMessage?.type,
);
const isTimelineResponse = serverMessage?.type === "fetch_agent_timeline_response";
if (isTimelineResponse) {
outboundMessage = failTimelineResponse(outboundMessage, failingTimelineAgentId);
}
const shouldForceTimelineReset =
forceTimelineEpochReset && serverMessage?.type === "fetch_agent_timeline_response";
outboundMessage = forceTimelineReset(outboundMessage, shouldForceTimelineReset);
if (shouldForceTimelineReset) forceTimelineEpochReset = false;
recordServerMessage(serverMessage);
if (isTimelineResponse && holdingTimelineAgentId) {
const payload = (serverMessage as { payload?: { agentId?: unknown } } | null)?.payload;
if (payload?.agentId === holdingTimelineAgentId) {
const forward = outboundMessage;
heldTimelineResponses.push(() => ws.send(forward));
for (const resolve of heldTimelineResponseWaiters) resolve();
heldTimelineResponseWaiters.clear();
return;
}
}
if (holdServerMessage({ browser: ws, message: outboundMessage, parsed: serverMessage }))
return;
if (holdReadyFileUpdate(ws, outboundMessage, fileMessage)) return;
@@ -560,6 +595,24 @@ export async function installDaemonWebSocketGate(page: Page) {
restore(): void {
acceptingConnections = true;
},
failTimelineResponses(agentId: string): void {
failingTimelineAgentId = agentId;
},
allowTimelineResponses(): void {
failingTimelineAgentId = null;
},
holdTimelineResponses(agentId: string): void {
holdingTimelineAgentId = agentId;
},
async waitForHeldTimelineResponse(): Promise<void> {
while (heldTimelineResponses.length === 0) {
await new Promise<void>((resolve) => heldTimelineResponseWaiters.add(resolve));
}
},
releaseHeldTimelineResponses(): void {
holdingTimelineAgentId = null;
for (const forward of heldTimelineResponses.splice(0)) forward();
},
restoreFresh(): void {
reconnectWithFreshClient = true;
acceptingConnections = true;
packages/app/src/hooks/use-agent-screen-state-machine.test.ts
+16
View File
@@ -317,6 +317,22 @@ describe("deriveAgentScreenViewState", () => {
expect(ready.agent.id).toBe("agent-1");
});
it("marks the sync error as retrying while a user-requested retry is in flight", () => {
const memory = createBaseMemory({
hasRenderedReady: true,
lastReadyAgent: createAgent("agent-1"),
});
const input: AgentScreenMachineInput = {
...createBaseInput(),
visibilityCatchUpStatus: "retrying",
};
const result = deriveAgentScreenViewState({ input, memory });
const ready = expectReadyState(result.state);
expect(ready.sync).toEqual({ status: "sync_error", isRetrying: true });
});
it("returns blocking error before first paint when refresh fails", () => {
const memory = createBaseMemory();
const input: AgentScreenMachineInput = {
packages/app/src/hooks/use-agent-screen-state-machine.ts
+6 -5
View File
@@ -4,6 +4,7 @@ import type {
AgentFeature,
AgentProvider,
} from "@getpaseo/protocol/agent-types";
import type { ViewedTimelineStatus } from "@/timeline/viewed-timeline-sync";
export interface AgentScreenAgent {
serverId: string;
@@ -47,7 +48,7 @@ export interface AgentScreenMachineInput {
isArchivingCurrentAgent: boolean;
isHistorySyncing: boolean;
needsAuthoritativeSync: boolean;
visibilityCatchUpStatus: "ready" | "pending" | "error";
visibilityCatchUpStatus: ViewedTimelineStatus;
continuity: AgentScreenContinuity;
hasHydratedHistoryBefore: boolean;
}
@@ -82,7 +83,7 @@ export type AgentScreenReadySyncState =
status: "catching_up";
ui: "overlay" | "silent";
}
| { status: "sync_error" };
| { status: "sync_error"; isRetrying: boolean };
export type AgentScreenViewState =
| {
@@ -173,10 +174,10 @@ function resolveAgentScreenSync(args: {
return { status: "reconnecting" };
}
if (input.missingAgentState.kind === "error") {
return { status: "sync_error" };
return { status: "sync_error", isRetrying: input.visibilityCatchUpStatus === "retrying" };
}
if (input.visibilityCatchUpStatus === "error") {
return { status: "sync_error" };
if (input.visibilityCatchUpStatus === "error" || input.visibilityCatchUpStatus === "retrying") {
return { status: "sync_error", isRetrying: input.visibilityCatchUpStatus === "retrying" };
}
if (
input.visibilityCatchUpStatus === "pending" ||
packages/app/src/i18n/resources/ar.ts
+2 -1
View File
@@ -215,7 +215,8 @@ export const ar: TranslationResources = {
notFound: "لم يتم العثور على Agent",
failedToLoad: "فشل تحميل الوكيل",
reconnecting: "جارٍ إعادة الاتصال",
timelineSyncFailed: "تعذر تحديث سجل الوكيل. جارٍ إعادة المحاولة…",
timelineSyncFailed: "تعذر تحديث سجل الوكيل.",
timelineSyncRetrying: "جارٍ إعادة المحاولة…",
archivingTitle: "وكيل الارشيف...",
archivingSubtitle: "الرجاء الانتظار بينما نقوم بأرشفة هذا الوكيل.",
},
packages/app/src/i18n/resources/en.ts
+2 -1
View File
@@ -212,7 +212,8 @@ export const en = {
notFound: "Agent not found",
failedToLoad: "Failed to load agent",
reconnecting: "Reconnecting",
timelineSyncFailed: "Couldn't refresh agent history. Retrying…",
timelineSyncFailed: "Couldn't refresh agent history.",
timelineSyncRetrying: "Retrying…",
archivingTitle: "Archiving agent...",
archivingSubtitle: "Please wait while we archive this agent.",
},
packages/app/src/i18n/resources/es.ts
+2 -1
View File
@@ -215,7 +215,8 @@ export const es: TranslationResources = {
notFound: "Agentno encontrado",
failedToLoad: "No se pudo cargar el agente",
reconnecting: "Reconectando",
timelineSyncFailed: "No se pudo actualizar el historial del agente. Reintentando…",
timelineSyncFailed: "No se pudo actualizar el historial del agente.",
timelineSyncRetrying: "Reintentando…",
archivingTitle: "Agente de archivo...",
archivingSubtitle: "Espere mientras archivamos este agente.",
},
packages/app/src/i18n/resources/fr.ts
+2 -1
View File
@@ -217,7 +217,8 @@ export const fr: TranslationResources = {
notFound: "Agentintrouvable",
failedToLoad: "Échec du chargement de l'agent",
reconnecting: "Reconnexion",
timelineSyncFailed: "Impossible d’actualiser l’historique de l’agent. Nouvelle tentative…",
timelineSyncFailed: "Impossible d’actualiser l’historique de l’agent.",
timelineSyncRetrying: "Nouvelle tentative…",
archivingTitle: "Agent d'archivage...",
archivingSubtitle: "Veuillez patienter pendant que nous archivons cet agent.",
},
packages/app/src/i18n/resources/ja.ts
+2 -1
View File
@@ -215,7 +215,8 @@ export const ja: TranslationResources = {
notFound: "エージェントが見つかりません",
failedToLoad: "エージェントの読み込みに失敗しました",
reconnecting: "再接続中",
timelineSyncFailed: "エージェントの履歴を更新できませんでした。再試行しています…",
timelineSyncFailed: "エージェントの履歴を更新できませんでした。",
timelineSyncRetrying: "再試行しています…",
archivingTitle: "エージェントをアーカイブ中...",
archivingSubtitle: "このエージェントをアーカイブするまでお待ちください。",
},
packages/app/src/i18n/resources/ko.ts
+2 -1
View File
@@ -215,7 +215,8 @@ export const ko: TranslationResources = {
notFound: "에이전트를 찾을 수 없습니다",
failedToLoad: "에이전트를 불러오지 못했습니다",
reconnecting: "다시 연결하는 중",
timelineSyncFailed: "에이전트 기록을 새로고침할 수 없습니다. 재시도 중…",
timelineSyncFailed: "에이전트 기록을 새로고침할 수 없습니다.",
timelineSyncRetrying: "재시도 중…",
archivingTitle: "에이전트 보관 중...",
archivingSubtitle: "이 에이전트를 보관하는 동안 잠시 기다려 주세요.",
},
packages/app/src/i18n/resources/pt-BR.ts
+2 -1
View File
@@ -215,7 +215,8 @@ export const ptBR: TranslationResources = {
notFound: "Agente não encontrado",
failedToLoad: "Falha ao carregar agente",
reconnecting: "Reconectando",
timelineSyncFailed: "Não foi possível atualizar o histórico do agente. Tentando novamente…",
timelineSyncFailed: "Não foi possível atualizar o histórico do agente.",
timelineSyncRetrying: "Tentando novamente…",
archivingTitle: "Arquivando agente...",
archivingSubtitle: "Aguarde enquanto arquivamos este agente.",
},
packages/app/src/i18n/resources/ru.ts
+2 -1
View File
@@ -215,7 +215,8 @@ export const ru: TranslationResources = {
notFound: "Agent не найден",
failedToLoad: "Не удалось загрузить агент",
reconnecting: "Повторное подключение",
timelineSyncFailed: "Не удалось обновить историю агента. Повторная попытка…",
timelineSyncFailed: "Не удалось обновить историю агента.",
timelineSyncRetrying: "Повторная попытка…",
archivingTitle: "Архивный агент...",
archivingSubtitle: "Пожалуйста, подождите, пока мы архивируем этого агента.",
},
packages/app/src/i18n/resources/zh-CN.ts
+2 -1
View File
@@ -215,7 +215,8 @@ export const zhCN: TranslationResources = {
notFound: "未找到 Agent",
failedToLoad: "加载 Agent 失败",
reconnecting: "正在重连",
timelineSyncFailed: "无法刷新代理历史记录。正在重试…",
timelineSyncFailed: "无法刷新代理历史记录。",
timelineSyncRetrying: "正在重试…",
archivingTitle: "正在归档 Agent...",
archivingSubtitle: "请稍候，我们正在归档这个 Agent。",
},
packages/app/src/panels/agent-panel.tsx
+95 -24
View File
@@ -1,3 +1,4 @@
import { Button } from "@/components/ui/button";
import { LoadingSpinner } from "@/components/ui/loading-spinner";
import type { DaemonClient } from "@getpaseo/client/internal/daemon-client";
import type { TFunction } from "i18next";
@@ -23,7 +24,6 @@ import { AgentStreamView, type AgentStreamViewHandle } from "@/agent-stream/view
import { ArchivedAgentCallout } from "@/components/archived-agent-callout";
import { FileDropZone } from "@/components/file-drop/file-drop-zone";
import { useRetainedPanelActive } from "@/components/retained-panel";
import { SidebarCallout } from "@/components/sidebar-callout";
import { Composer } from "@/composer";
import { resolveComposerTrackTailClearance } from "@/composer/pill-styles";
import { getActiveMessageSubmissions } from "@/composer/submission/model";
@@ -37,7 +37,11 @@ import {
} from "@/components/toast-host";
import type { WorkspaceComposerAttachment } from "@/attachments/types";
import { useWorkspaceAttachmentScopeKey } from "@/attachments/workspace-attachments-store";
import { COMPACT_FORM_FACTOR_WIDTH, useIsCompactFormFactor } from "@/constants/layout";
import {
COMPACT_FORM_FACTOR_WIDTH,
MAX_CONTENT_WIDTH,
useIsCompactFormFactor,
} from "@/constants/layout";
import { isWeb } from "@/constants/platform";
import { useAgentAttentionClear } from "@/hooks/use-agent-attention-clear";
import { useAgentInitialization } from "@/hooks/use-agent-initialization";
@@ -197,12 +201,30 @@ function buildChatAgentFromState(
};
}
function AgentLoadErrorView({ message, onRetry }: { message: string; onRetry: () => void }) {
const { t } = useTranslation();
return (
<View style={styles.container} testID="agent-load-error">
<View style={styles.errorContainer}>
<Text style={styles.errorText}>{t("agentPanel.states.failedToLoad")}</Text>
<Text style={styles.statusText}>{message}</Text>
<View style={styles.errorAction}>
<Button size="sm" variant="secondary" onPress={onRetry} testID="agent-load-error-retry">
{t("common.actions.retry")}
</Button>
</View>
</View>
</View>
);
}
function renderChatAgentNonReadyView(args: {
viewState: AgentScreenViewState;
effectiveAgent: AgentScreenAgent | null;
t: TFunction;
onRetryLoad: () => void;
}): React.ReactElement | null {
const { viewState, effectiveAgent, t } = args;
const { viewState, effectiveAgent, t, onRetryLoad } = args;
if (viewState.tag === "not_found") {
return (
<View style={styles.container} testID="agent-not-found">
@@ -213,14 +235,7 @@ function renderChatAgentNonReadyView(args: {
);
}
if (viewState.tag === "error") {
return (
<View style={styles.container} testID="agent-load-error">
<View style={styles.errorContainer}>
<Text style={styles.errorText}>{t("agentPanel.states.failedToLoad")}</Text>
<Text style={styles.statusText}>{viewState.message}</Text>
</View>
</View>
);
return <AgentLoadErrorView message={viewState.message} onRetry={onRetryLoad} />;
}
if (viewState.tag === "boot" || !effectiveAgent) {
return (
@@ -603,6 +618,7 @@ function AgentPanelBody({
);
const [lookupState, setLookupState] = useState<AgentLookupState>({ tag: "idle" });
const lookupAttemptTokenRef = useRef(0);
const retryAgentLookup = useCallback(() => setLookupState({ tag: "idle" }), []);
const workspaceKey = buildWorkspaceTabPersistenceKey({ serverId, workspaceId });
const resolvePendingAgent = useWorkspaceLayoutStore((state) => state.resolvePendingAgent);
@@ -695,14 +711,7 @@ function AgentPanelBody({
}
if (lookupState.tag === "error") {
return (
<View style={styles.container} testID="agent-load-error">
<View style={styles.errorContainer}>
<Text style={styles.errorText}>{t("agentPanel.states.failedToLoad")}</Text>
<Text style={styles.statusText}>{lookupState.message}</Text>
</View>
</View>
);
return <AgentLoadErrorView message={lookupState.message} onRetry={retryAgentLookup} />;
}
const agent: AgentScreenAgent | null =
@@ -1029,6 +1038,13 @@ function ChatAgentContent({
streamViewRef.current?.scrollToBottom("message-sent");
}, [agentId]);
const retryAgentLoad = useCallback(() => setMissingAgentState({ kind: "idle" }), []);
const retryTimelineSync = useCallback(() => {
if (!agentId || !viewedTimelineSync) return;
viewedTimelineSync.retryVisibleAgentTimeline(agentId);
}, [agentId, viewedTimelineSync]);
useEffect(() => {
initAttemptTokenRef.current += 1;
setMissingAgentState({ kind: "idle" });
@@ -1127,6 +1143,7 @@ function ChatAgentContent({
viewState,
effectiveAgent,
t,
onRetryLoad: retryAgentLoad,
});
if (nonReadyView) return nonReadyView;
invariant(agentId, "agent id is defined when agent content is ready");
@@ -1138,6 +1155,10 @@ function ChatAgentContent({
viewState.sync.status === "catching_up" &&
viewState.sync.ui === "overlay";
const showHistorySyncError = viewState.tag === "ready" && viewState.sync.status === "sync_error";
const isRetryingHistorySync =
viewState.tag === "ready" &&
viewState.sync.status === "sync_error" &&
viewState.sync.isRetrying;
return (
<ChatAgentReadyContent
@@ -1158,7 +1179,9 @@ function ChatAgentContent({
handleMessageSent={handleMessageSent}
showHistorySyncOverlay={showHistorySyncOverlay}
showHistorySyncError={showHistorySyncError}
isRetryingHistorySync={isRetryingHistorySync}
cwd={agentCwd}
retryTimelineSync={retryTimelineSync}
onAttentionInputFocus={attentionController.clearOnInputFocus}
onAttentionPromptSend={attentionController.clearOnPromptSend}
onOpenWorkspaceFile={onOpenWorkspaceFile}
@@ -1184,6 +1207,8 @@ const ChatAgentReadyContent = memo(function ChatAgentReadyContent({
handleMessageSent,
showHistorySyncOverlay,
showHistorySyncError,
isRetryingHistorySync,
retryTimelineSync,
cwd,
onAttentionInputFocus,
onAttentionPromptSend,
@@ -1206,6 +1231,8 @@ const ChatAgentReadyContent = memo(function ChatAgentReadyContent({
handleMessageSent: () => void;
showHistorySyncOverlay: boolean;
showHistorySyncError: boolean;
isRetryingHistorySync: boolean;
retryTimelineSync: () => void;
cwd: string;
onAttentionInputFocus: () => void;
onAttentionPromptSend: () => void;
@@ -1333,11 +1360,26 @@ const ChatAgentReadyContent = memo(function ChatAgentReadyContent({
{contentContainer}
{showHistorySyncError ? (
<SidebarCallout
title={t("agentPanel.states.timelineSyncFailed")}
variant="error"
testID="agent-timeline-sync-error"
/>
<View style={styles.timelineSyncCalloutRail}>
<View style={styles.timelineSyncCalloutContent}>
<View style={styles.timelineSyncCallout} testID="agent-timeline-sync-error">
<Text style={styles.timelineSyncCalloutText}>
{t("agentPanel.states.timelineSyncFailed")}
</Text>
<Button
size="sm"
variant="secondary"
onPress={retryTimelineSync}
disabled={isRetryingHistorySync}
testID="agent-timeline-sync-retry"
>
{isRetryingHistorySync
? t("agentPanel.states.timelineSyncRetrying")
: t("common.actions.retry")}
</Button>
</View>
</View>
</View>
) : null}
{composerSection}
@@ -1745,6 +1787,32 @@ const styles = StyleSheet.create((theme) => ({
overflow: "hidden",
...(isWeb ? { userSelect: "none" as const } : {}),
},
timelineSyncCalloutRail: {
width: "100%",
alignItems: "center",
paddingHorizontal: theme.spacing[4],
paddingTop: theme.spacing[2],
},
timelineSyncCalloutContent: {
width: "100%",
maxWidth: MAX_CONTENT_WIDTH,
},
timelineSyncCallout: {
flexDirection: "row",
alignItems: "center",
justifyContent: "center",
gap: theme.spacing[3],
backgroundColor: theme.colors.surface1,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.statusDanger,
borderRadius: theme.borderRadius["2xl"],
paddingVertical: theme.spacing[2],
paddingHorizontal: theme.spacing[4],
},
timelineSyncCalloutText: {
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.base,
},
historySyncOverlay: {
position: "absolute",
top: 0,
@@ -1807,6 +1875,9 @@ const styles = StyleSheet.create((theme) => ({
color: theme.colors.foregroundMuted,
textAlign: "center",
},
errorAction: {
marginTop: theme.spacing[4],
},
statusText: {
marginTop: theme.spacing[2],
textAlign: "center",
packages/app/src/timeline/viewed-timeline-sync.test.ts
+107 -7
View File
@@ -79,10 +79,11 @@ class TimelineWorld {
schedule: (task, delayMs) => {
const scheduled = { task, delayMs };
this.scheduled.push(scheduled);
const waiter = this.retryWaiters.shift();
if (waiter && delayMs === 1_000) {
const waiterIndex = this.retryWaiters.findIndex((waiter) => waiter.delayMs === delayMs);
if (waiterIndex >= 0) {
const [waiter] = this.retryWaiters.splice(waiterIndex, 1);
this.scheduled.splice(this.scheduled.indexOf(scheduled), 1);
waiter(task);
waiter.resolve(task);
}
return () => {
const index = this.scheduled.indexOf(scheduled);
@@ -100,7 +101,10 @@ class TimelineWorld {
}> = [];
private readonly errorWaiters: Array<(message: string) => void> = [];
private readonly scheduled: Array<{ task: () => void; delayMs: number }> = [];
private readonly retryWaiters: Array<(retry: () => void) => void> = [];
private readonly retryWaiters: Array<{
delayMs: number;
resolve(retry: () => void): void;
}> = [];
private fetchTimeline(
agentId: string,
@@ -147,10 +151,10 @@ class TimelineWorld {
return new Promise((resolve) => this.errorWaiters.push(resolve));
}
nextRetry(): Promise<() => void> {
const index = this.scheduled.findIndex((entry) => entry.delayMs === 1_000);
nextRetry(delayMs = 1_000): Promise<() => void> {
const index = this.scheduled.findIndex((entry) => entry.delayMs === delayMs);
if (index >= 0) return Promise.resolve(this.scheduled.splice(index, 1)[0].task);
return new Promise((resolve) => this.retryWaiters.push(resolve));
return new Promise((resolve) => this.retryWaiters.push({ delayMs, resolve }));
}
elapse(elapsedMs: number): void {
@@ -411,6 +415,74 @@ test("a failed catch-up reports once and retries through the explicit retry poli
world.expectNoPendingMembership();
});
test("a failed catch-up retries with exponential backoff", async () => {
const world = new TimelineWorld();
world.sync.setConnected(true);
world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
const membership = await world.nextMembership();
membership.succeed();
const first = await world.nextFetch("agent-a");
first.fail("timeline unavailable");
const [firstError, retryAfterFirstFailure] = await Promise.all([
world.nextError(),
world.nextRetry(),
]);
expect(firstError).toBe("timeline unavailable");
retryAfterFirstFailure();
const second = await world.nextFetch("agent-a");
second.fail("timeline unavailable");
await world.nextError();
const retryAfterSecondFailure = await world.nextRetry(2_000);
expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error");
retryAfterSecondFailure();
const third = await world.nextFetch("agent-a");
third.respond({ hasNewer: false });
await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready"));
});
test("manual retries can immediately re-attempt a failed catch-up", async () => {
const world = new TimelineWorld();
world.sync.setConnected(true);
world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
const membership = await world.nextMembership();
membership.succeed();
const failed = await world.nextFetch("agent-a");
failed.fail("timeline unavailable");
await world.nextRetry();
expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error");
world.sync.retryVisibleAgentTimeline("agent-a");
expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("retrying");
const retry = await world.nextFetch("agent-a");
retry.respond({ hasNewer: false });
await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready"));
});
test("a manual retry that fails returns to the error state", async () => {
const world = new TimelineWorld();
world.sync.setConnected(true);
world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
const membership = await world.nextMembership();
membership.succeed();
const failed = await world.nextFetch("agent-a");
failed.fail("timeline unavailable");
await world.nextRetry();
world.sync.retryVisibleAgentTimeline("agent-a");
expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("retrying");
const retry = await world.nextFetch("agent-a");
retry.fail("timeline still unavailable");
await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error"));
});
test("gap recovery supersedes completed catch-up and pages through the current tail", async () => {
const world = new TimelineWorld();
world.sync.setConnected(true);
@@ -486,6 +558,34 @@ test("membership failure autonomously retries without another visibility declara
});
});
test("membership failures retry with exponential backoff", async () => {
const world = new TimelineWorld();
world.sync.setConnected(true);
world.sync.replaceVisibleAgentIds("workspace", ["agent-a"]);
const first = await world.nextMembership();
first.fail("subscription unavailable");
const [firstError, retryAfterFirstFailure] = await Promise.all([
world.nextError(),
world.nextRetry(),
]);
expect(firstError).toBe("subscription unavailable");
retryAfterFirstFailure();
const second = await world.nextMembership();
second.fail("subscription unavailable again");
const retryAfterSecondFailure = await world.nextRetry(2_000);
expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("error");
retryAfterSecondFailure();
const retry = await world.nextMembership();
retry.succeed();
const catchUp = await world.nextFetch("agent-a");
catchUp.respond({ hasNewer: false });
await vi.waitFor(() => expect(world.sync.getAgentTimelineStatus("agent-a")).toBe("ready"));
});
test("backgrounding preserves the hot membership without catch-up on return", async () => {
const world = new TimelineWorld();
world.sync.setConnected(true);
packages/app/src/timeline/viewed-timeline-sync.ts
+69 -6
View File
@@ -23,12 +23,13 @@ interface ViewedTimelineSyncPorts {
}
export type TimelineDeliveryMode = "legacy" | "selective";
export type ViewedTimelineStatus = "ready" | "pending" | "error";
export type ViewedTimelineStatus = "ready" | "pending" | "error" | "retrying";
export interface ViewedTimelineUiBridge {
replaceVisibleAgentIds(sourceId: string, agentIds: string[]): void;
subscribe(listener: () => void): () => void;
getAgentTimelineStatus(agentId: string): ViewedTimelineStatus;
retryVisibleAgentTimeline(agentId: string): void;
}
export interface ViewedTimelineSync extends ViewedTimelineUiBridge {
@@ -40,6 +41,7 @@ export interface ViewedTimelineSync extends ViewedTimelineUiBridge {
}
const RETRY_DELAY_MS = 1_000;
const MAX_RETRY_DELAY_MS = 30_000;
const VIEWED_TIMELINE_HOT_AGENT_LIMIT = 5;
type CatchUpStatus = "running" | "complete" | "error";
@@ -49,8 +51,16 @@ interface CatchUpState {
status: CatchUpStatus;
request?: ProjectedTimelineForwardFetchPlan;
cancelRetry?: () => void;
retryDelayMs?: number;
}
const getNextRetryDelayMs = (previousDelayMs: number | undefined): number => {
if (previousDelayMs == null) {
return RETRY_DELAY_MS;
}
return Math.min(previousDelayMs * 2, MAX_RETRY_DELAY_MS);
};
function isSameCatchUpRequest(
left: ProjectedTimelineForwardFetchPlan | undefined,
right: ProjectedTimelineForwardFetchPlan | undefined,
@@ -102,6 +112,9 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
const pendingCatchUps = new Map<string, ProjectedTimelineForwardFetchPlan>();
const visibilityCatchUpPending = new Set<string>();
const visibilityCatchUpErrors = new Set<string>();
// User-initiated retries only. Background retries stay silent; a retry the user asked for
// owes them a pending state until it settles.
const manualRetries = new Set<string>();
const listeners = new Set<() => void>();
let active = true;
let connected = false;
@@ -113,6 +126,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
let reconciling = false;
let reconcileRequested = false;
let membershipNeedsRetry = false;
let membershipRetryDelayMs: number | undefined;
let cancelMembershipRetry: (() => void) | null = null;
let recentlyViewedAgentIds: string[] = [];
@@ -152,12 +166,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
const setVisibilityCatchUpReady = (agentId: string) => {
const wasPending = visibilityCatchUpPending.delete(agentId);
const hadError = visibilityCatchUpErrors.delete(agentId);
if (wasPending || hadError) notifyListeners();
const wasRetrying = manualRetries.delete(agentId);
if (wasPending || hadError || wasRetrying) notifyListeners();
};
const setVisibilityCatchUpError = (agentIds: string[]) => {
let changed = false;
for (const agentId of agentIds) {
if (manualRetries.delete(agentId)) changed = true;
if (!visibilityCatchUpPending.delete(agentId)) continue;
visibilityCatchUpErrors.add(agentId);
changed = true;
@@ -210,12 +226,19 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
setVisibilityCatchUpReady(agentId);
} catch (error) {
if (catchUps.get(agentId)?.generation === generation) {
const nextRetryDelayMs = getNextRetryDelayMs(catchUps.get(agentId)?.retryDelayMs);
const cancelRetry = ports.schedule(() => {
const current = catchUps.get(agentId);
if (current?.generation !== generation || current.status !== "error") return;
startCatchUp(agentId);
}, RETRY_DELAY_MS);
catchUps.set(agentId, { generation, status: "error", cancelRetry });
}, nextRetryDelayMs);
catchUps.set(agentId, {
generation,
status: "error",
request,
cancelRetry,
retryDelayMs: nextRetryDelayMs,
});
setVisibilityCatchUpError([agentId]);
ports.reportError(error);
}
@@ -247,7 +270,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
current?.cancelRetry?.();
const generation = (catchUpGenerations.get(agentId) ?? 0) + 1;
catchUpGenerations.set(agentId, generation);
catchUps.set(agentId, { generation, status: "running", request: nextRequest });
const retryDelayMs =
supersede || current?.status !== "error" ? undefined : current.retryDelayMs;
catchUps.set(agentId, {
generation,
status: "running",
request: nextRequest,
retryDelayMs,
});
pendingCatchUps.delete(agentId);
void fetchUntilCurrent(
agentId,
@@ -279,6 +309,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
membershipNeedsRetry = true;
setVisibilityCatchUpError(requested);
cancelMembershipRetry?.();
const nextRetryDelayMs = getNextRetryDelayMs(membershipRetryDelayMs);
cancelMembershipRetry = ports.schedule(() => {
cancelMembershipRetry = null;
if (
@@ -290,12 +321,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
return;
}
void reconcileMembership();
}, RETRY_DELAY_MS);
}, nextRetryDelayMs);
membershipRetryDelayMs = nextRetryDelayMs;
ports.reportError(error);
return;
}
cancelMembershipRetry?.();
cancelMembershipRetry = null;
membershipRetryDelayMs = undefined;
if (disposed || !connected || deliveryMode !== "selective") return;
acknowledged = requested;
if (generation !== membershipGeneration) {
@@ -338,6 +371,25 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
}
};
const retryVisibleAgentTimeline = (agentId: string) => {
if (!isDesired(agentId) || manualRetries.has(agentId)) return;
const catchUp = catchUps.get(agentId);
const membershipRetryable = deliveryMode === "selective" && membershipNeedsRetry && connected;
if (catchUp?.status !== "error" && !membershipRetryable) return;
manualRetries.add(agentId);
notifyListeners();
if (catchUp?.status === "error") {
catchUp.cancelRetry?.();
startCatchUp(agentId, { request: catchUp.request, supersede: true });
return;
}
cancelMembershipRetry?.();
cancelMembershipRetry = null;
membershipRetryDelayMs = undefined;
membershipNeedsRetry = false;
void reconcileMembership();
};
const commitDesiredMembership = (
nextDesired: string[],
options: { resetCatchUpStatus?: boolean } = {},
@@ -350,6 +402,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
statusChanged = true;
}
if (visibilityCatchUpErrors.delete(agentId)) statusChanged = true;
if (manualRetries.delete(agentId)) statusChanged = true;
}
}
if (sameAgentIds(nextDesired, desired)) {
@@ -364,12 +417,14 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
cancelCatchUp(agentId);
visibilityCatchUpPending.delete(agentId);
visibilityCatchUpErrors.delete(agentId);
manualRetries.delete(agentId);
}
}
for (const agentId of nextDesired) {
if (!desired.includes(agentId)) {
visibilityCatchUpPending.add(agentId);
visibilityCatchUpErrors.delete(agentId);
manualRetries.delete(agentId);
}
}
cancelMembershipRetry?.();
@@ -403,6 +458,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
return () => listeners.delete(listener);
},
getAgentTimelineStatus(agentId) {
if (manualRetries.has(agentId)) return "retrying";
if (visibilityCatchUpErrors.has(agentId)) return "error";
if (!isDesired(agentId) || visibilityCatchUpPending.has(agentId)) return "pending";
return "ready";
@@ -427,6 +483,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
commitDesiredMembership(visible, { resetCatchUpStatus: true });
cancelMembershipRetry?.();
cancelMembershipRetry = null;
membershipRetryDelayMs = undefined;
acknowledged = [];
membershipGeneration += 1;
for (const agentId of desired) cancelCatchUp(agentId);
@@ -445,6 +502,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
deliveryMode = nextMode;
cancelMembershipRetry?.();
cancelMembershipRetry = null;
membershipRetryDelayMs = undefined;
membershipNeedsRetry = false;
membershipGeneration += 1;
for (const agentId of desired) cancelCatchUp(agentId);
@@ -453,6 +511,7 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
desired = visible;
visibilityCatchUpPending.clear();
visibilityCatchUpErrors.clear();
manualRetries.clear();
for (const agentId of desired) visibilityCatchUpPending.add(agentId);
acknowledged = deliveryMode === "legacy" && connected ? desired : [];
notifyListeners();
@@ -470,6 +529,8 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
disposed = true;
cancelMembershipRetry?.();
cancelMembershipRetry = null;
membershipNeedsRetry = false;
membershipRetryDelayMs = undefined;
sources.clear();
membershipGeneration += 1;
for (const agentId of desired) cancelCatchUp(agentId);
@@ -478,8 +539,10 @@ export function createViewedTimelineSync(ports: ViewedTimelineSyncPorts): Viewed
recentlyViewedAgentIds = [];
visibilityCatchUpPending.clear();
visibilityCatchUpErrors.clear();
manualRetries.clear();
notifyListeners();
listeners.clear();
},
retryVisibleAgentTimeline,
};
}