jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Keep connections alive through brief daemon slowdowns (#1581)

Browse Source 
* Keep connections alive through brief daemon slowdowns

The active connection's liveness check was a side effect of the
candidate-probe loop: a single 5s latency measurement doubled as the
death timeout, so a daemon that was alive but briefly stalled (e.g. a
busy event loop) was misread as dead and the connection was torn down.

Give DaemonClient its own self-scheduling heartbeat that solely owns
liveness (15s timeout, 2 consecutive misses), and make the probe
read-only on the active connection — it reads the heartbeat's last RTT
instead of pinging. Split the shared ping into a pure measurement path
(measureLatency) and a private liveness path, so nothing outside the
client can drive teardown.

* Don't let a candidate latency timeout count as a liveness failure

The heartbeat (livenessPing) and candidate ranking (measureLatency) share
one in-flight ping slot. If a heartbeat tick lands while a candidate
measurement is still in flight, the heartbeat dedupes onto it — and a
measurement timeout was then recorded as a liveness failure, nudging the
connection one step closer to a spurious teardown.

Tag each ping with whether its timeout may drive a liveness failure
(only the heartbeat sets it) and gate recordLivenessFailure on that flag,
so an adopted measurement probe can never contribute to teardown.
This commit is contained in:
Mohamed Boudra
2026-06-18 21:10:08 +08:00
committed by GitHub
parent eb628765d7
commit ea2f768d3f
4 changed files with 667 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/client/src/daemon-client.ts
+117 -33
View File
@@ -737,10 +737,26 @@ class DaemonRpcError extends Error {
}
}
class PingTimeoutError extends Error {
constructor(readonly timeoutMs: number) {
super(`Ping timed out (${timeoutMs}ms)`);
this.name = "PingTimeoutError";
}
}
function toTimeoutError(error: unknown, label: string, timeoutMs: number): Error {
if (error instanceof PingTimeoutError) {
return new Error(`${label} timed out (${timeoutMs}ms)`);
}
return error instanceof Error ? error : new Error(String(error));
}
const DEFAULT_RECONNECT_BASE_DELAY_MS = 1500;
const DEFAULT_RECONNECT_MAX_DELAY_MS = 30000;
const DEFAULT_CONNECT_TIMEOUT_MS = 15000;
const DEFAULT_LIVENESS_TIMEOUT_MS = 5000;
const LIVENESS_HEARTBEAT_INTERVAL_MS = 10_000;
const LIVENESS_HEARTBEAT_TIMEOUT_MS = 15_000;
const LIVENESS_FAILURE_RECONNECT_THRESHOLD = 2;
/** Default timeout for waiting for connection before sending queued messages */
@@ -848,12 +864,16 @@ interface PendingSend {
timeoutHandle: ReturnType<typeof setTimeout>;
}
interface LivenessProbe {
promise: Promise<{ rttMs: number }>;
resolve: (value: { rttMs: number }) => void;
interface PingProbe {
promise: Promise<number>;
resolve: (value: number) => void;
reject: (error: Error) => void;
timeoutHandle: ReturnType<typeof setTimeout>;
startedAt: number;
// Whether a timeout on this ping should be recorded as a liveness failure. Only the
// heartbeat sets this; a latency measurement never drives teardown, even when a
// heartbeat tick shares (dedupes onto) an in-flight measurement ping.
drivesLivenessFailure: boolean;
}
export class DaemonClient {
@@ -899,7 +919,9 @@ export class DaemonClient {
private lastServerInfoMessage: ServerInfoStatusPayload | null = null;
private runtimeMetricsInterval: ReturnType<typeof setInterval> | null = null;
private runtimeMetrics: DaemonClientRuntimeMetrics | null = null;
private livenessProbe: LivenessProbe | null = null;
private pingProbe: PingProbe | null = null;
private livenessHeartbeatTimer: ReturnType<typeof setTimeout> | null = null;
private lastLivenessRttMs: number | null = null;
private consecutiveLivenessFailures = 0;
constructor(private config: DaemonClientConfig) {
@@ -1162,7 +1184,7 @@ export class DaemonClient {
this.disposeTransport(1000, "Client closed");
this.clearWaiters(new Error("Daemon client closed"));
this.rejectPendingSendQueue(new Error("Daemon client closed"));
this.rejectLivenessProbe(new Error("Daemon client closed"));
this.rejectPingProbe(new Error("Daemon client closed"));
this.terminalStreams.clearSlots();
this.lastServerInfoMessage = null;
if (this.runtimeMetricsInterval) {
@@ -1217,6 +1239,10 @@ export class DaemonClient {
return this.lastErrorValue;
}
getLastLivenessRttMs(): number | null {
return this.lastLivenessRttMs;
}
// ============================================================================
// Message Subscription
// ============================================================================
@@ -1627,54 +1653,108 @@ export class DaemonClient {
};
}
checkLiveness(params?: { timeoutMs?: number }): Promise<{ rttMs: number }> {
measureLatency(params?: { timeoutMs?: number }): Promise<number> {
const timeoutMs = Math.max(1, params?.timeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS);
return this.sendPingAwaitRtt({ timeoutMs, drivesLivenessFailure: false }).catch((error) => {
throw toTimeoutError(error, "Latency measurement", timeoutMs);
});
}
private async livenessPing(params?: { timeoutMs?: number }): Promise<number> {
const timeoutMs = Math.max(1, params?.timeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS);
try {
const rttMs = await this.sendPingAwaitRtt({ timeoutMs, drivesLivenessFailure: true });
this.lastLivenessRttMs = rttMs;
return rttMs;
} catch (error) {
throw toTimeoutError(error, "Liveness check", timeoutMs);
}
}
private sendPingAwaitRtt(params: {
timeoutMs: number;
drivesLivenessFailure: boolean;
}): Promise<number> {
if (this.connectionState.status !== "connected" || !this.transport) {
return Promise.reject(
new Error(`Transport not connected (status: ${this.connectionState.status})`),
);
}
if (this.livenessProbe) {
return this.livenessProbe.promise;
if (this.pingProbe) {
return this.pingProbe.promise;
}
const startedAt = perfNow();
const timeoutMs = Math.max(1, params?.timeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS);
let resolveProbe: ((value: { rttMs: number }) => void) | null = null;
const timeoutMs = params.timeoutMs;
let resolveProbe: ((value: number) => void) | null = null;
let rejectProbe: ((error: Error) => void) | null = null;
const promise = new Promise<{ rttMs: number }>((resolve, reject) => {
const promise = new Promise<number>((resolve, reject) => {
resolveProbe = resolve;
rejectProbe = reject;
});
const probe: LivenessProbe = {
const probe: PingProbe = {
promise,
resolve: (value) => resolveProbe?.(value),
reject: (error) => rejectProbe?.(error),
timeoutHandle: setTimeout(() => {
if (this.livenessProbe !== probe) {
if (this.pingProbe !== probe) {
return;
}
this.livenessProbe = null;
const error = new Error(`Liveness check timed out (${timeoutMs}ms)`);
this.pingProbe = null;
const error = new PingTimeoutError(timeoutMs);
probe.reject(error);
this.recordLivenessFailure(error);
if (probe.drivesLivenessFailure) {
this.recordLivenessFailure(toTimeoutError(error, "Liveness check", timeoutMs));
}
}, timeoutMs),
startedAt,
drivesLivenessFailure: params.drivesLivenessFailure,
};
this.livenessProbe = probe;
this.pingProbe = probe;
try {
this.transport.send(JSON.stringify({ type: "ping" }));
} catch (error) {
this.clearLivenessProbe();
const err = error instanceof Error ? error : new Error(String(error));
this.recordLivenessFailure(err);
return Promise.reject(err);
this.clearPingProbe();
const sendError = error instanceof Error ? error : new Error(String(error));
if (probe.drivesLivenessFailure) {
this.recordLivenessFailure(sendError);
}
return Promise.reject(sendError);
}
return promise;
}
private startLivenessHeartbeat(): void {
this.stopLivenessHeartbeat();
this.lastLivenessRttMs = null;
this.scheduleNextLivenessHeartbeat();
}
private stopLivenessHeartbeat(): void {
if (!this.livenessHeartbeatTimer) {
return;
}
clearTimeout(this.livenessHeartbeatTimer);
this.livenessHeartbeatTimer = null;
}
private scheduleNextLivenessHeartbeat(): void {
if (this.connectionState.status !== "connected" || this.livenessHeartbeatTimer) {
return;
}
this.livenessHeartbeatTimer = setTimeout(() => {
this.livenessHeartbeatTimer = null;
this.livenessPing({ timeoutMs: LIVENESS_HEARTBEAT_TIMEOUT_MS })
.catch(() => {})
.finally(() => {
this.scheduleNextLivenessHeartbeat();
});
}, LIVENESS_HEARTBEAT_INTERVAL_MS);
}
// ============================================================================
// Agent RPCs (requestId-correlated)
// ============================================================================
@@ -4463,6 +4543,7 @@ export class DaemonClient {
}
private disposeTransport(code = 1001, reason = "Reconnecting"): void {
this.stopLivenessHeartbeat();
this.cleanupTransport();
if (this.transport) {
try {
@@ -4556,7 +4637,7 @@ export class DaemonClient {
this.consecutiveLivenessFailures = 0;
if (parsed.data.type === "pong") {
this.resolveLivenessProbe();
this.resolvePingProbe();
this.runtimeMetrics?.recordMessage("pong", bytes, perfNow() - startMs);
return;
}
@@ -4572,6 +4653,7 @@ export class DaemonClient {
private tryHandleBinaryFrame(rawBytes: Uint8Array): boolean {
const fileFrame = decodeFileTransferFrame(rawBytes);
if (fileFrame) {
this.consecutiveLivenessFailures = 0;
this.handleFileTransferFrame(fileFrame);
this.runtimeMetrics?.recordBinaryFrame("other", rawBytes.byteLength, 0);
return true;
@@ -4581,6 +4663,7 @@ export class DaemonClient {
if (!frame) {
return false;
}
this.consecutiveLivenessFailures = 0;
const binaryStartMs = perfNow();
this.terminalStreams.handleFrame(frame);
let frameKind: "output" | "snapshot" | "other" = "other";
@@ -4707,7 +4790,7 @@ export class DaemonClient {
// and responses from the previous connection will never arrive.
this.clearWaiters(new Error(reason ?? "Connection lost"));
this.rejectPendingSendQueue(new Error(reason ?? "Connection lost"));
this.rejectLivenessProbe(new Error(reason ?? "Connection lost"));
this.rejectPingProbe(new Error(reason ?? "Connection lost"));
this.terminalStreams.clearSlots();
this.lastServerInfoMessage = null;
@@ -4756,31 +4839,31 @@ export class DaemonClient {
}, delay);
}
private resolveLivenessProbe(): void {
const probe = this.livenessProbe;
private resolvePingProbe(): void {
const probe = this.pingProbe;
if (!probe) {
return;
}
this.livenessProbe = null;
this.pingProbe = null;
clearTimeout(probe.timeoutHandle);
probe.resolve({ rttMs: perfNow() - probe.startedAt });
probe.resolve(perfNow() - probe.startedAt);
}
private clearLivenessProbe(): void {
const probe = this.livenessProbe;
private clearPingProbe(): void {
const probe = this.pingProbe;
if (!probe) {
return;
}
this.livenessProbe = null;
this.pingProbe = null;
clearTimeout(probe.timeoutHandle);
}
private rejectLivenessProbe(error: Error): void {
const probe = this.livenessProbe;
private rejectPingProbe(error: Error): void {
const probe = this.pingProbe;
if (!probe) {
return;
}
this.livenessProbe = null;
this.pingProbe = null;
clearTimeout(probe.timeoutHandle);
probe.reject(error);
}
@@ -4809,6 +4892,7 @@ export class DaemonClient {
this.resetConnectTimeout();
this.reconnectAttempt = 0;
this.updateConnectionState({ status: "connected" }, { event: "HELLO_SERVER_INFO" });
this.startLivenessHeartbeat();
this.resubscribeCheckoutDiffSubscriptions();
this.resubscribeTerminalDirectorySubscriptions();
this.flushPendingSendQueue();