Log daemon supervisor exit reasons

This commit is contained in:
Mohamed Boudra
2026-06-08 23:03:56 +07:00
parent 9d9f6905c6
commit b0954d1616
2 changed files with 65 additions and 11 deletions
+13 -1
View File
@@ -191,11 +191,23 @@ export function runSupervisor(options: SupervisorOptions): void {
const heartbeat = setInterval(() => {
const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" };
if (currentChild.connected) {
currentChild.send?.(message, () => undefined);
currentChild.send?.(message, (error) => {
if (error) {
writeLifecycleLog("Worker heartbeat IPC send failed", {
error: error instanceof Error ? error.message : String(error),
});
}
});
} else {
writeLifecycleLog("Worker heartbeat skipped because IPC channel is disconnected");
}
}, 1000);
heartbeat.unref();
child.on("disconnect", () => {
writeLifecycleLog("Worker IPC channel disconnected");
});
child.stdout?.on("data", (chunk: Buffer) => {
process.stdout.write(chunk);
writeDurableChunk(chunk);
+52 -10
View File
@@ -1,3 +1,5 @@
import { appendFileSync, mkdirSync } from "node:fs";
import path from "node:path";
import { createPaseoDaemon } from "./bootstrap.js";
import { loadConfig } from "./config.js";
import { resolvePaseoHome } from "./paseo-home.js";
@@ -41,6 +43,31 @@ function isPidAlive(pid: number): boolean {
}
}
function writeWorkerLifecycleLog(
paseoHome: string,
message: string,
fields: Record<string, unknown> = {},
): void {
try {
const logPath = path.join(paseoHome, "daemon.log");
mkdirSync(path.dirname(logPath), { recursive: true });
appendFileSync(
logPath,
`${JSON.stringify({
level: "warn",
time: new Date().toISOString(),
pid: process.pid,
name: "DaemonWorker",
msg: message,
...fields,
})}\n`,
"utf8",
);
} catch {
// Exit-reason logging must never prevent the worker from exiting.
}
}
function bootstrapFromEnvironment(): BootstrapResult {
try {
const paseoHome = resolvePaseoHome();
@@ -70,7 +97,7 @@ function applyCliFlagOverrides(config: ReturnType<typeof loadConfig>): void {
}
async function main() {
const { logger, config } = bootstrapFromEnvironment();
const { paseoHome, logger, config } = bootstrapFromEnvironment();
let daemon: Awaited<ReturnType<typeof createPaseoDaemon>> | null = null;
let shutdownPromise: Promise<number> | null = null;
let exitHookInstalled = false;
@@ -174,12 +201,20 @@ async function main() {
const supervisorPid = process.ppid;
let lastSupervisorHeartbeatAt = Date.now();
let supervisorExitRequested = false;
const exitAfterSupervisorLoss = () => {
const exitAfterSupervisorLoss = (reason: string) => {
if (supervisorExitRequested) {
return;
}
supervisorExitRequested = true;
writeWorkerLifecycleLog(paseoHome, "Supervisor liveness lost; worker exiting", {
reason,
supervisorPid,
currentParentPid: process.ppid,
ipcConnected: typeof process.connected === "boolean" ? process.connected : null,
heartbeatAgeMs: Date.now() - lastSupervisorHeartbeatAt,
});
// The supervisor owns the worker's stdout/stderr pipes. Once it is gone,
// logging during graceful shutdown can block on the broken pipe and leave
// the daemon orphaned, so supervisor loss is a hard process boundary.
@@ -196,19 +231,26 @@ async function main() {
lastSupervisorHeartbeatAt = Date.now();
}
});
process.on("disconnect", exitAfterSupervisorLoss);
process.on("disconnect", () => exitAfterSupervisorLoss("ipc_disconnect_event"));
const timer = setInterval(() => {
const ipcConnected = typeof process.connected === "boolean" ? process.connected : true;
const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500;
const supervisorChanged = process.ppid !== supervisorPid;
if (
ipcConnected === false ||
supervisorChanged ||
!isPidAlive(supervisorPid) ||
heartbeatExpired
) {
exitAfterSupervisorLoss();
if (ipcConnected === false) {
exitAfterSupervisorLoss("ipc_disconnected");
return;
}
if (supervisorChanged) {
exitAfterSupervisorLoss("supervisor_parent_pid_changed");
return;
}
if (!isPidAlive(supervisorPid)) {
exitAfterSupervisorLoss("supervisor_pid_dead");
return;
}
if (heartbeatExpired) {
exitAfterSupervisorLoss("supervisor_heartbeat_expired");
}
}, 1000);
timer.unref();