Log daemon supervisor exit reasons
This commit is contained in:
@@ -191,11 +191,23 @@ export function runSupervisor(options: SupervisorOptions): void {
|
||||
const heartbeat = setInterval(() => {
|
||||
const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" };
|
||||
if (currentChild.connected) {
|
||||
currentChild.send?.(message, () => undefined);
|
||||
currentChild.send?.(message, (error) => {
|
||||
if (error) {
|
||||
writeLifecycleLog("Worker heartbeat IPC send failed", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
writeLifecycleLog("Worker heartbeat skipped because IPC channel is disconnected");
|
||||
}
|
||||
}, 1000);
|
||||
heartbeat.unref();
|
||||
|
||||
child.on("disconnect", () => {
|
||||
writeLifecycleLog("Worker IPC channel disconnected");
|
||||
});
|
||||
|
||||
child.stdout?.on("data", (chunk: Buffer) => {
|
||||
process.stdout.write(chunk);
|
||||
writeDurableChunk(chunk);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { appendFileSync, mkdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createPaseoDaemon } from "./bootstrap.js";
|
||||
import { loadConfig } from "./config.js";
|
||||
import { resolvePaseoHome } from "./paseo-home.js";
|
||||
@@ -41,6 +43,31 @@ function isPidAlive(pid: number): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
function writeWorkerLifecycleLog(
|
||||
paseoHome: string,
|
||||
message: string,
|
||||
fields: Record<string, unknown> = {},
|
||||
): void {
|
||||
try {
|
||||
const logPath = path.join(paseoHome, "daemon.log");
|
||||
mkdirSync(path.dirname(logPath), { recursive: true });
|
||||
appendFileSync(
|
||||
logPath,
|
||||
`${JSON.stringify({
|
||||
level: "warn",
|
||||
time: new Date().toISOString(),
|
||||
pid: process.pid,
|
||||
name: "DaemonWorker",
|
||||
msg: message,
|
||||
...fields,
|
||||
})}\n`,
|
||||
"utf8",
|
||||
);
|
||||
} catch {
|
||||
// Exit-reason logging must never prevent the worker from exiting.
|
||||
}
|
||||
}
|
||||
|
||||
function bootstrapFromEnvironment(): BootstrapResult {
|
||||
try {
|
||||
const paseoHome = resolvePaseoHome();
|
||||
@@ -70,7 +97,7 @@ function applyCliFlagOverrides(config: ReturnType<typeof loadConfig>): void {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { logger, config } = bootstrapFromEnvironment();
|
||||
const { paseoHome, logger, config } = bootstrapFromEnvironment();
|
||||
let daemon: Awaited<ReturnType<typeof createPaseoDaemon>> | null = null;
|
||||
let shutdownPromise: Promise<number> | null = null;
|
||||
let exitHookInstalled = false;
|
||||
@@ -174,12 +201,20 @@ async function main() {
|
||||
const supervisorPid = process.ppid;
|
||||
let lastSupervisorHeartbeatAt = Date.now();
|
||||
let supervisorExitRequested = false;
|
||||
const exitAfterSupervisorLoss = () => {
|
||||
const exitAfterSupervisorLoss = (reason: string) => {
|
||||
if (supervisorExitRequested) {
|
||||
return;
|
||||
}
|
||||
supervisorExitRequested = true;
|
||||
|
||||
writeWorkerLifecycleLog(paseoHome, "Supervisor liveness lost; worker exiting", {
|
||||
reason,
|
||||
supervisorPid,
|
||||
currentParentPid: process.ppid,
|
||||
ipcConnected: typeof process.connected === "boolean" ? process.connected : null,
|
||||
heartbeatAgeMs: Date.now() - lastSupervisorHeartbeatAt,
|
||||
});
|
||||
|
||||
// The supervisor owns the worker's stdout/stderr pipes. Once it is gone,
|
||||
// logging during graceful shutdown can block on the broken pipe and leave
|
||||
// the daemon orphaned, so supervisor loss is a hard process boundary.
|
||||
@@ -196,19 +231,26 @@ async function main() {
|
||||
lastSupervisorHeartbeatAt = Date.now();
|
||||
}
|
||||
});
|
||||
process.on("disconnect", exitAfterSupervisorLoss);
|
||||
process.on("disconnect", () => exitAfterSupervisorLoss("ipc_disconnect_event"));
|
||||
|
||||
const timer = setInterval(() => {
|
||||
const ipcConnected = typeof process.connected === "boolean" ? process.connected : true;
|
||||
const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500;
|
||||
const supervisorChanged = process.ppid !== supervisorPid;
|
||||
if (
|
||||
ipcConnected === false ||
|
||||
supervisorChanged ||
|
||||
!isPidAlive(supervisorPid) ||
|
||||
heartbeatExpired
|
||||
) {
|
||||
exitAfterSupervisorLoss();
|
||||
if (ipcConnected === false) {
|
||||
exitAfterSupervisorLoss("ipc_disconnected");
|
||||
return;
|
||||
}
|
||||
if (supervisorChanged) {
|
||||
exitAfterSupervisorLoss("supervisor_parent_pid_changed");
|
||||
return;
|
||||
}
|
||||
if (!isPidAlive(supervisorPid)) {
|
||||
exitAfterSupervisorLoss("supervisor_pid_dead");
|
||||
return;
|
||||
}
|
||||
if (heartbeatExpired) {
|
||||
exitAfterSupervisorLoss("supervisor_heartbeat_expired");
|
||||
}
|
||||
}, 1000);
|
||||
timer.unref();
|
||||
|
||||
Reference in New Issue
Block a user