Log daemon supervisor exit reasons
This commit is contained in:
@@ -191,11 +191,23 @@ export function runSupervisor(options: SupervisorOptions): void {
|
|||||||
const heartbeat = setInterval(() => {
|
const heartbeat = setInterval(() => {
|
||||||
const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" };
|
const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" };
|
||||||
if (currentChild.connected) {
|
if (currentChild.connected) {
|
||||||
currentChild.send?.(message, () => undefined);
|
currentChild.send?.(message, (error) => {
|
||||||
|
if (error) {
|
||||||
|
writeLifecycleLog("Worker heartbeat IPC send failed", {
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
writeLifecycleLog("Worker heartbeat skipped because IPC channel is disconnected");
|
||||||
}
|
}
|
||||||
}, 1000);
|
}, 1000);
|
||||||
heartbeat.unref();
|
heartbeat.unref();
|
||||||
|
|
||||||
|
child.on("disconnect", () => {
|
||||||
|
writeLifecycleLog("Worker IPC channel disconnected");
|
||||||
|
});
|
||||||
|
|
||||||
child.stdout?.on("data", (chunk: Buffer) => {
|
child.stdout?.on("data", (chunk: Buffer) => {
|
||||||
process.stdout.write(chunk);
|
process.stdout.write(chunk);
|
||||||
writeDurableChunk(chunk);
|
writeDurableChunk(chunk);
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { appendFileSync, mkdirSync } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
import { createPaseoDaemon } from "./bootstrap.js";
|
import { createPaseoDaemon } from "./bootstrap.js";
|
||||||
import { loadConfig } from "./config.js";
|
import { loadConfig } from "./config.js";
|
||||||
import { resolvePaseoHome } from "./paseo-home.js";
|
import { resolvePaseoHome } from "./paseo-home.js";
|
||||||
@@ -41,6 +43,31 @@ function isPidAlive(pid: number): boolean {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function writeWorkerLifecycleLog(
|
||||||
|
paseoHome: string,
|
||||||
|
message: string,
|
||||||
|
fields: Record<string, unknown> = {},
|
||||||
|
): void {
|
||||||
|
try {
|
||||||
|
const logPath = path.join(paseoHome, "daemon.log");
|
||||||
|
mkdirSync(path.dirname(logPath), { recursive: true });
|
||||||
|
appendFileSync(
|
||||||
|
logPath,
|
||||||
|
`${JSON.stringify({
|
||||||
|
level: "warn",
|
||||||
|
time: new Date().toISOString(),
|
||||||
|
pid: process.pid,
|
||||||
|
name: "DaemonWorker",
|
||||||
|
msg: message,
|
||||||
|
...fields,
|
||||||
|
})}\n`,
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Exit-reason logging must never prevent the worker from exiting.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function bootstrapFromEnvironment(): BootstrapResult {
|
function bootstrapFromEnvironment(): BootstrapResult {
|
||||||
try {
|
try {
|
||||||
const paseoHome = resolvePaseoHome();
|
const paseoHome = resolvePaseoHome();
|
||||||
@@ -70,7 +97,7 @@ function applyCliFlagOverrides(config: ReturnType<typeof loadConfig>): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
const { logger, config } = bootstrapFromEnvironment();
|
const { paseoHome, logger, config } = bootstrapFromEnvironment();
|
||||||
let daemon: Awaited<ReturnType<typeof createPaseoDaemon>> | null = null;
|
let daemon: Awaited<ReturnType<typeof createPaseoDaemon>> | null = null;
|
||||||
let shutdownPromise: Promise<number> | null = null;
|
let shutdownPromise: Promise<number> | null = null;
|
||||||
let exitHookInstalled = false;
|
let exitHookInstalled = false;
|
||||||
@@ -174,12 +201,20 @@ async function main() {
|
|||||||
const supervisorPid = process.ppid;
|
const supervisorPid = process.ppid;
|
||||||
let lastSupervisorHeartbeatAt = Date.now();
|
let lastSupervisorHeartbeatAt = Date.now();
|
||||||
let supervisorExitRequested = false;
|
let supervisorExitRequested = false;
|
||||||
const exitAfterSupervisorLoss = () => {
|
const exitAfterSupervisorLoss = (reason: string) => {
|
||||||
if (supervisorExitRequested) {
|
if (supervisorExitRequested) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
supervisorExitRequested = true;
|
supervisorExitRequested = true;
|
||||||
|
|
||||||
|
writeWorkerLifecycleLog(paseoHome, "Supervisor liveness lost; worker exiting", {
|
||||||
|
reason,
|
||||||
|
supervisorPid,
|
||||||
|
currentParentPid: process.ppid,
|
||||||
|
ipcConnected: typeof process.connected === "boolean" ? process.connected : null,
|
||||||
|
heartbeatAgeMs: Date.now() - lastSupervisorHeartbeatAt,
|
||||||
|
});
|
||||||
|
|
||||||
// The supervisor owns the worker's stdout/stderr pipes. Once it is gone,
|
// The supervisor owns the worker's stdout/stderr pipes. Once it is gone,
|
||||||
// logging during graceful shutdown can block on the broken pipe and leave
|
// logging during graceful shutdown can block on the broken pipe and leave
|
||||||
// the daemon orphaned, so supervisor loss is a hard process boundary.
|
// the daemon orphaned, so supervisor loss is a hard process boundary.
|
||||||
@@ -196,19 +231,26 @@ async function main() {
|
|||||||
lastSupervisorHeartbeatAt = Date.now();
|
lastSupervisorHeartbeatAt = Date.now();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
process.on("disconnect", exitAfterSupervisorLoss);
|
process.on("disconnect", () => exitAfterSupervisorLoss("ipc_disconnect_event"));
|
||||||
|
|
||||||
const timer = setInterval(() => {
|
const timer = setInterval(() => {
|
||||||
const ipcConnected = typeof process.connected === "boolean" ? process.connected : true;
|
const ipcConnected = typeof process.connected === "boolean" ? process.connected : true;
|
||||||
const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500;
|
const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500;
|
||||||
const supervisorChanged = process.ppid !== supervisorPid;
|
const supervisorChanged = process.ppid !== supervisorPid;
|
||||||
if (
|
if (ipcConnected === false) {
|
||||||
ipcConnected === false ||
|
exitAfterSupervisorLoss("ipc_disconnected");
|
||||||
supervisorChanged ||
|
return;
|
||||||
!isPidAlive(supervisorPid) ||
|
}
|
||||||
heartbeatExpired
|
if (supervisorChanged) {
|
||||||
) {
|
exitAfterSupervisorLoss("supervisor_parent_pid_changed");
|
||||||
exitAfterSupervisorLoss();
|
return;
|
||||||
|
}
|
||||||
|
if (!isPidAlive(supervisorPid)) {
|
||||||
|
exitAfterSupervisorLoss("supervisor_pid_dead");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (heartbeatExpired) {
|
||||||
|
exitAfterSupervisorLoss("supervisor_heartbeat_expired");
|
||||||
}
|
}
|
||||||
}, 1000);
|
}, 1000);
|
||||||
timer.unref();
|
timer.unref();
|
||||||
|
|||||||
Reference in New Issue
Block a user