Daemon hardening epic

packages/server/scripts/supervisor.ts

@@ -1,4 +1,15 @@
 import { fork, spawn, type ChildProcess } from "child_process";
+import { mkdirSync } from "node:fs";
+import path from "node:path";
+import { createStream as createRotatingFileStream } from "rotating-file-stream";
+
+interface SupervisorLogFileOptions {
+  path: string;
+  rotate: {
+    maxSize: string;
+    maxFiles: number;
+  };
+}
 
 type WorkerLifecycleMessage =
   | {
@@ -28,6 +39,7 @@ interface SupervisorOptions {
   onWorkerReady?: (message: { listen: string }) => Promise<void> | void;
   restartOnCrash?: boolean;
   onSupervisorExit?: () => Promise<void> | void;
+  logFile?: SupervisorLogFileOptions;
 }
 
 function describeExit(code: number | null, signal: NodeJS.Signals | null): string {
@@ -59,6 +71,31 @@ function parseLifecycleMessage(msg: unknown): WorkerLifecycleMessage | null {
   return null;
 }
 
+function toRotatingFileStreamSize(size: string): string {
+  const trimmed = size.trim();
+  const match = trimmed.match(/^(\d+)\s*([bBkKmMgG])?$/);
+  if (!match) {
+    return trimmed;
+  }
+
+  const value = match[1];
+  const unit = (match[2] ?? "M").toUpperCase();
+  return `${value}${unit}`;
+}
+
+function createSupervisorLogStream(options: SupervisorLogFileOptions | undefined) {
+  if (!options) {
+    return null;
+  }
+
+  mkdirSync(path.dirname(options.path), { recursive: true });
+  return createRotatingFileStream(path.basename(options.path), {
+    path: path.dirname(options.path),
+    size: toRotatingFileStreamSize(options.rotate.maxSize),
+    maxFiles: options.rotate.maxFiles,
+  });
+}
+
 export function runSupervisor(options: SupervisorOptions): void {
   const restartOnCrash = options.restartOnCrash ?? false;
   const workerArgs = options.workerArgs ?? process.argv.slice(2);
@@ -70,11 +107,39 @@ export function runSupervisor(options: SupervisorOptions): void {
   let restarting = false;
   let shuttingDown = false;
   let exiting = false;
+  const logStream = createSupervisorLogStream(options.logFile);
+
+  const writeDurableChunk = (chunk: string | Buffer): void => {
+    logStream?.write(chunk);
+  };
+
+  const writeLifecycleLog = (message: string, fields: Record<string, unknown> = {}): void => {
+    writeDurableChunk(
+      `${JSON.stringify({
+        level: "info",
+        time: new Date().toISOString(),
+        pid: process.pid,
+        name: options.name,
+        msg: message,
+        ...fields,
+      })}\n`,
+    );
+  };
 
   const log = (message: string): void => {
     process.stderr.write(`[${options.name}] ${message}\n`);
+    writeLifecycleLog(message);
   };
 
+  const closeLogStream = (): Promise<void> =>
+    new Promise((resolve) => {
+      if (!logStream) {
+        resolve();
+        return;
+      }
+      logStream.end(resolve);
+    });
+
   const exitSupervisor = (code: number): void => {
     if (exiting) {
       return;
@@ -85,6 +150,7 @@ export function runSupervisor(options: SupervisorOptions): void {
         const message = error instanceof Error ? error.message : String(error);
         log(`Supervisor exit cleanup failed: ${message}`);
       })
+      .then(closeLogStream)
       .finally(() => {
         process.exit(code);
       });
@@ -103,19 +169,30 @@ export function runSupervisor(options: SupervisorOptions): void {
     }
 
     const spawnSpec = resolveWorkerSpawnSpec?.(workerEntry) ?? null;
+    writeLifecycleLog("Spawning worker", { workerEntry });
     if (spawnSpec) {
       child = spawn(spawnSpec.command, spawnSpec.args, {
-        stdio: ["inherit", "inherit", "inherit", "ipc"],
+        stdio: ["inherit", "pipe", "pipe", "ipc"],
         env: spawnSpec.env ?? workerEnv,
       });
     } else {
       child = fork(workerEntry, workerArgs, {
-        stdio: "inherit",
+        stdio: ["inherit", "pipe", "pipe", "ipc"],
         env: workerEnv,
         execArgv: workerExecArgv,
       });
     }
 
+    child.stdout?.on("data", (chunk: Buffer) => {
+      process.stdout.write(chunk);
+      writeDurableChunk(chunk);
+    });
+
+    child.stderr?.on("data", (chunk: Buffer) => {
+      process.stderr.write(chunk);
+      writeDurableChunk(chunk);
+    });
+
     child.on("message", (msg: unknown) => {
       const lifecycleMessage = parseLifecycleMessage(msg);
       if (!lifecycleMessage) {
@@ -123,6 +200,7 @@ export function runSupervisor(options: SupervisorOptions): void {
       }
 
       if (lifecycleMessage.type === "paseo:ready") {
+        writeLifecycleLog("Worker ready", { listen: lifecycleMessage.listen });
         Promise.resolve(options.onWorkerReady?.({ listen: lifecycleMessage.listen })).catch(
           (error) => {
             const message = error instanceof Error ? error.message : String(error);
@@ -133,15 +211,21 @@ export function runSupervisor(options: SupervisorOptions): void {
       }
 
       if (lifecycleMessage.type === "paseo:shutdown") {
+        writeLifecycleLog("Worker requested shutdown");
         requestShutdown("Shutdown requested by worker");
         return;
       }
 
+      writeLifecycleLog(
+        "Worker requested restart",
+        lifecycleMessage.reason ? { reason: lifecycleMessage.reason } : {},
+      );
       requestRestart("Restart requested by worker");
     });
 
-    child.on("exit", (code, signal) => {
+    child.on("close", (code, signal) => {
       const exitDescriptor = describeExit(code, signal);
+      writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor });
 
       if (shuttingDown) {
         log(`Worker exited (${exitDescriptor}). Supervisor shutting down.`);
@@ -151,17 +235,21 @@ export function runSupervisor(options: SupervisorOptions): void {
 
       const crashed =
         restartOnCrash &&
-        ((code !== 0 && code !== null) || (signal !== null && signal === "SIGKILL"));
+        ((code !== 0 && code !== null) || (signal !== null && signal !== "SIGTERM"));
 
       if (restarting || crashed) {
         restarting = false;
-        log(`Worker exited (${exitDescriptor}). Restarting worker...`);
+        log(
+          crashed
+            ? `Worker crashed (${exitDescriptor}). Restarting worker...`
+            : `Worker exited (${exitDescriptor}). Restarting worker...`,
+        );
         spawnWorker();
         return;
       }
 
       log(`Worker exited (${exitDescriptor}). Supervisor exiting.`);
-      exitSupervisor(typeof code === "number" ? code : 0);
+      exitSupervisor(typeof code === "number" ? code : 1);
     });
   };
 
@@ -170,6 +258,7 @@ export function runSupervisor(options: SupervisorOptions): void {
       return;
     }
     restarting = true;
+    writeLifecycleLog("Restart requested", { reason });
     log(`${reason}. Stopping worker for restart...`);
     child.kill("SIGTERM");
   };
@@ -180,6 +269,7 @@ export function runSupervisor(options: SupervisorOptions): void {
     }
     shuttingDown = true;
     restarting = false;
+    writeLifecycleLog("Supervisor shutdown requested", { reason });
     log(`${reason}. Stopping worker...`);
     if (!child) {
       exitSupervisor(0);
@@ -196,5 +286,6 @@ export function runSupervisor(options: SupervisorOptions): void {
   process.on("SIGTERM", () => forwardSignal("SIGTERM"));
 
   process.stdout.write(`[${options.name}] ${options.startupMessage}\n`);
+  writeLifecycleLog(options.startupMessage);
   spawnWorker();
 }