Recover daemon workers stalled by workspace Git pressure (#3033)

* fix(server): bound workspace git pipelines

* fix(server): bound workspace git watcher setup

* fix(server): recover unresponsive daemon workers

* fix(server): tolerate transient daemon heartbeat delays

A five-second watchdog threshold can replace a healthy worker during a short event-loop delay. Keep recovery bounded while allowing a 15-second heartbeat gap, and preserve process-level coverage for workers that ignore graceful termination.

* test(server): normalize workspace fixture paths

Workspace Git entry points resolve paths before invoking dependencies. Keep admission-control fixtures keyed by the same canonical paths so their assertions and deferred gates work on Windows.

---------

Co-authored-by: Timi Ajiboye <trypio00@gmail.com>
Co-authored-by: Mohamed Boudra <boudra.moha@gmail.com>

packages/server/scripts/supervisor.ts

@@ -2,6 +2,11 @@ import { fork, spawn, type ChildProcess } from "child_process";
 import { mkdirSync } from "node:fs";
 import path from "node:path";
 import { createStream as createRotatingFileStream } from "rotating-file-stream";
+import { signalProcessTree } from "../src/utils/tree-kill.js";
+
+const WORKER_HEARTBEAT_INTERVAL_MS = 1_000;
+const WORKER_HEARTBEAT_TIMEOUT_MS = 15_000;
+const WORKER_TERMINATION_GRACE_MS = 10_000;
 
 interface SupervisorLogFileOptions {
   path: string;
@@ -29,6 +34,10 @@ interface SupervisorHeartbeatMessage {
   type: "paseo:supervisor-heartbeat";
 }
 
+interface WorkerHeartbeatMessage {
+  type: "paseo:worker-heartbeat";
+}
+
 interface SupervisorOptions {
   name: string;
   startupMessage: string;
@@ -84,6 +93,15 @@ function parseLifecycleMessage(msg: unknown): WorkerLifecycleMessage | null {
   return null;
 }
 
+function isWorkerHeartbeatMessage(msg: unknown): msg is WorkerHeartbeatMessage {
+  return (
+    typeof msg === "object" &&
+    msg !== null &&
+    "type" in msg &&
+    (msg as { type?: unknown }).type === "paseo:worker-heartbeat"
+  );
+}
+
 function toRotatingFileStreamSize(size: string): string {
   const trimmed = size.trim();
   const match = trimmed.match(/^(\d+)\s*([bBkKmMgG])?$/);
@@ -120,6 +138,7 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
   let restarting = false;
   let shuttingDown = false;
   let exiting = false;
+  let forceKillTimer: NodeJS.Timeout | null = null;
   const logStream = createSupervisorLogStream(options.logFile);
 
   const writeDurableChunk = (chunk: string | Buffer): void => {
@@ -169,6 +188,40 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
       });
   };
 
+  const clearForceKillTimer = (): void => {
+    if (forceKillTimer) {
+      clearTimeout(forceKillTimer);
+      forceKillTimer = null;
+    }
+  };
+
+  const scheduleForceKill = (reason: string): void => {
+    if (!child) {
+      return;
+    }
+    const currentChild = child;
+    clearForceKillTimer();
+    forceKillTimer = setTimeout(() => {
+      forceKillTimer = null;
+      if (child !== currentChild) {
+        return;
+      }
+      writeLifecycleLog("Worker did not exit after SIGTERM; forcing SIGKILL", {
+        reason,
+        supervisorPid: process.pid,
+        workerPid: currentChild.pid ?? null,
+      });
+      void signalProcessTree(currentChild, "SIGKILL").catch((error) => {
+        writeLifecycleLog("Failed to force-kill worker process tree", {
+          error: error instanceof Error ? error.message : String(error),
+          supervisorPid: process.pid,
+          workerPid: currentChild.pid ?? null,
+        });
+      });
+    }, WORKER_TERMINATION_GRACE_MS);
+    forceKillTimer.unref();
+  };
+
   const spawnWorker = () => {
     let workerEntry: string;
     try {
@@ -197,6 +250,7 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
     }
 
     const currentChild = child;
+    let lastWorkerHeartbeatAt = Date.now();
     const heartbeat = setInterval(() => {
       const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" };
       if (currentChild.connected) {
@@ -210,9 +264,26 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
       } else {
         writeLifecycleLog("Worker heartbeat skipped because IPC channel is disconnected");
       }
-    }, 1000);
+    }, WORKER_HEARTBEAT_INTERVAL_MS);
     heartbeat.unref();
 
+    const workerWatchdog = setInterval(() => {
+      if (child !== currentChild || restarting || shuttingDown) {
+        return;
+      }
+      const heartbeatAgeMs = Date.now() - lastWorkerHeartbeatAt;
+      if (heartbeatAgeMs < WORKER_HEARTBEAT_TIMEOUT_MS) {
+        return;
+      }
+      writeLifecycleLog("Worker heartbeat timed out; restarting worker", {
+        heartbeatAgeMs,
+        supervisorPid: process.pid,
+        workerPid: currentChild.pid ?? null,
+      });
+      requestRestart("worker_heartbeat_timeout");
+    }, WORKER_HEARTBEAT_INTERVAL_MS);
+    workerWatchdog.unref();
+
     child.on("disconnect", () => {
       writeLifecycleLog("Worker IPC channel disconnected");
     });
@@ -228,6 +299,10 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
     });
 
     child.on("message", (msg: unknown) => {
+      if (isWorkerHeartbeatMessage(msg)) {
+        lastWorkerHeartbeatAt = Date.now();
+        return;
+      }
       const lifecycleMessage = parseLifecycleMessage(msg);
       if (!lifecycleMessage) {
         return;
@@ -256,8 +331,10 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
       requestRestart(reason);
     });
 
-    child.on("close", (code, signal) => {
+    child.on("exit", (code, signal) => {
       clearInterval(heartbeat);
+      clearInterval(workerWatchdog);
+      clearForceKillTimer();
       const exitDescriptor = describeExit(code, signal);
       writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor });
 
@@ -308,6 +385,7 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
     writeLifecycleLog("Restart requested", { reason });
     log(`${reason}. Stopping worker for restart...`);
     signalWorker("SIGTERM", reason);
+    scheduleForceKill(reason);
   };
 
   const requestShutdown = (reason: string) => {
@@ -323,6 +401,7 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
       return;
     }
     signalWorker("SIGTERM", reason);
+    scheduleForceKill(reason);
   };
 
   const forwardSignal = (signal: NodeJS.Signals) => {