Recover daemon workers stalled by workspace Git pressure (#3033)

* fix(server): bound workspace git pipelines

* fix(server): bound workspace git watcher setup

* fix(server): recover unresponsive daemon workers

* fix(server): tolerate transient daemon heartbeat delays

A five-second watchdog threshold can replace a healthy worker during a short event-loop delay. Keep recovery bounded while allowing a 15-second heartbeat gap, and preserve process-level coverage for workers that ignore graceful termination.

* test(server): normalize workspace fixture paths

Workspace Git entry points resolve paths before invoking dependencies. Keep admission-control fixtures keyed by the same canonical paths so their assertions and deferred gates work on Windows.

---------

Co-authored-by: Timi Ajiboye <trypio00@gmail.com>
Co-authored-by: Mohamed Boudra <boudra.moha@gmail.com>

packages/server/scripts/supervisor.logging.test.ts

@@ -13,9 +13,11 @@ const supervisorPath = fileURLToPath(new URL("./supervisor.ts", import.meta.url)
 async function runSupervisorFixture(options: {
   workerSource: string;
   restartOnCrash?: boolean;
+  timeoutMs?: number;
 }): Promise<{
   code: number | null;
   signal: NodeJS.Signals | null;
+  elapsedMs: number;
   log: string;
   stdout: string;
   stderr: string;
@@ -47,6 +49,7 @@ async function runSupervisorFixture(options: {
     `,
   );
 
+  const startedAt = Date.now();
   const child = spawn(process.execPath, ["--import", "tsx", runnerPath], {
     cwd: repoRoot,
     env: { ...process.env },
@@ -71,7 +74,7 @@ async function runSupervisorFixture(options: {
     const timeout = setTimeout(() => {
       child.kill("SIGKILL");
       reject(new Error("supervisor fixture timed out"));
-    }, 10000);
+    }, options.timeoutMs ?? 10_000);
 
     child.on("error", (error) => {
       clearTimeout(timeout);
@@ -84,7 +87,7 @@ async function runSupervisorFixture(options: {
   });
 
   const log = await readFile(logPath, "utf8");
-  return { code, signal, log, stdout, stderr };
+  return { code, signal, elapsedMs: Date.now() - startedAt, log, stdout, stderr };
 }
 
 describe("supervisor durable logging", () => {
@@ -186,6 +189,152 @@ describe("supervisor durable logging", () => {
     expect(result.log).toContain('"workerPid":');
   });
 
+  test("keeps a worker alive while heartbeats continue", async () => {
+    const result = await runSupervisorFixture({
+      timeoutMs: 10_000,
+      workerSource: `
+        setInterval(() => {
+          process.send?.({ type: "paseo:worker-heartbeat" });
+        }, 250);
+        setTimeout(() => {
+          process.send?.({ type: "paseo:shutdown", reason: "healthy_heartbeat_test_complete" });
+        }, 6000);
+      `,
+    });
+
+    expect(result.code).toBe(0);
+    expect(result.signal).toBeNull();
+    expect(result.log).toContain('"reason":"healthy_heartbeat_test_complete"');
+    expect(result.log).not.toContain('"msg":"Worker heartbeat timed out; restarting worker"');
+  }, 10_000);
+
+  test("tolerates a transient seven-second heartbeat pause", async () => {
+    const result = await runSupervisorFixture({
+      timeoutMs: 10_000,
+      workerSource: `
+        import { existsSync, writeFileSync } from "node:fs";
+
+        const marker = process.argv[1] + ".started";
+        if (!existsSync(marker)) {
+          writeFileSync(marker, "started");
+          let heartbeatCount = 0;
+          const heartbeat = setInterval(() => {
+            process.send?.({ type: "paseo:worker-heartbeat" });
+            heartbeatCount += 1;
+            if (heartbeatCount === 3) clearInterval(heartbeat);
+          }, 100);
+          setTimeout(() => {
+            process.send?.({ type: "paseo:shutdown", reason: "heartbeat_pause_tolerated" });
+          }, 7_000);
+          setInterval(() => {}, 1_000);
+        } else {
+          process.send?.({ type: "paseo:shutdown", reason: "unexpected_heartbeat_restart" });
+          setInterval(() => {}, 1_000);
+        }
+      `,
+    });
+
+    expect(result.code).toBe(0);
+    expect(result.signal).toBeNull();
+    expect(result.log).toContain('"reason":"heartbeat_pause_tolerated"');
+    expect(result.log).not.toContain('"reason":"unexpected_heartbeat_restart"');
+    expect(result.log).not.toContain('"msg":"Worker heartbeat timed out; restarting worker"');
+  }, 10_000);
+
+  test.skipIf(isPlatform("win32"))(
+    "forces shutdown when a worker ignores SIGTERM",
+    async () => {
+      const result = await runSupervisorFixture({
+        timeoutMs: 15_000,
+        workerSource: `
+          process.on("SIGTERM", () => {});
+          process.send?.({ type: "paseo:shutdown", reason: "stalled_worker_shutdown" });
+          setInterval(() => {}, 1_000);
+        `,
+      });
+
+      expect(result.code).toBe(0);
+      expect(result.signal).toBeNull();
+      expect(result.log).toContain('"reason":"stalled_worker_shutdown"');
+      expect(result.log).toContain('"msg":"Worker did not exit after SIGTERM; forcing SIGKILL"');
+      expect(result.log).toContain('"signal":"SIGKILL"');
+    },
+    20_000,
+  );
+
+  // POSIX-only: the watchdog uses SIGKILL after its graceful shutdown window.
+  test.skipIf(isPlatform("win32"))(
+    "restarts a worker that stops heartbeating",
+    async () => {
+      const result = await runSupervisorFixture({
+        timeoutMs: 35_000,
+        workerSource: `
+          import { existsSync, writeFileSync } from "node:fs";
+
+          const marker = process.argv[1] + ".started";
+          if (!existsSync(marker)) {
+            writeFileSync(marker, "started");
+            let heartbeatCount = 0;
+            const heartbeat = setInterval(() => {
+              process.send?.({ type: "paseo:worker-heartbeat" });
+              heartbeatCount += 1;
+              if (heartbeatCount === 3) clearInterval(heartbeat);
+            }, 100);
+            process.on("SIGTERM", () => {});
+            setInterval(() => {}, 1000);
+          } else {
+            process.send?.({ type: "paseo:shutdown", reason: "watchdog_test_complete" });
+            setInterval(() => {}, 1000);
+          }
+        `,
+      });
+
+      expect(result.code).toBe(0);
+      expect(result.signal).toBeNull();
+      expect(result.log).toContain('"msg":"Worker heartbeat timed out; restarting worker"');
+      expect(result.log).toContain('"msg":"Worker did not exit after SIGTERM; forcing SIGKILL"');
+      expect(result.log).toContain('"signal":"SIGKILL"');
+    },
+    40_000,
+  );
+
+  test.skipIf(isPlatform("win32"))(
+    "restarts after worker exit while a descendant retains the worker stdio",
+    async () => {
+      const result = await runSupervisorFixture({
+        timeoutMs: 7_000,
+        workerSource: `
+          import { spawn } from "node:child_process";
+          import { existsSync, writeFileSync } from "node:fs";
+
+          const marker = process.argv[1] + ".started";
+          if (!existsSync(marker)) {
+            writeFileSync(marker, "started");
+            const descendant = spawn(
+              process.execPath,
+              ["-e", "process.on('SIGTERM', () => {}); setTimeout(() => process.exit(0), 4000)"],
+              { detached: true, stdio: ["ignore", "inherit", "inherit"] },
+            );
+            descendant.unref();
+            process.on("SIGTERM", () => process.exit(0));
+            process.send?.({ type: "paseo:restart", reason: "stdio_descendant" });
+            setInterval(() => {}, 1000);
+          } else {
+            process.send?.({ type: "paseo:shutdown", reason: "stdio_restart_complete" });
+            setInterval(() => {}, 1000);
+          }
+        `,
+      });
+
+      expect(result.code).toBe(0);
+      expect(result.signal).toBeNull();
+      expect(result.elapsedMs).toBeLessThan(2_500);
+      expect(result.log).toContain('"reason":"stdio_descendant"');
+      expect(result.log).toContain("Restarting worker");
+    },
+    7_000,
+  );
+
   // POSIX-only: Windows reports the worker self-kill as an exit code, not SIGKILL.
   test.skipIf(isPlatform("win32"))(
     "logs worker signal exits even when the worker cannot log",