fix(server): centralize git subprocesses with p-limit throttling (#310)

* Fix uncaught timeout rejection when archiving agents * fix(server): centralize git subprocesses under runGitCommand with p-limit throttling Supersedes #299. Same problem (unbounded git subprocess concurrency causing zombie accumulation), different approach: a single `runGitCommand(args, opts)` function backed by `p-limit` instead of a custom semaphore pool. - Always uses `spawn` (one code path, no exec/execFile split) - Concurrency configurable via PASEO_GIT_CONCURRENCY env var (default 8) - 30s default timeout for read-only ops, 120s for mutations - Centralized error formatting, stdout truncation, process reaping - Runtime tests verifying throttling, timeout kills, deadlock prevention * fix: update lockfile signatures and Nix hash * fix(server): add Windows compatibility to runGitCommand - shell: true on Windows (resolves .cmd/.bat git shims) - windowsHide: true always (prevents console window flashes) - quoteForCmd for args with spaces when routing through cmd.exe - Tests verify shell/windowsHide behavior per platform * refactor: use spawnProcess instead of duplicating Windows handling spawnProcess from spawn.ts already handles shell: true on Windows, arg quoting for cmd.exe, and windowsHide: true. No need to reimplement. * fix: non-null assertions for stdio streams in strict build --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-04-13 13:03:01 +08:00
parent e6ff15a1e0
commit 964bf3e13f
16 changed files with 770 additions and 254 deletions
@@ -0,0 +1,140 @@
+import pLimit from "p-limit";
+import { spawnProcess } from "./spawn.js";
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_MAX_OUTPUT_BYTES = 20 * 1024 * 1024; // 20MB
+const DEFAULT_STDERR_LIMIT = 2048;
+
+const gitConcurrency = parseInt(process.env.PASEO_GIT_CONCURRENCY ?? "8", 10) || 8;
+const gitLimit = pLimit(gitConcurrency);
+
+export interface GitCommandOptions {
+  cwd: string;
+  env?: NodeJS.ProcessEnv;
+  timeout?: number;
+  maxOutputBytes?: number;
+  acceptExitCodes?: number[];
+}
+
+export interface GitCommandResult {
+  stdout: string;
+  stderr: string;
+  truncated: boolean;
+  exitCode: number | null;
+  signal: NodeJS.Signals | null;
+}
+
+export function runGitCommand(
+  args: string[],
+  options: GitCommandOptions,
+): Promise<GitCommandResult> {
+  return gitLimit(
+    () =>
+      new Promise<GitCommandResult>((resolve, reject) => {
+        const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
+        const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+        const acceptExitCodes = options.acceptExitCodes ?? [0];
+        const command = formatGitCommand(args);
+
+        const child = spawnProcess("git", args, {
+          cwd: options.cwd,
+          env: options.env,
+          stdio: ["ignore", "pipe", "pipe"],
+        });
+
+        let settled = false;
+        let truncated = false;
+        let stdoutBytes = 0;
+        let stderrBytes = 0;
+        const stdoutChunks: Buffer[] = [];
+        const stderrChunks: Buffer[] = [];
+
+        const settle = (callback: () => void) => {
+          if (settled) return;
+          settled = true;
+          clearTimeout(timer);
+          callback();
+        };
+
+        const timer = setTimeout(() => {
+          const error = new Error(`Git command timed out after ${timeout}ms: ${command}`);
+          child.kill("SIGKILL");
+          settle(() => reject(error));
+        }, timeout);
+
+        child.stdout!.on("data", (chunk: Buffer | string) => {
+          if (settled || truncated) return;
+
+          const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+          const remainingBytes = maxOutputBytes - stdoutBytes;
+
+          if (remainingBytes <= 0) {
+            truncated = true;
+            child.kill("SIGKILL");
+            return;
+          }
+
+          if (buffer.length > remainingBytes) {
+            stdoutChunks.push(buffer.subarray(0, remainingBytes));
+            stdoutBytes += remainingBytes;
+            truncated = true;
+            child.kill("SIGKILL");
+            return;
+          }
+
+          stdoutChunks.push(buffer);
+          stdoutBytes += buffer.length;
+        });
+
+        child.stderr!.on("data", (chunk: Buffer | string) => {
+          if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return;
+
+          const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+          const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes;
+
+          if (buffer.length > remainingBytes) {
+            stderrChunks.push(buffer.subarray(0, remainingBytes));
+            stderrBytes += remainingBytes;
+            return;
+          }
+
+          stderrChunks.push(buffer);
+          stderrBytes += buffer.length;
+        });
+
+        child.on("error", (error) => {
+          settle(() => reject(error));
+        });
+
+        child.on("close", (exitCode, signal) => {
+          const result: GitCommandResult = {
+            stdout: Buffer.concat(stdoutChunks).toString("utf8"),
+            stderr: Buffer.concat(stderrChunks).toString("utf8"),
+            truncated,
+            exitCode,
+            signal,
+          };
+
+          if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) {
+            const stderrPreview = result.stderr.trim() || "(no stderr)";
+            const truncationNote = result.truncated ? " (stdout truncated)" : "";
+
+            settle(() =>
+              reject(
+                new Error(
+                  `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`,
+                ),
+              ),
+            );
+            return;
+          }
+
+          settle(() => resolve(result));
+        });
+      }),
+  );
+}
+
+function formatGitCommand(args: string[]): string {
+  return ["git", ...args].join(" ");
+}