Limit daemon-wide Git process pressure (#2797)

* perf(server): bound Git process starts Git refresh bursts could keep creating short-lived processes at the concurrency ceiling, causing sustained process churn on constrained hosts. Apply one daemon-global start-rate and concurrency policy to every Git command while preserving the legacy concurrency environment variable. * build(nix): update npm dependency hash * fix(server): hold Git slots until process exit * fix(server): enforce Git limits across process lifetime * fix(server): align Git metrics with process exit * Fix Git limiter test deadlocks * Tune default Git process rate
2026-08-02 19:59:43 +02:00
parent f682156302
commit 2e188787ff
23 changed files with 822 additions and 225 deletions
@@ -1,5 +1,4 @@
 import { existsSync } from "node:fs";
-import pLimit from "p-limit";
 import type { Logger } from "pino";
 import type { ProcessEnvRecord } from "../server/paseo-env.js";
 import {
@@ -13,14 +12,30 @@ import {
  submitGitCommandTrace,
 } from "./git-command-trace.js";
 import { spawnProcess } from "./spawn.js";
+import {
+  GitProcessScheduler,
+  resolveGitProcessPolicy,
+  type GitProcessPolicy,
+} from "./git-process-scheduler.js";

 const DEFAULT_TIMEOUT_MS = 30_000;
 const DEFAULT_MAX_OUTPUT_BYTES = 20 * 1024 * 1024; // 20MB
 const DEFAULT_STDERR_LIMIT = 2048;

-const gitConcurrency = parseInt(process.env.PASEO_GIT_CONCURRENCY ?? "8", 10) || 8;
-const gitLimit = pLimit(gitConcurrency);
-const gitRuntimeMetrics = new GitCommandRuntimeMetricsWindow(gitConcurrency);
+let gitProcessScheduler = new GitProcessScheduler(resolveGitProcessPolicy({ env: process.env }));
+let gitRuntimeMetrics = createGitCommandRuntimeMetricsWindow(gitProcessScheduler.policy);
+
+function createGitCommandRuntimeMetricsWindow(policy: GitProcessPolicy) {
+  return new GitCommandRuntimeMetricsWindow({
+    concurrencyLimit: policy.maxProcessConcurrency,
+    maxProcessesPerSecond: policy.maxProcessesPerSecond,
+  });
+}
+
+export function configureGitProcessPolicy(policy: GitProcessPolicy): void {
+  gitProcessScheduler = new GitProcessScheduler(policy);
+  gitRuntimeMetrics = createGitCommandRuntimeMetricsWindow(policy);
+}

 export interface GitCommandOptions {
  cwd: string;
@@ -94,8 +109,8 @@ export function stopGitCommandMetrics(): GitCommandMetricsSnapshot {

 export function snapshotGitCommandRuntimeMetrics(): GitCommandRuntimeMetricsSnapshot {
  return gitRuntimeMetrics.snapshotAndReset({
-    active: gitLimit.activeCount,
-    pending: gitLimit.pendingCount,
+    active: gitProcessScheduler.activeCount,
+    pending: gitProcessScheduler.pendingCount,
  });
 }

@@ -142,232 +157,271 @@ export function runGitCommand(
  options: GitCommandOptions,
 ): Promise<GitCommandResult> {
  const commandTrace = submitGitCommandTrace(args, options.cwd, {
-    active: gitLimit.activeCount,
-    pending: gitLimit.pendingCount,
+    active: gitProcessScheduler.activeCount,
+    pending: gitProcessScheduler.pendingCount,
  });
  const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args));
-  const promise = gitLimit(
-    () =>
-      new Promise<GitCommandResult>((resolve, reject) => {
-        startGitCommandTrace(commandTrace, {
-          active: gitLimit.activeCount,
-          pending: gitLimit.pendingCount,
+  const promise = gitProcessScheduler.run(() => {
+    let releaseProcessSlot!: () => void;
+    const exited = new Promise<void>((resolve) => {
+      releaseProcessSlot = resolve;
+    });
+    const resultPromise = new Promise<GitCommandResult>((resolve, reject) => {
+      startGitCommandTrace(commandTrace, {
+        active: gitProcessScheduler.activeCount,
+        pending: gitProcessScheduler.pendingCount,
+      });
+      gitRuntimeMetrics.start(runtimeMetric);
+      const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
+      const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+      const acceptExitCodes = options.acceptExitCodes ?? [0];
+      const command = formatGitCommand(args);
+      const envOverlay = mergeEnvOverlays(options.env, options.envOverlay);
+      const startedAt = Date.now();
+      const metricsState = beginGitCommandMetric();
+      const logger = typeof options.logger?.trace === "function" ? options.logger : undefined;
+      const traceContext = logger
+        ? {
+            command: "git",
+            args,
+            cwd: options.cwd,
+            cwdExists: existsSync(options.cwd),
+            timeout,
+            maxOutputBytes,
+            acceptExitCodes,
+            envOverlayKeys: getEnvOverlayKeys(envOverlay),
+          }
+        : null;
+
+      if (logger && traceContext) {
+        logger.trace(traceContext, "Spawning git command");
+      }
+
+      let settled = false;
+      let metricFinished = false;
+      let processError: Error | null = null;
+      let processExit: { exitCode: number | null; signal: NodeJS.Signals | null } | null = null;
+      let timeoutError: Error | null = null;
+      let truncated = false;
+      let stdoutBytes = 0;
+      let stderrBytes = 0;
+      const stdoutChunks: Buffer[] = [];
+      const stderrChunks: Buffer[] = [];
+      let timer: NodeJS.Timeout | undefined;
+
+      const settle = (callback: () => void) => {
+        if (settled) return;
+        settled = true;
+        if (timer) clearTimeout(timer);
+        callback();
+      };
+
+      const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => {
+        if (metricFinished) return;
+        metricFinished = true;
+        finishGitCommandMetric(metricsState, metric);
+        gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut });
+      };
+
+      const settleTimeoutTrace = (exitCode: number | null, signal: NodeJS.Signals | null) => {
+        settleGitCommandTrace(commandTrace, {
+          outcome: "timed_out",
+          exitCode,
+          signal,
        });
-        gitRuntimeMetrics.start(runtimeMetric);
-        const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
-        const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
-        const acceptExitCodes = options.acceptExitCodes ?? [0];
-        const command = formatGitCommand(args);
-        const envOverlay = mergeEnvOverlays(options.env, options.envOverlay);
-        const startedAt = Date.now();
-        const metricsState = beginGitCommandMetric();
-        const logger = typeof options.logger?.trace === "function" ? options.logger : undefined;
-        const traceContext = logger
-          ? {
-              command: "git",
-              args,
-              cwd: options.cwd,
-              cwdExists: existsSync(options.cwd),
-              timeout,
-              maxOutputBytes,
-              acceptExitCodes,
-              envOverlayKeys: getEnvOverlayKeys(envOverlay),
-            }
-          : null;
+      };

-        if (logger && traceContext) {
-          logger.trace(traceContext, "Spawning git command");
+      const markProcessExited = (exitCode: number | null, signal: NodeJS.Signals | null) => {
+        if (processExit) return;
+        processExit = { exitCode, signal };
+        const timedOut = timeoutError !== null;
+        finishMetricOnce(
+          {
+            args,
+            cwd: options.cwd,
+            startedAtMs: startedAt,
+            durationMs: Date.now() - startedAt,
+            exitCode,
+            signal,
+            success:
+              !timedOut && !processError && (truncated || acceptExitCodes.includes(exitCode ?? -1)),
+          },
+          timedOut,
+        );
+        releaseProcessSlot();
+        if (timedOut) {
+          settleTimeoutTrace(exitCode, signal);
        }
+      };

+      const rejectSpawnFailure = (error: unknown) => {
+        processError = error instanceof Error ? error : new Error(String(error));
+        markProcessExited(null, null);
+        settleGitCommandTrace(commandTrace, {
+          outcome: "spawn_error",
+          exitCode: null,
+          signal: null,
+        });
+        settle(() => reject(processError));
+      };
+
+      let child: ReturnType<typeof spawnProcess>;
+      try {
        // `core.quotepath=false` makes git emit raw UTF-8 paths instead of
        // octal-escaping non-ASCII bytes (e.g. `测试文件.txt` vs `"\346\265\213..."`).
-        const child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], {
+        child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], {
          cwd: options.cwd,
          envOverlay,
          shell: false,
          stdio: ["ignore", "pipe", "pipe"],
        });
        spawnGitCommandTrace(commandTrace, child.pid);
+      } catch (error) {
+        rejectSpawnFailure(error);
+        return;
+      }

-        let settled = false;
-        let metricFinished = false;
-        let truncated = false;
-        let stdoutBytes = 0;
-        let stderrBytes = 0;
-        const stdoutChunks: Buffer[] = [];
-        const stderrChunks: Buffer[] = [];
+      const stdout = child.stdout;
+      const stderr = child.stderr;
+      if (!stdout || !stderr) {
+        child.kill("SIGKILL");
+        rejectSpawnFailure(new Error("Git process did not expose piped stdout and stderr"));
+        return;
+      }

-        const settle = (callback: () => void) => {
-          if (settled) return;
-          settled = true;
-          clearTimeout(timer);
-          callback();
-        };
+      timer = setTimeout(() => {
+        timeoutError = new Error(`Git command timed out after ${timeout}ms: ${command}`);
+        child.kill("SIGKILL");
+        settle(() => reject(timeoutError));
+        if (processExit) {
+          settleTimeoutTrace(processExit.exitCode, processExit.signal);
+        }
+      }, timeout);

-        const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => {
-          if (metricFinished) return;
-          metricFinished = true;
-          finishGitCommandMetric(metricsState, metric);
-          gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut });
-        };
+      stdout.on("data", (chunk: Buffer | string) => {
+        if (settled || truncated) return;

-        const timer = setTimeout(() => {
-          const error = new Error(`Git command timed out after ${timeout}ms: ${command}`);
+        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        const remainingBytes = maxOutputBytes - stdoutBytes;
+
+        if (remainingBytes <= 0) {
+          truncated = true;
          child.kill("SIGKILL");
-          settleGitCommandTrace(commandTrace, {
-            outcome: "timed_out",
-            exitCode: null,
-            signal: "SIGKILL",
-          });
-          finishMetricOnce(
+          return;
+        }
+
+        if (buffer.length > remainingBytes) {
+          stdoutChunks.push(buffer.subarray(0, remainingBytes));
+          stdoutBytes += remainingBytes;
+          truncated = true;
+          child.kill("SIGKILL");
+          return;
+        }
+
+        stdoutChunks.push(buffer);
+        stdoutBytes += buffer.length;
+      });
+
+      stderr.on("data", (chunk: Buffer | string) => {
+        if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return;
+
+        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes;
+
+        if (buffer.length > remainingBytes) {
+          stderrChunks.push(buffer.subarray(0, remainingBytes));
+          stderrBytes += remainingBytes;
+          return;
+        }
+
+        stderrChunks.push(buffer);
+        stderrBytes += buffer.length;
+      });
+
+      child.on("error", (error) => {
+        processError = error;
+        if (logger && traceContext) {
+          logger.trace(
            {
-              args,
-              cwd: options.cwd,
-              startedAtMs: startedAt,
+              ...traceContext,
+              err: error,
              durationMs: Date.now() - startedAt,
-              exitCode: null,
-              signal: "SIGKILL",
-              success: false,
            },
-            true,
+            "Git command process error",
          );
-          settle(() => reject(error));
-        }, timeout);
+        }
+      });

-        child.stdout!.on("data", (chunk: Buffer | string) => {
-          if (settled || truncated) return;
+      child.on("exit", markProcessExited);

-          const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-          const remainingBytes = maxOutputBytes - stdoutBytes;
-
-          if (remainingBytes <= 0) {
-            truncated = true;
-            child.kill("SIGKILL");
-            return;
-          }
-
-          if (buffer.length > remainingBytes) {
-            stdoutChunks.push(buffer.subarray(0, remainingBytes));
-            stdoutBytes += remainingBytes;
-            truncated = true;
-            child.kill("SIGKILL");
-            return;
-          }
-
-          stdoutChunks.push(buffer);
-          stdoutBytes += buffer.length;
-        });
-
-        child.stderr!.on("data", (chunk: Buffer | string) => {
-          if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return;
-
-          const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-          const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes;
-
-          if (buffer.length > remainingBytes) {
-            stderrChunks.push(buffer.subarray(0, remainingBytes));
-            stderrBytes += remainingBytes;
-            return;
-          }
-
-          stderrChunks.push(buffer);
-          stderrBytes += buffer.length;
-        });
-
-        child.on("error", (error) => {
-          settleGitCommandTrace(commandTrace, {
-            outcome: "spawn_error",
-            exitCode: null,
-            signal: null,
-          });
-          finishMetricOnce({
-            args,
-            cwd: options.cwd,
-            startedAtMs: startedAt,
-            durationMs: Date.now() - startedAt,
-            exitCode: null,
-            signal: null,
-            success: false,
-          });
-          if (logger && traceContext) {
-            logger.trace(
-              {
-                ...traceContext,
-                err: error,
-                durationMs: Date.now() - startedAt,
-              },
-              "Git command process error",
-            );
-          }
-          settle(() => reject(error));
-        });
-
-        child.on("close", (exitCode, signal) => {
-          settleGitCommandTrace(commandTrace, {
-            outcome: "closed",
-            exitCode,
-            signal,
-            truncated,
-          });
-          const result: GitCommandResult = {
-            stdout: Buffer.concat(stdoutChunks).toString("utf8"),
-            stderr: Buffer.concat(stderrChunks).toString("utf8"),
-            truncated,
-            exitCode,
-            signal,
-          };
-          if (logger && traceContext) {
-            logger.trace(
-              {
-                ...traceContext,
-                durationMs: Date.now() - startedAt,
-                exitCode,
-                signal,
-                truncated,
-                stdoutBytes,
-                stderrBytes,
-              },
-              "Git command closed",
-            );
-          }
-
-          if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) {
-            finishMetricOnce({
-              args,
-              cwd: options.cwd,
-              startedAtMs: startedAt,
+      child.on("close", (exitCode, signal) => {
+        markProcessExited(exitCode, signal);
+        const result: GitCommandResult = {
+          stdout: Buffer.concat(stdoutChunks).toString("utf8"),
+          stderr: Buffer.concat(stderrChunks).toString("utf8"),
+          truncated,
+          exitCode,
+          signal,
+        };
+        if (logger && traceContext) {
+          logger.trace(
+            {
+              ...traceContext,
              durationMs: Date.now() - startedAt,
              exitCode,
              signal,
-              success: false,
-            });
-            const stderrPreview = result.stderr.trim() || "(no stderr)";
-            const truncationNote = result.truncated ? " (stdout truncated)" : "";
+              truncated,
+              stdoutBytes,
+              stderrBytes,
+            },
+            "Git command closed",
+          );
+        }

-            settle(() =>
-              reject(
-                new Error(
-                  `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`,
-                ),
-              ),
-            );
-            return;
-          }
+        if (timeoutError) {
+          return;
+        }

-          finishMetricOnce({
-            args,
-            cwd: options.cwd,
-            startedAtMs: startedAt,
-            durationMs: Date.now() - startedAt,
+        if (processError) {
+          settleGitCommandTrace(commandTrace, {
+            outcome: "spawn_error",
            exitCode,
            signal,
-            success: true,
          });
-          settle(() => resolve(result));
+          settle(() => reject(processError));
+          return;
+        }
+
+        settleGitCommandTrace(commandTrace, {
+          outcome: "closed",
+          exitCode,
+          signal,
+          truncated,
        });
-      }),
+
+        if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) {
+          const stderrPreview = result.stderr.trim() || "(no stderr)";
+          const truncationNote = result.truncated ? " (stdout truncated)" : "";
+
+          settle(() =>
+            reject(
+              new Error(
+                `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`,
+              ),
+            ),
+          );
+          return;
+        }
+
+        settle(() => resolve(result));
+      });
+    });
+    return { result: resultPromise, exited };
+  });
+  gitRuntimeMetrics.observeLimiter(
+    gitProcessScheduler.activeCount,
+    gitProcessScheduler.pendingCount,
  );
-  gitRuntimeMetrics.observeLimiter(gitLimit.activeCount, gitLimit.pendingCount);
  return promise;
 }