Recover desktop startup from stale daemon locks

Refs #1950 Refs #1904 Refs #769
2026-07-15 15:48:09 +02:00
parent 7f011d16c5
commit f06792ae89
6 changed files with 516 additions and 75 deletions
@@ -1,9 +1,17 @@
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdtemp, open, rm, utimes, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { describe, expect, test } from "vitest";

-import { acquirePidLock, getPidLockInfo, releasePidLock, updatePidLock } from "./pid-lock.js";
+import {
+  acquirePidLock,
+  getPidLockInfo,
+  isLocked,
+  PidLockError,
+  refreshPidLock,
+  releasePidLock,
+  updatePidLock,
+} from "./pid-lock.js";

 describe("pid-lock ownership", () => {
  test("writes and releases lock for explicit owner pid", async () => {
@@ -22,6 +30,7 @@ describe("pid-lock ownership", () => {
      const lock = await getPidLockInfo(paseoHome);
      expect(lock?.pid).toBe(ownerPid);
      expect(lock?.listen).toBeNull();
+      expect(lock?.heartbeat).toBe(true);

      await (
        updatePidLock as unknown as (
@@ -49,4 +58,200 @@ describe("pid-lock ownership", () => {
      await rm(paseoHome, { recursive: true, force: true });
    }
  });
+
+  test("keeps a stale heartbeat lock when the recorded pid is alive without a reachability check", async () => {
+    const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-stale-heartbeat-"));
+    const replacementOwnerPid = process.pid + 10_000;
+
+    try {
+      const pidPath = join(paseoHome, "paseo.pid");
+      await writeFile(
+        pidPath,
+        JSON.stringify({
+          pid: process.pid,
+          startedAt: "2026-01-01T00:00:00.000Z",
+          hostname: "old-host",
+          uid: process.getuid?.() ?? 0,
+          listen: "127.0.0.1:6767",
+          desktopManaged: true,
+          heartbeat: true,
+        }),
+      );
+      const staleTime = new Date(Date.now() - 10 * 60_000);
+      await utimes(pidPath, staleTime, staleTime);
+
+      await expect(isLocked(paseoHome)).resolves.toMatchObject({ locked: true });
+      await expect(
+        acquirePidLock(paseoHome, null, { ownerPid: replacementOwnerPid }),
+      ).rejects.toThrow("Another Paseo daemon is already running");
+
+      const lock = await getPidLockInfo(paseoHome);
+      expect(lock?.pid).toBe(process.pid);
+    } finally {
+      await rm(paseoHome, { recursive: true, force: true });
+    }
+  });
+
+  test("reclaims a stale desktop heartbeat lock after desktop confirms the daemon is unreachable", async () => {
+    const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-stale-desktop-heartbeat-"));
+    const replacementOwnerPid = process.pid + 10_000;
+
+    try {
+      const pidPath = join(paseoHome, "paseo.pid");
+      await writeFile(
+        pidPath,
+        JSON.stringify({
+          pid: process.pid,
+          startedAt: "2026-01-01T00:00:00.000Z",
+          hostname: "old-host",
+          uid: process.getuid?.() ?? 0,
+          listen: "127.0.0.1:6767",
+          desktopManaged: true,
+          heartbeat: true,
+        }),
+      );
+      const staleTime = new Date(Date.now() - 10 * 60_000);
+      await utimes(pidPath, staleTime, staleTime);
+
+      await acquirePidLock(paseoHome, null, {
+        ownerPid: replacementOwnerPid,
+        reclaimStaleDesktopLock: true,
+      });
+
+      const lock = await getPidLockInfo(paseoHome);
+      expect(lock?.pid).toBe(replacementOwnerPid);
+      expect(lock?.listen).toBeNull();
+    } finally {
+      await rm(paseoHome, { recursive: true, force: true });
+    }
+  });
+
+  test("keeps a stale live lock written by a pre-heartbeat daemon", async () => {
+    const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-legacy-live-"));
+    const pidPath = join(paseoHome, "paseo.pid");
+
+    try {
+      await writeFile(
+        pidPath,
+        JSON.stringify({
+          pid: process.pid,
+          startedAt: "2026-01-01T00:00:00.000Z",
+          hostname: "old-host",
+          uid: process.getuid?.() ?? 0,
+          listen: "127.0.0.1:6767",
+          desktopManaged: true,
+        }),
+      );
+      const staleTime = new Date(Date.now() - 10 * 60_000);
+      await utimes(pidPath, staleTime, staleTime);
+
+      await expect(
+        acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 }),
+      ).rejects.toThrow("Another Paseo daemon is already running");
+
+      const lock = await getPidLockInfo(paseoHome);
+      expect(lock?.pid).toBe(process.pid);
+    } finally {
+      await rm(paseoHome, { recursive: true, force: true });
+    }
+  });
+
+  test("reclaims a stale legacy desktop lock after desktop confirms the daemon is unreachable", async () => {
+    const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-legacy-desktop-"));
+    const replacementOwnerPid = process.pid + 10_000;
+    const pidPath = join(paseoHome, "paseo.pid");
+
+    try {
+      await writeFile(
+        pidPath,
+        JSON.stringify({
+          pid: process.pid,
+          startedAt: "2026-01-01T00:00:00.000Z",
+          hostname: "old-host",
+          uid: process.getuid?.() ?? 0,
+          listen: "127.0.0.1:6767",
+          desktopManaged: true,
+        }),
+      );
+      const staleTime = new Date(Date.now() - 10 * 60_000);
+      await utimes(pidPath, staleTime, staleTime);
+
+      await acquirePidLock(paseoHome, null, {
+        ownerPid: replacementOwnerPid,
+        reclaimStaleDesktopLock: true,
+      });
+
+      const lock = await getPidLockInfo(paseoHome);
+      expect(lock?.pid).toBe(replacementOwnerPid);
+      expect(lock?.heartbeat).toBe(true);
+    } finally {
+      await rm(paseoHome, { recursive: true, force: true });
+    }
+  });
+
+  test("rejects a heartbeat refresh after another supervisor takes ownership", async () => {
+    const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-refresh-owner-"));
+
+    try {
+      await acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 });
+
+      await expect(refreshPidLock(paseoHome, { ownerPid: process.pid })).rejects.toBeInstanceOf(
+        PidLockError,
+      );
+    } finally {
+      await rm(paseoHome, { recursive: true, force: true });
+    }
+  });
+
+  test("retries a heartbeat refresh while its owner is rewriting the lock", async () => {
+    const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-refresh-rewrite-"));
+    const pidPath = join(paseoHome, "paseo.pid");
+
+    try {
+      await acquirePidLock(paseoHome, null, { ownerPid: process.pid });
+      const lock = await getPidLockInfo(paseoHome);
+      expect(lock).not.toBeNull();
+
+      const rewriteHandle = await open(pidPath, "r+");
+      await rewriteHandle.truncate(0);
+
+      const refresh = refreshPidLock(paseoHome, { ownerPid: process.pid });
+      await new Promise((resolve) => setTimeout(resolve, 250));
+      await rewriteHandle.writeFile(JSON.stringify(lock));
+      await rewriteHandle.close();
+
+      await expect(refresh).resolves.toBeUndefined();
+    } finally {
+      await rm(paseoHome, { recursive: true, force: true });
+    }
+  });
+
+  test("keeps a fresh lock when the recorded pid is alive", async () => {
+    const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-fresh-heartbeat-"));
+
+    try {
+      await writeFile(
+        join(paseoHome, "paseo.pid"),
+        JSON.stringify({
+          pid: process.pid,
+          startedAt: new Date().toISOString(),
+          hostname: "current-host",
+          uid: process.getuid?.() ?? 0,
+          listen: "127.0.0.1:6767",
+          desktopManaged: true,
+          heartbeat: true,
+        }),
+      );
+
+      await expect(
+        acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 }),
+      ).rejects.toThrow("Another Paseo daemon is already running");
+
+      const lock = await getPidLockInfo(paseoHome);
+      expect(lock?.pid).toBe(process.pid);
+      expect(lock?.listen).toBe("127.0.0.1:6767");
+    } finally {
+      await rm(paseoHome, { recursive: true, force: true });
+    }
+  });
 });
@@ -1,4 +1,5 @@
-import { open, readFile, unlink, mkdir } from "node:fs/promises";
+import { open, readFile, stat, unlink, mkdir, utimes } from "node:fs/promises";
+import type { FileHandle } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { hostname } from "node:os";
@@ -11,6 +12,7 @@ export const pidLockInfoSchema = z.object({
  uid: z.number(),
  listen: z.string().nullable(),
  desktopManaged: z.boolean().optional(),
+  heartbeat: z.literal(true).optional(),
 });

 export interface PidLockInfo extends z.infer<typeof pidLockInfoSchema> {}
@@ -34,6 +36,12 @@ export class PidLockError extends Error {
  }
 }

+// Stale recovery is for abandoned locks, so keep this well above ordinary event-loop stalls.
+const PID_LOCK_STALE_MS = 5 * 60_000;
+const PID_LOCK_HEARTBEAT_INTERVAL_MS = 30_000;
+const PID_LOCK_READ_RETRY_ATTEMPTS = 10;
+const PID_LOCK_READ_RETRY_DELAY_MS = 50;
+
 function isPidRunning(pid: number): boolean {
  try {
    process.kill(pid, 0);
@@ -47,6 +55,29 @@ function getPidFilePath(paseoHome: string): string {
  return join(paseoHome, "paseo.pid");
 }

+async function isPidLockFresh(pidPath: string): Promise<boolean> {
+  try {
+    const lockStat = await stat(pidPath);
+    return lockStat.mtimeMs >= Date.now() - PID_LOCK_STALE_MS;
+  } catch {
+    return false;
+  }
+}
+
+async function touchPidLockFile(pidPath: string): Promise<void> {
+  const now = new Date();
+  await utimes(pidPath, now, now);
+}
+
+async function readPidLock(pidPath: string): Promise<PidLockInfo | null> {
+  try {
+    const content = await readFile(pidPath, "utf-8");
+    return parsePidLockInfo(JSON.parse(content));
+  } catch {
+    return null;
+  }
+}
+
 function resolveOwnerPid(ownerPid?: number): number {
  if (typeof ownerPid === "number" && Number.isInteger(ownerPid) && ownerPid > 0) {
    return ownerPid;
@@ -54,10 +85,91 @@ function resolveOwnerPid(ownerPid?: number): number {
  return process.pid;
 }

+interface AcquirePidLockOptions {
+  ownerPid?: number;
+  reclaimStaleDesktopLock?: boolean;
+}
+
+function canReclaimLiveLock(
+  lock: PidLockInfo,
+  options: AcquirePidLockOptions | undefined,
+): boolean {
+  // COMPAT(pidLockHeartbeat): v0.1.108 desktop startup has already confirmed the old daemon is
+  // unreachable before it launches the supervisor. Remove after 2027-01-15.
+  return options?.reclaimStaleDesktopLock === true && lock.desktopManaged === true;
+}
+
+function isSamePidLock(left: PidLockInfo, right: PidLockInfo): boolean {
+  return left.pid === right.pid && left.startedAt === right.startedAt;
+}
+
+function createLockHeldError(lock: PidLockInfo): PidLockError {
+  return new PidLockError(
+    `Another Paseo daemon is already running (PID ${lock.pid}, started ${lock.startedAt})`,
+    lock,
+  );
+}
+
+async function clearExistingPidLock(
+  pidPath: string,
+  existingLock: PidLockInfo,
+  lockOwnerPid: number,
+  options: AcquirePidLockOptions | undefined,
+): Promise<"already_owned" | "cleared"> {
+  const lockOwnerRunning = isPidRunning(existingLock.pid);
+  if (existingLock.pid === lockOwnerPid && lockOwnerRunning) {
+    await touchPidLockFile(pidPath);
+    return "already_owned";
+  }
+
+  if (lockOwnerRunning) {
+    const reclaimable = canReclaimLiveLock(existingLock, options);
+    if (!reclaimable || (await isPidLockFresh(pidPath))) {
+      throw createLockHeldError(existingLock);
+    }
+
+    // Re-read immediately before unlinking so a heartbeat at the stale boundary wins.
+    const confirmedLock = await readPidLock(pidPath);
+    if (
+      !confirmedLock ||
+      !isSamePidLock(existingLock, confirmedLock) ||
+      (await isPidLockFresh(pidPath))
+    ) {
+      throw new PidLockError("PID lock changed while checking whether it was abandoned");
+    }
+  }
+
+  await unlink(pidPath).catch(() => {});
+  return "cleared";
+}
+
+async function writeNewPidLock(pidPath: string, lockInfo: PidLockInfo): Promise<void> {
+  let fd;
+  try {
+    fd = await open(pidPath, "wx");
+    await fd.write(JSON.stringify(lockInfo));
+  } catch (error) {
+    if (!isErrnoException(error) || error.code !== "EEXIST") {
+      throw error;
+    }
+
+    const raceLock = await readPidLock(pidPath);
+    if (raceLock) {
+      throw new PidLockError(
+        `Another Paseo daemon is already running (PID ${raceLock.pid})`,
+        raceLock,
+      );
+    }
+    throw new PidLockError("Failed to acquire PID lock due to race condition");
+  } finally {
+    await fd?.close();
+  }
+}
+
 export async function acquirePidLock(
  paseoHome: string,
  listen: string | null,
-  options?: { ownerPid?: number },
+  options?: AcquirePidLockOptions,
 ): Promise<void> {
  const pidPath = getPidFilePath(paseoHome);

@@ -67,29 +179,15 @@ export async function acquirePidLock(
  }

  // Try to read existing lock
-  let existingLock: PidLockInfo | null = null;
-  try {
-    const content = await readFile(pidPath, "utf-8");
-    existingLock = parsePidLockInfo(JSON.parse(content));
-  } catch {
-    // No existing lock or invalid JSON - that's fine
-  }
+  const existingLock = await readPidLock(pidPath);

  // Check if existing lock is stale
  const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
  if (existingLock) {
-    if (isPidRunning(existingLock.pid)) {
-      if (existingLock.pid === lockOwnerPid) {
-        return;
-      }
-
-      throw new PidLockError(
-        `Another Paseo daemon is already running (PID ${existingLock.pid}, started ${existingLock.startedAt})`,
-        existingLock,
-      );
+    const result = await clearExistingPidLock(pidPath, existingLock, lockOwnerPid, options);
+    if (result === "already_owned") {
+      return;
    }
-    // Stale lock - remove it
-    await unlink(pidPath).catch(() => {});
  }

  // Create new lock with exclusive flag
@@ -99,36 +197,103 @@ export async function acquirePidLock(
    hostname: hostname(),
    uid: process.getuid?.() ?? 0,
    listen,
+    heartbeat: true,
    ...(process.env.PASEO_DESKTOP_MANAGED === "1" ? { desktopManaged: true } : {}),
  };

+  await writeNewPidLock(pidPath, lockInfo);
+}
+
+export async function refreshPidLock(
+  paseoHome: string,
+  options?: { ownerPid?: number },
+): Promise<void> {
+  const pidPath = getPidFilePath(paseoHome);
+  const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
  let fd;
  try {
-    fd = await open(pidPath, "wx");
-    await fd.write(JSON.stringify(lockInfo));
-  } catch (err) {
-    if (isErrnoException(err) && err.code === "EEXIST") {
-      // Race condition - another process created the file
-      // Re-read and check
-      try {
-        const content = await readFile(pidPath, "utf-8");
-        const raceLock = parsePidLockInfo(JSON.parse(content));
-        if (raceLock) {
-          throw new PidLockError(
-            `Another Paseo daemon is already running (PID ${raceLock.pid})`,
-            raceLock,
-          );
-        }
-        throw new PidLockError("Failed to acquire PID lock due to race condition");
-      } catch (innerErr) {
-        if (innerErr instanceof PidLockError) throw innerErr;
-        throw new PidLockError("Failed to acquire PID lock due to race condition");
-      }
+    fd = await open(pidPath, "r+");
+  } catch (error) {
+    if (isErrnoException(error) && error.code === "ENOENT") {
+      throw new PidLockError("Cannot refresh PID lock: lock file is missing");
    }
-    throw err;
-  } finally {
-    await fd?.close();
+    throw error;
  }
+
+  try {
+    const lock = await readPidLockFromHandleWithRetry(fd);
+    if (!lock) {
+      throw new PidLockError("Cannot refresh PID lock: invalid lock file");
+    }
+    if (lock.pid !== lockOwnerPid) {
+      throw new PidLockError(`Cannot refresh PID lock owned by PID ${lock.pid}`, lock);
+    }
+    const now = new Date();
+    await fd.utimes(now, now);
+  } finally {
+    await fd.close();
+  }
+}
+
+async function readPidLockFromHandle(fd: FileHandle): Promise<PidLockInfo | null> {
+  try {
+    const { size } = await fd.stat();
+    if (size === 0) {
+      return null;
+    }
+    const content = Buffer.alloc(size);
+    const { bytesRead } = await fd.read(content, 0, size, 0);
+    return parsePidLockInfo(JSON.parse(content.subarray(0, bytesRead).toString("utf-8")));
+  } catch {
+    return null;
+  }
+}
+
+async function readPidLockFromHandleWithRetry(fd: FileHandle): Promise<PidLockInfo | null> {
+  for (let attempt = 0; attempt < PID_LOCK_READ_RETRY_ATTEMPTS; attempt += 1) {
+    const lock = await readPidLockFromHandle(fd);
+    if (lock) {
+      return lock;
+    }
+    if (attempt < PID_LOCK_READ_RETRY_ATTEMPTS - 1) {
+      await new Promise((resolve) => setTimeout(resolve, PID_LOCK_READ_RETRY_DELAY_MS));
+    }
+  }
+  return null;
+}
+
+export function startPidLockHeartbeat(
+  paseoHome: string,
+  options?: {
+    ownerPid?: number;
+    intervalMs?: number;
+    onError?: (error: unknown) => void;
+  },
+): () => void {
+  const intervalMs = options?.intervalMs ?? PID_LOCK_HEARTBEAT_INTERVAL_MS;
+  let refreshing = false;
+
+  const timer = setInterval(() => {
+    if (refreshing) {
+      return;
+    }
+    refreshing = true;
+    refreshPidLock(paseoHome, { ownerPid: options?.ownerPid })
+      .catch((error) => {
+        if (options?.onError) {
+          options.onError(error);
+          return;
+        }
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`PID lock heartbeat failed: ${message}\n`);
+      })
+      .finally(() => {
+        refreshing = false;
+      });
+  }, intervalMs);
+  timer.unref();
+
+  return () => clearInterval(timer);
 }

 export async function updatePidLock(
@@ -138,23 +303,23 @@ export async function updatePidLock(
 ): Promise<void> {
  const pidPath = getPidFilePath(paseoHome);
  const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
-  const content = await readFile(pidPath, "utf-8");
-  const existingLock = parsePidLockInfo(JSON.parse(content));
-  if (!existingLock) {
-    throw new PidLockError("Cannot update PID lock: invalid lock file");
-  }
-
-  if (existingLock.pid !== lockOwnerPid) {
-    throw new PidLockError(`Cannot update PID lock owned by PID ${existingLock.pid}`, existingLock);
-  }
-
-  const updatedLock: PidLockInfo = {
-    ...existingLock,
-    ...patch,
-  };
-
  const fd = await open(pidPath, "r+");
  try {
+    const existingLock = await readPidLockFromHandleWithRetry(fd);
+    if (!existingLock) {
+      throw new PidLockError("Cannot update PID lock: invalid lock file");
+    }
+    if (existingLock.pid !== lockOwnerPid) {
+      throw new PidLockError(
+        `Cannot update PID lock owned by PID ${existingLock.pid}`,
+        existingLock,
+      );
+    }
+
+    const updatedLock: PidLockInfo = {
+      ...existingLock,
+      ...patch,
+    };
    await fd.truncate(0);
    await fd.writeFile(JSON.stringify(updatedLock));
  } finally {
@@ -182,12 +347,7 @@ export async function releasePidLock(

 export async function getPidLockInfo(paseoHome: string): Promise<PidLockInfo | null> {
  const pidPath = getPidFilePath(paseoHome);
-  try {
-    const content = await readFile(pidPath, "utf-8");
-    return parsePidLockInfo(JSON.parse(content));
-  } catch {
-    return null;
-  }
+  return readPidLock(pidPath);
 }

 export async function isLocked(