Recover desktop startup from stale daemon locks
Refs #1950 Refs #1904 Refs #769
This commit is contained in:
@@ -16,6 +16,11 @@ const mocks = vi.hoisted(() => ({
|
|||||||
},
|
},
|
||||||
runExternalCliJsonCommand: vi.fn(),
|
runExternalCliJsonCommand: vi.fn(),
|
||||||
runExternalCliTextCommand: vi.fn(),
|
runExternalCliTextCommand: vi.fn(),
|
||||||
|
createNodeEntrypointInvocation: vi.fn(() => ({
|
||||||
|
command: "node",
|
||||||
|
args: [],
|
||||||
|
env: {},
|
||||||
|
})),
|
||||||
spawnProcess: vi.fn(),
|
spawnProcess: vi.fn(),
|
||||||
logInfo: vi.fn(),
|
logInfo: vi.fn(),
|
||||||
logError: vi.fn(),
|
logError: vi.fn(),
|
||||||
@@ -59,11 +64,7 @@ vi.mock("../settings/desktop-settings-electron.js", () => ({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("./runtime-paths.js", () => ({
|
vi.mock("./runtime-paths.js", () => ({
|
||||||
createNodeEntrypointInvocation: vi.fn(() => ({
|
createNodeEntrypointInvocation: mocks.createNodeEntrypointInvocation,
|
||||||
command: "node",
|
|
||||||
args: [],
|
|
||||||
env: {},
|
|
||||||
})),
|
|
||||||
resolveDaemonRunnerEntrypoint: vi.fn(() => ({
|
resolveDaemonRunnerEntrypoint: vi.fn(() => ({
|
||||||
entryPath: "/tmp/daemon.js",
|
entryPath: "/tmp/daemon.js",
|
||||||
execArgv: [],
|
execArgv: [],
|
||||||
@@ -112,6 +113,8 @@ describe("daemon-manager commands", () => {
|
|||||||
mocks.settings = DEFAULT_DESKTOP_SETTINGS;
|
mocks.settings = DEFAULT_DESKTOP_SETTINGS;
|
||||||
mocks.runExternalCliJsonCommand.mockReset();
|
mocks.runExternalCliJsonCommand.mockReset();
|
||||||
mocks.runExternalCliTextCommand.mockReset();
|
mocks.runExternalCliTextCommand.mockReset();
|
||||||
|
mocks.createNodeEntrypointInvocation.mockReset();
|
||||||
|
mocks.createNodeEntrypointInvocation.mockReturnValue({ command: "node", args: [], env: {} });
|
||||||
mocks.spawnProcess.mockReset();
|
mocks.spawnProcess.mockReset();
|
||||||
mocks.logInfo.mockReset();
|
mocks.logInfo.mockReset();
|
||||||
mocks.logError.mockReset();
|
mocks.logError.mockReset();
|
||||||
@@ -439,6 +442,9 @@ describe("daemon-manager commands", () => {
|
|||||||
expect(message).toContain("Daemon failed to start: exit code 1");
|
expect(message).toContain("Daemon failed to start: exit code 1");
|
||||||
expect(recentLogsLabel?.split(/[\\/]/).at(-1)).toBe("daemon.log");
|
expect(recentLogsLabel?.split(/[\\/]/).at(-1)).toBe("daemon.log");
|
||||||
expect(message).toContain("recent daemon failure");
|
expect(message).toContain("recent daemon failure");
|
||||||
|
expect(mocks.createNodeEntrypointInvocation).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ args: [] }),
|
||||||
|
);
|
||||||
expect(mocks.spawnProcess).toHaveBeenCalledWith(
|
expect(mocks.spawnProcess).toHaveBeenCalledWith(
|
||||||
"node",
|
"node",
|
||||||
[],
|
[],
|
||||||
@@ -450,6 +456,47 @@ describe("daemon-manager commands", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("passes stale lock reclaim only after a live desktop daemon is confirmed unresponsive", async () => {
|
||||||
|
mocks.runExternalCliJsonCommand.mockResolvedValue({
|
||||||
|
localDaemon: "unresponsive",
|
||||||
|
connectedDaemon: "unreachable",
|
||||||
|
serverId: "",
|
||||||
|
pid: 7675,
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
desktopManaged: true,
|
||||||
|
});
|
||||||
|
mocks.spawnProcess.mockImplementation(() => {
|
||||||
|
const child = createMockChildProcess();
|
||||||
|
scheduleFailedStartup(child);
|
||||||
|
return child;
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(createDaemonCommandHandlers().start_desktop_daemon()).rejects.toThrow(
|
||||||
|
"Daemon failed to start: exit code 1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(mocks.createNodeEntrypointInvocation).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ args: ["--reclaim-stale-pid-lock"] }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not pass stale lock reclaim when the status command fails", async () => {
|
||||||
|
mocks.runExternalCliJsonCommand.mockRejectedValue(new Error("status command failed"));
|
||||||
|
mocks.spawnProcess.mockImplementation(() => {
|
||||||
|
const child = createMockChildProcess();
|
||||||
|
scheduleFailedStartup(child);
|
||||||
|
return child;
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(createDaemonCommandHandlers().start_desktop_daemon()).rejects.toThrow(
|
||||||
|
"Daemon failed to start: exit code 1",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(mocks.createNodeEntrypointInvocation).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ args: [] }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("returns the Electron main-process log tail from electron-log", () => {
|
it("returns the Electron main-process log tail from electron-log", () => {
|
||||||
writeFileSync(
|
writeFileSync(
|
||||||
mocks.appLogPath,
|
mocks.appLogPath,
|
||||||
|
|||||||
@@ -385,10 +385,12 @@ async function startDaemon(): Promise<DesktopDaemonStatus> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const daemonRunner = resolveDaemonRunnerEntrypoint();
|
const daemonRunner = resolveDaemonRunnerEntrypoint();
|
||||||
|
const reclaimStalePidLock =
|
||||||
|
current.status === "errored" && current.desktopManaged && current.error === null;
|
||||||
const invocation = createNodeEntrypointInvocation({
|
const invocation = createNodeEntrypointInvocation({
|
||||||
entrypoint: daemonRunner,
|
entrypoint: daemonRunner,
|
||||||
argvMode: "node-script",
|
argvMode: "node-script",
|
||||||
args: [],
|
args: reclaimStalePidLock ? ["--reclaim-stale-pid-lock"] : [],
|
||||||
baseEnv: process.env,
|
baseEnv: process.env,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
acquirePidLock,
|
acquirePidLock,
|
||||||
PidLockError,
|
PidLockError,
|
||||||
releasePidLock,
|
releasePidLock,
|
||||||
|
startPidLockHeartbeat,
|
||||||
updatePidLock,
|
updatePidLock,
|
||||||
} from "../src/server/pid-lock.js";
|
} from "../src/server/pid-lock.js";
|
||||||
import { resolvePaseoHome } from "../src/server/paseo-home.js";
|
import { resolvePaseoHome } from "../src/server/paseo-home.js";
|
||||||
@@ -17,11 +18,13 @@ process.title = "Paseo Supervisor";
|
|||||||
|
|
||||||
interface DaemonRunnerConfig {
|
interface DaemonRunnerConfig {
|
||||||
devMode: boolean;
|
devMode: boolean;
|
||||||
|
reclaimStalePidLock: boolean;
|
||||||
workerArgs: string[];
|
workerArgs: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
function parseConfig(argv: string[]): DaemonRunnerConfig {
|
function parseConfig(argv: string[]): DaemonRunnerConfig {
|
||||||
let devMode = false;
|
let devMode = false;
|
||||||
|
let reclaimStalePidLock = false;
|
||||||
const workerArgs: string[] = [];
|
const workerArgs: string[] = [];
|
||||||
|
|
||||||
for (const arg of argv) {
|
for (const arg of argv) {
|
||||||
@@ -29,10 +32,14 @@ function parseConfig(argv: string[]): DaemonRunnerConfig {
|
|||||||
devMode = true;
|
devMode = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (arg === "--reclaim-stale-pid-lock") {
|
||||||
|
reclaimStalePidLock = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
workerArgs.push(arg);
|
workerArgs.push(arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { devMode, workerArgs };
|
return { devMode, reclaimStalePidLock, workerArgs };
|
||||||
}
|
}
|
||||||
|
|
||||||
function resolveWorkerEntry(): string {
|
function resolveWorkerEntry(): string {
|
||||||
@@ -109,6 +116,7 @@ async function main(): Promise<void> {
|
|||||||
try {
|
try {
|
||||||
await acquirePidLock(paseoHome, null, {
|
await acquirePidLock(paseoHome, null, {
|
||||||
ownerPid: process.pid,
|
ownerPid: process.pid,
|
||||||
|
reclaimStaleDesktopLock: config.reclaimStalePidLock,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof PidLockError) {
|
if (error instanceof PidLockError) {
|
||||||
@@ -120,17 +128,29 @@ async function main(): Promise<void> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let lockReleased = false;
|
let lockReleased = false;
|
||||||
|
let requestSupervisorShutdown: ((reason: string) => void) | null = null;
|
||||||
|
const stopLockHeartbeat = startPidLockHeartbeat(paseoHome, {
|
||||||
|
ownerPid: process.pid,
|
||||||
|
onError: (error) => {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
process.stderr.write(`PID lock heartbeat failed: ${message}\n`);
|
||||||
|
if (error instanceof PidLockError) {
|
||||||
|
requestSupervisorShutdown?.("pid_lock_ownership_lost");
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
const releaseLock = async (): Promise<void> => {
|
const releaseLock = async (): Promise<void> => {
|
||||||
if (lockReleased) {
|
if (lockReleased) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
lockReleased = true;
|
lockReleased = true;
|
||||||
|
stopLockHeartbeat();
|
||||||
await releasePidLock(paseoHome, {
|
await releasePidLock(paseoHome, {
|
||||||
ownerPid: process.pid,
|
ownerPid: process.pid,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
runSupervisor({
|
const supervisor = runSupervisor({
|
||||||
name: "DaemonRunner",
|
name: "DaemonRunner",
|
||||||
startupMessage: "Starting daemon worker (IPC restart and crash restart enabled)",
|
startupMessage: "Starting daemon worker (IPC restart and crash restart enabled)",
|
||||||
resolveWorkerEntry: () => workerEntry,
|
resolveWorkerEntry: () => workerEntry,
|
||||||
@@ -159,6 +179,7 @@ async function main(): Promise<void> {
|
|||||||
},
|
},
|
||||||
onSupervisorExit: releaseLock,
|
onSupervisorExit: releaseLock,
|
||||||
});
|
});
|
||||||
|
requestSupervisorShutdown = supervisor.requestShutdown;
|
||||||
}
|
}
|
||||||
|
|
||||||
void main().catch((error) => {
|
void main().catch((error) => {
|
||||||
|
|||||||
@@ -47,6 +47,10 @@ interface SupervisorOptions {
|
|||||||
logFile?: SupervisorLogFileOptions;
|
logFile?: SupervisorLogFileOptions;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SupervisorController {
|
||||||
|
requestShutdown(reason: string): void;
|
||||||
|
}
|
||||||
|
|
||||||
function describeExit(code: number | null, signal: NodeJS.Signals | null): string {
|
function describeExit(code: number | null, signal: NodeJS.Signals | null): string {
|
||||||
return signal ?? (typeof code === "number" ? `code ${code}` : "unknown");
|
return signal ?? (typeof code === "number" ? `code ${code}` : "unknown");
|
||||||
}
|
}
|
||||||
@@ -105,7 +109,7 @@ function createSupervisorLogStream(options: SupervisorLogFileOptions | undefined
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export function runSupervisor(options: SupervisorOptions): void {
|
export function runSupervisor(options: SupervisorOptions): SupervisorController {
|
||||||
const restartOnCrash = options.restartOnCrash ?? false;
|
const restartOnCrash = options.restartOnCrash ?? false;
|
||||||
const workerArgs = options.workerArgs ?? process.argv.slice(2);
|
const workerArgs = options.workerArgs ?? process.argv.slice(2);
|
||||||
const workerEnv = options.workerEnv ?? process.env;
|
const workerEnv = options.workerEnv ?? process.env;
|
||||||
@@ -331,4 +335,6 @@ export function runSupervisor(options: SupervisorOptions): void {
|
|||||||
process.stdout.write(`[${options.name}] ${options.startupMessage}\n`);
|
process.stdout.write(`[${options.name}] ${options.startupMessage}\n`);
|
||||||
writeLifecycleLog(options.startupMessage);
|
writeLifecycleLog(options.startupMessage);
|
||||||
spawnWorker();
|
spawnWorker();
|
||||||
|
|
||||||
|
return { requestShutdown };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,17 @@
|
|||||||
import { mkdtemp, rm } from "node:fs/promises";
|
import { mkdtemp, open, rm, utimes, writeFile } from "node:fs/promises";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { describe, expect, test } from "vitest";
|
import { describe, expect, test } from "vitest";
|
||||||
|
|
||||||
import { acquirePidLock, getPidLockInfo, releasePidLock, updatePidLock } from "./pid-lock.js";
|
import {
|
||||||
|
acquirePidLock,
|
||||||
|
getPidLockInfo,
|
||||||
|
isLocked,
|
||||||
|
PidLockError,
|
||||||
|
refreshPidLock,
|
||||||
|
releasePidLock,
|
||||||
|
updatePidLock,
|
||||||
|
} from "./pid-lock.js";
|
||||||
|
|
||||||
describe("pid-lock ownership", () => {
|
describe("pid-lock ownership", () => {
|
||||||
test("writes and releases lock for explicit owner pid", async () => {
|
test("writes and releases lock for explicit owner pid", async () => {
|
||||||
@@ -22,6 +30,7 @@ describe("pid-lock ownership", () => {
|
|||||||
const lock = await getPidLockInfo(paseoHome);
|
const lock = await getPidLockInfo(paseoHome);
|
||||||
expect(lock?.pid).toBe(ownerPid);
|
expect(lock?.pid).toBe(ownerPid);
|
||||||
expect(lock?.listen).toBeNull();
|
expect(lock?.listen).toBeNull();
|
||||||
|
expect(lock?.heartbeat).toBe(true);
|
||||||
|
|
||||||
await (
|
await (
|
||||||
updatePidLock as unknown as (
|
updatePidLock as unknown as (
|
||||||
@@ -49,4 +58,200 @@ describe("pid-lock ownership", () => {
|
|||||||
await rm(paseoHome, { recursive: true, force: true });
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("keeps a stale heartbeat lock when the recorded pid is alive without a reachability check", async () => {
|
||||||
|
const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-stale-heartbeat-"));
|
||||||
|
const replacementOwnerPid = process.pid + 10_000;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const pidPath = join(paseoHome, "paseo.pid");
|
||||||
|
await writeFile(
|
||||||
|
pidPath,
|
||||||
|
JSON.stringify({
|
||||||
|
pid: process.pid,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
hostname: "old-host",
|
||||||
|
uid: process.getuid?.() ?? 0,
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
desktopManaged: true,
|
||||||
|
heartbeat: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const staleTime = new Date(Date.now() - 10 * 60_000);
|
||||||
|
await utimes(pidPath, staleTime, staleTime);
|
||||||
|
|
||||||
|
await expect(isLocked(paseoHome)).resolves.toMatchObject({ locked: true });
|
||||||
|
await expect(
|
||||||
|
acquirePidLock(paseoHome, null, { ownerPid: replacementOwnerPid }),
|
||||||
|
).rejects.toThrow("Another Paseo daemon is already running");
|
||||||
|
|
||||||
|
const lock = await getPidLockInfo(paseoHome);
|
||||||
|
expect(lock?.pid).toBe(process.pid);
|
||||||
|
} finally {
|
||||||
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("reclaims a stale desktop heartbeat lock after desktop confirms the daemon is unreachable", async () => {
|
||||||
|
const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-stale-desktop-heartbeat-"));
|
||||||
|
const replacementOwnerPid = process.pid + 10_000;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const pidPath = join(paseoHome, "paseo.pid");
|
||||||
|
await writeFile(
|
||||||
|
pidPath,
|
||||||
|
JSON.stringify({
|
||||||
|
pid: process.pid,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
hostname: "old-host",
|
||||||
|
uid: process.getuid?.() ?? 0,
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
desktopManaged: true,
|
||||||
|
heartbeat: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const staleTime = new Date(Date.now() - 10 * 60_000);
|
||||||
|
await utimes(pidPath, staleTime, staleTime);
|
||||||
|
|
||||||
|
await acquirePidLock(paseoHome, null, {
|
||||||
|
ownerPid: replacementOwnerPid,
|
||||||
|
reclaimStaleDesktopLock: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const lock = await getPidLockInfo(paseoHome);
|
||||||
|
expect(lock?.pid).toBe(replacementOwnerPid);
|
||||||
|
expect(lock?.listen).toBeNull();
|
||||||
|
} finally {
|
||||||
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("keeps a stale live lock written by a pre-heartbeat daemon", async () => {
|
||||||
|
const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-legacy-live-"));
|
||||||
|
const pidPath = join(paseoHome, "paseo.pid");
|
||||||
|
|
||||||
|
try {
|
||||||
|
await writeFile(
|
||||||
|
pidPath,
|
||||||
|
JSON.stringify({
|
||||||
|
pid: process.pid,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
hostname: "old-host",
|
||||||
|
uid: process.getuid?.() ?? 0,
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
desktopManaged: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const staleTime = new Date(Date.now() - 10 * 60_000);
|
||||||
|
await utimes(pidPath, staleTime, staleTime);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 }),
|
||||||
|
).rejects.toThrow("Another Paseo daemon is already running");
|
||||||
|
|
||||||
|
const lock = await getPidLockInfo(paseoHome);
|
||||||
|
expect(lock?.pid).toBe(process.pid);
|
||||||
|
} finally {
|
||||||
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("reclaims a stale legacy desktop lock after desktop confirms the daemon is unreachable", async () => {
|
||||||
|
const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-legacy-desktop-"));
|
||||||
|
const replacementOwnerPid = process.pid + 10_000;
|
||||||
|
const pidPath = join(paseoHome, "paseo.pid");
|
||||||
|
|
||||||
|
try {
|
||||||
|
await writeFile(
|
||||||
|
pidPath,
|
||||||
|
JSON.stringify({
|
||||||
|
pid: process.pid,
|
||||||
|
startedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
hostname: "old-host",
|
||||||
|
uid: process.getuid?.() ?? 0,
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
desktopManaged: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const staleTime = new Date(Date.now() - 10 * 60_000);
|
||||||
|
await utimes(pidPath, staleTime, staleTime);
|
||||||
|
|
||||||
|
await acquirePidLock(paseoHome, null, {
|
||||||
|
ownerPid: replacementOwnerPid,
|
||||||
|
reclaimStaleDesktopLock: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const lock = await getPidLockInfo(paseoHome);
|
||||||
|
expect(lock?.pid).toBe(replacementOwnerPid);
|
||||||
|
expect(lock?.heartbeat).toBe(true);
|
||||||
|
} finally {
|
||||||
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects a heartbeat refresh after another supervisor takes ownership", async () => {
|
||||||
|
const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-refresh-owner-"));
|
||||||
|
|
||||||
|
try {
|
||||||
|
await acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 });
|
||||||
|
|
||||||
|
await expect(refreshPidLock(paseoHome, { ownerPid: process.pid })).rejects.toBeInstanceOf(
|
||||||
|
PidLockError,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("retries a heartbeat refresh while its owner is rewriting the lock", async () => {
|
||||||
|
const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-refresh-rewrite-"));
|
||||||
|
const pidPath = join(paseoHome, "paseo.pid");
|
||||||
|
|
||||||
|
try {
|
||||||
|
await acquirePidLock(paseoHome, null, { ownerPid: process.pid });
|
||||||
|
const lock = await getPidLockInfo(paseoHome);
|
||||||
|
expect(lock).not.toBeNull();
|
||||||
|
|
||||||
|
const rewriteHandle = await open(pidPath, "r+");
|
||||||
|
await rewriteHandle.truncate(0);
|
||||||
|
|
||||||
|
const refresh = refreshPidLock(paseoHome, { ownerPid: process.pid });
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||||
|
await rewriteHandle.writeFile(JSON.stringify(lock));
|
||||||
|
await rewriteHandle.close();
|
||||||
|
|
||||||
|
await expect(refresh).resolves.toBeUndefined();
|
||||||
|
} finally {
|
||||||
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("keeps a fresh lock when the recorded pid is alive", async () => {
|
||||||
|
const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-fresh-heartbeat-"));
|
||||||
|
|
||||||
|
try {
|
||||||
|
await writeFile(
|
||||||
|
join(paseoHome, "paseo.pid"),
|
||||||
|
JSON.stringify({
|
||||||
|
pid: process.pid,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
hostname: "current-host",
|
||||||
|
uid: process.getuid?.() ?? 0,
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
desktopManaged: true,
|
||||||
|
heartbeat: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 }),
|
||||||
|
).rejects.toThrow("Another Paseo daemon is already running");
|
||||||
|
|
||||||
|
const lock = await getPidLockInfo(paseoHome);
|
||||||
|
expect(lock?.pid).toBe(process.pid);
|
||||||
|
expect(lock?.listen).toBe("127.0.0.1:6767");
|
||||||
|
} finally {
|
||||||
|
await rm(paseoHome, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { open, readFile, unlink, mkdir } from "node:fs/promises";
|
import { open, readFile, stat, unlink, mkdir, utimes } from "node:fs/promises";
|
||||||
|
import type { FileHandle } from "node:fs/promises";
|
||||||
import { existsSync } from "node:fs";
|
import { existsSync } from "node:fs";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { hostname } from "node:os";
|
import { hostname } from "node:os";
|
||||||
@@ -11,6 +12,7 @@ export const pidLockInfoSchema = z.object({
|
|||||||
uid: z.number(),
|
uid: z.number(),
|
||||||
listen: z.string().nullable(),
|
listen: z.string().nullable(),
|
||||||
desktopManaged: z.boolean().optional(),
|
desktopManaged: z.boolean().optional(),
|
||||||
|
heartbeat: z.literal(true).optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export interface PidLockInfo extends z.infer<typeof pidLockInfoSchema> {}
|
export interface PidLockInfo extends z.infer<typeof pidLockInfoSchema> {}
|
||||||
@@ -34,6 +36,12 @@ export class PidLockError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stale recovery is for abandoned locks, so keep this well above ordinary event-loop stalls.
|
||||||
|
const PID_LOCK_STALE_MS = 5 * 60_000;
|
||||||
|
const PID_LOCK_HEARTBEAT_INTERVAL_MS = 30_000;
|
||||||
|
const PID_LOCK_READ_RETRY_ATTEMPTS = 10;
|
||||||
|
const PID_LOCK_READ_RETRY_DELAY_MS = 50;
|
||||||
|
|
||||||
function isPidRunning(pid: number): boolean {
|
function isPidRunning(pid: number): boolean {
|
||||||
try {
|
try {
|
||||||
process.kill(pid, 0);
|
process.kill(pid, 0);
|
||||||
@@ -47,6 +55,29 @@ function getPidFilePath(paseoHome: string): string {
|
|||||||
return join(paseoHome, "paseo.pid");
|
return join(paseoHome, "paseo.pid");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function isPidLockFresh(pidPath: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const lockStat = await stat(pidPath);
|
||||||
|
return lockStat.mtimeMs >= Date.now() - PID_LOCK_STALE_MS;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function touchPidLockFile(pidPath: string): Promise<void> {
|
||||||
|
const now = new Date();
|
||||||
|
await utimes(pidPath, now, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readPidLock(pidPath: string): Promise<PidLockInfo | null> {
|
||||||
|
try {
|
||||||
|
const content = await readFile(pidPath, "utf-8");
|
||||||
|
return parsePidLockInfo(JSON.parse(content));
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function resolveOwnerPid(ownerPid?: number): number {
|
function resolveOwnerPid(ownerPid?: number): number {
|
||||||
if (typeof ownerPid === "number" && Number.isInteger(ownerPid) && ownerPid > 0) {
|
if (typeof ownerPid === "number" && Number.isInteger(ownerPid) && ownerPid > 0) {
|
||||||
return ownerPid;
|
return ownerPid;
|
||||||
@@ -54,10 +85,91 @@ function resolveOwnerPid(ownerPid?: number): number {
|
|||||||
return process.pid;
|
return process.pid;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface AcquirePidLockOptions {
|
||||||
|
ownerPid?: number;
|
||||||
|
reclaimStaleDesktopLock?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function canReclaimLiveLock(
|
||||||
|
lock: PidLockInfo,
|
||||||
|
options: AcquirePidLockOptions | undefined,
|
||||||
|
): boolean {
|
||||||
|
// COMPAT(pidLockHeartbeat): v0.1.108 desktop startup has already confirmed the old daemon is
|
||||||
|
// unreachable before it launches the supervisor. Remove after 2027-01-15.
|
||||||
|
return options?.reclaimStaleDesktopLock === true && lock.desktopManaged === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSamePidLock(left: PidLockInfo, right: PidLockInfo): boolean {
|
||||||
|
return left.pid === right.pid && left.startedAt === right.startedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createLockHeldError(lock: PidLockInfo): PidLockError {
|
||||||
|
return new PidLockError(
|
||||||
|
`Another Paseo daemon is already running (PID ${lock.pid}, started ${lock.startedAt})`,
|
||||||
|
lock,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function clearExistingPidLock(
|
||||||
|
pidPath: string,
|
||||||
|
existingLock: PidLockInfo,
|
||||||
|
lockOwnerPid: number,
|
||||||
|
options: AcquirePidLockOptions | undefined,
|
||||||
|
): Promise<"already_owned" | "cleared"> {
|
||||||
|
const lockOwnerRunning = isPidRunning(existingLock.pid);
|
||||||
|
if (existingLock.pid === lockOwnerPid && lockOwnerRunning) {
|
||||||
|
await touchPidLockFile(pidPath);
|
||||||
|
return "already_owned";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lockOwnerRunning) {
|
||||||
|
const reclaimable = canReclaimLiveLock(existingLock, options);
|
||||||
|
if (!reclaimable || (await isPidLockFresh(pidPath))) {
|
||||||
|
throw createLockHeldError(existingLock);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-read immediately before unlinking so a heartbeat at the stale boundary wins.
|
||||||
|
const confirmedLock = await readPidLock(pidPath);
|
||||||
|
if (
|
||||||
|
!confirmedLock ||
|
||||||
|
!isSamePidLock(existingLock, confirmedLock) ||
|
||||||
|
(await isPidLockFresh(pidPath))
|
||||||
|
) {
|
||||||
|
throw new PidLockError("PID lock changed while checking whether it was abandoned");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await unlink(pidPath).catch(() => {});
|
||||||
|
return "cleared";
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeNewPidLock(pidPath: string, lockInfo: PidLockInfo): Promise<void> {
|
||||||
|
let fd;
|
||||||
|
try {
|
||||||
|
fd = await open(pidPath, "wx");
|
||||||
|
await fd.write(JSON.stringify(lockInfo));
|
||||||
|
} catch (error) {
|
||||||
|
if (!isErrnoException(error) || error.code !== "EEXIST") {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const raceLock = await readPidLock(pidPath);
|
||||||
|
if (raceLock) {
|
||||||
|
throw new PidLockError(
|
||||||
|
`Another Paseo daemon is already running (PID ${raceLock.pid})`,
|
||||||
|
raceLock,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw new PidLockError("Failed to acquire PID lock due to race condition");
|
||||||
|
} finally {
|
||||||
|
await fd?.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function acquirePidLock(
|
export async function acquirePidLock(
|
||||||
paseoHome: string,
|
paseoHome: string,
|
||||||
listen: string | null,
|
listen: string | null,
|
||||||
options?: { ownerPid?: number },
|
options?: AcquirePidLockOptions,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const pidPath = getPidFilePath(paseoHome);
|
const pidPath = getPidFilePath(paseoHome);
|
||||||
|
|
||||||
@@ -67,29 +179,15 @@ export async function acquirePidLock(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Try to read existing lock
|
// Try to read existing lock
|
||||||
let existingLock: PidLockInfo | null = null;
|
const existingLock = await readPidLock(pidPath);
|
||||||
try {
|
|
||||||
const content = await readFile(pidPath, "utf-8");
|
|
||||||
existingLock = parsePidLockInfo(JSON.parse(content));
|
|
||||||
} catch {
|
|
||||||
// No existing lock or invalid JSON - that's fine
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if existing lock is stale
|
// Check if existing lock is stale
|
||||||
const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
|
const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
|
||||||
if (existingLock) {
|
if (existingLock) {
|
||||||
if (isPidRunning(existingLock.pid)) {
|
const result = await clearExistingPidLock(pidPath, existingLock, lockOwnerPid, options);
|
||||||
if (existingLock.pid === lockOwnerPid) {
|
if (result === "already_owned") {
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
|
|
||||||
throw new PidLockError(
|
|
||||||
`Another Paseo daemon is already running (PID ${existingLock.pid}, started ${existingLock.startedAt})`,
|
|
||||||
existingLock,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
// Stale lock - remove it
|
|
||||||
await unlink(pidPath).catch(() => {});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create new lock with exclusive flag
|
// Create new lock with exclusive flag
|
||||||
@@ -99,36 +197,103 @@ export async function acquirePidLock(
|
|||||||
hostname: hostname(),
|
hostname: hostname(),
|
||||||
uid: process.getuid?.() ?? 0,
|
uid: process.getuid?.() ?? 0,
|
||||||
listen,
|
listen,
|
||||||
|
heartbeat: true,
|
||||||
...(process.env.PASEO_DESKTOP_MANAGED === "1" ? { desktopManaged: true } : {}),
|
...(process.env.PASEO_DESKTOP_MANAGED === "1" ? { desktopManaged: true } : {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
await writeNewPidLock(pidPath, lockInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function refreshPidLock(
|
||||||
|
paseoHome: string,
|
||||||
|
options?: { ownerPid?: number },
|
||||||
|
): Promise<void> {
|
||||||
|
const pidPath = getPidFilePath(paseoHome);
|
||||||
|
const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
|
||||||
let fd;
|
let fd;
|
||||||
try {
|
try {
|
||||||
fd = await open(pidPath, "wx");
|
fd = await open(pidPath, "r+");
|
||||||
await fd.write(JSON.stringify(lockInfo));
|
} catch (error) {
|
||||||
} catch (err) {
|
if (isErrnoException(error) && error.code === "ENOENT") {
|
||||||
if (isErrnoException(err) && err.code === "EEXIST") {
|
throw new PidLockError("Cannot refresh PID lock: lock file is missing");
|
||||||
// Race condition - another process created the file
|
|
||||||
// Re-read and check
|
|
||||||
try {
|
|
||||||
const content = await readFile(pidPath, "utf-8");
|
|
||||||
const raceLock = parsePidLockInfo(JSON.parse(content));
|
|
||||||
if (raceLock) {
|
|
||||||
throw new PidLockError(
|
|
||||||
`Another Paseo daemon is already running (PID ${raceLock.pid})`,
|
|
||||||
raceLock,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
throw new PidLockError("Failed to acquire PID lock due to race condition");
|
|
||||||
} catch (innerErr) {
|
|
||||||
if (innerErr instanceof PidLockError) throw innerErr;
|
|
||||||
throw new PidLockError("Failed to acquire PID lock due to race condition");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
throw err;
|
throw error;
|
||||||
} finally {
|
|
||||||
await fd?.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const lock = await readPidLockFromHandleWithRetry(fd);
|
||||||
|
if (!lock) {
|
||||||
|
throw new PidLockError("Cannot refresh PID lock: invalid lock file");
|
||||||
|
}
|
||||||
|
if (lock.pid !== lockOwnerPid) {
|
||||||
|
throw new PidLockError(`Cannot refresh PID lock owned by PID ${lock.pid}`, lock);
|
||||||
|
}
|
||||||
|
const now = new Date();
|
||||||
|
await fd.utimes(now, now);
|
||||||
|
} finally {
|
||||||
|
await fd.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readPidLockFromHandle(fd: FileHandle): Promise<PidLockInfo | null> {
|
||||||
|
try {
|
||||||
|
const { size } = await fd.stat();
|
||||||
|
if (size === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const content = Buffer.alloc(size);
|
||||||
|
const { bytesRead } = await fd.read(content, 0, size, 0);
|
||||||
|
return parsePidLockInfo(JSON.parse(content.subarray(0, bytesRead).toString("utf-8")));
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readPidLockFromHandleWithRetry(fd: FileHandle): Promise<PidLockInfo | null> {
|
||||||
|
for (let attempt = 0; attempt < PID_LOCK_READ_RETRY_ATTEMPTS; attempt += 1) {
|
||||||
|
const lock = await readPidLockFromHandle(fd);
|
||||||
|
if (lock) {
|
||||||
|
return lock;
|
||||||
|
}
|
||||||
|
if (attempt < PID_LOCK_READ_RETRY_ATTEMPTS - 1) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, PID_LOCK_READ_RETRY_DELAY_MS));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function startPidLockHeartbeat(
|
||||||
|
paseoHome: string,
|
||||||
|
options?: {
|
||||||
|
ownerPid?: number;
|
||||||
|
intervalMs?: number;
|
||||||
|
onError?: (error: unknown) => void;
|
||||||
|
},
|
||||||
|
): () => void {
|
||||||
|
const intervalMs = options?.intervalMs ?? PID_LOCK_HEARTBEAT_INTERVAL_MS;
|
||||||
|
let refreshing = false;
|
||||||
|
|
||||||
|
const timer = setInterval(() => {
|
||||||
|
if (refreshing) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
refreshing = true;
|
||||||
|
refreshPidLock(paseoHome, { ownerPid: options?.ownerPid })
|
||||||
|
.catch((error) => {
|
||||||
|
if (options?.onError) {
|
||||||
|
options.onError(error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
process.stderr.write(`PID lock heartbeat failed: ${message}\n`);
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
refreshing = false;
|
||||||
|
});
|
||||||
|
}, intervalMs);
|
||||||
|
timer.unref();
|
||||||
|
|
||||||
|
return () => clearInterval(timer);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function updatePidLock(
|
export async function updatePidLock(
|
||||||
@@ -138,23 +303,23 @@ export async function updatePidLock(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const pidPath = getPidFilePath(paseoHome);
|
const pidPath = getPidFilePath(paseoHome);
|
||||||
const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
|
const lockOwnerPid = resolveOwnerPid(options?.ownerPid);
|
||||||
const content = await readFile(pidPath, "utf-8");
|
|
||||||
const existingLock = parsePidLockInfo(JSON.parse(content));
|
|
||||||
if (!existingLock) {
|
|
||||||
throw new PidLockError("Cannot update PID lock: invalid lock file");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (existingLock.pid !== lockOwnerPid) {
|
|
||||||
throw new PidLockError(`Cannot update PID lock owned by PID ${existingLock.pid}`, existingLock);
|
|
||||||
}
|
|
||||||
|
|
||||||
const updatedLock: PidLockInfo = {
|
|
||||||
...existingLock,
|
|
||||||
...patch,
|
|
||||||
};
|
|
||||||
|
|
||||||
const fd = await open(pidPath, "r+");
|
const fd = await open(pidPath, "r+");
|
||||||
try {
|
try {
|
||||||
|
const existingLock = await readPidLockFromHandleWithRetry(fd);
|
||||||
|
if (!existingLock) {
|
||||||
|
throw new PidLockError("Cannot update PID lock: invalid lock file");
|
||||||
|
}
|
||||||
|
if (existingLock.pid !== lockOwnerPid) {
|
||||||
|
throw new PidLockError(
|
||||||
|
`Cannot update PID lock owned by PID ${existingLock.pid}`,
|
||||||
|
existingLock,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const updatedLock: PidLockInfo = {
|
||||||
|
...existingLock,
|
||||||
|
...patch,
|
||||||
|
};
|
||||||
await fd.truncate(0);
|
await fd.truncate(0);
|
||||||
await fd.writeFile(JSON.stringify(updatedLock));
|
await fd.writeFile(JSON.stringify(updatedLock));
|
||||||
} finally {
|
} finally {
|
||||||
@@ -182,12 +347,7 @@ export async function releasePidLock(
|
|||||||
|
|
||||||
export async function getPidLockInfo(paseoHome: string): Promise<PidLockInfo | null> {
|
export async function getPidLockInfo(paseoHome: string): Promise<PidLockInfo | null> {
|
||||||
const pidPath = getPidFilePath(paseoHome);
|
const pidPath = getPidFilePath(paseoHome);
|
||||||
try {
|
return readPidLock(pidPath);
|
||||||
const content = await readFile(pidPath, "utf-8");
|
|
||||||
return parsePidLockInfo(JSON.parse(content));
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function isLocked(
|
export async function isLocked(
|
||||||
|
|||||||
Reference in New Issue
Block a user