jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Recover stale supervised daemons

Browse Source 
Teach daemon stop to use lifecycle shutdown when the API is reachable even if the owner pid is stale, then wait for the API to disappear and clean the stale pidfile.

Launch desktop-managed daemons detached from desktop stdio, preserve stale reachable daemon ownership for version checks, and allow desktop stop/restart to use the CLI recovery path.

Add supervisor heartbeats so supervised workers shut down when their supervisor disappears instead of surviving as orphaned reachable daemons.
This commit is contained in:
Mohamed Boudra
2026-06-07 23:18:48 +07:00
parent 7c8b290e2f
commit 893e3376b0
7 changed files with 769 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/desktop/src/daemon/daemon-manager.test.ts
+143 -23
View File
@@ -1,10 +1,12 @@
import { EventEmitter } from "node:events";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_DESKTOP_SETTINGS } from "../settings/desktop-settings";
import { createDaemonCommandHandlers } from "./daemon-manager";
const mocks = vi.hoisted(() => ({
paseoHome: "/tmp/paseo-desktop-daemon-manager-test-home",
settings: {
releaseChannel: "stable",
daemon: {
@@ -21,7 +23,7 @@ vi.mock("electron", () => ({
app: {
getPath: vi.fn(() => "/tmp/paseo-user-data"),
getVersion: vi.fn(() => "1.2.3"),
isPackaged: false,
isPackaged: true,
},
ipcMain: { handle: vi.fn() },
powerMonitor: { getSystemIdleTime: vi.fn(() => 0) },
@@ -32,7 +34,7 @@ vi.mock("electron-log/main", () => ({
}));
vi.mock("@getpaseo/server", () => ({
resolvePaseoHome: vi.fn(() => "/tmp/paseo-home"),
resolvePaseoHome: vi.fn(() => mocks.paseoHome),
spawnProcess: mocks.spawnProcess,
}));
@@ -72,8 +74,6 @@ function desktopSettingsWithManagement(enabled: boolean) {
}
type MockChildProcess = EventEmitter & {
stdout: EventEmitter;
stderr: EventEmitter;
pid: number;
spawnfile: string;
spawnargs: string[];
@@ -82,8 +82,6 @@ type MockChildProcess = EventEmitter & {
function createMockChildProcess(): MockChildProcess {
const child = new EventEmitter() as MockChildProcess;
child.stdout = new EventEmitter();
child.stderr = new EventEmitter();
child.pid = 1234;
child.spawnfile = "node";
child.spawnargs = ["node", "daemon.js"];
@@ -91,10 +89,8 @@ function createMockChildProcess(): MockChildProcess {
return child;
}
function scheduleFailedStartupOutput(child: MockChildProcess): void {
function scheduleFailedStartup(child: MockChildProcess): void {
setImmediate(() => {
child.stdout.emit("data", Buffer.from(`${"x".repeat(80_000)}stdout-tail`));
child.stderr.emit("data", Buffer.from(`${"y".repeat(80_000)}stderr-tail`));
child.emit("exit", 1, null);
});
}
@@ -105,6 +101,11 @@ describe("daemon-manager commands", () => {
mocks.runExternalCliJsonCommand.mockReset();
mocks.runExternalCliTextCommand.mockReset();
mocks.spawnProcess.mockReset();
rmSync(mocks.paseoHome, { recursive: true, force: true });
});
afterEach(() => {
rmSync(mocks.paseoHome, { recursive: true, force: true });
});
it("refuses start and restart while built-in daemon management is disabled", async () => {
@@ -136,7 +137,7 @@ describe("daemon-manager commands", () => {
listen: null,
hostname: null,
pid: null,
home: "/tmp/paseo-home",
home: mocks.paseoHome,
version: null,
desktopManaged: false,
error: null,
@@ -167,7 +168,7 @@ describe("daemon-manager commands", () => {
listen: null,
hostname: null,
pid: null,
home: "/tmp/paseo-home",
home: mocks.paseoHome,
version: null,
desktopManaged: false,
error: null,
@@ -195,7 +196,50 @@ describe("daemon-manager commands", () => {
]);
});
it("uses a reachable daemon when the PID file is stale", async () => {
it("routes stale reachable desktop daemon stops through external CLI daemon stop", async () => {
mocks.runExternalCliJsonCommand
.mockResolvedValueOnce({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
serverId: "server-1",
pid: 7675,
listen: "127.0.0.1:6767",
daemonVersion: "1.2.2",
desktopManaged: true,
})
.mockResolvedValueOnce({ action: "stopped" })
.mockResolvedValueOnce({
localDaemon: "stopped",
connectedDaemon: "unreachable",
serverId: "",
});
const handlers = createDaemonCommandHandlers();
await expect(handlers.stop_desktop_daemon()).resolves.toEqual({
serverId: "",
status: "stopped",
listen: null,
hostname: null,
pid: null,
home: mocks.paseoHome,
version: null,
desktopManaged: false,
error: null,
});
expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(2, [
"daemon",
"stop",
"--json",
"--timeout",
"5",
"--force",
"--kill-timeout",
"5",
]);
});
it("uses a stale reachable desktop daemon when the version matches", async () => {
mocks.runExternalCliJsonCommand.mockResolvedValue({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
@@ -203,7 +247,7 @@ describe("daemon-manager commands", () => {
pid: 7675,
listen: "127.0.0.1:6767",
hostname: "dev-host",
daemonVersion: "1.2.2",
daemonVersion: "1.2.3",
desktopManaged: true,
});
const handlers = createDaemonCommandHandlers();
@@ -214,16 +258,86 @@ describe("daemon-manager commands", () => {
listen: "127.0.0.1:6767",
hostname: "dev-host",
pid: null,
home: "/tmp/paseo-home",
version: "1.2.2",
desktopManaged: false,
home: mocks.paseoHome,
version: "1.2.3",
desktopManaged: true,
error: null,
});
expect(mocks.spawnProcess).not.toHaveBeenCalled();
});
it("bounds captured daemon startup output", async () => {
it("restarts a stale reachable desktop daemon when the version differs", async () => {
mocks.runExternalCliJsonCommand
.mockResolvedValueOnce({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
serverId: "server-1",
pid: 7675,
listen: "127.0.0.1:6767",
hostname: "dev-host",
daemonVersion: "1.2.2",
desktopManaged: true,
})
.mockResolvedValueOnce({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
serverId: "server-1",
pid: 7675,
listen: "127.0.0.1:6767",
daemonVersion: "1.2.2",
desktopManaged: true,
})
.mockResolvedValueOnce({ action: "stopped" })
.mockResolvedValueOnce({
localDaemon: "stopped",
connectedDaemon: "unreachable",
serverId: "",
})
.mockResolvedValueOnce({
localDaemon: "running",
connectedDaemon: "reachable",
serverId: "server-2",
pid: 8888,
listen: "127.0.0.1:6767",
hostname: "dev-host",
daemonVersion: "1.2.3",
desktopManaged: true,
});
mocks.spawnProcess.mockReturnValue(createMockChildProcess());
const handlers = createDaemonCommandHandlers();
await expect(handlers.start_desktop_daemon()).resolves.toEqual({
serverId: "server-2",
status: "running",
listen: "127.0.0.1:6767",
hostname: "dev-host",
pid: 8888,
home: mocks.paseoHome,
version: "1.2.3",
desktopManaged: true,
error: null,
});
expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(3, [
"daemon",
"stop",
"--json",
"--timeout",
"5",
"--force",
"--kill-timeout",
"5",
]);
expect(mocks.spawnProcess).toHaveBeenCalled();
});
it("starts the managed daemon detached from desktop stdio and reports daemon log failures", async () => {
mkdirSync(mocks.paseoHome, { recursive: true });
writeFileSync(
`${mocks.paseoHome}/daemon.log`,
["old log line", "recent daemon failure"].join("\n"),
);
mocks.runExternalCliJsonCommand.mockResolvedValue({
localDaemon: "stopped",
connectedDaemon: "unreachable",
@@ -231,7 +345,7 @@ describe("daemon-manager commands", () => {
});
mocks.spawnProcess.mockImplementation(() => {
const child = createMockChildProcess();
scheduleFailedStartupOutput(child);
scheduleFailedStartup(child);
return child;
});
const handlers = createDaemonCommandHandlers();
@@ -246,9 +360,15 @@ describe("daemon-manager commands", () => {
expect(thrown).toBeInstanceOf(Error);
const message = thrown?.message ?? "";
expect(message).toContain("Daemon failed to start: exit code 1");
expect(message).toContain("output truncated to the last 65536 chars");
expect(message).toContain("stdout-tail");
expect(message).toContain("stderr-tail");
expect(message.length).toBeLessThan(150_000);
expect(message).toContain(`Recent logs (${mocks.paseoHome}/daemon.log):`);
expect(message).toContain("recent daemon failure");
expect(mocks.spawnProcess).toHaveBeenCalledWith(
"node",
[],
expect.objectContaining({
detached: true,
stdio: ["ignore", "ignore", "ignore"],
}),
);
});
});
packages/desktop/src/daemon/daemon-manager.ts
+10 -54
View File
@@ -43,7 +43,6 @@ const DAEMON_LOG_FILENAME = "daemon.log";
const STARTUP_POLL_INTERVAL_MS = 200;
const STARTUP_POLL_MAX_ATTEMPTS = 150;
const DETACHED_STARTUP_GRACE_MS = 1200;
const STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS = 64 * 1024;
type DesktopDaemonState = "starting" | "running" | "stopped" | "errored";
@@ -70,11 +69,6 @@ interface DesktopPairingOffer {
qr: string | null;
}
interface StartupOutputCapture {
text: string;
truncated: boolean;
}
function parseReleaseChannel(
args: Record<string, unknown> | undefined,
): AppReleaseChannel | undefined {
@@ -151,30 +145,6 @@ function tailFile(filePath: string, lines = 50): string {
}
}
function createStartupOutputCapture(): StartupOutputCapture {
return { text: "", truncated: false };
}
function appendStartupOutput(capture: StartupOutputCapture, chunk: Buffer): StartupOutputCapture {
const nextText = capture.text + chunk.toString();
if (nextText.length <= STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS) {
return { text: nextText, truncated: capture.truncated };
}
return {
text: nextText.slice(-STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS),
truncated: true,
};
}
function formatStartupOutput(capture: StartupOutputCapture): string {
if (!capture.truncated) {
return capture.text;
}
return `[output truncated to the last ${STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS} chars]\n${capture.text}`;
}
function logDesktopDaemonLifecycle(message: string, details?: Record<string, unknown>): void {
log.info("[desktop daemon]", message, {
pid: process.pid,
@@ -231,6 +201,7 @@ export async function resolveDesktopDaemonStatus(): Promise<DesktopDaemonStatus>
typeof payload.connectedDaemon === "string" ? payload.connectedDaemon : "not_probed";
const hasRunningLocalProcess = localDaemon === "running";
const hasLocalProcess = hasRunningLocalProcess || localDaemon === "unresponsive";
const desktopManaged = payload.desktopManaged === true;
const apiReachable = connectedDaemon === "reachable";
let status: DesktopDaemonState = "stopped";
if (apiReachable || hasRunningLocalProcess) {
@@ -248,7 +219,7 @@ export async function resolveDesktopDaemonStatus(): Promise<DesktopDaemonStatus>
pid: hasLocalProcess && typeof payload.pid === "number" ? payload.pid : null,
home,
version: typeof payload.daemonVersion === "string" ? payload.daemonVersion : null,
desktopManaged: hasRunningLocalProcess && payload.desktopManaged === true,
desktopManaged,
error: null,
};
} catch (error) {
@@ -287,19 +258,15 @@ function assertBuiltInDaemonManagementEnabled(settings: DesktopSettings): void {
}
}
function buildStartupFailureError(
result: { code: number | null; signal: string | null; error?: Error },
stdout: StartupOutputCapture,
stderr: StartupOutputCapture,
): Error {
function buildStartupFailureError(result: {
code: number | null;
signal: string | null;
error?: Error;
}): Error {
const reason = result.error
? result.error.message
: `exit code ${result.code ?? "unknown"}${result.signal ? ` (${result.signal})` : ""}`;
const parts = [`Daemon failed to start: ${reason}`];
const formattedStderr = formatStartupOutput(stderr).trim();
const formattedStdout = formatStartupOutput(stdout).trim();
if (formattedStderr) parts.push(`stderr:\n${formattedStderr}`);
if (formattedStdout) parts.push(`stdout:\n${formattedStdout}`);
const logs = tailFile(logFilePath(), 15);
if (logs) parts.push(`Recent logs (${logFilePath()}):\n${logs}`);
return new Error(parts.join("\n\n"));
@@ -377,16 +344,7 @@ async function startDaemon(): Promise<DesktopDaemonStatus> {
envMode: "internal",
env: invocation.env,
envOverlay: { PASEO_DESKTOP_MANAGED: "1" },
stdio: ["ignore", "pipe", "pipe"],
});
let stdout = createStartupOutputCapture();
let stderr = createStartupOutputCapture();
child.stdout!.on("data", (data: Buffer) => {
stdout = appendStartupOutput(stdout, data);
});
child.stderr!.on("data", (data: Buffer) => {
stderr = appendStartupOutput(stderr, data);
stdio: ["ignore", "ignore", "ignore"],
});
logDesktopDaemonLifecycle("detached spawn returned", {
@@ -424,8 +382,6 @@ async function startDaemon(): Promise<DesktopDaemonStatus> {
logDesktopDaemonLifecycle("detached startup grace period completed", {
childPid: child.pid ?? null,
exitedEarly: result.exitedEarly,
stdout: formatStartupOutput(stdout).slice(0, 2000),
stderr: formatStartupOutput(stderr).slice(0, 2000),
...(result.exitedEarly
? {
exitCode: result.code,
@@ -436,7 +392,7 @@ async function startDaemon(): Promise<DesktopDaemonStatus> {
});
if (result.exitedEarly) {
throw buildStartupFailureError(result, stdout, stderr);
throw buildStartupFailureError(result);
}
return pollForRunningDaemon();
@@ -444,7 +400,7 @@ async function startDaemon(): Promise<DesktopDaemonStatus> {
export async function stopDesktopDaemon(): Promise<DesktopDaemonStatus> {
const status = await resolveDesktopDaemonStatus();
if (status.status !== "running" || !status.pid) return status;
if (status.status !== "running") return status;
await stopDesktopDaemonViaCli();
return await resolveDesktopDaemonStatus();