jb/paseo
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Recover stale supervised daemons

Browse Source 
Teach daemon stop to use lifecycle shutdown when the API is reachable even if the owner pid is stale, then wait for the API to disappear and clean the stale pidfile.

Launch desktop-managed daemons detached from desktop stdio, preserve stale reachable daemon ownership for version checks, and allow desktop stop/restart to use the CLI recovery path.

Add supervisor heartbeats so supervised workers shut down when their supervisor disappears instead of surviving as orphaned reachable daemons.
This commit is contained in:
Mohamed Boudra
2026-06-07 23:18:48 +07:00
parent 7c8b290e2f
commit 893e3376b0
7 changed files with 769 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/desktop/src/daemon/daemon-manager.test.ts
+143 -23
View File
@@ -1,10 +1,12 @@
import { EventEmitter } from "node:events";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_DESKTOP_SETTINGS } from "../settings/desktop-settings";
import { createDaemonCommandHandlers } from "./daemon-manager";
const mocks = vi.hoisted(() => ({
paseoHome: "/tmp/paseo-desktop-daemon-manager-test-home",
settings: {
releaseChannel: "stable",
daemon: {
@@ -21,7 +23,7 @@ vi.mock("electron", () => ({
app: {
getPath: vi.fn(() => "/tmp/paseo-user-data"),
getVersion: vi.fn(() => "1.2.3"),
isPackaged: false,
isPackaged: true,
},
ipcMain: { handle: vi.fn() },
powerMonitor: { getSystemIdleTime: vi.fn(() => 0) },
@@ -32,7 +34,7 @@ vi.mock("electron-log/main", () => ({
}));
vi.mock("@getpaseo/server", () => ({
resolvePaseoHome: vi.fn(() => "/tmp/paseo-home"),
resolvePaseoHome: vi.fn(() => mocks.paseoHome),
spawnProcess: mocks.spawnProcess,
}));
@@ -72,8 +74,6 @@ function desktopSettingsWithManagement(enabled: boolean) {
}
type MockChildProcess = EventEmitter & {
stdout: EventEmitter;
stderr: EventEmitter;
pid: number;
spawnfile: string;
spawnargs: string[];
@@ -82,8 +82,6 @@ type MockChildProcess = EventEmitter & {
function createMockChildProcess(): MockChildProcess {
const child = new EventEmitter() as MockChildProcess;
child.stdout = new EventEmitter();
child.stderr = new EventEmitter();
child.pid = 1234;
child.spawnfile = "node";
child.spawnargs = ["node", "daemon.js"];
@@ -91,10 +89,8 @@ function createMockChildProcess(): MockChildProcess {
return child;
}
function scheduleFailedStartupOutput(child: MockChildProcess): void {
function scheduleFailedStartup(child: MockChildProcess): void {
setImmediate(() => {
child.stdout.emit("data", Buffer.from(`${"x".repeat(80_000)}stdout-tail`));
child.stderr.emit("data", Buffer.from(`${"y".repeat(80_000)}stderr-tail`));
child.emit("exit", 1, null);
});
}
@@ -105,6 +101,11 @@ describe("daemon-manager commands", () => {
mocks.runExternalCliJsonCommand.mockReset();
mocks.runExternalCliTextCommand.mockReset();
mocks.spawnProcess.mockReset();
rmSync(mocks.paseoHome, { recursive: true, force: true });
});
afterEach(() => {
rmSync(mocks.paseoHome, { recursive: true, force: true });
});
it("refuses start and restart while built-in daemon management is disabled", async () => {
@@ -136,7 +137,7 @@ describe("daemon-manager commands", () => {
listen: null,
hostname: null,
pid: null,
home: "/tmp/paseo-home",
home: mocks.paseoHome,
version: null,
desktopManaged: false,
error: null,
@@ -167,7 +168,7 @@ describe("daemon-manager commands", () => {
listen: null,
hostname: null,
pid: null,
home: "/tmp/paseo-home",
home: mocks.paseoHome,
version: null,
desktopManaged: false,
error: null,
@@ -195,7 +196,50 @@ describe("daemon-manager commands", () => {
]);
});
it("uses a reachable daemon when the PID file is stale", async () => {
it("routes stale reachable desktop daemon stops through external CLI daemon stop", async () => {
mocks.runExternalCliJsonCommand
.mockResolvedValueOnce({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
serverId: "server-1",
pid: 7675,
listen: "127.0.0.1:6767",
daemonVersion: "1.2.2",
desktopManaged: true,
})
.mockResolvedValueOnce({ action: "stopped" })
.mockResolvedValueOnce({
localDaemon: "stopped",
connectedDaemon: "unreachable",
serverId: "",
});
const handlers = createDaemonCommandHandlers();
await expect(handlers.stop_desktop_daemon()).resolves.toEqual({
serverId: "",
status: "stopped",
listen: null,
hostname: null,
pid: null,
home: mocks.paseoHome,
version: null,
desktopManaged: false,
error: null,
});
expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(2, [
"daemon",
"stop",
"--json",
"--timeout",
"5",
"--force",
"--kill-timeout",
"5",
]);
});
it("uses a stale reachable desktop daemon when the version matches", async () => {
mocks.runExternalCliJsonCommand.mockResolvedValue({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
@@ -203,7 +247,7 @@ describe("daemon-manager commands", () => {
pid: 7675,
listen: "127.0.0.1:6767",
hostname: "dev-host",
daemonVersion: "1.2.2",
daemonVersion: "1.2.3",
desktopManaged: true,
});
const handlers = createDaemonCommandHandlers();
@@ -214,16 +258,86 @@ describe("daemon-manager commands", () => {
listen: "127.0.0.1:6767",
hostname: "dev-host",
pid: null,
home: "/tmp/paseo-home",
version: "1.2.2",
desktopManaged: false,
home: mocks.paseoHome,
version: "1.2.3",
desktopManaged: true,
error: null,
});
expect(mocks.spawnProcess).not.toHaveBeenCalled();
});
it("bounds captured daemon startup output", async () => {
it("restarts a stale reachable desktop daemon when the version differs", async () => {
mocks.runExternalCliJsonCommand
.mockResolvedValueOnce({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
serverId: "server-1",
pid: 7675,
listen: "127.0.0.1:6767",
hostname: "dev-host",
daemonVersion: "1.2.2",
desktopManaged: true,
})
.mockResolvedValueOnce({
localDaemon: "stale_pid",
connectedDaemon: "reachable",
serverId: "server-1",
pid: 7675,
listen: "127.0.0.1:6767",
daemonVersion: "1.2.2",
desktopManaged: true,
})
.mockResolvedValueOnce({ action: "stopped" })
.mockResolvedValueOnce({
localDaemon: "stopped",
connectedDaemon: "unreachable",
serverId: "",
})
.mockResolvedValueOnce({
localDaemon: "running",
connectedDaemon: "reachable",
serverId: "server-2",
pid: 8888,
listen: "127.0.0.1:6767",
hostname: "dev-host",
daemonVersion: "1.2.3",
desktopManaged: true,
});
mocks.spawnProcess.mockReturnValue(createMockChildProcess());
const handlers = createDaemonCommandHandlers();
await expect(handlers.start_desktop_daemon()).resolves.toEqual({
serverId: "server-2",
status: "running",
listen: "127.0.0.1:6767",
hostname: "dev-host",
pid: 8888,
home: mocks.paseoHome,
version: "1.2.3",
desktopManaged: true,
error: null,
});
expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(3, [
"daemon",
"stop",
"--json",
"--timeout",
"5",
"--force",
"--kill-timeout",
"5",
]);
expect(mocks.spawnProcess).toHaveBeenCalled();
});
it("starts the managed daemon detached from desktop stdio and reports daemon log failures", async () => {
mkdirSync(mocks.paseoHome, { recursive: true });
writeFileSync(
`${mocks.paseoHome}/daemon.log`,
["old log line", "recent daemon failure"].join("\n"),
);
mocks.runExternalCliJsonCommand.mockResolvedValue({
localDaemon: "stopped",
connectedDaemon: "unreachable",
@@ -231,7 +345,7 @@ describe("daemon-manager commands", () => {
});
mocks.spawnProcess.mockImplementation(() => {
const child = createMockChildProcess();
scheduleFailedStartupOutput(child);
scheduleFailedStartup(child);
return child;
});
const handlers = createDaemonCommandHandlers();
@@ -246,9 +360,15 @@ describe("daemon-manager commands", () => {
expect(thrown).toBeInstanceOf(Error);
const message = thrown?.message ?? "";
expect(message).toContain("Daemon failed to start: exit code 1");
expect(message).toContain("output truncated to the last 65536 chars");
expect(message).toContain("stdout-tail");
expect(message).toContain("stderr-tail");
expect(message.length).toBeLessThan(150_000);
expect(message).toContain(`Recent logs (${mocks.paseoHome}/daemon.log):`);
expect(message).toContain("recent daemon failure");
expect(mocks.spawnProcess).toHaveBeenCalledWith(
"node",
[],
expect.objectContaining({
detached: true,
stdio: ["ignore", "ignore", "ignore"],
}),
);
});
});