Fix desktop daemon stale PID startup (#913)
This commit is contained in:
@@ -1,3 +1,4 @@
|
|||||||
|
import { EventEmitter } from "node:events";
|
||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
import { DEFAULT_DESKTOP_SETTINGS } from "../settings/desktop-settings";
|
import { DEFAULT_DESKTOP_SETTINGS } from "../settings/desktop-settings";
|
||||||
@@ -70,6 +71,34 @@ function desktopSettingsWithManagement(enabled: boolean) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type MockChildProcess = EventEmitter & {
|
||||||
|
stdout: EventEmitter;
|
||||||
|
stderr: EventEmitter;
|
||||||
|
pid: number;
|
||||||
|
spawnfile: string;
|
||||||
|
spawnargs: string[];
|
||||||
|
unref: ReturnType<typeof vi.fn>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function createMockChildProcess(): MockChildProcess {
|
||||||
|
const child = new EventEmitter() as MockChildProcess;
|
||||||
|
child.stdout = new EventEmitter();
|
||||||
|
child.stderr = new EventEmitter();
|
||||||
|
child.pid = 1234;
|
||||||
|
child.spawnfile = "node";
|
||||||
|
child.spawnargs = ["node", "daemon.js"];
|
||||||
|
child.unref = vi.fn();
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleFailedStartupOutput(child: MockChildProcess): void {
|
||||||
|
setImmediate(() => {
|
||||||
|
child.stdout.emit("data", Buffer.from(`${"x".repeat(80_000)}stdout-tail`));
|
||||||
|
child.stderr.emit("data", Buffer.from(`${"y".repeat(80_000)}stderr-tail`));
|
||||||
|
child.emit("exit", 1, null);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
describe("daemon-manager commands", () => {
|
describe("daemon-manager commands", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
mocks.settings = DEFAULT_DESKTOP_SETTINGS;
|
mocks.settings = DEFAULT_DESKTOP_SETTINGS;
|
||||||
@@ -165,4 +194,61 @@ describe("daemon-manager commands", () => {
|
|||||||
"--json",
|
"--json",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses a reachable daemon when the PID file is stale", async () => {
|
||||||
|
mocks.runExternalCliJsonCommand.mockResolvedValue({
|
||||||
|
localDaemon: "stale_pid",
|
||||||
|
connectedDaemon: "reachable",
|
||||||
|
serverId: "server-1",
|
||||||
|
pid: 7675,
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
hostname: "dev-host",
|
||||||
|
daemonVersion: "1.2.2",
|
||||||
|
desktopManaged: true,
|
||||||
|
});
|
||||||
|
const handlers = createDaemonCommandHandlers();
|
||||||
|
|
||||||
|
await expect(handlers.start_desktop_daemon()).resolves.toEqual({
|
||||||
|
serverId: "server-1",
|
||||||
|
status: "running",
|
||||||
|
listen: "127.0.0.1:6767",
|
||||||
|
hostname: "dev-host",
|
||||||
|
pid: null,
|
||||||
|
home: "/tmp/paseo-home",
|
||||||
|
version: "1.2.2",
|
||||||
|
desktopManaged: false,
|
||||||
|
error: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mocks.spawnProcess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bounds captured daemon startup output", async () => {
|
||||||
|
mocks.runExternalCliJsonCommand.mockResolvedValue({
|
||||||
|
localDaemon: "stopped",
|
||||||
|
connectedDaemon: "unreachable",
|
||||||
|
serverId: "",
|
||||||
|
});
|
||||||
|
mocks.spawnProcess.mockImplementation(() => {
|
||||||
|
const child = createMockChildProcess();
|
||||||
|
scheduleFailedStartupOutput(child);
|
||||||
|
return child;
|
||||||
|
});
|
||||||
|
const handlers = createDaemonCommandHandlers();
|
||||||
|
|
||||||
|
let thrown: Error | null = null;
|
||||||
|
try {
|
||||||
|
await handlers.start_desktop_daemon();
|
||||||
|
} catch (error) {
|
||||||
|
thrown = error instanceof Error ? error : new Error(String(error));
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(thrown).toBeInstanceOf(Error);
|
||||||
|
const message = thrown?.message ?? "";
|
||||||
|
expect(message).toContain("Daemon failed to start: exit code 1");
|
||||||
|
expect(message).toContain("output truncated to the last 65536 chars");
|
||||||
|
expect(message).toContain("stdout-tail");
|
||||||
|
expect(message).toContain("stderr-tail");
|
||||||
|
expect(message.length).toBeLessThan(150_000);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ const DAEMON_LOG_FILENAME = "daemon.log";
|
|||||||
const STARTUP_POLL_INTERVAL_MS = 200;
|
const STARTUP_POLL_INTERVAL_MS = 200;
|
||||||
const STARTUP_POLL_MAX_ATTEMPTS = 150;
|
const STARTUP_POLL_MAX_ATTEMPTS = 150;
|
||||||
const DETACHED_STARTUP_GRACE_MS = 1200;
|
const DETACHED_STARTUP_GRACE_MS = 1200;
|
||||||
|
const STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS = 64 * 1024;
|
||||||
|
|
||||||
type DesktopDaemonState = "starting" | "running" | "stopped" | "errored";
|
type DesktopDaemonState = "starting" | "running" | "stopped" | "errored";
|
||||||
|
|
||||||
@@ -69,6 +70,11 @@ interface DesktopPairingOffer {
|
|||||||
qr: string | null;
|
qr: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface StartupOutputCapture {
|
||||||
|
text: string;
|
||||||
|
truncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
function parseReleaseChannel(
|
function parseReleaseChannel(
|
||||||
args: Record<string, unknown> | undefined,
|
args: Record<string, unknown> | undefined,
|
||||||
): AppReleaseChannel | undefined {
|
): AppReleaseChannel | undefined {
|
||||||
@@ -145,6 +151,30 @@ function tailFile(filePath: string, lines = 50): string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createStartupOutputCapture(): StartupOutputCapture {
|
||||||
|
return { text: "", truncated: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
function appendStartupOutput(capture: StartupOutputCapture, chunk: Buffer): StartupOutputCapture {
|
||||||
|
const nextText = capture.text + chunk.toString();
|
||||||
|
if (nextText.length <= STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS) {
|
||||||
|
return { text: nextText, truncated: capture.truncated };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
text: nextText.slice(-STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS),
|
||||||
|
truncated: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatStartupOutput(capture: StartupOutputCapture): string {
|
||||||
|
if (!capture.truncated) {
|
||||||
|
return capture.text;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `[output truncated to the last ${STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS} chars]\n${capture.text}`;
|
||||||
|
}
|
||||||
|
|
||||||
function logDesktopDaemonLifecycle(message: string, details?: Record<string, unknown>): void {
|
function logDesktopDaemonLifecycle(message: string, details?: Record<string, unknown>): void {
|
||||||
log.info("[desktop daemon]", message, {
|
log.info("[desktop daemon]", message, {
|
||||||
pid: process.pid,
|
pid: process.pid,
|
||||||
@@ -197,17 +227,28 @@ export async function resolveDesktopDaemonStatus(): Promise<DesktopDaemonStatus>
|
|||||||
unknown
|
unknown
|
||||||
>;
|
>;
|
||||||
const localDaemon = typeof payload.localDaemon === "string" ? payload.localDaemon : "stopped";
|
const localDaemon = typeof payload.localDaemon === "string" ? payload.localDaemon : "stopped";
|
||||||
const running = localDaemon === "running";
|
const connectedDaemon =
|
||||||
|
typeof payload.connectedDaemon === "string" ? payload.connectedDaemon : "not_probed";
|
||||||
|
const hasRunningLocalProcess = localDaemon === "running";
|
||||||
|
const hasLocalProcess = hasRunningLocalProcess || localDaemon === "unresponsive";
|
||||||
|
const apiReachable = connectedDaemon === "reachable";
|
||||||
|
let status: DesktopDaemonState = "stopped";
|
||||||
|
if (apiReachable || hasRunningLocalProcess) {
|
||||||
|
status = "running";
|
||||||
|
} else if (localDaemon === "unresponsive") {
|
||||||
|
status = "errored";
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
serverId: typeof payload.serverId === "string" ? payload.serverId : "",
|
serverId: typeof payload.serverId === "string" ? payload.serverId : "",
|
||||||
status: running ? "running" : "stopped",
|
status,
|
||||||
listen: typeof payload.listen === "string" ? payload.listen : null,
|
listen: typeof payload.listen === "string" ? payload.listen : null,
|
||||||
hostname: running && typeof payload.hostname === "string" ? payload.hostname : null,
|
hostname:
|
||||||
pid: running && typeof payload.pid === "number" ? payload.pid : null,
|
status === "running" && typeof payload.hostname === "string" ? payload.hostname : null,
|
||||||
|
pid: hasLocalProcess && typeof payload.pid === "number" ? payload.pid : null,
|
||||||
home,
|
home,
|
||||||
version: typeof payload.daemonVersion === "string" ? payload.daemonVersion : null,
|
version: typeof payload.daemonVersion === "string" ? payload.daemonVersion : null,
|
||||||
desktopManaged: payload.desktopManaged === true,
|
desktopManaged: hasRunningLocalProcess && payload.desktopManaged === true,
|
||||||
error: null,
|
error: null,
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -248,15 +289,17 @@ function assertBuiltInDaemonManagementEnabled(settings: DesktopSettings): void {
|
|||||||
|
|
||||||
function buildStartupFailureError(
|
function buildStartupFailureError(
|
||||||
result: { code: number | null; signal: string | null; error?: Error },
|
result: { code: number | null; signal: string | null; error?: Error },
|
||||||
stdout: string,
|
stdout: StartupOutputCapture,
|
||||||
stderr: string,
|
stderr: StartupOutputCapture,
|
||||||
): Error {
|
): Error {
|
||||||
const reason = result.error
|
const reason = result.error
|
||||||
? result.error.message
|
? result.error.message
|
||||||
: `exit code ${result.code ?? "unknown"}${result.signal ? ` (${result.signal})` : ""}`;
|
: `exit code ${result.code ?? "unknown"}${result.signal ? ` (${result.signal})` : ""}`;
|
||||||
const parts = [`Daemon failed to start: ${reason}`];
|
const parts = [`Daemon failed to start: ${reason}`];
|
||||||
if (stderr.trim()) parts.push(`stderr:\n${stderr.trim()}`);
|
const formattedStderr = formatStartupOutput(stderr).trim();
|
||||||
if (stdout.trim()) parts.push(`stdout:\n${stdout.trim()}`);
|
const formattedStdout = formatStartupOutput(stdout).trim();
|
||||||
|
if (formattedStderr) parts.push(`stderr:\n${formattedStderr}`);
|
||||||
|
if (formattedStdout) parts.push(`stdout:\n${formattedStdout}`);
|
||||||
const logs = tailFile(logFilePath(), 15);
|
const logs = tailFile(logFilePath(), 15);
|
||||||
if (logs) parts.push(`Recent logs (${logFilePath()}):\n${logs}`);
|
if (logs) parts.push(`Recent logs (${logFilePath()}):\n${logs}`);
|
||||||
return new Error(parts.join("\n\n"));
|
return new Error(parts.join("\n\n"));
|
||||||
@@ -337,13 +380,13 @@ async function startDaemon(): Promise<DesktopDaemonStatus> {
|
|||||||
stdio: ["ignore", "pipe", "pipe"],
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
});
|
});
|
||||||
|
|
||||||
let stdout = "";
|
let stdout = createStartupOutputCapture();
|
||||||
let stderr = "";
|
let stderr = createStartupOutputCapture();
|
||||||
child.stdout!.on("data", (data: Buffer) => {
|
child.stdout!.on("data", (data: Buffer) => {
|
||||||
stdout += data.toString();
|
stdout = appendStartupOutput(stdout, data);
|
||||||
});
|
});
|
||||||
child.stderr!.on("data", (data: Buffer) => {
|
child.stderr!.on("data", (data: Buffer) => {
|
||||||
stderr += data.toString();
|
stderr = appendStartupOutput(stderr, data);
|
||||||
});
|
});
|
||||||
|
|
||||||
logDesktopDaemonLifecycle("detached spawn returned", {
|
logDesktopDaemonLifecycle("detached spawn returned", {
|
||||||
@@ -381,8 +424,8 @@ async function startDaemon(): Promise<DesktopDaemonStatus> {
|
|||||||
logDesktopDaemonLifecycle("detached startup grace period completed", {
|
logDesktopDaemonLifecycle("detached startup grace period completed", {
|
||||||
childPid: child.pid ?? null,
|
childPid: child.pid ?? null,
|
||||||
exitedEarly: result.exitedEarly,
|
exitedEarly: result.exitedEarly,
|
||||||
stdout: stdout.slice(0, 2000),
|
stdout: formatStartupOutput(stdout).slice(0, 2000),
|
||||||
stderr: stderr.slice(0, 2000),
|
stderr: formatStartupOutput(stderr).slice(0, 2000),
|
||||||
...(result.exitedEarly
|
...(result.exitedEarly
|
||||||
? {
|
? {
|
||||||
exitCode: result.code,
|
exitCode: result.code,
|
||||||
|
|||||||
Reference in New Issue
Block a user