Files
paseo/packages/server/src/server/agent/providers/opencode/server-manager.ts
T
Mohamed BoudraandGitHub 79be6d8dba Stop OpenCode helper servers from leaking (#1697)
* Fix OpenCode helper server ownership

Own OpenCode helper generations as soon as spawn returns so startup timeout and shutdown clean up through the manager/reaper path. Collapse the OpenCode runtime wrapper so the provider talks to the server manager directly.

* Protect dedicated OpenCode server startup
2026-06-24 14:44:43 +08:00

514 lines
16 KiB
TypeScript

import type { ChildProcess } from "node:child_process";
import net from "node:net";
import os from "node:os";
import type { Logger } from "pino";
import { findExecutable } from "../../../../executable-resolution/executable-resolution.js";
import { spawnProcess, type SpawnProcessOptions } from "../../../../utils/spawn.js";
import { terminateWithTreeKill, type ProcessTerminator } from "../../../../utils/tree-kill.js";
import type { ManagedProcessRegistry } from "../../../managed-processes/managed-processes.js";
import {
createProviderEnvSpec,
resolveProviderCommandPrefix,
type ProviderRuntimeSettings,
} from "../../provider-launch-config.js";
const OPENCODE_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 5_000;
const OPENCODE_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS = 1_000;
export interface OpenCodeServerAcquisition {
server: { port: number; url: string };
release: () => void;
}
export interface OpenCodeServerManagerLike {
ensureRunning(): Promise<{ port: number; url: string }>;
acquireCurrent(): Promise<OpenCodeServerAcquisition>;
acquireNew(): Promise<OpenCodeServerAcquisition>;
acquireDedicated(env: Record<string, string>): Promise<OpenCodeServerAcquisition>;
shutdown(): Promise<void>;
}
export interface OpenCodeServerGeneration {
process: ChildProcess;
port: number;
url: string;
refCount: number;
retired: boolean;
ready: Promise<void>;
managedProcessId?: string;
managedProcessRecord?: Promise<{ id: string } | null>;
}
export type OpenCodePortAllocator = () => Promise<number>;
export type OpenCodeCommandPrefixResolver = () => Promise<{ command: string; args: string[] }>;
export type OpenCodeServerProcessSpawner = (
command: string,
args: string[],
options: SpawnProcessOptions,
) => ChildProcess;
export interface OpenCodeServerManagerOptions {
logger: Logger;
runtimeSettings?: ProviderRuntimeSettings;
managedProcesses?: ManagedProcessRegistry;
terminateProcess?: ProcessTerminator;
portAllocator?: OpenCodePortAllocator;
resolveCommandPrefix?: OpenCodeCommandPrefixResolver;
spawnServerProcess?: OpenCodeServerProcessSpawner;
}
export class OpenCodeServerManager implements OpenCodeServerManagerLike {
private static instance: OpenCodeServerManager | null = null;
private static exitHandlerRegistered = false;
private currentServer: OpenCodeServerGeneration | null = null;
private retiredServers = new Set<OpenCodeServerGeneration>();
private startPromise: Promise<OpenCodeServerGeneration> | null = null;
private newServerPromise: Promise<OpenCodeServerGeneration> | null = null;
private readonly logger: Logger;
private readonly runtimeSettings?: ProviderRuntimeSettings;
private readonly runtimeSettingsKey: string;
private readonly managedProcesses?: ManagedProcessRegistry;
private readonly terminateProcess: ProcessTerminator;
private readonly portAllocator: OpenCodePortAllocator;
private readonly resolveCommandPrefix: OpenCodeCommandPrefixResolver;
private readonly spawnServerProcess: OpenCodeServerProcessSpawner;
constructor(options: OpenCodeServerManagerOptions) {
this.logger = options.logger;
this.runtimeSettings = options.runtimeSettings;
this.runtimeSettingsKey = JSON.stringify(this.runtimeSettings ?? {});
this.managedProcesses = options.managedProcesses;
this.terminateProcess = options.terminateProcess ?? terminateWithTreeKill;
this.portAllocator = options.portAllocator ?? findAvailablePort;
this.resolveCommandPrefix =
options.resolveCommandPrefix ??
(() => resolveProviderCommandPrefix(this.runtimeSettings?.command, resolveOpenCodeBinary));
this.spawnServerProcess = options.spawnServerProcess ?? spawnProcess;
}
static getInstance(
logger: Logger,
runtimeSettings?: ProviderRuntimeSettings,
options: Omit<OpenCodeServerManagerOptions, "logger" | "runtimeSettings"> = {},
): OpenCodeServerManager {
const nextSettingsKey = JSON.stringify(runtimeSettings ?? {});
if (!OpenCodeServerManager.instance) {
OpenCodeServerManager.instance = new OpenCodeServerManager({
logger,
runtimeSettings,
...options,
});
OpenCodeServerManager.registerExitHandler();
} else if (OpenCodeServerManager.instance.runtimeSettingsKey !== nextSettingsKey) {
logger.warn(
{
existingRuntimeSettings: OpenCodeServerManager.instance.runtimeSettingsKey,
requestedRuntimeSettings: nextSettingsKey,
},
"OpenCode server manager already initialized with different runtime settings",
);
}
return OpenCodeServerManager.instance;
}
private static registerExitHandler(): void {
if (OpenCodeServerManager.exitHandlerRegistered) {
return;
}
OpenCodeServerManager.exitHandlerRegistered = true;
const cleanup = () => {
const instance = OpenCodeServerManager.instance;
void instance?.shutdown();
};
process.on("exit", cleanup);
process.on("SIGTERM", cleanup);
process.on("SIGINT", cleanup);
}
async ensureRunning(): Promise<{ port: number; url: string }> {
const acquisition = await this.acquireCurrent();
acquisition.release();
return acquisition.server;
}
async acquireCurrent(): Promise<OpenCodeServerAcquisition> {
const server = await this.getCurrentServer();
return this.acquireServer(server);
}
async acquireNew(): Promise<OpenCodeServerAcquisition> {
const server = await this.getNewServer();
return this.acquireServer(server);
}
async acquireDedicated(env: Record<string, string>): Promise<OpenCodeServerAcquisition> {
const server = await this.startServer(env);
server.retired = true;
this.retiredServers.add(server);
const acquisition = this.acquireServer(server);
try {
await server.ready;
return acquisition;
} catch (error) {
acquisition.release();
throw error;
}
}
private acquireServer(server: OpenCodeServerGeneration): OpenCodeServerAcquisition {
server.refCount += 1;
let released = false;
return {
server: { port: server.port, url: server.url },
release: () => {
if (released) {
return;
}
released = true;
server.refCount -= 1;
this.cleanupRetiredServers();
},
};
}
private async getNewServer(): Promise<OpenCodeServerGeneration> {
if (this.newServerPromise) {
return this.newServerPromise;
}
this.newServerPromise = Promise.resolve()
.then(async () => {
await this.rotateCurrentServer();
const server = await this.startServer();
if (!server.retired) {
this.currentServer = server;
}
await server.ready;
return server;
})
.finally(() => {
this.newServerPromise = null;
});
return this.newServerPromise;
}
private async getCurrentServer(): Promise<OpenCodeServerGeneration> {
if (this.newServerPromise) {
return this.newServerPromise;
}
if (this.startPromise) {
const server = await this.startPromise;
await server.ready;
return server;
}
if (this.currentServer && !this.currentServer.process.killed) {
await this.currentServer.ready;
return this.currentServer;
}
this.startPromise = this.startServer().then((server) => {
if (!server.retired) {
this.currentServer = server;
}
return server;
});
const currentStart = this.startPromise;
const result = await currentStart.finally(() => {
if (this.startPromise === currentStart) {
this.startPromise = null;
}
});
await result.ready;
return result;
}
private async rotateCurrentServer(): Promise<void> {
const existing = this.currentServer;
if (existing) {
existing.retired = true;
this.retiredServers.add(existing);
this.currentServer = null;
this.cleanupRetiredServers();
}
if (this.startPromise) {
const pending = await this.startPromise;
pending.retired = true;
this.retiredServers.add(pending);
this.currentServer = null;
this.cleanupRetiredServers();
}
}
private async startServer(launchEnv?: Record<string, string>): Promise<OpenCodeServerGeneration> {
const port = await this.portAllocator();
const url = `http://127.0.0.1:${port}`;
const launchPrefix = await this.resolveCommandPrefix();
const serverArgs = [...launchPrefix.args, "serve", "--port", String(port)];
const serverCwd = os.homedir();
const serverProcess = this.spawnServerProcess(launchPrefix.command, serverArgs, {
cwd: serverCwd,
detached: process.platform !== "win32",
stdio: ["ignore", "pipe", "pipe"],
...createProviderEnvSpec({
runtimeSettings: this.runtimeSettings,
overlays: [launchEnv],
}),
});
const managedProcessRecord = this.recordManagedServerProcess({
process: serverProcess,
command: launchPrefix.command,
args: serverArgs,
port,
});
const server: OpenCodeServerGeneration = {
process: serverProcess,
port,
url,
refCount: 0,
retired: false,
ready: Promise.resolve(),
managedProcessRecord,
};
void managedProcessRecord.then((record) => {
if (record && server.managedProcessRecord === managedProcessRecord) {
server.managedProcessId = record.id;
}
return undefined;
});
let started = false;
let settled = false;
let stderrBuffer = "";
let stdoutBuffer = "";
const STARTUP_BUFFER_CAP = 8192;
const appendCapped = (current: string, chunk: string): string => {
if (current.length >= STARTUP_BUFFER_CAP) {
return current;
}
const remaining = STARTUP_BUFFER_CAP - current.length;
return current + chunk.slice(0, remaining);
};
const buildStartupErrorMessage = (headline: string): string => {
const sections = [headline];
const stderrTrimmed = stderrBuffer.trim();
if (stderrTrimmed.length > 0) {
sections.push(`stderr: ${stderrTrimmed}`);
}
const stdoutTrimmed = stdoutBuffer.trim();
if (stdoutTrimmed.length > 0) {
sections.push(`stdout: ${stdoutTrimmed}`);
}
return sections.join("\n");
};
const ready = new Promise<void>((resolve, reject) => {
let timeout: ReturnType<typeof setTimeout>;
const failStartup = (error: Error) => {
if (settled) {
return;
}
settled = true;
clearTimeout(timeout);
reject(error);
};
timeout = setTimeout(() => {
if (!started) {
failStartup(new Error(buildStartupErrorMessage("OpenCode server startup timeout")));
}
}, 30_000);
serverProcess.stdout?.on("data", (data: Buffer) => {
const output = data.toString();
stdoutBuffer = appendCapped(stdoutBuffer, output);
if (output.includes("listening on") && !settled) {
started = true;
settled = true;
clearTimeout(timeout);
resolve();
}
});
serverProcess.stderr?.on("data", (data: Buffer) => {
const output = data.toString();
stderrBuffer = appendCapped(stderrBuffer, output);
this.logger.error({ stderr: output.trim() }, "OpenCode server stderr");
});
serverProcess.on("error", (error) => {
const headline = error instanceof Error ? error.message : String(error);
failStartup(new Error(buildStartupErrorMessage(headline)));
});
serverProcess.on("exit", (code) => {
this.removeManagedServerRecord(server);
if (!started) {
failStartup(
new Error(buildStartupErrorMessage(`OpenCode server exited with code ${code}`)),
);
}
if (this.currentServer?.process === serverProcess) {
this.currentServer = null;
}
for (const retired of Array.from(this.retiredServers)) {
if (retired.process === serverProcess) {
this.retiredServers.delete(retired);
}
}
});
});
server.ready = ready.catch(async (error) => {
await this.killServer(server);
if (this.currentServer === server) {
this.currentServer = null;
}
this.retiredServers.delete(server);
throw error;
});
return server;
}
async shutdown(): Promise<void> {
const servers = [
...(this.currentServer ? [this.currentServer] : []),
...Array.from(this.retiredServers),
];
await Promise.all(servers.map((server) => this.killServer(server)));
this.currentServer = null;
this.retiredServers.clear();
}
private cleanupRetiredServers(): void {
for (const server of Array.from(this.retiredServers)) {
if (server.refCount === 0) {
this.retiredServers.delete(server);
void this.killServer(server);
}
}
}
private async killServer(server: OpenCodeServerGeneration): Promise<void> {
if (
(server.process.exitCode !== null && server.process.exitCode !== undefined) ||
(server.process.signalCode !== null && server.process.signalCode !== undefined)
) {
return;
}
const result = await this.terminateProcess(server.process, {
gracefulTimeoutMs: OPENCODE_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_MS,
forceTimeoutMs: OPENCODE_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS,
onForceSignal: () => {
this.logger.warn(
{ timeoutMs: OPENCODE_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_MS },
"OpenCode server did not exit after SIGTERM; sending SIGKILL",
);
},
});
if (result === "kill-timeout") {
this.logger.warn(
{ timeoutMs: OPENCODE_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS },
"OpenCode server did not report exit after SIGKILL",
);
}
if (server.managedProcessId) {
await this.removeManagedProcessId(server.managedProcessId);
server.managedProcessId = undefined;
server.managedProcessRecord = undefined;
} else {
this.removeManagedServerRecord(server);
}
}
private async recordManagedServerProcess(options: {
process: ChildProcess;
command: string;
args: string[];
port: number;
}): Promise<{ id: string } | null> {
const pid = options.process.pid;
if (!this.managedProcesses || typeof pid !== "number" || pid <= 0) {
return null;
}
try {
return await this.managedProcesses.record({
owner: { provider: "opencode", kind: "helper-server" },
pid,
command: options.command,
args: options.args,
metadata: { port: options.port },
});
} catch (error) {
this.logger.warn(
{ err: error, pid, port: options.port },
"Failed to record OpenCode helper process",
);
return null;
}
}
private removeManagedProcessRecordWhenResolved(record: Promise<{ id: string } | null>): void {
void record.then((resolved) => {
if (resolved) {
return this.removeManagedProcessId(resolved.id);
}
return undefined;
});
}
private removeManagedServerRecord(server: OpenCodeServerGeneration): void {
const record = server.managedProcessRecord;
server.managedProcessRecord = undefined;
if (server.managedProcessId) {
void this.removeManagedProcessId(server.managedProcessId);
server.managedProcessId = undefined;
return;
}
if (record) {
this.removeManagedProcessRecordWhenResolved(record);
}
}
private async removeManagedProcessId(id: string): Promise<void> {
try {
await this.managedProcesses?.remove(id);
} catch (error) {
this.logger.warn({ err: error, id }, "Failed to remove OpenCode helper process record");
}
}
}
async function resolveOpenCodeBinary(): Promise<string> {
const found = await findExecutable("opencode");
if (found) {
return found;
}
throw new Error(
"OpenCode binary not found. Install OpenCode (https://github.com/opencode-ai/opencode) and ensure it is available in your shell PATH.",
);
}
function findAvailablePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = net.createServer();
server.listen(0, "127.0.0.1", () => {
const address = server.address();
server.close(() => {
if (typeof address === "object" && address) {
resolve(address.port);
} else {
reject(new Error("Failed to allocate port"));
}
});
});
server.on("error", reject);
});
}