* Fix OpenCode helper server ownership Own OpenCode helper generations as soon as spawn returns so startup timeout and shutdown clean up through the manager/reaper path. Collapse the OpenCode runtime wrapper so the provider talks to the server manager directly. * Protect dedicated OpenCode server startup
514 lines
16 KiB
TypeScript
514 lines
16 KiB
TypeScript
import type { ChildProcess } from "node:child_process";
|
|
import net from "node:net";
|
|
import os from "node:os";
|
|
import type { Logger } from "pino";
|
|
|
|
import { findExecutable } from "../../../../executable-resolution/executable-resolution.js";
|
|
import { spawnProcess, type SpawnProcessOptions } from "../../../../utils/spawn.js";
|
|
import { terminateWithTreeKill, type ProcessTerminator } from "../../../../utils/tree-kill.js";
|
|
import type { ManagedProcessRegistry } from "../../../managed-processes/managed-processes.js";
|
|
import {
|
|
createProviderEnvSpec,
|
|
resolveProviderCommandPrefix,
|
|
type ProviderRuntimeSettings,
|
|
} from "../../provider-launch-config.js";
|
|
|
|
const OPENCODE_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_MS = 5_000;
|
|
const OPENCODE_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS = 1_000;
|
|
|
|
export interface OpenCodeServerAcquisition {
|
|
server: { port: number; url: string };
|
|
release: () => void;
|
|
}
|
|
|
|
export interface OpenCodeServerManagerLike {
|
|
ensureRunning(): Promise<{ port: number; url: string }>;
|
|
acquireCurrent(): Promise<OpenCodeServerAcquisition>;
|
|
acquireNew(): Promise<OpenCodeServerAcquisition>;
|
|
acquireDedicated(env: Record<string, string>): Promise<OpenCodeServerAcquisition>;
|
|
shutdown(): Promise<void>;
|
|
}
|
|
|
|
export interface OpenCodeServerGeneration {
|
|
process: ChildProcess;
|
|
port: number;
|
|
url: string;
|
|
refCount: number;
|
|
retired: boolean;
|
|
ready: Promise<void>;
|
|
managedProcessId?: string;
|
|
managedProcessRecord?: Promise<{ id: string } | null>;
|
|
}
|
|
|
|
export type OpenCodePortAllocator = () => Promise<number>;
|
|
export type OpenCodeCommandPrefixResolver = () => Promise<{ command: string; args: string[] }>;
|
|
export type OpenCodeServerProcessSpawner = (
|
|
command: string,
|
|
args: string[],
|
|
options: SpawnProcessOptions,
|
|
) => ChildProcess;
|
|
|
|
export interface OpenCodeServerManagerOptions {
|
|
logger: Logger;
|
|
runtimeSettings?: ProviderRuntimeSettings;
|
|
managedProcesses?: ManagedProcessRegistry;
|
|
terminateProcess?: ProcessTerminator;
|
|
portAllocator?: OpenCodePortAllocator;
|
|
resolveCommandPrefix?: OpenCodeCommandPrefixResolver;
|
|
spawnServerProcess?: OpenCodeServerProcessSpawner;
|
|
}
|
|
|
|
export class OpenCodeServerManager implements OpenCodeServerManagerLike {
|
|
private static instance: OpenCodeServerManager | null = null;
|
|
private static exitHandlerRegistered = false;
|
|
private currentServer: OpenCodeServerGeneration | null = null;
|
|
private retiredServers = new Set<OpenCodeServerGeneration>();
|
|
private startPromise: Promise<OpenCodeServerGeneration> | null = null;
|
|
private newServerPromise: Promise<OpenCodeServerGeneration> | null = null;
|
|
private readonly logger: Logger;
|
|
private readonly runtimeSettings?: ProviderRuntimeSettings;
|
|
private readonly runtimeSettingsKey: string;
|
|
private readonly managedProcesses?: ManagedProcessRegistry;
|
|
private readonly terminateProcess: ProcessTerminator;
|
|
private readonly portAllocator: OpenCodePortAllocator;
|
|
private readonly resolveCommandPrefix: OpenCodeCommandPrefixResolver;
|
|
private readonly spawnServerProcess: OpenCodeServerProcessSpawner;
|
|
|
|
constructor(options: OpenCodeServerManagerOptions) {
|
|
this.logger = options.logger;
|
|
this.runtimeSettings = options.runtimeSettings;
|
|
this.runtimeSettingsKey = JSON.stringify(this.runtimeSettings ?? {});
|
|
this.managedProcesses = options.managedProcesses;
|
|
this.terminateProcess = options.terminateProcess ?? terminateWithTreeKill;
|
|
this.portAllocator = options.portAllocator ?? findAvailablePort;
|
|
this.resolveCommandPrefix =
|
|
options.resolveCommandPrefix ??
|
|
(() => resolveProviderCommandPrefix(this.runtimeSettings?.command, resolveOpenCodeBinary));
|
|
this.spawnServerProcess = options.spawnServerProcess ?? spawnProcess;
|
|
}
|
|
|
|
static getInstance(
|
|
logger: Logger,
|
|
runtimeSettings?: ProviderRuntimeSettings,
|
|
options: Omit<OpenCodeServerManagerOptions, "logger" | "runtimeSettings"> = {},
|
|
): OpenCodeServerManager {
|
|
const nextSettingsKey = JSON.stringify(runtimeSettings ?? {});
|
|
if (!OpenCodeServerManager.instance) {
|
|
OpenCodeServerManager.instance = new OpenCodeServerManager({
|
|
logger,
|
|
runtimeSettings,
|
|
...options,
|
|
});
|
|
OpenCodeServerManager.registerExitHandler();
|
|
} else if (OpenCodeServerManager.instance.runtimeSettingsKey !== nextSettingsKey) {
|
|
logger.warn(
|
|
{
|
|
existingRuntimeSettings: OpenCodeServerManager.instance.runtimeSettingsKey,
|
|
requestedRuntimeSettings: nextSettingsKey,
|
|
},
|
|
"OpenCode server manager already initialized with different runtime settings",
|
|
);
|
|
}
|
|
return OpenCodeServerManager.instance;
|
|
}
|
|
|
|
private static registerExitHandler(): void {
|
|
if (OpenCodeServerManager.exitHandlerRegistered) {
|
|
return;
|
|
}
|
|
OpenCodeServerManager.exitHandlerRegistered = true;
|
|
|
|
const cleanup = () => {
|
|
const instance = OpenCodeServerManager.instance;
|
|
void instance?.shutdown();
|
|
};
|
|
|
|
process.on("exit", cleanup);
|
|
process.on("SIGTERM", cleanup);
|
|
process.on("SIGINT", cleanup);
|
|
}
|
|
|
|
async ensureRunning(): Promise<{ port: number; url: string }> {
|
|
const acquisition = await this.acquireCurrent();
|
|
acquisition.release();
|
|
return acquisition.server;
|
|
}
|
|
|
|
async acquireCurrent(): Promise<OpenCodeServerAcquisition> {
|
|
const server = await this.getCurrentServer();
|
|
return this.acquireServer(server);
|
|
}
|
|
|
|
async acquireNew(): Promise<OpenCodeServerAcquisition> {
|
|
const server = await this.getNewServer();
|
|
return this.acquireServer(server);
|
|
}
|
|
|
|
async acquireDedicated(env: Record<string, string>): Promise<OpenCodeServerAcquisition> {
|
|
const server = await this.startServer(env);
|
|
server.retired = true;
|
|
this.retiredServers.add(server);
|
|
const acquisition = this.acquireServer(server);
|
|
try {
|
|
await server.ready;
|
|
return acquisition;
|
|
} catch (error) {
|
|
acquisition.release();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private acquireServer(server: OpenCodeServerGeneration): OpenCodeServerAcquisition {
|
|
server.refCount += 1;
|
|
let released = false;
|
|
return {
|
|
server: { port: server.port, url: server.url },
|
|
release: () => {
|
|
if (released) {
|
|
return;
|
|
}
|
|
released = true;
|
|
server.refCount -= 1;
|
|
this.cleanupRetiredServers();
|
|
},
|
|
};
|
|
}
|
|
|
|
private async getNewServer(): Promise<OpenCodeServerGeneration> {
|
|
if (this.newServerPromise) {
|
|
return this.newServerPromise;
|
|
}
|
|
|
|
this.newServerPromise = Promise.resolve()
|
|
.then(async () => {
|
|
await this.rotateCurrentServer();
|
|
const server = await this.startServer();
|
|
if (!server.retired) {
|
|
this.currentServer = server;
|
|
}
|
|
await server.ready;
|
|
return server;
|
|
})
|
|
.finally(() => {
|
|
this.newServerPromise = null;
|
|
});
|
|
return this.newServerPromise;
|
|
}
|
|
|
|
private async getCurrentServer(): Promise<OpenCodeServerGeneration> {
|
|
if (this.newServerPromise) {
|
|
return this.newServerPromise;
|
|
}
|
|
|
|
if (this.startPromise) {
|
|
const server = await this.startPromise;
|
|
await server.ready;
|
|
return server;
|
|
}
|
|
|
|
if (this.currentServer && !this.currentServer.process.killed) {
|
|
await this.currentServer.ready;
|
|
return this.currentServer;
|
|
}
|
|
|
|
this.startPromise = this.startServer().then((server) => {
|
|
if (!server.retired) {
|
|
this.currentServer = server;
|
|
}
|
|
return server;
|
|
});
|
|
const currentStart = this.startPromise;
|
|
const result = await currentStart.finally(() => {
|
|
if (this.startPromise === currentStart) {
|
|
this.startPromise = null;
|
|
}
|
|
});
|
|
await result.ready;
|
|
return result;
|
|
}
|
|
|
|
private async rotateCurrentServer(): Promise<void> {
|
|
const existing = this.currentServer;
|
|
if (existing) {
|
|
existing.retired = true;
|
|
this.retiredServers.add(existing);
|
|
this.currentServer = null;
|
|
this.cleanupRetiredServers();
|
|
}
|
|
if (this.startPromise) {
|
|
const pending = await this.startPromise;
|
|
pending.retired = true;
|
|
this.retiredServers.add(pending);
|
|
this.currentServer = null;
|
|
this.cleanupRetiredServers();
|
|
}
|
|
}
|
|
|
|
private async startServer(launchEnv?: Record<string, string>): Promise<OpenCodeServerGeneration> {
|
|
const port = await this.portAllocator();
|
|
const url = `http://127.0.0.1:${port}`;
|
|
const launchPrefix = await this.resolveCommandPrefix();
|
|
const serverArgs = [...launchPrefix.args, "serve", "--port", String(port)];
|
|
const serverCwd = os.homedir();
|
|
|
|
const serverProcess = this.spawnServerProcess(launchPrefix.command, serverArgs, {
|
|
cwd: serverCwd,
|
|
detached: process.platform !== "win32",
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
...createProviderEnvSpec({
|
|
runtimeSettings: this.runtimeSettings,
|
|
overlays: [launchEnv],
|
|
}),
|
|
});
|
|
const managedProcessRecord = this.recordManagedServerProcess({
|
|
process: serverProcess,
|
|
command: launchPrefix.command,
|
|
args: serverArgs,
|
|
port,
|
|
});
|
|
const server: OpenCodeServerGeneration = {
|
|
process: serverProcess,
|
|
port,
|
|
url,
|
|
refCount: 0,
|
|
retired: false,
|
|
ready: Promise.resolve(),
|
|
managedProcessRecord,
|
|
};
|
|
void managedProcessRecord.then((record) => {
|
|
if (record && server.managedProcessRecord === managedProcessRecord) {
|
|
server.managedProcessId = record.id;
|
|
}
|
|
return undefined;
|
|
});
|
|
|
|
let started = false;
|
|
let settled = false;
|
|
let stderrBuffer = "";
|
|
let stdoutBuffer = "";
|
|
const STARTUP_BUFFER_CAP = 8192;
|
|
const appendCapped = (current: string, chunk: string): string => {
|
|
if (current.length >= STARTUP_BUFFER_CAP) {
|
|
return current;
|
|
}
|
|
const remaining = STARTUP_BUFFER_CAP - current.length;
|
|
return current + chunk.slice(0, remaining);
|
|
};
|
|
const buildStartupErrorMessage = (headline: string): string => {
|
|
const sections = [headline];
|
|
const stderrTrimmed = stderrBuffer.trim();
|
|
if (stderrTrimmed.length > 0) {
|
|
sections.push(`stderr: ${stderrTrimmed}`);
|
|
}
|
|
const stdoutTrimmed = stdoutBuffer.trim();
|
|
if (stdoutTrimmed.length > 0) {
|
|
sections.push(`stdout: ${stdoutTrimmed}`);
|
|
}
|
|
return sections.join("\n");
|
|
};
|
|
|
|
const ready = new Promise<void>((resolve, reject) => {
|
|
let timeout: ReturnType<typeof setTimeout>;
|
|
const failStartup = (error: Error) => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
settled = true;
|
|
clearTimeout(timeout);
|
|
reject(error);
|
|
};
|
|
timeout = setTimeout(() => {
|
|
if (!started) {
|
|
failStartup(new Error(buildStartupErrorMessage("OpenCode server startup timeout")));
|
|
}
|
|
}, 30_000);
|
|
|
|
serverProcess.stdout?.on("data", (data: Buffer) => {
|
|
const output = data.toString();
|
|
stdoutBuffer = appendCapped(stdoutBuffer, output);
|
|
if (output.includes("listening on") && !settled) {
|
|
started = true;
|
|
settled = true;
|
|
clearTimeout(timeout);
|
|
resolve();
|
|
}
|
|
});
|
|
|
|
serverProcess.stderr?.on("data", (data: Buffer) => {
|
|
const output = data.toString();
|
|
stderrBuffer = appendCapped(stderrBuffer, output);
|
|
this.logger.error({ stderr: output.trim() }, "OpenCode server stderr");
|
|
});
|
|
|
|
serverProcess.on("error", (error) => {
|
|
const headline = error instanceof Error ? error.message : String(error);
|
|
failStartup(new Error(buildStartupErrorMessage(headline)));
|
|
});
|
|
|
|
serverProcess.on("exit", (code) => {
|
|
this.removeManagedServerRecord(server);
|
|
if (!started) {
|
|
failStartup(
|
|
new Error(buildStartupErrorMessage(`OpenCode server exited with code ${code}`)),
|
|
);
|
|
}
|
|
if (this.currentServer?.process === serverProcess) {
|
|
this.currentServer = null;
|
|
}
|
|
for (const retired of Array.from(this.retiredServers)) {
|
|
if (retired.process === serverProcess) {
|
|
this.retiredServers.delete(retired);
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
server.ready = ready.catch(async (error) => {
|
|
await this.killServer(server);
|
|
if (this.currentServer === server) {
|
|
this.currentServer = null;
|
|
}
|
|
this.retiredServers.delete(server);
|
|
throw error;
|
|
});
|
|
|
|
return server;
|
|
}
|
|
|
|
async shutdown(): Promise<void> {
|
|
const servers = [
|
|
...(this.currentServer ? [this.currentServer] : []),
|
|
...Array.from(this.retiredServers),
|
|
];
|
|
await Promise.all(servers.map((server) => this.killServer(server)));
|
|
this.currentServer = null;
|
|
this.retiredServers.clear();
|
|
}
|
|
|
|
private cleanupRetiredServers(): void {
|
|
for (const server of Array.from(this.retiredServers)) {
|
|
if (server.refCount === 0) {
|
|
this.retiredServers.delete(server);
|
|
void this.killServer(server);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async killServer(server: OpenCodeServerGeneration): Promise<void> {
|
|
if (
|
|
(server.process.exitCode !== null && server.process.exitCode !== undefined) ||
|
|
(server.process.signalCode !== null && server.process.signalCode !== undefined)
|
|
) {
|
|
return;
|
|
}
|
|
const result = await this.terminateProcess(server.process, {
|
|
gracefulTimeoutMs: OPENCODE_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_MS,
|
|
forceTimeoutMs: OPENCODE_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS,
|
|
onForceSignal: () => {
|
|
this.logger.warn(
|
|
{ timeoutMs: OPENCODE_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT_MS },
|
|
"OpenCode server did not exit after SIGTERM; sending SIGKILL",
|
|
);
|
|
},
|
|
});
|
|
if (result === "kill-timeout") {
|
|
this.logger.warn(
|
|
{ timeoutMs: OPENCODE_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS },
|
|
"OpenCode server did not report exit after SIGKILL",
|
|
);
|
|
}
|
|
if (server.managedProcessId) {
|
|
await this.removeManagedProcessId(server.managedProcessId);
|
|
server.managedProcessId = undefined;
|
|
server.managedProcessRecord = undefined;
|
|
} else {
|
|
this.removeManagedServerRecord(server);
|
|
}
|
|
}
|
|
|
|
private async recordManagedServerProcess(options: {
|
|
process: ChildProcess;
|
|
command: string;
|
|
args: string[];
|
|
port: number;
|
|
}): Promise<{ id: string } | null> {
|
|
const pid = options.process.pid;
|
|
if (!this.managedProcesses || typeof pid !== "number" || pid <= 0) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
return await this.managedProcesses.record({
|
|
owner: { provider: "opencode", kind: "helper-server" },
|
|
pid,
|
|
command: options.command,
|
|
args: options.args,
|
|
metadata: { port: options.port },
|
|
});
|
|
} catch (error) {
|
|
this.logger.warn(
|
|
{ err: error, pid, port: options.port },
|
|
"Failed to record OpenCode helper process",
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private removeManagedProcessRecordWhenResolved(record: Promise<{ id: string } | null>): void {
|
|
void record.then((resolved) => {
|
|
if (resolved) {
|
|
return this.removeManagedProcessId(resolved.id);
|
|
}
|
|
return undefined;
|
|
});
|
|
}
|
|
|
|
private removeManagedServerRecord(server: OpenCodeServerGeneration): void {
|
|
const record = server.managedProcessRecord;
|
|
server.managedProcessRecord = undefined;
|
|
if (server.managedProcessId) {
|
|
void this.removeManagedProcessId(server.managedProcessId);
|
|
server.managedProcessId = undefined;
|
|
return;
|
|
}
|
|
if (record) {
|
|
this.removeManagedProcessRecordWhenResolved(record);
|
|
}
|
|
}
|
|
|
|
private async removeManagedProcessId(id: string): Promise<void> {
|
|
try {
|
|
await this.managedProcesses?.remove(id);
|
|
} catch (error) {
|
|
this.logger.warn({ err: error, id }, "Failed to remove OpenCode helper process record");
|
|
}
|
|
}
|
|
}
|
|
|
|
async function resolveOpenCodeBinary(): Promise<string> {
|
|
const found = await findExecutable("opencode");
|
|
if (found) {
|
|
return found;
|
|
}
|
|
throw new Error(
|
|
"OpenCode binary not found. Install OpenCode (https://github.com/opencode-ai/opencode) and ensure it is available in your shell PATH.",
|
|
);
|
|
}
|
|
|
|
function findAvailablePort(): Promise<number> {
|
|
return new Promise((resolve, reject) => {
|
|
const server = net.createServer();
|
|
server.listen(0, "127.0.0.1", () => {
|
|
const address = server.address();
|
|
server.close(() => {
|
|
if (typeof address === "object" && address) {
|
|
resolve(address.port);
|
|
} else {
|
|
reject(new Error("Failed to allocate port"));
|
|
}
|
|
});
|
|
});
|
|
server.on("error", reject);
|
|
});
|
|
}
|