diff --git a/docs/agent-lifecycle.md b/docs/agent-lifecycle.md index 78a985574..4b550c972 100644 --- a/docs/agent-lifecycle.md +++ b/docs/agent-lifecycle.md @@ -24,6 +24,15 @@ primed. Idle agents remain resident indefinitely. Runtime closure happens only through an explicit lifecycle action such as archive, replacement, reload, workspace teardown, or daemon shutdown. +A provider runtime can still die on its own — crash, OOM kill, host suspend. Work the agent parked +inside that process dies with it: Claude Code's background Bash shells, `Monitor` watches, and +workflows all live in the CLI process, and the completion notification that would have woken the +agent never arrives. A runtime that dies mid-turn is reported by whatever is draining its stream, but +between turns nothing is watching, so the agent sits at `idle` looking healthy while its background +work is gone. Report that exit as a turn failure so the agent lands in `error` with a timeline entry. +Only the Claude provider does this today; the others still report a death only when a turn happens to +be in flight. + ### Cancellation Cancellation changes lifecycle state only after the provider acknowledges the interrupt or emits a terminal turn event. If the interrupt is rejected or times out, the agent remains `running` with its active foreground turn intact. Follow-up actions such as replacement, reload, rewind, and Stop must report that failure instead of accepting work they cannot perform. Synthesizing a local cancellation without provider acknowledgment creates a split-brain session: Paseo accepts a new prompt while the provider still owns the previous foreground turn. diff --git a/packages/server/src/server/agent/providers/claude/agent.runtime-exit.test.ts b/packages/server/src/server/agent/providers/claude/agent.runtime-exit.test.ts new file mode 100644 index 000000000..859c1859c --- /dev/null +++ b/packages/server/src/server/agent/providers/claude/agent.runtime-exit.test.ts @@ -0,0 +1,281 @@ +import { EventEmitter } from "node:events"; +import type { ChildProcess } from "node:child_process"; +import type { + Options, + Query, + SpawnOptions as ClaudeSpawnOptions, +} from "@anthropic-ai/claude-agent-sdk"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { createTestLogger } from "../../../../test-utils/test-logger.js"; +import * as spawnUtils from "../../../../utils/spawn.js"; +import type { AgentStreamEvent } from "../../agent-sdk-types.js"; +import { ClaudeAgentClient } from "./agent.js"; +import type { ClaudeQueryInput } from "./query.js"; + +interface QueryMockOptions { + // Runs while the session retires this query, modelling a process that dies as + // part of the retirement handshake. + onReturn?: () => void; + // Awaited once the scripted events run out, so the query stays open the way a + // real one does. Resolving it with a value delivers one more event. + tail?: Promise; +} + +function createQueryMock(events: unknown[], options: QueryMockOptions = {}): Query { + let index = 0; + return { + next: vi.fn(async () => { + if (index < events.length) { + return { done: false, value: events[index++] }; + } + const late = await options.tail; + if (late !== undefined) { + options.tail = undefined; + return { done: false, value: late }; + } + return { done: true, value: undefined }; + }), + return: vi.fn(async () => { + options.onReturn?.(); + return { done: true, value: undefined }; + }), + interrupt: vi.fn(async () => undefined), + close: vi.fn(() => undefined), + setPermissionMode: vi.fn(async () => undefined), + setModel: vi.fn(async () => undefined), + supportedModels: vi.fn(async () => [{ value: "opus", displayName: "Opus" }]), + supportedCommands: vi.fn(async () => []), + rewindFiles: vi.fn(async () => ({ canRewind: true })), + [Symbol.asyncIterator]() { + return this; + }, + } as Query; +} + +function createChildProcessStub(): ChildProcess & { killSignals: (NodeJS.Signals | number)[] } { + const child = new EventEmitter() as ChildProcess & { + killSignals: (NodeJS.Signals | number)[]; + }; + child.stderr = new EventEmitter() as ChildProcess["stderr"]; + child.killSignals = []; + // A real child dies when signalled; without this the teardown path waits out + // its full graceful + force timeout on every test. + child.kill = ((signal?: NodeJS.Signals | number) => { + child.killSignals.push(signal ?? "SIGTERM"); + child.emit("exit", null, typeof signal === "string" ? signal : "SIGTERM"); + return true; + }) as ChildProcess["kill"]; + return child; +} + +const COMPLETED_TURN_EVENTS = [ + { + type: "system", + subtype: "init", + session_id: "claude-runtime-exit-session", + permissionMode: "default", + model: "opus", + }, + { type: "assistant", message: { content: "SPAWNED" } }, + { + type: "result", + subtype: "success", + usage: { input_tokens: 1, cache_read_input_tokens: 0, output_tokens: 1 }, + total_cost_usd: 0, + }, +]; + +const MISSING_RESUMED_CONVERSATION_RESULT = { + type: "result", + subtype: "error_during_execution", + errors: ["No conversation found with session ID: claude-runtime-exit-session"], +}; + +const SPAWN_OPTIONS: ClaudeSpawnOptions = { + command: "node", + args: ["claude.js"], + cwd: process.cwd(), + env: {}, + signal: new AbortController().signal, +}; + +describe("Claude runtime exit", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + test("reports a turn failure when the process dies while the session is idle", async () => { + let capturedOptions: Options | undefined; + const queryFactory = vi.fn(({ options }: ClaudeQueryInput) => { + capturedOptions = options; + return createQueryMock(COMPLETED_TURN_EVENTS); + }); + const child = createChildProcessStub(); + vi.spyOn(spawnUtils, "spawnProcess").mockReturnValue(child); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + queryFactory, + resolveBinary: async () => "/test/claude/bin", + }); + const session = await client.createSession({ provider: "claude", cwd: process.cwd() }); + + try { + await session.run("start a background shell"); + capturedOptions?.spawnClaudeCodeProcess?.(SPAWN_OPTIONS); + + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + child.emit("exit", 1, null); + + const failure = events.find((event) => event.type === "turn_failed"); + expect(failure).toBeDefined(); + expect(failure && "error" in failure ? failure.error : "").toContain("background shells"); + } finally { + await session.close(); + } + }); + + test("stays quiet when the process exits during an intentional teardown", async () => { + let capturedOptions: Options | undefined; + const queryFactory = vi.fn(({ options }: ClaudeQueryInput) => { + capturedOptions = options; + return createQueryMock(COMPLETED_TURN_EVENTS); + }); + const child = createChildProcessStub(); + vi.spyOn(spawnUtils, "spawnProcess").mockReturnValue(child); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + queryFactory, + resolveBinary: async () => "/test/claude/bin", + }); + const session = await client.createSession({ provider: "claude", cwd: process.cwd() }); + + await session.run("start a background shell"); + capturedOptions?.spawnClaudeCodeProcess?.(SPAWN_OPTIONS); + + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + await session.close(); + child.emit("exit", 0, null); + + expect(events.some((event) => event.type === "turn_failed")).toBe(false); + }); + + test("stays quiet when a query restart retires the process", async () => { + let capturedOptions: Options | undefined; + const child = createChildProcessStub(); + const queryFactory = vi.fn(({ options }: ClaudeQueryInput) => { + capturedOptions = options; + // A real query stays open between turns, so hold it open; the process + // dies when the session retires it, as ending its stdin does in practice. + return createQueryMock(COMPLETED_TURN_EVENTS, { + tail: new Promise(() => undefined), + onReturn: () => child.emit("exit", 0, null), + }); + }); + vi.spyOn(spawnUtils, "spawnProcess").mockReturnValue(child); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + queryFactory, + resolveBinary: async () => "/test/claude/bin", + }); + const session = await client.createSession({ provider: "claude", cwd: process.cwd() }); + + try { + await session.run("first turn"); + capturedOptions?.spawnClaudeCodeProcess?.(SPAWN_OPTIONS); + + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + // Restarts the query on the next call, which retires the current process + // while no turn is running. + await session.setThinkingOption(null); + await session.listCommands(); + + expect(events.some((event) => event.type === "turn_failed")).toBe(false); + } finally { + await session.close(); + } + }); + + test("tree-kills the retired process when the resumed conversation is gone", async () => { + let capturedOptions: Options | undefined; + let deliverMissingConversation: ((event: unknown) => void) | undefined; + // A single query throughout, so the only path that can kill the child is the + // missing-conversation recovery — not the restart path in ensureQuery(). + const queryFactory = vi.fn(({ options }: ClaudeQueryInput) => { + capturedOptions = options; + return createQueryMock(COMPLETED_TURN_EVENTS, { + tail: new Promise((resolve) => { + deliverMissingConversation = resolve; + }), + }); + }); + const child = createChildProcessStub(); + vi.spyOn(spawnUtils, "spawnProcess").mockReturnValue(child); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + queryFactory, + resolveBinary: async () => "/test/claude/bin", + }); + const session = await client.createSession({ provider: "claude", cwd: process.cwd() }); + + try { + // Establishes the claude session id the next result fails to resume. + await session.run("first turn"); + capturedOptions?.spawnClaudeCodeProcess?.(SPAWN_OPTIONS); + + deliverMissingConversation?.(MISSING_RESUMED_CONVERSATION_RESULT); + + // MCP children of the retired process outlive it unless the tree is killed. + await vi.waitFor(() => expect(child.killSignals.length).toBeGreaterThan(0)); + expect(queryFactory).toHaveBeenCalledTimes(1); + } finally { + await session.close(); + } + }); + + test("leaves a mid-turn crash to the query pump", async () => { + let rejectStream: ((error: Error) => void) | undefined; + const tail = new Promise((_resolve, reject) => { + rejectStream = reject; + }); + let capturedOptions: Options | undefined; + const queryFactory = vi.fn(({ options }: ClaudeQueryInput) => { + capturedOptions = options; + return createQueryMock(COMPLETED_TURN_EVENTS.slice(0, 2), { tail }); + }); + const child = createChildProcessStub(); + vi.spyOn(spawnUtils, "spawnProcess").mockReturnValue(child); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + queryFactory, + resolveBinary: async () => "/test/claude/bin", + }); + const session = await client.createSession({ provider: "claude", cwd: process.cwd() }); + + try { + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + const turn = session.run("start a background shell"); + await vi.waitFor(() => expect(capturedOptions?.spawnClaudeCodeProcess).toBeDefined()); + capturedOptions?.spawnClaudeCodeProcess?.(SPAWN_OPTIONS); + + child.emit("exit", 1, null); + rejectStream?.(new Error("Claude Code process exited with code 1")); + await expect(turn).rejects.toThrow("exited with code 1"); + + const failures = events.filter((event) => event.type === "turn_failed"); + expect(failures).toHaveLength(1); + expect(failures[0] && "error" in failures[0] ? failures[0].error : "").toContain( + "exited with code 1", + ); + } finally { + await session.close(); + } + }); +}); diff --git a/packages/server/src/server/agent/providers/claude/agent.ts b/packages/server/src/server/agent/providers/claude/agent.ts index ab7a0cf1f..bc79c586c 100644 --- a/packages/server/src/server/agent/providers/claude/agent.ts +++ b/packages/server/src/server/agent/providers/claude/agent.ts @@ -2904,6 +2904,10 @@ class ClaudeAgentSession implements AgentSession { this.input = null; this.queryPumpPromise = null; this.queryRestartNeeded = false; + // Ending the input retires the process on purpose. Detach first so its + // exit is not reported as a crash. + const retiredChild = this.childProcess; + this.childProcess = null; oldInput?.end(); oldQuery.close?.(); try { @@ -2914,14 +2918,13 @@ class ClaudeAgentSession implements AgentSession { // Tree-kill the old process tree now that the SDK has cleaned up. // If we skip this, MCP children of the previous claude process can // survive as orphans when the session spawns a replacement query. - if (this.childProcess) { - await terminateWithTreeKill(this.childProcess, { + if (retiredChild) { + await terminateWithTreeKill(retiredChild, { gracefulTimeoutMs: 2_000, forceTimeoutMs: 2_000, }).catch(() => { /* process may already be dead */ }); - this.childProcess = null; } } @@ -2941,6 +2944,7 @@ class ClaudeAgentSession implements AgentSession { queryFactory: this.queryFactory, onChildProcess: (child) => { this.childProcess = child; + child.once("exit", (code, signal) => this.handleRuntimeExit(child, code, signal)); }, }, ); @@ -3429,6 +3433,41 @@ class ClaudeAgentSession implements AgentSession { } } + // Background Bash shells, Monitor watches and workflows live inside the Claude + // Code process, so they die with it. When the process exits mid-turn the query + // pump reports it, but when it exits between turns nothing observes the death: + // the agent stays "idle" and the wake-back that would have continued the work + // never arrives. + private handleRuntimeExit( + child: ChildProcess, + code: number | null, + signal: NodeJS.Signals | null, + ): void { + if (this.closed || this.childProcess !== child) { + return; + } + this.childProcess = null; + this.logger.warn( + { agentId: this.agentId, pid: child.pid, code, signal }, + "Claude runtime exited unexpectedly", + ); + if (this.activeForegroundTurnId || this.autonomousTurn) { + // The pump is about to throw. It waits for stderr to flush and reports the + // real cause; reporting here first would replace that with a bare exit code + // and detach this.query, which is how the pump recognizes its own stream. + return; + } + // Drop the dead handles so the next write spawns a fresh process instead of + // failing against a dead transport forever. + this.query = null; + this.input = null; + this.dispatchEvents([ + this.buildTurnFailedEvent( + `Claude stopped unexpectedly (${signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`}). Any background shells, monitors or other work it had running were terminated with it.`, + ), + ]); + } + private startQueryPump(): void { if (this.closed || this.queryPumpPromise) { return; @@ -3683,11 +3722,25 @@ class ClaudeAgentSession implements AgentSession { ); this.failActiveTurns(staleResumeError); + // Ending the input retires the process on purpose. Detach first so its exit + // is not reported as a crash. + const retiredChild = this.childProcess; + this.childProcess = null; this.input?.end(); await this.awaitWithTimeout( activeQuery.return?.(), "query pump return on missing resumed conversation", ); + // Tree-kill for the same reason the restart path does: MCP children of the + // retired claude process outlive it otherwise. + if (retiredChild) { + await terminateWithTreeKill(retiredChild, { + gracefulTimeoutMs: 2_000, + forceTimeoutMs: 2_000, + }).catch(() => { + /* process may already be dead */ + }); + } if (this.query === activeQuery) { this.query = null; this.input = null;