feat(pi-workflow): improve workflow usability and recovery
This commit is contained in:
@@ -29,6 +29,7 @@ import {
|
||||
type HostCommandExecutor,
|
||||
type StartWorkflowCommand,
|
||||
type WorkflowCommand,
|
||||
type WorkflowView,
|
||||
type WorkspaceManager,
|
||||
} from "./engine.ts";
|
||||
import { loadAgentProfiles } from "./profiles.ts";
|
||||
@@ -113,10 +114,12 @@ class FakeWorkspace implements WorkspaceManager {
|
||||
branchExists = false;
|
||||
createError?: Error;
|
||||
reconcileState: WorkflowRun["workspace"]["state"] = "ready";
|
||||
beforePrepare?: () => Promise<void>;
|
||||
beforeCreate?: (plan: WorkspacePlan) => Promise<void>;
|
||||
|
||||
async prepare(options: PrepareWorkspaceOptions): Promise<WorkspacePlan> {
|
||||
this.prepareCalls++;
|
||||
await this.beforePrepare?.();
|
||||
return {
|
||||
sourceRoot: options.sourceRoot,
|
||||
worktreeRoot: path.join(options.sourceRoot, options.configuredRoot),
|
||||
@@ -128,6 +131,9 @@ class FakeWorkspace implements WorkspaceManager {
|
||||
branch: `feature/${options.featureName}`,
|
||||
baseHead: "abc123",
|
||||
branchExists: this.branchExists,
|
||||
sourceChanges: [],
|
||||
sourceChangesOmitted: 0,
|
||||
dirtySourceAllowed: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -316,6 +322,449 @@ function design(uiImpact: "none" | "minor" | "non-trivial" = "minor") {
|
||||
};
|
||||
}
|
||||
|
||||
async function createClaimedContextHandoff(
|
||||
setup: Fixture,
|
||||
): Promise<WorkflowView> {
|
||||
setup.executor.enqueue("discover", {
|
||||
...discoveryWithQuestions(),
|
||||
questions: [],
|
||||
});
|
||||
setup.executor.enqueue("design", design("minor"));
|
||||
const planned = await setup.engine().execute(startCommand());
|
||||
const approved = await setup.engine().execute({
|
||||
action: "decide",
|
||||
runId: planned.runId,
|
||||
gateId: planned.openGate?.id ?? "missing",
|
||||
expectedRevision: planned.revision,
|
||||
decision: {
|
||||
kind: "approve-plan",
|
||||
approved: true,
|
||||
acceptedUiImpact: "minor",
|
||||
},
|
||||
idempotencyKey: "approve-claimed-context",
|
||||
});
|
||||
const selected = await setup.engine().execute({
|
||||
action: "decide",
|
||||
runId: approved.runId,
|
||||
gateId: approved.openGate?.id ?? "missing",
|
||||
expectedRevision: approved.revision,
|
||||
decision: {
|
||||
kind: "select-implementation-context",
|
||||
strategy: "fresh-session",
|
||||
},
|
||||
idempotencyKey: "select-claimed-context",
|
||||
});
|
||||
const intent = selected.implementationContext;
|
||||
assert.ok(intent);
|
||||
const continued = await setup.engine().execute({
|
||||
action: "continue-context",
|
||||
runId: selected.runId,
|
||||
gateId: intent.gateId,
|
||||
preparationId: intent.preparationId,
|
||||
planRevision: intent.planRevision,
|
||||
contextRevision: intent.runRevision,
|
||||
expectedRevision: selected.revision,
|
||||
sourceSessionFile: "/sessions/source.jsonl",
|
||||
});
|
||||
const pending = continued.implementationContext;
|
||||
assert.ok(pending);
|
||||
return setup.engine().execute({
|
||||
action: "claim-context-handoff",
|
||||
runId: continued.runId,
|
||||
gateId: pending.gateId,
|
||||
preparationId: pending.preparationId,
|
||||
planRevision: pending.planRevision,
|
||||
contextRevision: pending.runRevision,
|
||||
expectedRevision: continued.revision,
|
||||
sourceSessionFile: "/sessions/source.jsonl",
|
||||
targetSessionFile: "/sessions/target.jsonl",
|
||||
reason: "new",
|
||||
});
|
||||
}
|
||||
|
||||
test("non-trivial UI design fails with the exact missing-prototype validation error", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
setup.executor.enqueue("discover", {
|
||||
...discoveryWithQuestions(),
|
||||
questions: [],
|
||||
});
|
||||
const missingPrototype = design("non-trivial");
|
||||
missingPrototype.assignments = missingPrototype.assignments.filter(
|
||||
(assignment) => assignment.role !== "prototype",
|
||||
);
|
||||
setup.executor.enqueue("design", missingPrototype);
|
||||
|
||||
const failed = await setup.engine().execute(startCommand());
|
||||
assert.equal(failed.status, "failed");
|
||||
assert.equal(failed.error?.code, "invalid-design-output");
|
||||
assert.equal(
|
||||
failed.error?.message,
|
||||
"Design must select exactly one prototype Assignment",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("feature designer contract requires the non-trivial prototype assignment", async () => {
|
||||
const instructions = await readFile(
|
||||
new URL("./agents/feature-designer.md", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(
|
||||
instructions,
|
||||
/For `non-trivial` UI impact, also emit exactly one eligible `prototype` assignment/,
|
||||
);
|
||||
assert.match(instructions, /owned paths isolate prototype artifacts/);
|
||||
assert.match(instructions, /one eligible `implement` assignment/);
|
||||
assert.match(instructions, /one eligible `verify` assignment/);
|
||||
assert.match(instructions, /one eligible independent `review` assignment/);
|
||||
});
|
||||
|
||||
test("design Assignment execution times out with persisted evidence", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
setup.executor.enqueue("discover", {
|
||||
...discoveryWithQuestions(),
|
||||
questions: [],
|
||||
});
|
||||
const blocking = new BlockingExecutor(design("minor"));
|
||||
const operation = setup.engine("session-1", {
|
||||
executor: {
|
||||
async run(request) {
|
||||
if (request.role === "discover") {
|
||||
return setup.executor.run(request);
|
||||
}
|
||||
return blocking.run(request);
|
||||
},
|
||||
},
|
||||
assignmentExecutionTimeoutMs: 20,
|
||||
assignmentAbortTimeoutMs: 5,
|
||||
}).execute(startCommand());
|
||||
await blocking.started;
|
||||
const failed = await operation;
|
||||
blocking.release();
|
||||
|
||||
assert.equal(failed.status, "failed");
|
||||
assert.equal(failed.error?.code, "assignment-timeout");
|
||||
assert.match(failed.error?.message ?? "", /Assignment design timed out after 20ms/);
|
||||
const persisted = await setup.store.loadRun(failed.runId);
|
||||
const designAssignment = persisted.assignments.find(
|
||||
(assignment) => assignment.role === "design",
|
||||
);
|
||||
assert.equal(designAssignment?.status, "interrupted");
|
||||
assert.match(designAssignment?.error ?? "", /timed out after 20ms/);
|
||||
});
|
||||
});
|
||||
|
||||
test("executor rejection clears its deadline before transcript persistence", async (context) => {
|
||||
await withTempDir(async (directory) => {
|
||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||
const setup = await fixture(directory);
|
||||
let markStarted = () => {};
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
let rejectExecution = (_error: Error) => {};
|
||||
const execution = new Promise<never>((_resolve, reject) => {
|
||||
rejectExecution = reject;
|
||||
});
|
||||
const executor: AssignmentExecutor = {
|
||||
run() {
|
||||
markStarted();
|
||||
return execution;
|
||||
},
|
||||
};
|
||||
const saveTranscript = setup.store.saveAssignmentTranscript.bind(setup.store);
|
||||
let markTranscriptStarted = () => {};
|
||||
const transcriptStarted = new Promise<void>((resolve) => {
|
||||
markTranscriptStarted = resolve;
|
||||
});
|
||||
let finishTranscript = () => {};
|
||||
const transcriptMayFinish = new Promise<void>((resolve) => {
|
||||
finishTranscript = resolve;
|
||||
});
|
||||
setup.store.saveAssignmentTranscript = async (
|
||||
runId,
|
||||
assignmentId,
|
||||
transcript,
|
||||
) => {
|
||||
markTranscriptStarted();
|
||||
await transcriptMayFinish;
|
||||
return saveTranscript(runId, assignmentId, transcript);
|
||||
};
|
||||
|
||||
const operation = setup.engine("session-1", {
|
||||
executor,
|
||||
assignmentExecutionTimeoutMs: 10,
|
||||
}).execute(startCommand());
|
||||
await started;
|
||||
context.mock.timers.tick(9);
|
||||
rejectExecution(Object.assign(new Error("provider rejected near deadline"), {
|
||||
assignmentMetadata: {
|
||||
transcript: [{ role: "assistant", text: "partial response" }],
|
||||
},
|
||||
}));
|
||||
await transcriptStarted;
|
||||
context.mock.timers.tick(1);
|
||||
finishTranscript();
|
||||
|
||||
const failed = await operation;
|
||||
assert.equal(failed.status, "failed");
|
||||
assert.equal(failed.error?.code, "assignment-failed");
|
||||
assert.match(failed.error?.message ?? "", /provider rejected near deadline/);
|
||||
assert.equal(failed.assignments[0]?.status, "failed");
|
||||
});
|
||||
});
|
||||
|
||||
test("assignment timeout retains its mutation fence until the executor settles", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
setup.executor.enqueue("discover", {
|
||||
...discoveryWithQuestions(),
|
||||
questions: [],
|
||||
});
|
||||
setup.executor.enqueue("design", design("minor"));
|
||||
const blocking = new BlockingExecutor({
|
||||
summary: "late implementation",
|
||||
changedPaths: ["src/index.ts"],
|
||||
focusedTests: ["late behavior"],
|
||||
});
|
||||
const executor: AssignmentExecutor = {
|
||||
async run(request) {
|
||||
return request.role === "implement"
|
||||
? blocking.run(request)
|
||||
: setup.executor.run(request);
|
||||
},
|
||||
};
|
||||
const engine = setup.engine("session-1", {
|
||||
executor,
|
||||
assignmentExecutionTimeoutMs: 20,
|
||||
assignmentAbortTimeoutMs: 5,
|
||||
});
|
||||
const planned = await engine.execute(startCommand());
|
||||
const approved = await engine.execute({
|
||||
action: "decide",
|
||||
runId: planned.runId,
|
||||
gateId: planned.openGate?.id ?? "missing",
|
||||
expectedRevision: planned.revision,
|
||||
decision: {
|
||||
kind: "approve-plan",
|
||||
approved: true,
|
||||
acceptedUiImpact: "minor",
|
||||
},
|
||||
idempotencyKey: "approve-timeout-plan",
|
||||
});
|
||||
const operation = engine.execute({
|
||||
action: "decide",
|
||||
runId: approved.runId,
|
||||
gateId: approved.openGate?.id ?? "missing",
|
||||
expectedRevision: approved.revision,
|
||||
decision: {
|
||||
kind: "select-implementation-context",
|
||||
strategy: "current-session",
|
||||
},
|
||||
idempotencyKey: "start-timeout-implementation",
|
||||
});
|
||||
await blocking.started;
|
||||
const timedOut = await operation;
|
||||
assert.equal(timedOut.status, "waiting");
|
||||
assert.equal(timedOut.openGate?.kind, "recovery");
|
||||
assert.equal(timedOut.error?.code, "mutation-recovery-required");
|
||||
const interrupted = timedOut.assignments.find(
|
||||
(assignment) => assignment.role === "implement",
|
||||
);
|
||||
assert.equal(interrupted?.status, "interrupted");
|
||||
assert.ok(interrupted?.executionToken);
|
||||
assert.ok(
|
||||
await setup.store.loadAssignmentLease(
|
||||
timedOut.runId,
|
||||
interrupted?.id ?? "missing",
|
||||
),
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
engine.execute({
|
||||
action: "decide",
|
||||
runId: timedOut.runId,
|
||||
gateId: timedOut.openGate?.id ?? "missing",
|
||||
expectedRevision: timedOut.revision,
|
||||
decision: { kind: "recover", resolution: "retry" },
|
||||
idempotencyKey: "recover-before-timeout-settlement",
|
||||
}),
|
||||
/still active/,
|
||||
);
|
||||
|
||||
blocking.release();
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
const lease = await setup.store.loadAssignmentLease(
|
||||
timedOut.runId,
|
||||
interrupted?.id ?? "missing",
|
||||
);
|
||||
if (!lease) break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
}
|
||||
assert.equal(
|
||||
await setup.store.loadAssignmentLease(
|
||||
timedOut.runId,
|
||||
interrupted?.id ?? "missing",
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
|
||||
setup.executor.enqueue("remediate", {
|
||||
summary: "recovered implementation",
|
||||
changedPaths: ["src/index.ts"],
|
||||
focusedTests: ["recovered behavior"],
|
||||
});
|
||||
setup.executor.enqueue("verify", {
|
||||
succeeded: true,
|
||||
focusedBehavioralTests: true,
|
||||
summary: "verified after settlement",
|
||||
});
|
||||
setup.executor.enqueue("review", {
|
||||
specification: { verdict: "approved", findings: [] },
|
||||
standards: { verdict: "approved", findings: [] },
|
||||
});
|
||||
const recovered = await engine.execute({
|
||||
action: "decide",
|
||||
runId: timedOut.runId,
|
||||
gateId: timedOut.openGate?.id ?? "missing",
|
||||
expectedRevision: timedOut.revision,
|
||||
decision: { kind: "recover", resolution: "retry" },
|
||||
idempotencyKey: "recover-after-timeout-settlement",
|
||||
});
|
||||
assert.equal(recovered.status, "completed", recovered.error?.message);
|
||||
});
|
||||
});
|
||||
|
||||
test("old heartbeat never permits takeover from a live run owner", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
setup.executor.enqueue("discover", {
|
||||
...discoveryWithQuestions(),
|
||||
questions: [],
|
||||
});
|
||||
setup.executor.enqueue("design", design("minor"));
|
||||
const planned = await setup.engine().execute(startCommand());
|
||||
const approved = await setup.engine().execute({
|
||||
action: "decide",
|
||||
runId: planned.runId,
|
||||
gateId: planned.openGate?.id ?? "missing",
|
||||
expectedRevision: planned.revision,
|
||||
decision: {
|
||||
kind: "approve-plan",
|
||||
approved: true,
|
||||
acceptedUiImpact: "minor",
|
||||
},
|
||||
idempotencyKey: "approve-before-live-owner-recovery",
|
||||
});
|
||||
const owned = await setup.store.loadRun(approved.runId);
|
||||
assert.ok(owned.implementationContext);
|
||||
owned.lease = {
|
||||
processId: 901,
|
||||
activeParentSessionId: "session-1",
|
||||
heartbeatAt: 1,
|
||||
worktreeIdentity: owned.workspace.root,
|
||||
};
|
||||
owned.revision++;
|
||||
owned.implementationContext.runRevision = owned.revision;
|
||||
await setup.store.saveRun(owned, owned.revision - 1);
|
||||
|
||||
assert.deepEqual(
|
||||
await setup.engine("session-2", {
|
||||
now: () => 100_000,
|
||||
processId: 902,
|
||||
isProcessAlive: (processId) => processId === 901,
|
||||
}).recover(),
|
||||
[],
|
||||
);
|
||||
const fenced = await setup.store.loadRun(owned.runId);
|
||||
assert.equal(fenced.lease?.activeParentSessionId, "session-1");
|
||||
assert.equal(
|
||||
fenced.implementationContext?.activeParentSessionId,
|
||||
"session-1",
|
||||
);
|
||||
assert.equal(fenced.lease?.processId, 901);
|
||||
assert.equal(fenced.parentSessionIds.includes("session-2"), false);
|
||||
});
|
||||
});
|
||||
|
||||
test("startup recovery fails closed for a fresh live owner and atomically rotates dead ownership", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
setup.executor.enqueue("discover", {
|
||||
...discoveryWithQuestions(),
|
||||
questions: [],
|
||||
});
|
||||
setup.executor.enqueue("design", design("minor"));
|
||||
const planned = await setup.engine().execute(startCommand());
|
||||
const approved = await setup.engine().execute({
|
||||
action: "decide",
|
||||
runId: planned.runId,
|
||||
gateId: planned.openGate!.id,
|
||||
expectedRevision: planned.revision,
|
||||
decision: {
|
||||
kind: "approve-plan",
|
||||
approved: true,
|
||||
acceptedUiImpact: "minor",
|
||||
},
|
||||
idempotencyKey: "approve-before-owner-recovery",
|
||||
});
|
||||
const abandoned = await setup.store.loadRun(approved.runId);
|
||||
assert.ok(abandoned.implementationContext);
|
||||
abandoned.lease = {
|
||||
processId: 901,
|
||||
activeParentSessionId: "session-1",
|
||||
heartbeatAt: 1,
|
||||
worktreeIdentity: abandoned.workspace.root,
|
||||
};
|
||||
abandoned.revision++;
|
||||
abandoned.implementationContext.runRevision = abandoned.revision;
|
||||
await setup.store.saveRun(abandoned, abandoned.revision - 1);
|
||||
|
||||
const liveRecovery = await setup.engine("session-2", {
|
||||
processId: 902,
|
||||
isProcessAlive: (processId) => processId === 901,
|
||||
}).recover();
|
||||
assert.deepEqual(liveRecovery, []);
|
||||
const stillOwned = await setup.store.loadRun(abandoned.runId);
|
||||
assert.equal(stillOwned.lease?.activeParentSessionId, "session-1");
|
||||
assert.equal(
|
||||
stillOwned.implementationContext?.activeParentSessionId,
|
||||
"session-1",
|
||||
);
|
||||
|
||||
await setup.engine("session-2", {
|
||||
processId: 902,
|
||||
isProcessAlive: () => false,
|
||||
}).recover();
|
||||
const adopted = await setup.store.loadRun(abandoned.runId);
|
||||
assert.equal(adopted.lease?.activeParentSessionId, "session-2");
|
||||
assert.equal(
|
||||
adopted.implementationContext?.activeParentSessionId,
|
||||
"session-2",
|
||||
);
|
||||
assert.equal(adopted.lease?.processId, 902);
|
||||
assert.ok(adopted.parentSessionIds.includes("session-2"));
|
||||
|
||||
adopted.lease = undefined;
|
||||
adopted.revision++;
|
||||
adopted.implementationContext!.runRevision = adopted.revision;
|
||||
await setup.store.saveRun(adopted, adopted.revision - 1);
|
||||
await setup.engine("session-3", {
|
||||
processId: 903,
|
||||
isProcessAlive: () => false,
|
||||
}).recover();
|
||||
const adoptedFromAbsent = await setup.store.loadRun(abandoned.runId);
|
||||
assert.equal(adoptedFromAbsent.lease?.activeParentSessionId, "session-3");
|
||||
assert.equal(
|
||||
adoptedFromAbsent.implementationContext?.activeParentSessionId,
|
||||
"session-3",
|
||||
);
|
||||
assert.equal(adoptedFromAbsent.lease?.processId, 903);
|
||||
});
|
||||
});
|
||||
|
||||
test("analysis scripts run persisted read-only Assignments with tool intersection", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
@@ -1032,6 +1481,98 @@ test("fresh-session handoff is revision-bound and rotates the active parent only
|
||||
});
|
||||
});
|
||||
|
||||
test("expired context recovery rejects another live process without mutation", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
const claimed = await createClaimedContextHandoff(setup);
|
||||
const claim = claimed.implementationContext;
|
||||
assert.ok(claim);
|
||||
const before = await setup.store.loadRun(claimed.runId);
|
||||
|
||||
await assert.rejects(
|
||||
setup.engine("unrelated-session", {
|
||||
now: () => 310_001,
|
||||
processId: 200,
|
||||
isProcessAlive: (processId) => processId === 100,
|
||||
}).execute({
|
||||
action: "recover-context-intent",
|
||||
runId: claimed.runId,
|
||||
gateId: claim.gateId,
|
||||
preparationId: claim.preparationId,
|
||||
planRevision: claim.planRevision,
|
||||
contextRevision: claim.runRevision,
|
||||
expectedRevision: claimed.revision,
|
||||
reason: "expired",
|
||||
}),
|
||||
/owned by live process 100/,
|
||||
);
|
||||
|
||||
assert.deepEqual(await setup.store.loadRun(claimed.runId), before);
|
||||
});
|
||||
});
|
||||
|
||||
test("expired context recovery succeeds when the owning process is dead", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
const claimed = await createClaimedContextHandoff(setup);
|
||||
const claim = claimed.implementationContext;
|
||||
assert.ok(claim);
|
||||
|
||||
const recovered = await setup.engine("replacement-session", {
|
||||
now: () => 310_001,
|
||||
processId: 200,
|
||||
isProcessAlive: () => false,
|
||||
}).execute({
|
||||
action: "recover-context-intent",
|
||||
runId: claimed.runId,
|
||||
gateId: claim.gateId,
|
||||
preparationId: claim.preparationId,
|
||||
planRevision: claim.planRevision,
|
||||
contextRevision: claim.runRevision,
|
||||
expectedRevision: claimed.revision,
|
||||
reason: "expired",
|
||||
});
|
||||
|
||||
assert.equal(recovered.revision, claimed.revision + 1);
|
||||
assert.equal(recovered.status, "waiting");
|
||||
assert.equal(recovered.implementationContext?.status, "handoff-pending");
|
||||
assert.equal(recovered.implementationContext?.sourceSessionFile, undefined);
|
||||
assert.equal(recovered.implementationContext?.targetSessionFile, undefined);
|
||||
assert.match(recovered.implementationContext?.error ?? "", /during expired/);
|
||||
});
|
||||
});
|
||||
|
||||
test("expired context recovery succeeds for the owning process", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
const claimed = await createClaimedContextHandoff(setup);
|
||||
const claim = claimed.implementationContext;
|
||||
assert.ok(claim);
|
||||
|
||||
const recovered = await setup.engine("session-1", {
|
||||
now: () => 310_001,
|
||||
processId: 100,
|
||||
isProcessAlive: (processId) => processId === 100,
|
||||
}).execute({
|
||||
action: "recover-context-intent",
|
||||
runId: claimed.runId,
|
||||
gateId: claim.gateId,
|
||||
preparationId: claim.preparationId,
|
||||
planRevision: claim.planRevision,
|
||||
contextRevision: claim.runRevision,
|
||||
expectedRevision: claimed.revision,
|
||||
reason: "expired",
|
||||
});
|
||||
|
||||
assert.equal(recovered.revision, claimed.revision + 1);
|
||||
assert.equal(recovered.status, "waiting");
|
||||
assert.equal(recovered.implementationContext?.status, "handoff-pending");
|
||||
assert.equal(recovered.implementationContext?.sourceSessionFile, undefined);
|
||||
assert.equal(recovered.implementationContext?.targetSessionFile, undefined);
|
||||
assert.match(recovered.implementationContext?.error ?? "", /during expired/);
|
||||
});
|
||||
});
|
||||
|
||||
test("smart compaction reports are fenced and unavailable alone enables fallback", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
@@ -1190,7 +1731,6 @@ test("recovery marks an interrupted normal compaction intent failed", async () =
|
||||
});
|
||||
const recovered = await setup.engine("session-1", {
|
||||
now: () => 100_000,
|
||||
leaseStaleMs: 1,
|
||||
processId: 200,
|
||||
isProcessAlive: () => false,
|
||||
}).recover();
|
||||
@@ -1602,7 +2142,7 @@ test("Recovery retry reruns failed verification after the remediation limit", as
|
||||
hostCommands,
|
||||
effectiveConfig,
|
||||
processId: 200,
|
||||
isProcessAlive: () => false,
|
||||
isProcessAlive: (processId) => processId === 100,
|
||||
});
|
||||
assert.deepEqual(await transientRecovery.recover(), []);
|
||||
const untouched = await setup.store.loadRun(waiting.runId);
|
||||
@@ -2340,6 +2880,114 @@ test("failed execution metadata and transcripts are persisted", async () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("start cancellation during slow preflight settles before rejecting without publication", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
let markPreparing = () => {};
|
||||
let releasePrepare = () => {};
|
||||
const preparing = new Promise<void>((resolve) => {
|
||||
markPreparing = resolve;
|
||||
});
|
||||
const prepareBlocked = new Promise<void>((resolve) => {
|
||||
releasePrepare = resolve;
|
||||
});
|
||||
setup.workspace.beforePrepare = async () => {
|
||||
markPreparing();
|
||||
await prepareBlocked;
|
||||
};
|
||||
const engine = setup.engine();
|
||||
const operation = engine.execute(startCommand());
|
||||
await preparing;
|
||||
const cancellation = engine.cancelStart(
|
||||
"start-key",
|
||||
"cancel-slow-preflight",
|
||||
new Error("cancelled during slow preflight"),
|
||||
);
|
||||
let cancellationSettled = false;
|
||||
void cancellation.then(() => {
|
||||
cancellationSettled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
assert.equal(cancellationSettled, false);
|
||||
releasePrepare();
|
||||
|
||||
await assert.rejects(operation, /cancelled during slow preflight/);
|
||||
await cancellation;
|
||||
assert.equal(setup.workspace.createCalls, 0);
|
||||
assert.equal(setup.executor.requests.length, 0);
|
||||
assert.deepEqual(await setup.store.listRuns(), []);
|
||||
});
|
||||
});
|
||||
|
||||
test("start cancellation after reservation prevents later run and worktree publication", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
const reserveStart = setup.store.reserveStart.bind(setup.store);
|
||||
let markReserved = () => {};
|
||||
let releaseReservation = () => {};
|
||||
const reserved = new Promise<void>((resolve) => {
|
||||
markReserved = resolve;
|
||||
});
|
||||
const reservationBlocked = new Promise<void>((resolve) => {
|
||||
releaseReservation = resolve;
|
||||
});
|
||||
setup.store.reserveStart = async (key, fingerprint, runId) => {
|
||||
const reservation = await reserveStart(key, fingerprint, runId);
|
||||
markReserved();
|
||||
await reservationBlocked;
|
||||
return reservation;
|
||||
};
|
||||
const engine = setup.engine();
|
||||
const operation = engine.execute(startCommand());
|
||||
await reserved;
|
||||
const cancellation = engine.cancelStart(
|
||||
"start-key",
|
||||
"cancel-reserved-start",
|
||||
new Error("cancelled after reservation"),
|
||||
);
|
||||
let cancellationSettled = false;
|
||||
void cancellation.then(() => {
|
||||
cancellationSettled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
assert.equal(cancellationSettled, false);
|
||||
releaseReservation();
|
||||
|
||||
await assert.rejects(operation, /cancelled after reservation/);
|
||||
await cancellation;
|
||||
assert.equal(setup.workspace.createCalls, 0);
|
||||
assert.equal(setup.executor.requests.length, 0);
|
||||
assert.deepEqual(await setup.store.listRuns(), []);
|
||||
});
|
||||
});
|
||||
|
||||
test("start cancellation after publication durably aborts a settled run", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
setup.executor.enqueue("discover", discoveryWithQuestions());
|
||||
const engine = setup.engine();
|
||||
const started = await engine.execute(startCommand());
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
await engine.cancelStart(
|
||||
"start-key",
|
||||
"cancel-published-start",
|
||||
new Error("cancelled after publication"),
|
||||
);
|
||||
|
||||
const aborted = await setup.store.loadRun(started.runId);
|
||||
assert.equal(aborted.status, "aborted");
|
||||
assert.equal(
|
||||
(await setup.store.getStart("start-key"))?.status,
|
||||
"completed",
|
||||
);
|
||||
assert.equal(
|
||||
((await setup.store.getStart("start-key"))?.result as WorkflowView).status,
|
||||
"aborted",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("start and decision idempotency prevent replay and reject changed commands", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
@@ -2353,6 +3001,10 @@ test("start and decision idempotency prevent replay and reject changed commands"
|
||||
assert.equal(setup.workspace.prepareCalls, 1);
|
||||
assert.equal(setup.workspace.createCalls, 1);
|
||||
assert.equal(setup.executor.requests.length, 1);
|
||||
await assert.rejects(
|
||||
engine.execute({ ...command, allowDirtySource: true }),
|
||||
/another start command/,
|
||||
);
|
||||
|
||||
const decide = {
|
||||
action: "decide" as const,
|
||||
@@ -2508,7 +3160,6 @@ test("live workspace creation cannot be stolen after its heartbeat ages", async
|
||||
now: () => now,
|
||||
createId,
|
||||
processId,
|
||||
leaseStaleMs: 10,
|
||||
isProcessAlive: () => true,
|
||||
});
|
||||
const owner = makeEngine(
|
||||
@@ -2823,7 +3474,6 @@ test("active assignment leases heartbeat and block concurrent recovery", async (
|
||||
executor: blocking,
|
||||
now: () => now,
|
||||
processId: 100,
|
||||
leaseStaleMs: 10,
|
||||
assignmentHeartbeatMs: 2,
|
||||
});
|
||||
const start = engine.execute(startCommand());
|
||||
@@ -2843,7 +3493,6 @@ test("active assignment leases heartbeat and block concurrent recovery", async (
|
||||
.engine("session-2", {
|
||||
now: () => now,
|
||||
processId: 100,
|
||||
leaseStaleMs: 10,
|
||||
isProcessAlive: () => false,
|
||||
})
|
||||
.recover(),
|
||||
@@ -2854,7 +3503,7 @@ test("active assignment leases heartbeat and block concurrent recovery", async (
|
||||
});
|
||||
});
|
||||
|
||||
test("a stale heartbeat fences a live read-only assignment during recovery", async () => {
|
||||
test("a live read-only assignment remains fenced when its heartbeat is old", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
const blocking = new BlockingExecutor(discoveryWithQuestions());
|
||||
@@ -2864,27 +3513,21 @@ test("a stale heartbeat fences a live read-only assignment during recovery", asy
|
||||
executor: blocking,
|
||||
now: () => now,
|
||||
processId: 100,
|
||||
leaseStaleMs: 10,
|
||||
assignmentHeartbeatMs: 1_000,
|
||||
})
|
||||
.execute(startCommand());
|
||||
const startOutcome = start.then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
await blocking.started;
|
||||
now = 100_000;
|
||||
const recovery = setup.engine("session-2", {
|
||||
now: () => now,
|
||||
processId: 200,
|
||||
leaseStaleMs: 10,
|
||||
isProcessAlive: () => true,
|
||||
});
|
||||
const [recovered] = await recovery.recover();
|
||||
assert.equal(recovered?.openGate?.kind, "recovery");
|
||||
assert.equal(recovered?.assignments[0]?.status, "interrupted");
|
||||
assert.deepEqual(await recovery.recover(), []);
|
||||
const [running] = await setup.store.listRuns();
|
||||
assert.equal(running?.assignments[0]?.status, "running");
|
||||
blocking.release();
|
||||
assert.match(String(await startOutcome), /lost its execution lease/);
|
||||
assert.equal((await start).openGate?.kind, "questions");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2916,7 +3559,6 @@ test("a live mutating assignment is not reclaimed on heartbeat age alone", async
|
||||
const recovery = setup.engine("session-2", {
|
||||
now: () => 100_000,
|
||||
processId: 200,
|
||||
leaseStaleMs: 10,
|
||||
isProcessAlive: () => true,
|
||||
});
|
||||
assert.deepEqual(await recovery.recover(), []);
|
||||
@@ -3423,6 +4065,65 @@ test("abort fails closed for an Assignment owned by another live process", async
|
||||
});
|
||||
});
|
||||
|
||||
test("cross-instance abort fails closed for a live timed-out mutator lease", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
setup.executor.enqueue("discover", discoveryWithQuestions());
|
||||
const started = await setup.engine().execute(startCommand());
|
||||
const current = await setup.store.loadRun(started.runId);
|
||||
const profile = setup.profiles.find((candidate) =>
|
||||
candidate.roles.includes("implement"),
|
||||
);
|
||||
const evidence = current.preflightEligibility.find(
|
||||
(candidate) => candidate.role === "implement",
|
||||
);
|
||||
assert.ok(profile && evidence);
|
||||
current.assignments.push({
|
||||
id: "timed-out-external-mutator",
|
||||
stageId: `${current.runId}:implement`,
|
||||
role: "implement",
|
||||
profileName: profile.name,
|
||||
profileDigest: profile.digest,
|
||||
selection: {
|
||||
eligibleProfiles: structuredClone(evidence.candidates),
|
||||
selectedProfileName: profile.name,
|
||||
rationale: "Timed-out mutator owned by another engine instance",
|
||||
},
|
||||
status: "interrupted",
|
||||
mutates: true,
|
||||
ownedPaths: ["src"],
|
||||
executionToken: "timed-out-external-token",
|
||||
error: "Assignment implement timed out",
|
||||
});
|
||||
current.status = "waiting";
|
||||
current.revision++;
|
||||
await setup.store.saveRun(current, current.revision - 1);
|
||||
await setup.store.createAssignmentLease(
|
||||
current.runId,
|
||||
"timed-out-external-mutator",
|
||||
{
|
||||
token: "timed-out-external-token",
|
||||
processId: 200,
|
||||
heartbeatAt: 10_000,
|
||||
},
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
setup.engine("session-1", {
|
||||
processId: 100,
|
||||
isProcessAlive: (processId) => processId === 200,
|
||||
}).execute({
|
||||
action: "abort",
|
||||
runId: current.runId,
|
||||
expectedRevision: current.revision,
|
||||
idempotencyKey: "abort-timed-out-external",
|
||||
}),
|
||||
/Cannot safely abort Assignment timed-out-external-mutator owned by live process 200/,
|
||||
);
|
||||
assert.equal((await setup.store.loadRun(current.runId)).status, "waiting");
|
||||
});
|
||||
});
|
||||
|
||||
test("abort is revision checked and idempotent", async () => {
|
||||
await withTempDir(async (directory) => {
|
||||
const setup = await fixture(directory);
|
||||
|
||||
Reference in New Issue
Block a user