import assert from "node:assert/strict"; import test from "node:test"; import { runSteps, validateSteps, TextGenerationError, type ControllerDeps } from "./controller.ts"; import { BrowserOperationError } from "./contracts.ts"; import type { ActRequest, Observation, Step } from "./contracts.ts"; const observation: Observation = { observationId: "snap1", url: "http://localhost:8099/form", title: "Fixture", candidates: [ { id: "field1", role: "textbox", name: "Display name", actions: ["fill"] }, { id: "button1", role: "button", name: "Continue", actions: ["click"] }, ], }; const fill: Step = { instruction: "Find the display name input", action: "fill", text: "Ada" }; const click: Step = { instruction: "Continue", action: "click", postcondition: { kind: "text", value: "Next page" } }; function setup() { const actions: ActRequest[] = []; let decisions = 0; let generated = 0; let observed = 0; const deps: ControllerDeps = { browser: { async observe() { observed++; return structuredClone(observation); }, async act(request) { actions.push(request); return { verified: true, evidence: "field value matched" }; }, async close() {}, }, async decide() { decisions++; return { candidateId: "field1", confidence: 0.95, model: "jev-test" }; }, async generateText() { generated++; return "Generated text"; }, }; return { deps, actions, counts: () => ({ decisions, generated, observed }) }; } test("literal fill uses JEV but never text generation", async () => { const s = setup(); const result = await runSteps([fill], s.deps); assert.equal(result.status, "completed"); assert.equal(s.actions[0].text, "Ada"); assert.deepEqual(s.counts(), { decisions: 1, generated: 0, observed: 1 }); assert.equal(result.steps[0].confidence, 0.95); assert.equal(JSON.stringify(result).includes("Ada"), false); }); test("unique exact accessible name bypasses JEV", async () => { const s = setup(); assert.equal((await runSteps([click], s.deps)).status, "completed"); assert.equal(s.counts().decisions, 0); assert.equal(s.actions[0].candidateId, "button1"); }); test("composition is generated once and inserted without interactive callbacks", async () => { const s = setup(); const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write a name" }], s.deps); assert.equal(result.status, "completed"); assert.equal(s.actions[0].text, "Generated text"); assert.equal(s.counts().generated, 1); }); test("known generation failures remain actionable and cause no mutation", async () => { const s = setup(); s.deps.generateText = async () => { throw new TextGenerationError("Configured auxiliary model is not supported by the current ChatGPT account"); }; const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write a greeting" }], s.deps); assert.match(result.reason, /not supported by the current ChatGPT account/); assert.equal(result.actionMayHaveExecuted, false); assert.equal(s.actions.length, 0); }); test("missing generation model fails before mutation", async () => { const s = setup(); delete s.deps.generateText; const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write" }], s.deps); assert.match(result.reason, /not configured/); assert.equal(s.actions.length, 0); }); for (const [name, decision] of [ ["abstention", { candidateId: null, confidence: 0.99, model: "jev" }], ["low confidence", { candidateId: "field1", confidence: 0.4, model: "jev" }], ["unknown candidate", { candidateId: "made-up", confidence: 0.99, model: "jev" }], ["ineligible candidate", { candidateId: "button1", confidence: 0.99, model: "jev" }], ["NaN confidence", { candidateId: "field1", confidence: NaN, model: "jev" }], ] as const) { test(`${name} never executes`, async () => { const s = setup(); s.deps.decide = async () => decision; const result = await runSteps([fill], s.deps); assert.equal(result.status, "stopped"); assert.equal(s.actions.length, 0); }); } test("credential values only reach the browser, never selection, generation or traces", async () => { const s = setup(); s.deps.resolveCredential = async (name) => { assert.equal(name, "account.password"); return "test-secret-value"; }; s.deps.decide = async (input) => { assert.equal(JSON.stringify(input).includes("test-secret-value"), false); assert.equal(JSON.stringify(input).includes("account.password"), false); return { candidateId: "field1", confidence: 1, model: "test" }; }; const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "account.password" }], s.deps); assert.equal(result.status, "completed"); assert.equal(s.actions[0].text, "test-secret-value"); assert.equal(s.counts().generated, 0); assert.equal(JSON.stringify(result).includes("test-secret-value"), false); }); test("missing credential resolver stops without mutation", async () => { const s = setup(); const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "password" }], s.deps); assert.match(result.reason, /No credential resolver/); assert.equal(s.actions.length, 0); }); test("unverified mutation is never retried or followed by another step", async () => { const s = setup(); s.deps.browser.act = async (r) => { s.actions.push(r); return { verified: false, evidence: "not visible" }; }; const result = await runSteps([fill, click], s.deps); assert.equal(result.steps[0].status, "unverified"); assert.equal(s.actions.length, 1); assert.equal(s.counts().observed, 1); }); test("throw after mutation is not retried and private exception text is not returned", async () => { const s = setup(); s.deps.browser.act = async (r) => { s.actions.push(r); throw new Error("secret-user-data"); }; const result = await runSteps([fill, click], s.deps); assert.equal(s.actions.length, 1); assert.equal(result.status, "stopped"); assert.equal(JSON.stringify(result).includes("secret-user-data"), false); }); test("deadline bounds a non-cooperative decision and prevents later action", async () => { const s = setup(); let release!: () => void; s.deps.decide = async () => { await new Promise((r) => { release = r; }); return { candidateId: "field1", confidence: 1, model: "jev" }; }; const result = await runSteps([fill], s.deps, { timeoutMs: 20 }); assert.match(result.reason, /deadline/); release(); await new Promise((resolve) => setImmediate(resolve)); assert.equal(s.actions.length, 0); }); test("cancellation while resolving credentials cannot produce a late mutation", async () => { const s = setup(); const abort = new AbortController(); s.deps.resolveCredential = async () => { abort.abort(); return "secret"; }; const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "password" }], s.deps, { signal: abort.signal }); assert.match(result.reason, /cancelled/); assert.equal(s.actions.length, 0); }); test("already cancelled run does not observe", async () => { const s = setup(); const abort = new AbortController(); abort.abort(); assert.equal((await runSteps([fill], s.deps, { signal: abort.signal })).status, "stopped"); assert.equal(s.counts().observed, 0); }); test("fresh observation required for every step", async () => { const s = setup(); assert.equal((await runSteps([fill, click], s.deps)).status, "completed"); assert.equal(s.counts().observed, 2); }); test("malformed or duplicate browser handles fail closed", async () => { const s = setup(); s.deps.browser.observe = async () => ({ ...observation, candidates: [observation.candidates[0], observation.candidates[0]] }); assert.equal((await runSteps([fill], s.deps)).status, "stopped"); assert.equal(s.actions.length, 0); }); test("empty candidate list abstains without model call", async () => { const s = setup(); s.deps.browser.observe = async () => ({ ...observation, candidates: [] }); assert.equal((await runSteps([fill], s.deps)).steps[0].status, "abstained"); assert.equal(s.counts().decisions, 0); }); test("cancel during a non-cooperative action explicitly reports an unknown mutation", async () => { const s = setup(); let captures = 0; s.deps.captureEvidence = async () => { captures++; return { status: "unavailable" }; }; let release!: () => void; s.deps.browser.act = async (request) => { await new Promise((resolve) => { release = resolve; }); s.actions.push(request); return { verified: true, evidence: "field value verified" }; }; const result = await runSteps([fill], s.deps, { timeoutMs: 20 }); assert.equal(result.status, "stopped"); assert.equal(result.actionMayHaveExecuted, true); assert.equal(result.steps[0].mutationState, "unknown"); assert.equal(captures, 1); assert.match(result.steps[0].evidence ?? "", /Failure evidence unavailable/); release(); await new Promise((resolve) => setImmediate(resolve)); assert.equal(s.actions.length, 1); // A foreign adapter cannot be undone by a promise race. assert.equal(result.steps[0].mutationState, "unknown"); }); test("typed pre-dispatch action errors remain safe and not attempted", async () => { const s = setup(); s.deps.browser.act = async () => { throw new BrowserOperationError({ code: "policy_denied", operation: "act", phase: "policy", message: "browser origin policy denied the operation", mutationState: "not_attempted", retry: "never" }); }; const result = await runSteps([fill], s.deps); assert.equal(result.actionMayHaveExecuted, false); assert.equal(result.steps[0].mutationState, "not_attempted"); }); test("non-cooperative failure evidence is bounded and preserves the original failure", async () => { const s = setup(); s.deps.browser.act = async () => { throw new Error("action failed safely"); }; s.deps.captureEvidence = async () => new Promise(() => {}); const started = Date.now(); const result = await runSteps([fill], s.deps); assert.ok(Date.now() - started >= 1_900); assert.match(result.reason, /operation failed/); assert.match(result.steps[0].evidence ?? "", /Failure evidence unavailable/); assert.equal(result.actionMayHaveExecuted, true); }); test("failure evidence is best effort and cannot replace the original action failure", async () => { const s = setup(); let captures = 0; s.deps.captureEvidence = async (request) => { captures++; assert.equal(request.reason, "failure"); throw new Error("artifact writer failed"); }; s.deps.browser.act = async () => { throw new Error("private action details"); }; const result = await runSteps([fill], s.deps, { runId: "offline-run" }); assert.equal(captures, 1); assert.match(result.reason, /operation failed/); assert.equal(JSON.stringify(result).includes("private action details"), false); }); test("unknown action evidence cannot leak field values or terminal controls", async () => { const s = setup(); s.deps.browser.act = async () => ({ verified: true, evidence: "secret-value\\u001b[2J" }); const result = await runSteps([fill], s.deps); assert.equal(result.actionMayHaveExecuted, false); assert.equal(result.steps[0].mutationState, "verified"); assert.equal(result.steps[0].evidence, "Postcondition verified"); assert.equal(JSON.stringify(result).includes("secret-value"), false); }); test("parallel controllers keep credential values and browser actions isolated", async () => { const first = setup(); const second = setup(); first.deps.resolveCredential = async () => "first-account-secret"; second.deps.resolveCredential = async () => "second-account-secret"; const plan: Step[] = [{ instruction: fill.instruction, action: "fill", credential: "password" }]; const results = await Promise.all([runSteps(plan, first.deps), runSteps(plan, second.deps)]); assert(results.every((result) => result.status === "completed")); assert.equal(first.actions[0].text, "first-account-secret"); assert.equal(second.actions[0].text, "second-account-secret"); assert.equal(JSON.stringify(results).includes("account-secret"), false); }); test("step schema rejects unbounded, mixed or unsupported actions", () => { for (const steps of [ [], Array(11).fill(fill), [{ ...fill, compose: "write" }], [{ instruction: "send", action: "click" }], [{ ...fill, action: "evaluate" }], [{ ...fill, text: "x".repeat(8001) }], [{ ...fill, credential: "password" }], [{ ...click, credential: "password" }], [{ instruction: "Password", action: "fill", credential: "invalid ref" }], [{ ...click, postcondition: { kind: "url", value: "file:///etc/passwd" } }], ]) assert.throws(() => validateSteps(steps as Step[])); validateSteps([{ ...fill, text: "" }]); // Clearing a field is a valid bounded action. });