263 lines
13 KiB
TypeScript
263 lines
13 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { runSteps, validateSteps, TextGenerationError, type ControllerDeps } from "./controller.ts";
|
|
import { BrowserOperationError } from "./contracts.ts";
|
|
import type { ActRequest, Observation, Step } from "./contracts.ts";
|
|
|
|
const observation: Observation = {
|
|
observationId: "snap1", url: "http://localhost:8099/form", title: "Fixture",
|
|
candidates: [
|
|
{ id: "field1", role: "textbox", name: "Display name", actions: ["fill"] },
|
|
{ id: "button1", role: "button", name: "Continue", actions: ["click"] },
|
|
],
|
|
};
|
|
const fill: Step = { instruction: "Find the display name input", action: "fill", text: "Ada" };
|
|
const click: Step = { instruction: "Continue", action: "click", postcondition: { kind: "text", value: "Next page" } };
|
|
function setup() {
|
|
const actions: ActRequest[] = [];
|
|
let decisions = 0;
|
|
let generated = 0;
|
|
let observed = 0;
|
|
const deps: ControllerDeps = {
|
|
browser: {
|
|
async observe() { observed++; return structuredClone(observation); },
|
|
async act(request) { actions.push(request); return { verified: true, evidence: "field value matched" }; },
|
|
async close() {},
|
|
},
|
|
async decide() { decisions++; return { candidateId: "field1", confidence: 0.95, model: "jev-test" }; },
|
|
async generateText() { generated++; return "Generated text"; },
|
|
};
|
|
return { deps, actions, counts: () => ({ decisions, generated, observed }) };
|
|
}
|
|
|
|
test("literal fill uses JEV but never text generation", async () => {
|
|
const s = setup();
|
|
const result = await runSteps([fill], s.deps);
|
|
assert.equal(result.status, "completed");
|
|
assert.equal(s.actions[0].text, "Ada");
|
|
assert.deepEqual(s.counts(), { decisions: 1, generated: 0, observed: 1 });
|
|
assert.equal(result.steps[0].confidence, 0.95);
|
|
assert.equal(JSON.stringify(result).includes("Ada"), false);
|
|
});
|
|
|
|
test("unique exact accessible name bypasses JEV", async () => {
|
|
const s = setup();
|
|
assert.equal((await runSteps([click], s.deps)).status, "completed");
|
|
assert.equal(s.counts().decisions, 0);
|
|
assert.equal(s.actions[0].candidateId, "button1");
|
|
});
|
|
|
|
test("composition is generated once and inserted without interactive callbacks", async () => {
|
|
const s = setup();
|
|
const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write a name" }], s.deps);
|
|
assert.equal(result.status, "completed");
|
|
assert.equal(s.actions[0].text, "Generated text");
|
|
assert.equal(s.counts().generated, 1);
|
|
});
|
|
|
|
test("known generation failures remain actionable and cause no mutation", async () => {
|
|
const s = setup();
|
|
s.deps.generateText = async () => { throw new TextGenerationError("Configured auxiliary model is not supported by the current ChatGPT account"); };
|
|
const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write a greeting" }], s.deps);
|
|
assert.match(result.reason, /not supported by the current ChatGPT account/);
|
|
assert.equal(result.actionMayHaveExecuted, false);
|
|
assert.equal(s.actions.length, 0);
|
|
});
|
|
|
|
test("missing generation model fails before mutation", async () => {
|
|
const s = setup(); delete s.deps.generateText;
|
|
const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write" }], s.deps);
|
|
assert.match(result.reason, /not configured/);
|
|
assert.equal(s.actions.length, 0);
|
|
});
|
|
|
|
for (const [name, decision] of [
|
|
["abstention", { candidateId: null, confidence: 0.99, model: "jev" }],
|
|
["low confidence", { candidateId: "field1", confidence: 0.4, model: "jev" }],
|
|
["unknown candidate", { candidateId: "made-up", confidence: 0.99, model: "jev" }],
|
|
["ineligible candidate", { candidateId: "button1", confidence: 0.99, model: "jev" }],
|
|
["NaN confidence", { candidateId: "field1", confidence: NaN, model: "jev" }],
|
|
] as const) {
|
|
test(`${name} never executes`, async () => {
|
|
const s = setup(); s.deps.decide = async () => decision;
|
|
const result = await runSteps([fill], s.deps);
|
|
assert.equal(result.status, "stopped");
|
|
assert.equal(s.actions.length, 0);
|
|
});
|
|
}
|
|
|
|
test("credential values only reach the browser, never selection, generation or traces", async () => {
|
|
const s = setup();
|
|
s.deps.resolveCredential = async (name) => { assert.equal(name, "account.password"); return "test-secret-value"; };
|
|
s.deps.decide = async (input) => {
|
|
assert.equal(JSON.stringify(input).includes("test-secret-value"), false);
|
|
assert.equal(JSON.stringify(input).includes("account.password"), false);
|
|
return { candidateId: "field1", confidence: 1, model: "test" };
|
|
};
|
|
const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "account.password" }], s.deps);
|
|
assert.equal(result.status, "completed");
|
|
assert.equal(s.actions[0].text, "test-secret-value");
|
|
assert.equal(s.counts().generated, 0);
|
|
assert.equal(JSON.stringify(result).includes("test-secret-value"), false);
|
|
});
|
|
|
|
test("missing credential resolver stops without mutation", async () => {
|
|
const s = setup();
|
|
const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "password" }], s.deps);
|
|
assert.match(result.reason, /No credential resolver/);
|
|
assert.equal(s.actions.length, 0);
|
|
});
|
|
|
|
test("unverified mutation is never retried or followed by another step", async () => {
|
|
const s = setup();
|
|
s.deps.browser.act = async (r) => { s.actions.push(r); return { verified: false, evidence: "not visible" }; };
|
|
const result = await runSteps([fill, click], s.deps);
|
|
assert.equal(result.steps[0].status, "unverified");
|
|
assert.equal(s.actions.length, 1);
|
|
assert.equal(s.counts().observed, 1);
|
|
});
|
|
|
|
test("throw after mutation is not retried and private exception text is not returned", async () => {
|
|
const s = setup();
|
|
s.deps.browser.act = async (r) => { s.actions.push(r); throw new Error("secret-user-data"); };
|
|
const result = await runSteps([fill, click], s.deps);
|
|
assert.equal(s.actions.length, 1);
|
|
assert.equal(result.status, "stopped");
|
|
assert.equal(JSON.stringify(result).includes("secret-user-data"), false);
|
|
});
|
|
|
|
test("deadline bounds a non-cooperative decision and prevents later action", async () => {
|
|
const s = setup();
|
|
let release!: () => void;
|
|
s.deps.decide = async () => {
|
|
await new Promise<void>((r) => { release = r; });
|
|
return { candidateId: "field1", confidence: 1, model: "jev" };
|
|
};
|
|
const result = await runSteps([fill], s.deps, { timeoutMs: 20 });
|
|
assert.match(result.reason, /deadline/);
|
|
release();
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
assert.equal(s.actions.length, 0);
|
|
});
|
|
|
|
test("cancellation while resolving credentials cannot produce a late mutation", async () => {
|
|
const s = setup();
|
|
const abort = new AbortController();
|
|
s.deps.resolveCredential = async () => { abort.abort(); return "secret"; };
|
|
const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "password" }], s.deps, { signal: abort.signal });
|
|
assert.match(result.reason, /cancelled/);
|
|
assert.equal(s.actions.length, 0);
|
|
});
|
|
|
|
test("already cancelled run does not observe", async () => {
|
|
const s = setup(); const abort = new AbortController(); abort.abort();
|
|
assert.equal((await runSteps([fill], s.deps, { signal: abort.signal })).status, "stopped");
|
|
assert.equal(s.counts().observed, 0);
|
|
});
|
|
|
|
test("fresh observation required for every step", async () => {
|
|
const s = setup();
|
|
assert.equal((await runSteps([fill, click], s.deps)).status, "completed");
|
|
assert.equal(s.counts().observed, 2);
|
|
});
|
|
|
|
test("malformed or duplicate browser handles fail closed", async () => {
|
|
const s = setup();
|
|
s.deps.browser.observe = async () => ({ ...observation, candidates: [observation.candidates[0], observation.candidates[0]] });
|
|
assert.equal((await runSteps([fill], s.deps)).status, "stopped");
|
|
assert.equal(s.actions.length, 0);
|
|
});
|
|
|
|
test("empty candidate list abstains without model call", async () => {
|
|
const s = setup(); s.deps.browser.observe = async () => ({ ...observation, candidates: [] });
|
|
assert.equal((await runSteps([fill], s.deps)).steps[0].status, "abstained");
|
|
assert.equal(s.counts().decisions, 0);
|
|
});
|
|
|
|
test("cancel during a non-cooperative action explicitly reports an unknown mutation", async () => {
|
|
const s = setup();
|
|
let captures = 0;
|
|
s.deps.captureEvidence = async () => { captures++; return { status: "unavailable" }; };
|
|
let release!: () => void;
|
|
s.deps.browser.act = async (request) => {
|
|
await new Promise<void>((resolve) => { release = resolve; });
|
|
s.actions.push(request);
|
|
return { verified: true, evidence: "field value verified" };
|
|
};
|
|
const result = await runSteps([fill], s.deps, { timeoutMs: 20 });
|
|
assert.equal(result.status, "stopped");
|
|
assert.equal(result.actionMayHaveExecuted, true);
|
|
assert.equal(result.steps[0].mutationState, "unknown");
|
|
assert.equal(captures, 1);
|
|
assert.match(result.steps[0].evidence ?? "", /Failure evidence unavailable/);
|
|
release();
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
assert.equal(s.actions.length, 1); // A foreign adapter cannot be undone by a promise race.
|
|
assert.equal(result.steps[0].mutationState, "unknown");
|
|
});
|
|
|
|
test("typed pre-dispatch action errors remain safe and not attempted", async () => {
|
|
const s = setup();
|
|
s.deps.browser.act = async () => { throw new BrowserOperationError({ code: "policy_denied", operation: "act", phase: "policy", message: "browser origin policy denied the operation", mutationState: "not_attempted", retry: "never" }); };
|
|
const result = await runSteps([fill], s.deps);
|
|
assert.equal(result.actionMayHaveExecuted, false);
|
|
assert.equal(result.steps[0].mutationState, "not_attempted");
|
|
});
|
|
|
|
test("non-cooperative failure evidence is bounded and preserves the original failure", async () => {
|
|
const s = setup();
|
|
s.deps.browser.act = async () => { throw new Error("action failed safely"); };
|
|
s.deps.captureEvidence = async () => new Promise<unknown>(() => {});
|
|
const started = Date.now();
|
|
const result = await runSteps([fill], s.deps);
|
|
assert.ok(Date.now() - started >= 1_900);
|
|
assert.match(result.reason, /operation failed/);
|
|
assert.match(result.steps[0].evidence ?? "", /Failure evidence unavailable/);
|
|
assert.equal(result.actionMayHaveExecuted, true);
|
|
});
|
|
|
|
test("failure evidence is best effort and cannot replace the original action failure", async () => {
|
|
const s = setup();
|
|
let captures = 0;
|
|
s.deps.captureEvidence = async (request) => { captures++; assert.equal(request.reason, "failure"); throw new Error("artifact writer failed"); };
|
|
s.deps.browser.act = async () => { throw new Error("private action details"); };
|
|
const result = await runSteps([fill], s.deps, { runId: "offline-run" });
|
|
assert.equal(captures, 1);
|
|
assert.match(result.reason, /operation failed/);
|
|
assert.equal(JSON.stringify(result).includes("private action details"), false);
|
|
});
|
|
|
|
test("unknown action evidence cannot leak field values or terminal controls", async () => {
|
|
const s = setup();
|
|
s.deps.browser.act = async () => ({ verified: true, evidence: "secret-value\\u001b[2J" });
|
|
const result = await runSteps([fill], s.deps);
|
|
assert.equal(result.actionMayHaveExecuted, false);
|
|
assert.equal(result.steps[0].mutationState, "verified");
|
|
assert.equal(result.steps[0].evidence, "Postcondition verified");
|
|
assert.equal(JSON.stringify(result).includes("secret-value"), false);
|
|
});
|
|
|
|
test("parallel controllers keep credential values and browser actions isolated", async () => {
|
|
const first = setup();
|
|
const second = setup();
|
|
first.deps.resolveCredential = async () => "first-account-secret";
|
|
second.deps.resolveCredential = async () => "second-account-secret";
|
|
const plan: Step[] = [{ instruction: fill.instruction, action: "fill", credential: "password" }];
|
|
const results = await Promise.all([runSteps(plan, first.deps), runSteps(plan, second.deps)]);
|
|
assert(results.every((result) => result.status === "completed"));
|
|
assert.equal(first.actions[0].text, "first-account-secret");
|
|
assert.equal(second.actions[0].text, "second-account-secret");
|
|
assert.equal(JSON.stringify(results).includes("account-secret"), false);
|
|
});
|
|
|
|
test("step schema rejects unbounded, mixed or unsupported actions", () => {
|
|
for (const steps of [
|
|
[], Array(11).fill(fill), [{ ...fill, compose: "write" }], [{ instruction: "send", action: "click" }],
|
|
[{ ...fill, action: "evaluate" }], [{ ...fill, text: "x".repeat(8001) }],
|
|
[{ ...fill, credential: "password" }], [{ ...click, credential: "password" }],
|
|
[{ instruction: "Password", action: "fill", credential: "invalid ref" }],
|
|
[{ ...click, postcondition: { kind: "url", value: "file:///etc/passwd" } }],
|
|
]) assert.throws(() => validateSteps(steps as Step[]));
|
|
validateSteps([{ ...fill, text: "" }]); // Clearing a field is a valid bounded action.
|
|
});
|