fast-browser-with-jev
This commit is contained in:
@@ -0,0 +1,262 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { runSteps, validateSteps, TextGenerationError, type ControllerDeps } from "./controller.ts";
|
||||
import { BrowserOperationError } from "./contracts.ts";
|
||||
import type { ActRequest, Observation, Step } from "./contracts.ts";
|
||||
|
||||
const observation: Observation = {
|
||||
observationId: "snap1", url: "http://localhost:8099/form", title: "Fixture",
|
||||
candidates: [
|
||||
{ id: "field1", role: "textbox", name: "Display name", actions: ["fill"] },
|
||||
{ id: "button1", role: "button", name: "Continue", actions: ["click"] },
|
||||
],
|
||||
};
|
||||
const fill: Step = { instruction: "Find the display name input", action: "fill", text: "Ada" };
|
||||
const click: Step = { instruction: "Continue", action: "click", postcondition: { kind: "text", value: "Next page" } };
|
||||
function setup() {
|
||||
const actions: ActRequest[] = [];
|
||||
let decisions = 0;
|
||||
let generated = 0;
|
||||
let observed = 0;
|
||||
const deps: ControllerDeps = {
|
||||
browser: {
|
||||
async observe() { observed++; return structuredClone(observation); },
|
||||
async act(request) { actions.push(request); return { verified: true, evidence: "field value matched" }; },
|
||||
async close() {},
|
||||
},
|
||||
async decide() { decisions++; return { candidateId: "field1", confidence: 0.95, model: "jev-test" }; },
|
||||
async generateText() { generated++; return "Generated text"; },
|
||||
};
|
||||
return { deps, actions, counts: () => ({ decisions, generated, observed }) };
|
||||
}
|
||||
|
||||
test("literal fill uses JEV but never text generation", async () => {
|
||||
const s = setup();
|
||||
const result = await runSteps([fill], s.deps);
|
||||
assert.equal(result.status, "completed");
|
||||
assert.equal(s.actions[0].text, "Ada");
|
||||
assert.deepEqual(s.counts(), { decisions: 1, generated: 0, observed: 1 });
|
||||
assert.equal(result.steps[0].confidence, 0.95);
|
||||
assert.equal(JSON.stringify(result).includes("Ada"), false);
|
||||
});
|
||||
|
||||
test("unique exact accessible name bypasses JEV", async () => {
|
||||
const s = setup();
|
||||
assert.equal((await runSteps([click], s.deps)).status, "completed");
|
||||
assert.equal(s.counts().decisions, 0);
|
||||
assert.equal(s.actions[0].candidateId, "button1");
|
||||
});
|
||||
|
||||
test("composition is generated once and inserted without interactive callbacks", async () => {
|
||||
const s = setup();
|
||||
const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write a name" }], s.deps);
|
||||
assert.equal(result.status, "completed");
|
||||
assert.equal(s.actions[0].text, "Generated text");
|
||||
assert.equal(s.counts().generated, 1);
|
||||
});
|
||||
|
||||
test("known generation failures remain actionable and cause no mutation", async () => {
|
||||
const s = setup();
|
||||
s.deps.generateText = async () => { throw new TextGenerationError("Configured auxiliary model is not supported by the current ChatGPT account"); };
|
||||
const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write a greeting" }], s.deps);
|
||||
assert.match(result.reason, /not supported by the current ChatGPT account/);
|
||||
assert.equal(result.actionMayHaveExecuted, false);
|
||||
assert.equal(s.actions.length, 0);
|
||||
});
|
||||
|
||||
test("missing generation model fails before mutation", async () => {
|
||||
const s = setup(); delete s.deps.generateText;
|
||||
const result = await runSteps([{ instruction: fill.instruction, action: "fill", compose: "Write" }], s.deps);
|
||||
assert.match(result.reason, /not configured/);
|
||||
assert.equal(s.actions.length, 0);
|
||||
});
|
||||
|
||||
for (const [name, decision] of [
|
||||
["abstention", { candidateId: null, confidence: 0.99, model: "jev" }],
|
||||
["low confidence", { candidateId: "field1", confidence: 0.4, model: "jev" }],
|
||||
["unknown candidate", { candidateId: "made-up", confidence: 0.99, model: "jev" }],
|
||||
["ineligible candidate", { candidateId: "button1", confidence: 0.99, model: "jev" }],
|
||||
["NaN confidence", { candidateId: "field1", confidence: NaN, model: "jev" }],
|
||||
] as const) {
|
||||
test(`${name} never executes`, async () => {
|
||||
const s = setup(); s.deps.decide = async () => decision;
|
||||
const result = await runSteps([fill], s.deps);
|
||||
assert.equal(result.status, "stopped");
|
||||
assert.equal(s.actions.length, 0);
|
||||
});
|
||||
}
|
||||
|
||||
test("credential values only reach the browser, never selection, generation or traces", async () => {
|
||||
const s = setup();
|
||||
s.deps.resolveCredential = async (name) => { assert.equal(name, "account.password"); return "test-secret-value"; };
|
||||
s.deps.decide = async (input) => {
|
||||
assert.equal(JSON.stringify(input).includes("test-secret-value"), false);
|
||||
assert.equal(JSON.stringify(input).includes("account.password"), false);
|
||||
return { candidateId: "field1", confidence: 1, model: "test" };
|
||||
};
|
||||
const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "account.password" }], s.deps);
|
||||
assert.equal(result.status, "completed");
|
||||
assert.equal(s.actions[0].text, "test-secret-value");
|
||||
assert.equal(s.counts().generated, 0);
|
||||
assert.equal(JSON.stringify(result).includes("test-secret-value"), false);
|
||||
});
|
||||
|
||||
test("missing credential resolver stops without mutation", async () => {
|
||||
const s = setup();
|
||||
const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "password" }], s.deps);
|
||||
assert.match(result.reason, /No credential resolver/);
|
||||
assert.equal(s.actions.length, 0);
|
||||
});
|
||||
|
||||
test("unverified mutation is never retried or followed by another step", async () => {
|
||||
const s = setup();
|
||||
s.deps.browser.act = async (r) => { s.actions.push(r); return { verified: false, evidence: "not visible" }; };
|
||||
const result = await runSteps([fill, click], s.deps);
|
||||
assert.equal(result.steps[0].status, "unverified");
|
||||
assert.equal(s.actions.length, 1);
|
||||
assert.equal(s.counts().observed, 1);
|
||||
});
|
||||
|
||||
test("throw after mutation is not retried and private exception text is not returned", async () => {
|
||||
const s = setup();
|
||||
s.deps.browser.act = async (r) => { s.actions.push(r); throw new Error("secret-user-data"); };
|
||||
const result = await runSteps([fill, click], s.deps);
|
||||
assert.equal(s.actions.length, 1);
|
||||
assert.equal(result.status, "stopped");
|
||||
assert.equal(JSON.stringify(result).includes("secret-user-data"), false);
|
||||
});
|
||||
|
||||
test("deadline bounds a non-cooperative decision and prevents later action", async () => {
|
||||
const s = setup();
|
||||
let release!: () => void;
|
||||
s.deps.decide = async () => {
|
||||
await new Promise<void>((r) => { release = r; });
|
||||
return { candidateId: "field1", confidence: 1, model: "jev" };
|
||||
};
|
||||
const result = await runSteps([fill], s.deps, { timeoutMs: 20 });
|
||||
assert.match(result.reason, /deadline/);
|
||||
release();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(s.actions.length, 0);
|
||||
});
|
||||
|
||||
test("cancellation while resolving credentials cannot produce a late mutation", async () => {
|
||||
const s = setup();
|
||||
const abort = new AbortController();
|
||||
s.deps.resolveCredential = async () => { abort.abort(); return "secret"; };
|
||||
const result = await runSteps([{ instruction: fill.instruction, action: "fill", credential: "password" }], s.deps, { signal: abort.signal });
|
||||
assert.match(result.reason, /cancelled/);
|
||||
assert.equal(s.actions.length, 0);
|
||||
});
|
||||
|
||||
test("already cancelled run does not observe", async () => {
|
||||
const s = setup(); const abort = new AbortController(); abort.abort();
|
||||
assert.equal((await runSteps([fill], s.deps, { signal: abort.signal })).status, "stopped");
|
||||
assert.equal(s.counts().observed, 0);
|
||||
});
|
||||
|
||||
test("fresh observation required for every step", async () => {
|
||||
const s = setup();
|
||||
assert.equal((await runSteps([fill, click], s.deps)).status, "completed");
|
||||
assert.equal(s.counts().observed, 2);
|
||||
});
|
||||
|
||||
test("malformed or duplicate browser handles fail closed", async () => {
|
||||
const s = setup();
|
||||
s.deps.browser.observe = async () => ({ ...observation, candidates: [observation.candidates[0], observation.candidates[0]] });
|
||||
assert.equal((await runSteps([fill], s.deps)).status, "stopped");
|
||||
assert.equal(s.actions.length, 0);
|
||||
});
|
||||
|
||||
test("empty candidate list abstains without model call", async () => {
|
||||
const s = setup(); s.deps.browser.observe = async () => ({ ...observation, candidates: [] });
|
||||
assert.equal((await runSteps([fill], s.deps)).steps[0].status, "abstained");
|
||||
assert.equal(s.counts().decisions, 0);
|
||||
});
|
||||
|
||||
test("cancel during a non-cooperative action explicitly reports an unknown mutation", async () => {
|
||||
const s = setup();
|
||||
let captures = 0;
|
||||
s.deps.captureEvidence = async () => { captures++; return { status: "unavailable" }; };
|
||||
let release!: () => void;
|
||||
s.deps.browser.act = async (request) => {
|
||||
await new Promise<void>((resolve) => { release = resolve; });
|
||||
s.actions.push(request);
|
||||
return { verified: true, evidence: "field value verified" };
|
||||
};
|
||||
const result = await runSteps([fill], s.deps, { timeoutMs: 20 });
|
||||
assert.equal(result.status, "stopped");
|
||||
assert.equal(result.actionMayHaveExecuted, true);
|
||||
assert.equal(result.steps[0].mutationState, "unknown");
|
||||
assert.equal(captures, 1);
|
||||
assert.match(result.steps[0].evidence ?? "", /Failure evidence unavailable/);
|
||||
release();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(s.actions.length, 1); // A foreign adapter cannot be undone by a promise race.
|
||||
assert.equal(result.steps[0].mutationState, "unknown");
|
||||
});
|
||||
|
||||
test("typed pre-dispatch action errors remain safe and not attempted", async () => {
|
||||
const s = setup();
|
||||
s.deps.browser.act = async () => { throw new BrowserOperationError({ code: "policy_denied", operation: "act", phase: "policy", message: "browser origin policy denied the operation", mutationState: "not_attempted", retry: "never" }); };
|
||||
const result = await runSteps([fill], s.deps);
|
||||
assert.equal(result.actionMayHaveExecuted, false);
|
||||
assert.equal(result.steps[0].mutationState, "not_attempted");
|
||||
});
|
||||
|
||||
test("non-cooperative failure evidence is bounded and preserves the original failure", async () => {
|
||||
const s = setup();
|
||||
s.deps.browser.act = async () => { throw new Error("action failed safely"); };
|
||||
s.deps.captureEvidence = async () => new Promise<unknown>(() => {});
|
||||
const started = Date.now();
|
||||
const result = await runSteps([fill], s.deps);
|
||||
assert.ok(Date.now() - started >= 1_900);
|
||||
assert.match(result.reason, /operation failed/);
|
||||
assert.match(result.steps[0].evidence ?? "", /Failure evidence unavailable/);
|
||||
assert.equal(result.actionMayHaveExecuted, true);
|
||||
});
|
||||
|
||||
test("failure evidence is best effort and cannot replace the original action failure", async () => {
|
||||
const s = setup();
|
||||
let captures = 0;
|
||||
s.deps.captureEvidence = async (request) => { captures++; assert.equal(request.reason, "failure"); throw new Error("artifact writer failed"); };
|
||||
s.deps.browser.act = async () => { throw new Error("private action details"); };
|
||||
const result = await runSteps([fill], s.deps, { runId: "offline-run" });
|
||||
assert.equal(captures, 1);
|
||||
assert.match(result.reason, /operation failed/);
|
||||
assert.equal(JSON.stringify(result).includes("private action details"), false);
|
||||
});
|
||||
|
||||
test("unknown action evidence cannot leak field values or terminal controls", async () => {
|
||||
const s = setup();
|
||||
s.deps.browser.act = async () => ({ verified: true, evidence: "secret-value\\u001b[2J" });
|
||||
const result = await runSteps([fill], s.deps);
|
||||
assert.equal(result.actionMayHaveExecuted, false);
|
||||
assert.equal(result.steps[0].mutationState, "verified");
|
||||
assert.equal(result.steps[0].evidence, "Postcondition verified");
|
||||
assert.equal(JSON.stringify(result).includes("secret-value"), false);
|
||||
});
|
||||
|
||||
test("parallel controllers keep credential values and browser actions isolated", async () => {
|
||||
const first = setup();
|
||||
const second = setup();
|
||||
first.deps.resolveCredential = async () => "first-account-secret";
|
||||
second.deps.resolveCredential = async () => "second-account-secret";
|
||||
const plan: Step[] = [{ instruction: fill.instruction, action: "fill", credential: "password" }];
|
||||
const results = await Promise.all([runSteps(plan, first.deps), runSteps(plan, second.deps)]);
|
||||
assert(results.every((result) => result.status === "completed"));
|
||||
assert.equal(first.actions[0].text, "first-account-secret");
|
||||
assert.equal(second.actions[0].text, "second-account-secret");
|
||||
assert.equal(JSON.stringify(results).includes("account-secret"), false);
|
||||
});
|
||||
|
||||
test("step schema rejects unbounded, mixed or unsupported actions", () => {
|
||||
for (const steps of [
|
||||
[], Array(11).fill(fill), [{ ...fill, compose: "write" }], [{ instruction: "send", action: "click" }],
|
||||
[{ ...fill, action: "evaluate" }], [{ ...fill, text: "x".repeat(8001) }],
|
||||
[{ ...fill, credential: "password" }], [{ ...click, credential: "password" }],
|
||||
[{ instruction: "Password", action: "fill", credential: "invalid ref" }],
|
||||
[{ ...click, postcondition: { kind: "url", value: "file:///etc/passwd" } }],
|
||||
]) assert.throws(() => validateSteps(steps as Step[]));
|
||||
validateSteps([{ ...fill, text: "" }]); // Clearing a field is a valid bounded action.
|
||||
});
|
||||
Reference in New Issue
Block a user