128 lines
6.1 KiB
TypeScript
128 lines
6.1 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { buildJevRequest, createJevDecider, parseJevDecision } from "./jev.ts";
|
|
import type { DecisionInput } from "./contracts.ts";
|
|
|
|
const input: DecisionInput = {
|
|
instruction: "Choose the search field", action: "fill",
|
|
observation: { observationId: "snapshot", url: "https://example.com/account?token=secret#private", title: "Page",
|
|
candidates: [
|
|
{ id: "opaque-field", role: "textbox", name: "Search", actions: ["fill"] },
|
|
{ id: "opaque-button", role: "button", name: "Search", actions: ["click"] },
|
|
],
|
|
},
|
|
};
|
|
const answer = () => ({
|
|
model: "jev-1.13.0",
|
|
answers: { target: { choice: "target_0", confidence: 0.96,
|
|
probabilities: { target_0: 0.99, none_of_the_above: 0.01 } } },
|
|
});
|
|
|
|
test("bounded request filters by action and omits query, hash and browser handles", () => {
|
|
const { body, mapping } = buildJevRequest(input, "jev-1.13.0");
|
|
assert.equal(body.state.controls.length, 1);
|
|
assert.equal(mapping.get("target_0"), "opaque-field");
|
|
const encoded = JSON.stringify(body);
|
|
for (const value of ["token=", "secret", "private", "opaque-field", "snapshot", "/account", '"title"']) assert.equal(encoded.includes(value), false);
|
|
assert.match(body.questions.target.instructions, /untrusted/);
|
|
assert.ok(body.questions.target.criteria.none_of_the_above);
|
|
});
|
|
|
|
test("valid option is translated back into original opaque handle", () => {
|
|
const { mapping } = buildJevRequest(input, "jev");
|
|
assert.deepEqual(parseJevDecision(answer(), mapping), { candidateId: "opaque-field", confidence: 0.96, model: "jev-1.13.0" });
|
|
});
|
|
|
|
test("none of the above is a real abstention", () => {
|
|
const value = answer(); value.answers.target.choice = "none_of_the_above";
|
|
value.answers.target.probabilities = { target_0: 0.01, none_of_the_above: 0.99 };
|
|
assert.equal(parseJevDecision(value, buildJevRequest(input, "jev").mapping).candidateId, null);
|
|
});
|
|
|
|
test("unknown, incomplete, nonnumeric or malformed responses fail closed", () => {
|
|
const { mapping } = buildJevRequest(input, "jev");
|
|
const bad = [null, {}, { model: "jev", answers: {} }];
|
|
for (const value of bad) assert.throws(() => parseJevDecision(value, mapping));
|
|
for (const replacement of [
|
|
{ choice: "execute_python" }, { confidence: "0.99" }, { confidence: NaN }, { confidence: 1.1 },
|
|
{ probabilities: { target_0: 1 } }, { probabilities: { target_0: 1, none_of_the_above: 1 } },
|
|
{ probabilities: { target_0: 0.8, none_of_the_above: 0.1, invented: 0.1 } },
|
|
]) {
|
|
const value = answer(); Object.assign(value.answers.target, replacement);
|
|
assert.throws(() => parseJevDecision(value, mapping));
|
|
}
|
|
});
|
|
|
|
test("API pins model, sends bearer key and disallows redirects", async () => {
|
|
let calls = 0;
|
|
const decide = createJevDecider({ apiKey: "test-key", fetch: (async (url, options) => {
|
|
calls++;
|
|
assert.equal(url, "https://api.typesafe.ai/v1/systemone");
|
|
assert.equal(options?.redirect, "error");
|
|
assert.equal((options?.headers as Record<string, string>).Authorization, "Bearer test-key");
|
|
assert.equal(JSON.parse(options?.body as string).model, "jev-1.13.0");
|
|
return Response.json(answer());
|
|
}) as typeof fetch });
|
|
assert.equal((await decide(input, new AbortController().signal)).candidateId, "opaque-field");
|
|
assert.equal(calls, 1);
|
|
});
|
|
|
|
test("provider failures have no retry and do not echo private response", async () => {
|
|
let calls = 0;
|
|
const decide = createJevDecider({ apiKey: "secret-key", fetch: (async () => {
|
|
calls++; return new Response("private response secret-key", { status: 429 });
|
|
}) as typeof fetch });
|
|
await assert.rejects(decide(input, new AbortController().signal), (error: Error) => {
|
|
assert.match(error.message, /429/); assert.doesNotMatch(error.message, /private|secret-key/); return true;
|
|
});
|
|
assert.equal(calls, 1);
|
|
});
|
|
|
|
test("oversized and non-JSON responses fail closed", async () => {
|
|
for (const body of ["x".repeat(256_001), "not-json"]) {
|
|
const decide = createJevDecider({ apiKey: "test", fetch: (async () => new Response(body)) as typeof fetch });
|
|
await assert.rejects(decide(input, new AbortController().signal));
|
|
}
|
|
});
|
|
|
|
test("contradictory winners and slightly nonnormalized probabilities fail closed", () => {
|
|
const { mapping } = buildJevRequest(input, "jev");
|
|
for (const probabilities of [
|
|
{ target_0: 0.01, none_of_the_above: 0.99 },
|
|
{ target_0: 0.98, none_of_the_above: 0.01 },
|
|
{ target_0: 0.999, none_of_the_above: 0.01 },
|
|
]) {
|
|
const value = answer(); value.answers.target.probabilities = probabilities;
|
|
assert.throws(() => parseJevDecision(value, mapping));
|
|
}
|
|
});
|
|
|
|
test("standalone JEV boundary rejects oversized inputs before HTTP", async () => {
|
|
const decide = createJevDecider({ apiKey: "test", fetch: (async () => assert.fail("must not fetch")) as typeof fetch });
|
|
await assert.rejects(decide({ ...input, instruction: "x".repeat(2001) }, new AbortController().signal));
|
|
assert.throws(() => buildJevRequest(input, "m".repeat(101)));
|
|
});
|
|
|
|
test("body read is cancellable and private stream errors are normalized", async () => {
|
|
const controller = new AbortController();
|
|
const decide = createJevDecider({ apiKey: "test", fetch: (async () => new Response(new ReadableStream({
|
|
start() { setTimeout(() => controller.abort(), 10); },
|
|
}))) as typeof fetch });
|
|
await assert.rejects(decide(input, controller.signal), /cancelled/);
|
|
const failure = createJevDecider({ apiKey: "test", fetch: (async () => new Response(new ReadableStream({
|
|
start(stream) { stream.error(new Error("private-input-token")); },
|
|
}))) as typeof fetch });
|
|
await assert.rejects(failure(input, new AbortController().signal), (error: Error) => {
|
|
assert.match(error.message, /stream failed/);
|
|
assert.doesNotMatch(error.message, /private-input-token/);
|
|
return true;
|
|
});
|
|
});
|
|
|
|
test("missing key and pre-aborted request perform no HTTP calls", async () => {
|
|
assert.throws(() => createJevDecider({ apiKey: " " }));
|
|
const decide = createJevDecider({ apiKey: "test", fetch: (async () => { assert.fail("must not fetch"); }) as typeof fetch });
|
|
const abort = new AbortController(); abort.abort();
|
|
await assert.rejects(decide(input, abort.signal));
|
|
});
|