fast-browser-with-jev

extensions/fast-browser/jev.test.ts

@@ -0,0 +1,127 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { buildJevRequest, createJevDecider, parseJevDecision } from "./jev.ts";
+import type { DecisionInput } from "./contracts.ts";
+
+const input: DecisionInput = {
+  instruction: "Choose the search field", action: "fill",
+  observation: { observationId: "snapshot", url: "https://example.com/account?token=secret#private", title: "Page",
+    candidates: [
+      { id: "opaque-field", role: "textbox", name: "Search", actions: ["fill"] },
+      { id: "opaque-button", role: "button", name: "Search", actions: ["click"] },
+    ],
+  },
+};
+const answer = () => ({
+  model: "jev-1.13.0",
+  answers: { target: { choice: "target_0", confidence: 0.96,
+    probabilities: { target_0: 0.99, none_of_the_above: 0.01 } } },
+});
+
+test("bounded request filters by action and omits query, hash and browser handles", () => {
+  const { body, mapping } = buildJevRequest(input, "jev-1.13.0");
+  assert.equal(body.state.controls.length, 1);
+  assert.equal(mapping.get("target_0"), "opaque-field");
+  const encoded = JSON.stringify(body);
+  for (const value of ["token=", "secret", "private", "opaque-field", "snapshot", "/account", '"title"']) assert.equal(encoded.includes(value), false);
+  assert.match(body.questions.target.instructions, /untrusted/);
+  assert.ok(body.questions.target.criteria.none_of_the_above);
+});
+
+test("valid option is translated back into original opaque handle", () => {
+  const { mapping } = buildJevRequest(input, "jev");
+  assert.deepEqual(parseJevDecision(answer(), mapping), { candidateId: "opaque-field", confidence: 0.96, model: "jev-1.13.0" });
+});
+
+test("none of the above is a real abstention", () => {
+  const value = answer(); value.answers.target.choice = "none_of_the_above";
+  value.answers.target.probabilities = { target_0: 0.01, none_of_the_above: 0.99 };
+  assert.equal(parseJevDecision(value, buildJevRequest(input, "jev").mapping).candidateId, null);
+});
+
+test("unknown, incomplete, nonnumeric or malformed responses fail closed", () => {
+  const { mapping } = buildJevRequest(input, "jev");
+  const bad = [null, {}, { model: "jev", answers: {} }];
+  for (const value of bad) assert.throws(() => parseJevDecision(value, mapping));
+  for (const replacement of [
+    { choice: "execute_python" }, { confidence: "0.99" }, { confidence: NaN }, { confidence: 1.1 },
+    { probabilities: { target_0: 1 } }, { probabilities: { target_0: 1, none_of_the_above: 1 } },
+    { probabilities: { target_0: 0.8, none_of_the_above: 0.1, invented: 0.1 } },
+  ]) {
+    const value = answer(); Object.assign(value.answers.target, replacement);
+    assert.throws(() => parseJevDecision(value, mapping));
+  }
+});
+
+test("API pins model, sends bearer key and disallows redirects", async () => {
+  let calls = 0;
+  const decide = createJevDecider({ apiKey: "test-key", fetch: (async (url, options) => {
+    calls++;
+    assert.equal(url, "https://api.typesafe.ai/v1/systemone");
+    assert.equal(options?.redirect, "error");
+    assert.equal((options?.headers as Record<string, string>).Authorization, "Bearer test-key");
+    assert.equal(JSON.parse(options?.body as string).model, "jev-1.13.0");
+    return Response.json(answer());
+  }) as typeof fetch });
+  assert.equal((await decide(input, new AbortController().signal)).candidateId, "opaque-field");
+  assert.equal(calls, 1);
+});
+
+test("provider failures have no retry and do not echo private response", async () => {
+  let calls = 0;
+  const decide = createJevDecider({ apiKey: "secret-key", fetch: (async () => {
+    calls++; return new Response("private response secret-key", { status: 429 });
+  }) as typeof fetch });
+  await assert.rejects(decide(input, new AbortController().signal), (error: Error) => {
+    assert.match(error.message, /429/); assert.doesNotMatch(error.message, /private|secret-key/); return true;
+  });
+  assert.equal(calls, 1);
+});
+
+test("oversized and non-JSON responses fail closed", async () => {
+  for (const body of ["x".repeat(256_001), "not-json"]) {
+    const decide = createJevDecider({ apiKey: "test", fetch: (async () => new Response(body)) as typeof fetch });
+    await assert.rejects(decide(input, new AbortController().signal));
+  }
+});
+
+test("contradictory winners and slightly nonnormalized probabilities fail closed", () => {
+  const { mapping } = buildJevRequest(input, "jev");
+  for (const probabilities of [
+    { target_0: 0.01, none_of_the_above: 0.99 },
+    { target_0: 0.98, none_of_the_above: 0.01 },
+    { target_0: 0.999, none_of_the_above: 0.01 },
+  ]) {
+    const value = answer(); value.answers.target.probabilities = probabilities;
+    assert.throws(() => parseJevDecision(value, mapping));
+  }
+});
+
+test("standalone JEV boundary rejects oversized inputs before HTTP", async () => {
+  const decide = createJevDecider({ apiKey: "test", fetch: (async () => assert.fail("must not fetch")) as typeof fetch });
+  await assert.rejects(decide({ ...input, instruction: "x".repeat(2001) }, new AbortController().signal));
+  assert.throws(() => buildJevRequest(input, "m".repeat(101)));
+});
+
+test("body read is cancellable and private stream errors are normalized", async () => {
+  const controller = new AbortController();
+  const decide = createJevDecider({ apiKey: "test", fetch: (async () => new Response(new ReadableStream({
+    start() { setTimeout(() => controller.abort(), 10); },
+  }))) as typeof fetch });
+  await assert.rejects(decide(input, controller.signal), /cancelled/);
+  const failure = createJevDecider({ apiKey: "test", fetch: (async () => new Response(new ReadableStream({
+    start(stream) { stream.error(new Error("private-input-token")); },
+  }))) as typeof fetch });
+  await assert.rejects(failure(input, new AbortController().signal), (error: Error) => {
+    assert.match(error.message, /stream failed/);
+    assert.doesNotMatch(error.message, /private-input-token/);
+    return true;
+  });
+});
+
+test("missing key and pre-aborted request perform no HTTP calls", async () => {
+  assert.throws(() => createJevDecider({ apiKey: " " }));
+  const decide = createJevDecider({ apiKey: "test", fetch: (async () => { assert.fail("must not fetch"); }) as typeof fetch });
+  const abort = new AbortController(); abort.abort();
+  await assert.rejects(decide(input, abort.signal));
+});