fast-browser-with-jev

2026-09-20 12:24:08 -04:00
parent fd0edb1511
commit fd63b0ccec
73 changed files with 21212 additions and 2 deletions
@@ -0,0 +1,199 @@
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import {
+  loadScenarioFiles,
+  parseScenarioDocument,
+  readScenarioFile,
+  ScenarioValidationError,
+} from "./scenarios.ts";
+
+const clickStep = {
+  instruction: "Continue",
+  action: "click",
+  postcondition: { kind: "text", value: "Fixture complete" },
+};
+
+function document(overrides: Record<string, unknown> = {}) {
+  return {
+    version: 1,
+    id: "local-form",
+    operations: [
+      { op: "step", step: { instruction: "Display name", action: "fill", text: "Ada" } },
+      { op: "step", step: clickStep },
+    ],
+    ...overrides,
+  };
+}
+
+async function temporary(fn: (directory: string) => Promise<void>): Promise<void> {
+  const directory = await mkdtemp(join(tmpdir(), "fast-browser-scenario-test-"));
+  try { await fn(directory); } finally { await rm(directory, { recursive: true, force: true }); }
+}
+
+test("parses the published ScenarioOperation union without contacting a runtime", () => {
+  const parsed = parseScenarioDocument(document({
+    description: "offline local fixture",
+    operations: [
+      { op: "navigate", request: { url: "http://127.0.0.1:8765/scenario-suite.html" } },
+      { op: "read", request: {} },
+      { op: "assert", request: { assertion: { kind: "visibleText", match: "contains", value: "Scenario fixture ready" }, wait: "bounded" } },
+      { op: "step", step: { instruction: "Continue", action: "click", postcondition: { kind: "text", value: "Fixture complete" } } },
+      { op: "listTabs" },
+      { op: "reload", request: {} },
+    ],
+    budgets: { maxSteps: 1, timeoutMs: 3_000, minConfidence: 1 },
+    evidence: { onFailure: true, onRequest: true },
+    hooks: {
+      reset: {
+        argv: ["node", "fixture/reset.mjs", "--local"],
+        cwd: ".",
+        env: { FIXTURE_MODE: { env: "FAST_BROWSER_FIXTURE_MODE" } },
+      },
+    },
+  }));
+  assert.equal(parsed.scenarios[0]?.id, "local-form");
+  assert.equal(parsed.scenarios[0]?.operations[0]?.op, "navigate");
+  assert.equal(parsed.scenarios[0]?.operations[2]?.op, "assert");
+  assert.deepEqual(parsed.scenarios[0]?.hooks?.reset?.args, ["fixture/reset.mjs", "--local"]);
+  assert.equal(parsed.scenarios[0]?.evidence?.onFailure, true);
+  assert.equal(parsed.scenarios[0]?.budgets?.maxSteps, 1);
+});
+
+test("parses opaque tab references and rejects forward, missing, ambiguous, and incompatible handles", () => {
+  const parsed = parseScenarioDocument(document({ operations: [
+    { op: "listTabs", id: "tabs", request: { select: { kind: "origin", origin: "http://fixture.test" } } },
+    { op: "selectTab", id: "selected", request: { tab: { source: "operation", operation: { id: "tabs" }, result: "listedTab", index: 0 } } },
+    { op: "waitPopup", id: "popup", request: { opener: { source: "initial", target: "selected" } } },
+    { op: "read", request: { tab: { source: "operation", operation: { id: "popup" }, result: "popup" } } },
+    { op: "closeTab", request: { tab: { source: "operation", operation: { id: "selected" }, result: "selectedTab" } } },
+  ] }));
+  assert.equal(parsed.scenarios[0]?.operations[1]?.id, "selected");
+  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "read", request: { tab: { source: "operation", operation: { index: 1 }, result: "listedTab", index: 0 } } }, { op: "listTabs" }] })), /earlier operation/);
+  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "read", request: { tab: { source: "operation", operation: { id: "missing" }, result: "popup" } } }] })), /earlier operation/);
+  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", id: "tabs" }, { op: "read", request: { tab: { source: "operation", operation: { id: "tabs" }, result: "selectedTab" } } }] })), /selectedTab requires/);
+  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", id: "same" }, { op: "listTabs", id: "same" }] })), /duplicated/);
+  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", request: { select: { kind: "index", index: 32 } } }] })), ScenarioValidationError);
+  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", id: "tabs" }, { op: "read", request: { tab: { source: "operation", operation: { index: 32 }, result: "listedTab", index: 0 } } }] })), ScenarioValidationError);
+  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", request: { select: { kind: "origin", origin: "http://fixture.test/path" } } }] })), /exact HTTP/);
+});
+
+test("parses bounded external and managed fixture readiness configuration", () => {
+  const parsed = parseScenarioDocument(document({
+    fixture: {
+      baseUrl: "http://127.0.0.1:8765",
+      server: {
+        mode: "managed",
+        start: { argv: ["python3", "-m", "http.server", "8765"], cwd: ".", timeoutMs: 2_000 },
+        readiness: { url: "http://127.0.0.1:8765/scenario-suite.html", expectedStatus: 200, timeoutMs: 3_000 },
+        shutdownTimeoutMs: 2_000,
+      },
+    },
+  }));
+  assert.equal(parsed.scenarios[0]?.fixture?.server?.mode, "managed");
+  const external = parseScenarioDocument(document({ fixture: { baseUrl: "http://127.0.0.1:8765", server: { mode: "external", readiness: { url: "http://127.0.0.1:8765/ready", expectedStatus: 204 } } } }));
+  assert.equal(external.scenarios[0]?.fixture?.server?.mode, "external");
+  assert.throws(() => parseScenarioDocument(document({ fixture: { baseUrl: "http://127.0.0.1:8765", server: { mode: "external", readiness: { url: "file:///tmp/unsafe" } } } })), /HTTP/);
+});
+
+test("accepts a bounded scenario matrix and rejects duplicate IDs", () => {
+  const parsed = parseScenarioDocument({
+    version: 1,
+    scenarios: [document({ id: "first" }), document({ id: "second" })],
+  });
+  assert.deepEqual(parsed.scenarios.map((scenario) => scenario.id), ["first", "second"]);
+  assert.throws(
+    () => parseScenarioDocument({ version: 1, scenarios: [document({ id: "same" }), document({ id: "same" })] }),
+    ScenarioValidationError,
+  );
+});
+
+test("rejects unbounded, shell-enabled, malformed, and unsafe hooks", () => {
+  const invalid = [
+    { hooks: { setup: { command: "node", shell: true } } },
+    { hooks: { setup: { command: "node", argv: ["node"] } } },
+    { hooks: { setup: { argv: ["node", "\nunsafe"] } } },
+    { hooks: { setup: { command: "node", env: { "BAD-NAME": "x" } } } },
+    { hooks: { setup: { command: "node", timeoutMs: 120_001 } } },
+  ];
+  for (const value of invalid) assert.throws(() => parseScenarioDocument(document(value)), ScenarioValidationError);
+});
+
+test("requires an explicit auxiliary model for composition and keeps exact-name steps provider-free", () => {
+  assert.doesNotThrow(() => parseScenarioDocument(document({
+    operations: [{ op: "step", step: { instruction: "Display name", action: "fill", text: "Ada" } }],
+  })));
+  assert.throws(() => parseScenarioDocument(document({
+    operations: [{ op: "step", step: { instruction: "Greeting", action: "fill", compose: "a short greeting" } }],
+  })), /settings.textModel/);
+  assert.doesNotThrow(() => parseScenarioDocument(document({
+    settings: { textModel: "fixture/small" },
+    operations: [{ op: "step", step: { instruction: "Greeting", action: "fill", compose: "a short greeting" } }],
+  })));
+});
+
+test("rejects extra step fields and invalid budgets before browser startup", () => {
+  assert.throws(
+    () => parseScenarioDocument(document({ operations: [{ op: "step", step: { ...clickStep, secret: "must-not-be-ignored" } }] })),
+    ScenarioValidationError,
+  );
+  assert.throws(
+    () => parseScenarioDocument(document({ budgets: { maxSteps: 1 } })),
+    ScenarioValidationError,
+  );
+  assert.throws(
+    () => parseScenarioDocument(document({ version: 2 })),
+    ScenarioValidationError,
+  );
+});
+
+test("rejects unsupported mutation/evidence operation arms and unsafe URLs", () => {
+  for (const operation of [
+    { op: "observe" },
+    { op: "close" },
+    { op: "captureEvidence", request: { runId: "run", reason: "requested" } },
+    { op: "navigate", request: { url: "https://user:password@example.test/" } },
+    { op: "assert", request: { assertion: { kind: "url", match: "origin", value: "file:///private" }, wait: "none" } },
+  ]) {
+    assert.throws(() => parseScenarioDocument(document({ operations: [operation] })), ScenarioValidationError);
+  }
+});
+
+test("loads only explicit bounded regular JSON files and merges unique IDs", async () => {
+  await temporary(async (directory) => {
+    const first = join(directory, "first.json");
+    const second = join(directory, "second.json");
+    await writeFile(first, JSON.stringify(document({ id: "first" })));
+    await writeFile(second, JSON.stringify(document({ id: "second" })));
+    const loaded = await loadScenarioFiles(["first.json", "second.json"], { cwd: directory });
+    assert.deepEqual(loaded.scenarios.map((entry) => entry.scenario.id), ["first", "second"]);
+    assert.equal(loaded.files[0], first);
+    await assert.rejects(readScenarioFile("-", directory), ScenarioValidationError);
+    await assert.rejects(readScenarioFile("missing.json", directory), ScenarioValidationError);
+  });
+});
+
+test("rejects oversized and nonregular scenario inputs before JSON parsing", async () => {
+  await temporary(async (directory) => {
+    const path = join(directory, "large.json");
+    await writeFile(path, "x".repeat(512 * 1024 + 1));
+    await assert.rejects(readScenarioFile(path), /bounded regular file|size limit/);
+    const folder = join(directory, "folder.json");
+    await mkdir(folder);
+    await assert.rejects(readScenarioFile(folder), /bounded regular file|could not be read/);
+  });
+});
+
+test("credential declarations are validated lazily without reading their values", () => {
+  const parsed = parseScenarioDocument(document({
+    credentials: { password: { env: "FAST_BROWSER_TEST_PASSWORD" } },
+    operations: [
+      { op: "step", step: { instruction: "Password", action: "fill", credential: "password" } },
+      { op: "step", step: clickStep },
+    ],
+  }));
+  assert.deepEqual(parsed.scenarios[0]?.credentials, { password: { env: "FAST_BROWSER_TEST_PASSWORD" } });
+  assert.throws(() => parseScenarioDocument(document({ credentials: { password: { env: "not valid" } } })), ScenarioValidationError);
+});