harness-tools/extensions/fast-browser/scenarios.test.ts

import assert from "node:assert/strict";
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import {
  loadScenarioFiles,
  parseScenarioDocument,
  readScenarioFile,
  ScenarioValidationError,
} from "./scenarios.ts";

const clickStep = {
  instruction: "Continue",
  action: "click",
  postcondition: { kind: "text", value: "Fixture complete" },
};

function document(overrides: Record<string, unknown> = {}) {
  return {
    version: 1,
    id: "local-form",
    operations: [
      { op: "step", step: { instruction: "Display name", action: "fill", text: "Ada" } },
      { op: "step", step: clickStep },
    ],
    ...overrides,
  };
}

async function temporary(fn: (directory: string) => Promise<void>): Promise<void> {
  const directory = await mkdtemp(join(tmpdir(), "fast-browser-scenario-test-"));
  try { await fn(directory); } finally { await rm(directory, { recursive: true, force: true }); }
}

test("parses the published ScenarioOperation union without contacting a runtime", () => {
  const parsed = parseScenarioDocument(document({
    description: "offline local fixture",
    operations: [
      { op: "navigate", request: { url: "http://127.0.0.1:8765/scenario-suite.html" } },
      { op: "read", request: {} },
      { op: "assert", request: { assertion: { kind: "visibleText", match: "contains", value: "Scenario fixture ready" }, wait: "bounded" } },
      { op: "step", step: { instruction: "Continue", action: "click", postcondition: { kind: "text", value: "Fixture complete" } } },
      { op: "listTabs" },
      { op: "reload", request: {} },
    ],
    budgets: { maxSteps: 1, timeoutMs: 3_000, minConfidence: 1 },
    evidence: { onFailure: true, onRequest: true },
    hooks: {
      reset: {
        argv: ["node", "fixture/reset.mjs", "--local"],
        cwd: ".",
        env: { FIXTURE_MODE: { env: "FAST_BROWSER_FIXTURE_MODE" } },
      },
    },
  }));
  assert.equal(parsed.scenarios[0]?.id, "local-form");
  assert.equal(parsed.scenarios[0]?.operations[0]?.op, "navigate");
  assert.equal(parsed.scenarios[0]?.operations[2]?.op, "assert");
  assert.deepEqual(parsed.scenarios[0]?.hooks?.reset?.args, ["fixture/reset.mjs", "--local"]);
  assert.equal(parsed.scenarios[0]?.evidence?.onFailure, true);
  assert.equal(parsed.scenarios[0]?.budgets?.maxSteps, 1);
});

test("parses opaque tab references and rejects forward, missing, ambiguous, and incompatible handles", () => {
  const parsed = parseScenarioDocument(document({ operations: [
    { op: "listTabs", id: "tabs", request: { select: { kind: "origin", origin: "http://fixture.test" } } },
    { op: "selectTab", id: "selected", request: { tab: { source: "operation", operation: { id: "tabs" }, result: "listedTab", index: 0 } } },
    { op: "waitPopup", id: "popup", request: { opener: { source: "initial", target: "selected" } } },
    { op: "read", request: { tab: { source: "operation", operation: { id: "popup" }, result: "popup" } } },
    { op: "closeTab", request: { tab: { source: "operation", operation: { id: "selected" }, result: "selectedTab" } } },
  ] }));
  assert.equal(parsed.scenarios[0]?.operations[1]?.id, "selected");
  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "read", request: { tab: { source: "operation", operation: { index: 1 }, result: "listedTab", index: 0 } } }, { op: "listTabs" }] })), /earlier operation/);
  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "read", request: { tab: { source: "operation", operation: { id: "missing" }, result: "popup" } } }] })), /earlier operation/);
  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", id: "tabs" }, { op: "read", request: { tab: { source: "operation", operation: { id: "tabs" }, result: "selectedTab" } } }] })), /selectedTab requires/);
  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", id: "same" }, { op: "listTabs", id: "same" }] })), /duplicated/);
  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", request: { select: { kind: "index", index: 32 } } }] })), ScenarioValidationError);
  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", id: "tabs" }, { op: "read", request: { tab: { source: "operation", operation: { index: 32 }, result: "listedTab", index: 0 } } }] })), ScenarioValidationError);
  assert.throws(() => parseScenarioDocument(document({ operations: [{ op: "listTabs", request: { select: { kind: "origin", origin: "http://fixture.test/path" } } }] })), /exact HTTP/);
});

test("parses bounded external and managed fixture readiness configuration", () => {
  const parsed = parseScenarioDocument(document({
    fixture: {
      baseUrl: "http://127.0.0.1:8765",
      server: {
        mode: "managed",
        start: { argv: ["python3", "-m", "http.server", "8765"], cwd: ".", timeoutMs: 2_000 },
        readiness: { url: "http://127.0.0.1:8765/scenario-suite.html", expectedStatus: 200, timeoutMs: 3_000 },
        shutdownTimeoutMs: 2_000,
      },
    },
  }));
  assert.equal(parsed.scenarios[0]?.fixture?.server?.mode, "managed");
  const external = parseScenarioDocument(document({ fixture: { baseUrl: "http://127.0.0.1:8765", server: { mode: "external", readiness: { url: "http://127.0.0.1:8765/ready", expectedStatus: 204 } } } }));
  assert.equal(external.scenarios[0]?.fixture?.server?.mode, "external");
  assert.throws(() => parseScenarioDocument(document({ fixture: { baseUrl: "http://127.0.0.1:8765", server: { mode: "external", readiness: { url: "file:///tmp/unsafe" } } } })), /HTTP/);
});

test("accepts a bounded scenario matrix and rejects duplicate IDs", () => {
  const parsed = parseScenarioDocument({
    version: 1,
    scenarios: [document({ id: "first" }), document({ id: "second" })],
  });
  assert.deepEqual(parsed.scenarios.map((scenario) => scenario.id), ["first", "second"]);
  assert.throws(
    () => parseScenarioDocument({ version: 1, scenarios: [document({ id: "same" }), document({ id: "same" })] }),
    ScenarioValidationError,
  );
});

test("rejects unbounded, shell-enabled, malformed, and unsafe hooks", () => {
  const invalid = [
    { hooks: { setup: { command: "node", shell: true } } },
    { hooks: { setup: { command: "node", argv: ["node"] } } },
    { hooks: { setup: { argv: ["node", "\nunsafe"] } } },
    { hooks: { setup: { command: "node", env: { "BAD-NAME": "x" } } } },
    { hooks: { setup: { command: "node", timeoutMs: 120_001 } } },
  ];
  for (const value of invalid) assert.throws(() => parseScenarioDocument(document(value)), ScenarioValidationError);
});

test("requires an explicit auxiliary model for composition and keeps exact-name steps provider-free", () => {
  assert.doesNotThrow(() => parseScenarioDocument(document({
    operations: [{ op: "step", step: { instruction: "Display name", action: "fill", text: "Ada" } }],
  })));
  assert.throws(() => parseScenarioDocument(document({
    operations: [{ op: "step", step: { instruction: "Greeting", action: "fill", compose: "a short greeting" } }],
  })), /settings.textModel/);
  assert.doesNotThrow(() => parseScenarioDocument(document({
    settings: { textModel: "fixture/small" },
    operations: [{ op: "step", step: { instruction: "Greeting", action: "fill", compose: "a short greeting" } }],
  })));
});

test("rejects extra step fields and invalid budgets before browser startup", () => {
  assert.throws(
    () => parseScenarioDocument(document({ operations: [{ op: "step", step: { ...clickStep, secret: "must-not-be-ignored" } }] })),
    ScenarioValidationError,
  );
  assert.throws(
    () => parseScenarioDocument(document({ budgets: { maxSteps: 1 } })),
    ScenarioValidationError,
  );
  assert.throws(
    () => parseScenarioDocument(document({ version: 2 })),
    ScenarioValidationError,
  );
});

test("rejects unsupported mutation/evidence operation arms and unsafe URLs", () => {
  for (const operation of [
    { op: "observe" },
    { op: "close" },
    { op: "captureEvidence", request: { runId: "run", reason: "requested" } },
    { op: "navigate", request: { url: "https://user:password@example.test/" } },
    { op: "assert", request: { assertion: { kind: "url", match: "origin", value: "file:///private" }, wait: "none" } },
  ]) {
    assert.throws(() => parseScenarioDocument(document({ operations: [operation] })), ScenarioValidationError);
  }
});

test("loads only explicit bounded regular JSON files and merges unique IDs", async () => {
  await temporary(async (directory) => {
    const first = join(directory, "first.json");
    const second = join(directory, "second.json");
    await writeFile(first, JSON.stringify(document({ id: "first" })));
    await writeFile(second, JSON.stringify(document({ id: "second" })));
    const loaded = await loadScenarioFiles(["first.json", "second.json"], { cwd: directory });
    assert.deepEqual(loaded.scenarios.map((entry) => entry.scenario.id), ["first", "second"]);
    assert.equal(loaded.files[0], first);
    await assert.rejects(readScenarioFile("-", directory), ScenarioValidationError);
    await assert.rejects(readScenarioFile("missing.json", directory), ScenarioValidationError);
  });
});

test("rejects oversized and nonregular scenario inputs before JSON parsing", async () => {
  await temporary(async (directory) => {
    const path = join(directory, "large.json");
    await writeFile(path, "x".repeat(512 * 1024 + 1));
    await assert.rejects(readScenarioFile(path), /bounded regular file|size limit/);
    const folder = join(directory, "folder.json");
    await mkdir(folder);
    await assert.rejects(readScenarioFile(folder), /bounded regular file|could not be read/);
  });
});

test("credential declarations are validated lazily without reading their values", () => {
  const parsed = parseScenarioDocument(document({
    credentials: { password: { env: "FAST_BROWSER_TEST_PASSWORD" } },
    operations: [
      { op: "step", step: { instruction: "Password", action: "fill", credential: "password" } },
      { op: "step", step: clickStep },
    ],
  }));
  assert.deepEqual(parsed.scenarios[0]?.credentials, { password: { env: "FAST_BROWSER_TEST_PASSWORD" } });
  assert.throws(() => parseScenarioDocument(document({ credentials: { password: { env: "not valid" } } })), ScenarioValidationError);
});