745 lines
40 KiB
TypeScript
745 lines
40 KiB
TypeScript
import { mkdtemp, rm } from "node:fs/promises";
|
|
import { join, resolve } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { createBrowserRuntimeFactory } from "./runtime.ts";
|
|
import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts";
|
|
import { createPrivateArtifactStore } from "./artifact-store.ts";
|
|
import { loadFastBrowserConfig, type ConfigOverrides } from "./config.ts";
|
|
import { createCredentialResolver } from "./credentials.ts";
|
|
import { createJevDecider, DecisionError } from "./jev.ts";
|
|
import { runOwnedFixtureHook, startOwnedFixtureProcess, type OwnedFixtureProcessHandle } from "./fixture-process.ts";
|
|
import { runSteps } from "./controller.ts";
|
|
import {
|
|
formatScenarioReport,
|
|
formatScenarioReportJson,
|
|
createScenarioReport,
|
|
evidenceReference,
|
|
sanitizeReportError,
|
|
writeScenarioReport,
|
|
type ScenarioEvidenceReference,
|
|
type ScenarioReportEnvelope,
|
|
} from "./reports.ts";
|
|
import { loadScenarioFiles, type LoadedScenarios, type DeclarativeScenario, type ScenarioHook } from "./scenarios.ts";
|
|
import {
|
|
BROWSER_EVIDENCE_OPERATIONS,
|
|
BrowserOperationError,
|
|
FAST_BROWSER_BUDGETS,
|
|
FAST_BROWSER_SCENARIO_SERVER_BOUNDS,
|
|
type BrowserEvidenceOperationName,
|
|
type AssertRequest,
|
|
type BrowserErrorDetails,
|
|
type BrowserTab,
|
|
type BrowserTabId,
|
|
type CloseTabRequest,
|
|
type DecisionInput,
|
|
type BrowserRuntime,
|
|
type BrowserRuntimeFactory,
|
|
type FailureEvidence,
|
|
type NavigateRequest,
|
|
type ReadRequest,
|
|
type ReloadRequest,
|
|
type ScenarioHttpReadiness,
|
|
type ScenarioOperation,
|
|
type ScenarioTabReference,
|
|
type SelectTabRequest,
|
|
type WaitPopupRequest,
|
|
type ScenarioOperationReport,
|
|
type Step,
|
|
} from "./contracts.ts";
|
|
|
|
export const SCENARIO_CLI_VERSION = "1" as const;
|
|
export const SCENARIO_CLI_USAGE = `Usage: scenario-cli.ts [options] <scenario.json>\n\nOptions:\n --scenario <path> Explicit scenario JSON path (repeatable)\n --report <path> Private machine-readable report output path\n --format human|json Output format (default: human)\n --headless Select headless execution mode (default)\n --visible Select visible execution mode\n --parallel <n> Bounded scenario matrix width (default: 1)\n --capture-on-request Enable explicitly requested local evidence capture\n --check Validate input only; do not start a browser\n --cwd <path> Resolve scenario paths from this directory\n --help Show this help\n`;
|
|
|
|
export interface ScenarioCliOptions {
|
|
scenarioPaths: string[];
|
|
reportPath?: string;
|
|
format: "human" | "json";
|
|
headless: boolean;
|
|
parallel: number;
|
|
captureOnRequest: boolean;
|
|
checkOnly: boolean;
|
|
cwd: string;
|
|
help: boolean;
|
|
}
|
|
|
|
export interface ScenarioCliResult {
|
|
exitCode: number;
|
|
options: ScenarioCliOptions;
|
|
loaded?: LoadedScenarios;
|
|
report?: ScenarioReportEnvelope;
|
|
}
|
|
|
|
/** Test/embedding seams use the published runtime factory and session APIs. */
|
|
export interface ScenarioCliDependencies {
|
|
runtimeFactory?: BrowserRuntimeFactory;
|
|
sessionManager?: BrowserSessionManager;
|
|
artifactStoreFactory?: typeof createPrivateArtifactStore;
|
|
/** Embedding/test cancellation seam; process signals remain supported by the CLI. */
|
|
signal?: AbortSignal;
|
|
}
|
|
|
|
export class ScenarioCliError extends Error {
|
|
override name = "ScenarioCliError";
|
|
}
|
|
|
|
class ScenarioExecutionError extends Error {
|
|
readonly phase: string;
|
|
readonly details?: BrowserErrorDetails;
|
|
constructor(phase: string, message: string, details?: BrowserErrorDetails) {
|
|
super(message);
|
|
this.phase = phase;
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
const MAX_PARALLEL = 20;
|
|
const MAX_PATH = 4_000;
|
|
const SAFE_ENVIRONMENT_NAMES = [
|
|
"PATH", "HOME", "USER", "LOGNAME", "LANG", "TMPDIR", "TMP", "TEMP",
|
|
"XDG_RUNTIME_DIR", "SYSTEMROOT", "COMSPEC",
|
|
];
|
|
|
|
function valueAfter(argv: readonly string[], index: number, option: string): string {
|
|
const value = argv[index + 1];
|
|
if (value === undefined || value === "" || value.startsWith("-")) throw new ScenarioCliError(`${option} requires a value`);
|
|
if (value.length > MAX_PATH || value.includes("\0")) throw new ScenarioCliError(`${option} value is too long`);
|
|
return value;
|
|
}
|
|
|
|
function parseParallel(value: string): number {
|
|
if (!/^[0-9]+$/.test(value)) throw new ScenarioCliError("--parallel must be a bounded positive integer");
|
|
const parsed = Number(value);
|
|
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > MAX_PARALLEL) {
|
|
throw new ScenarioCliError(`--parallel must be between 1 and ${MAX_PARALLEL}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
/** Parse only explicit command-line inputs; no environment or page data is consulted. */
|
|
export function parseScenarioCliArgs(argv: readonly string[] = process.argv.slice(2)): ScenarioCliOptions {
|
|
const scenarioPaths: string[] = [];
|
|
let reportPath: string | undefined;
|
|
let format: "human" | "json" = "human";
|
|
let headless = true;
|
|
let mode: "headless" | "visible" = "headless";
|
|
let parallel = 1;
|
|
let captureOnRequest = false;
|
|
let checkOnly = false;
|
|
let cwd = process.cwd();
|
|
let help = false;
|
|
let positional = true;
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const argument = argv[index];
|
|
if (argument === "--") { positional = true; continue; }
|
|
if (positional && argument && !argument.startsWith("-")) { scenarioPaths.push(argument); continue; }
|
|
if (argument === "--help" || argument === "-h") { help = true; continue; }
|
|
if (argument === "--scenario" || argument === "--scenarios") { scenarioPaths.push(valueAfter(argv, index++, argument)); continue; }
|
|
if (argument?.startsWith("--scenario=") || argument?.startsWith("--scenarios=")) {
|
|
const value = argument.slice(argument.indexOf("=") + 1);
|
|
if (!value || value.length > MAX_PATH || value.includes("\0")) throw new ScenarioCliError("--scenario value is invalid");
|
|
scenarioPaths.push(value); continue;
|
|
}
|
|
if (argument === "--report" || argument === "--report-json") { reportPath = valueAfter(argv, index++, argument); continue; }
|
|
if (argument?.startsWith("--report=") || argument?.startsWith("--report-json=")) {
|
|
const value = argument.slice(argument.indexOf("=") + 1);
|
|
if (!value || value.length > MAX_PATH || value.includes("\0")) throw new ScenarioCliError("--report value is invalid");
|
|
reportPath = value; continue;
|
|
}
|
|
if (argument === "--format") {
|
|
const value = valueAfter(argv, index++, argument);
|
|
if (value !== "human" && value !== "json") throw new ScenarioCliError("--format must be human or json");
|
|
format = value; continue;
|
|
}
|
|
if (argument?.startsWith("--format=")) {
|
|
const value = argument.slice("--format=".length);
|
|
if (value !== "human" && value !== "json") throw new ScenarioCliError("--format must be human or json");
|
|
format = value; continue;
|
|
}
|
|
if (argument === "--headless") {
|
|
if (mode === "visible") throw new ScenarioCliError("--headless and --visible are mutually exclusive");
|
|
headless = true; mode = "headless"; continue;
|
|
}
|
|
if (argument === "--visible") {
|
|
if (mode === "headless" && argv.slice(0, index).includes("--headless")) throw new ScenarioCliError("--headless and --visible are mutually exclusive");
|
|
headless = false; mode = "visible"; continue;
|
|
}
|
|
if (argument === "--parallel" || argument === "--matrix") { parallel = parseParallel(valueAfter(argv, index++, argument)); continue; }
|
|
if (argument?.startsWith("--parallel=") || argument?.startsWith("--matrix=")) { parallel = parseParallel(argument.slice(argument.indexOf("=") + 1)); continue; }
|
|
if (argument === "--capture-on-request") { captureOnRequest = true; continue; }
|
|
if (argument === "--check" || argument === "--validate") { checkOnly = true; continue; }
|
|
if (argument === "--cwd") { cwd = valueAfter(argv, index++, argument); continue; }
|
|
if (argument?.startsWith("--cwd=")) {
|
|
const value = argument.slice("--cwd=".length);
|
|
if (!value || value.length > MAX_PATH || value.includes("\0")) throw new ScenarioCliError("--cwd value is invalid");
|
|
cwd = value; continue;
|
|
}
|
|
if (argument === "--json") { format = "json"; continue; }
|
|
if (argument === "--human") { format = "human"; continue; }
|
|
if (argument?.startsWith("-")) throw new ScenarioCliError(`unknown option: ${argument}`);
|
|
throw new ScenarioCliError("scenario paths must be explicit files");
|
|
}
|
|
if (scenarioPaths.length === 0 && !help) throw new ScenarioCliError("at least one explicit scenario path is required");
|
|
return { scenarioPaths, ...(reportPath === undefined ? {} : { reportPath }), format, headless, parallel, captureOnRequest, checkOnly, cwd, help };
|
|
}
|
|
|
|
function assertCliCapabilities(loaded: LoadedScenarios): void {
|
|
for (const entry of loaded.scenarios) {
|
|
if (entry.scenario.operations.some((operation) => operation.op === "step" && operation.step.compose !== undefined)) {
|
|
throw new ScenarioCliError(`scenario ${entry.scenario.id} uses compose, but scenario-cli has no text-generation adapter; use literal text/credentials or run through Pi`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function printResult(result: ScenarioCliResult): void {
|
|
if (!result.report) return;
|
|
if (result.options.format === "json") process.stdout.write(formatScenarioReportJson(result.report));
|
|
else process.stdout.write(formatScenarioReport(result.report));
|
|
}
|
|
|
|
function safeHookEnvironment(hook: ScenarioHook): NodeJS.ProcessEnv {
|
|
const env: NodeJS.ProcessEnv = {};
|
|
for (const name of SAFE_ENVIRONMENT_NAMES) if (process.env[name] !== undefined) env[name] = process.env[name];
|
|
for (const [name, value] of Object.entries(hook.env ?? {})) {
|
|
if (typeof value === "string") env[name] = value;
|
|
else {
|
|
const resolved = process.env[value.env];
|
|
if (resolved === undefined) throw new ScenarioCliError(`fixture hook environment reference ${value.env} is not set`);
|
|
env[name] = resolved;
|
|
}
|
|
}
|
|
env.PYTHONNOUSERSITE = "1";
|
|
env.PYTHONUTF8 = "1";
|
|
return env;
|
|
}
|
|
|
|
function runHook(hook: ScenarioHook, baseCwd: string, signal: AbortSignal): Promise<void> {
|
|
const cwd = resolve(baseCwd, hook.cwd ?? ".");
|
|
return runOwnedFixtureHook({
|
|
argv: [hook.command, ...hook.args],
|
|
cwd,
|
|
env: safeHookEnvironment(hook),
|
|
...(hook.timeoutMs === undefined ? {} : { timeoutMs: hook.timeoutMs }),
|
|
}, signal).then(() => undefined).catch((error) => {
|
|
throw new ScenarioExecutionError("hook", error instanceof Error ? error.message : "fixture hook failed");
|
|
});
|
|
}
|
|
|
|
export async function waitForScenarioHttpReadiness(
|
|
readiness: ScenarioHttpReadiness,
|
|
signal: AbortSignal,
|
|
deadline: number = Date.now() + (readiness.timeoutMs ?? FAST_BROWSER_SCENARIO_SERVER_BOUNDS.readinessTimeoutMs.default),
|
|
fetcher: typeof globalThis.fetch = globalThis.fetch,
|
|
): Promise<void> {
|
|
const timeoutMs = Math.min(readiness.timeoutMs ?? FAST_BROWSER_SCENARIO_SERVER_BOUNDS.readinessTimeoutMs.default, FAST_BROWSER_SCENARIO_SERVER_BOUNDS.readinessTimeoutMs.hardCeiling);
|
|
const readinessDeadline = Math.min(deadline, Date.now() + timeoutMs);
|
|
const expectedStatus = readiness.expectedStatus ?? 200;
|
|
let lastFailure = "fixture readiness did not succeed";
|
|
while (Date.now() < readinessDeadline) {
|
|
signal.throwIfAborted();
|
|
const requestController = new AbortController();
|
|
const onAbort = () => requestController.abort(signal.reason);
|
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
const timer = setTimeout(() => requestController.abort(new Error("fixture readiness request timed out")), Math.max(1, readinessDeadline - Date.now()));
|
|
try {
|
|
const response = await fetcher(readiness.url, { method: "GET", redirect: "error", signal: requestController.signal });
|
|
await response.body?.cancel().catch(() => {});
|
|
if (response.status === expectedStatus) return;
|
|
lastFailure = `fixture readiness returned HTTP ${response.status}`;
|
|
} catch (error) {
|
|
if (signal.aborted) throw error;
|
|
lastFailure = error instanceof Error ? error.message : lastFailure;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
signal.removeEventListener("abort", onAbort);
|
|
}
|
|
const remaining = readinessDeadline - Date.now();
|
|
if (remaining <= 0) break;
|
|
await new Promise((resolve) => setTimeout(resolve, Math.min(FAST_BROWSER_SCENARIO_SERVER_BOUNDS.readinessPollMs.default, remaining)));
|
|
}
|
|
throw new ScenarioExecutionError("fixture", lastFailure);
|
|
}
|
|
|
|
function operationName(operation: ScenarioOperation): BrowserEvidenceOperationName {
|
|
const name = operation.op === "step" ? "act" : operation.op;
|
|
return BROWSER_EVIDENCE_OPERATIONS.includes(name as BrowserEvidenceOperationName) ? name as BrowserEvidenceOperationName : "observe";
|
|
}
|
|
|
|
function isCompletedBrowserSessionClose(error: unknown): boolean {
|
|
return error instanceof Error && /^Browser session is no longer available:/.test(error.message);
|
|
}
|
|
|
|
function operationFailure(error: unknown, operation: ScenarioOperation, fallbackPhase: string): BrowserErrorDetails {
|
|
if (error instanceof BrowserOperationError) return error.details;
|
|
if (error instanceof ScenarioExecutionError && error.details) return error.details;
|
|
const message = error instanceof ScenarioExecutionError ? error.message : sanitizeReportError(error);
|
|
return {
|
|
code: "worker_failure",
|
|
operation: operationName(operation),
|
|
phase: error instanceof ScenarioExecutionError ? error.phase : fallbackPhase,
|
|
message,
|
|
mutationState: "not_attempted",
|
|
retry: "never",
|
|
};
|
|
}
|
|
|
|
interface ScenarioReferenceContext {
|
|
initialSelectedTabId: BrowserTabId;
|
|
}
|
|
|
|
interface ScenarioOperationState {
|
|
index: number;
|
|
id?: string;
|
|
op: ScenarioOperation["op"];
|
|
listedTabs?: readonly BrowserTab[];
|
|
selectedTab?: BrowserTab;
|
|
popup?: BrowserTab;
|
|
}
|
|
|
|
function resolveReferenceState(reference: ScenarioTabReference, states: readonly ScenarioOperationState[]): ScenarioOperationState {
|
|
if (reference.source !== "operation") throw new ScenarioExecutionError("resolve", "initial tab reference requires runtime selection");
|
|
const operationReference = reference.operation;
|
|
let state: ScenarioOperationState | undefined;
|
|
if ("index" in operationReference) state = states.find((candidate) => candidate.index === operationReference.index);
|
|
else state = states.find((candidate) => candidate.id === operationReference.id);
|
|
if (!state) throw new ScenarioExecutionError("resolve", "scenario tab reference operation result is unavailable");
|
|
return state;
|
|
}
|
|
|
|
async function resolveTabReference(reference: ScenarioTabReference, runtime: BrowserRuntime, states: readonly ScenarioOperationState[], context: ScenarioReferenceContext, signal: AbortSignal): Promise<BrowserTabId> {
|
|
const tabs = await runtime.listTabs(signal);
|
|
let expected: BrowserTab | undefined;
|
|
if (reference.source === "initial") {
|
|
expected = tabs.find((tab) => tab.tabId === context.initialSelectedTabId);
|
|
if (!expected) throw new ScenarioExecutionError("resolve", "scenario initial selected tab is unavailable or closed");
|
|
} else {
|
|
const state = resolveReferenceState(reference, states);
|
|
if (reference.result === "selectedTab") expected = state.selectedTab;
|
|
else if (reference.result === "popup") expected = state.popup;
|
|
else if ("index" in reference) expected = state.listedTabs?.[reference.index];
|
|
if (!expected) throw new ScenarioExecutionError("resolve", "scenario tab reference result is unavailable");
|
|
}
|
|
const current = tabs.find((tab) => tab.tabId === expected!.tabId);
|
|
if (!current) throw new ScenarioExecutionError("resolve", "scenario tab reference points to a missing or closed tab");
|
|
return current.tabId;
|
|
}
|
|
|
|
async function executeOperation(operation: ScenarioOperation, runtime: BrowserRuntime, credentials: DeclarativeScenario["credentials"], config: Awaited<ReturnType<typeof loadFastBrowserConfig>>, cwd: string, states: readonly ScenarioOperationState[], referenceContext: ScenarioReferenceContext, signal: AbortSignal, options: { timeoutMs: number; minConfidence: number }): Promise<{ mutationState: BrowserErrorDetails["mutationState"]; state?: Omit<ScenarioOperationState, "index" | "id" | "op"> }> {
|
|
if (operation.op === "step") {
|
|
const resolveCredential = credentials === undefined ? undefined : createCredentialResolver(credentials, { cwd, env: process.env });
|
|
let decider: ReturnType<typeof createJevDecider> | undefined;
|
|
const resolveTypesafeKey = async (keySignal: AbortSignal): Promise<string> => {
|
|
if (config.typesafeKey) {
|
|
const resolver = createCredentialResolver({ __fast_browser_typesafe_key: config.typesafeKey }, { cwd, env: process.env });
|
|
return (await resolver("__fast_browser_typesafe_key", keySignal)).trim();
|
|
}
|
|
const key = process.env.TYPESAFE_API_KEY?.trim();
|
|
if (!key) throw new DecisionError("TYPESAFE_API_KEY is required for semantic browser selection");
|
|
return key;
|
|
};
|
|
const result = await runSteps([operation.step], {
|
|
browser: runtime,
|
|
decide: async (input: DecisionInput, decideSignal: AbortSignal) => {
|
|
if (!decider) decider = createJevDecider({ apiKey: await resolveTypesafeKey(decideSignal) });
|
|
return decider(input, decideSignal);
|
|
},
|
|
...(resolveCredential ? { resolveCredential } : {}),
|
|
}, { signal, maxSteps: 1, timeoutMs: options.timeoutMs, minConfidence: options.minConfidence });
|
|
const trace = result.steps[0];
|
|
if (result.status !== "completed") {
|
|
throw new ScenarioExecutionError("step", result.reason, {
|
|
code: "worker_failure",
|
|
operation: "act",
|
|
phase: "step",
|
|
message: result.reason,
|
|
mutationState: trace?.mutationState ?? "not_attempted",
|
|
retry: "never",
|
|
});
|
|
}
|
|
return { mutationState: trace?.mutationState ?? "verified" };
|
|
}
|
|
switch (operation.op) {
|
|
case "read": {
|
|
const request: ReadRequest = operation.request.tab === undefined ? {} : { tabId: await resolveTabReference(operation.request.tab, runtime, states, referenceContext, signal) };
|
|
await runtime.read(request, signal); return { mutationState: "not_attempted" };
|
|
}
|
|
case "assert": {
|
|
const request: AssertRequest = { assertion: operation.request.assertion, wait: operation.request.wait, ...(operation.request.tab === undefined ? {} : { tabId: await resolveTabReference(operation.request.tab, runtime, states, referenceContext, signal) }) };
|
|
const result = await runtime.assert(request, signal);
|
|
if (result.status !== "satisfied") throw new ScenarioExecutionError("assert", `assertion ${result.status}`);
|
|
return { mutationState: "not_attempted" };
|
|
}
|
|
case "navigate": {
|
|
const request: NavigateRequest = { url: operation.request.url, ...(operation.request.tab === undefined ? {} : { tabId: await resolveTabReference(operation.request.tab, runtime, states, referenceContext, signal) }) };
|
|
await runtime.navigate(request, signal); return { mutationState: "verified" };
|
|
}
|
|
case "reload": {
|
|
const request: ReloadRequest = operation.request.tab === undefined ? {} : { tabId: await resolveTabReference(operation.request.tab, runtime, states, referenceContext, signal) };
|
|
await runtime.reload(request, signal); return { mutationState: "verified" };
|
|
}
|
|
case "listTabs": {
|
|
const tabs = await runtime.listTabs(signal);
|
|
const selection = operation.request?.select;
|
|
let listedTabs = tabs;
|
|
if (selection?.kind === "index") {
|
|
const selected = tabs[selection.index];
|
|
if (!selected) throw new ScenarioExecutionError("resolve", "scenario listTabs index did not match a tab");
|
|
listedTabs = [selected];
|
|
}
|
|
if (selection?.kind === "origin") {
|
|
const matches = tabs.filter((tab) => tab.origin === selection.origin);
|
|
if (matches.length !== 1) throw new ScenarioExecutionError("resolve", "scenario listTabs origin did not match exactly one tab");
|
|
listedTabs = matches;
|
|
}
|
|
return { mutationState: "not_attempted", state: { listedTabs } };
|
|
}
|
|
case "selectTab": {
|
|
const request: SelectTabRequest = { tabId: await resolveTabReference(operation.request.tab, runtime, states, referenceContext, signal) };
|
|
const result = await runtime.selectTab(request, signal);
|
|
return { mutationState: "verified", state: { selectedTab: result.tab } };
|
|
}
|
|
case "waitPopup": {
|
|
const request: WaitPopupRequest = { openerTabId: await resolveTabReference(operation.request.opener, runtime, states, referenceContext, signal) };
|
|
const result = await runtime.waitPopup(request, signal);
|
|
if (result.status !== "found" || !result.tab) throw new ScenarioExecutionError("waitPopup", "popup wait timed out");
|
|
return { mutationState: "not_attempted", state: { popup: result.tab } };
|
|
}
|
|
case "closeTab": {
|
|
const request: CloseTabRequest = { tabId: await resolveTabReference(operation.request.tab, runtime, states, referenceContext, signal) };
|
|
await runtime.closeTab(request, signal); return { mutationState: "verified" };
|
|
}
|
|
}
|
|
}
|
|
|
|
interface AttachedFailureEvidence {
|
|
status: "complete" | "partial" | "unavailable";
|
|
reason?: string;
|
|
screenshot?: unknown;
|
|
diagnostics?: unknown;
|
|
}
|
|
|
|
function attachedFailureEvidence(error: unknown): AttachedFailureEvidence | undefined {
|
|
if (!(error instanceof BrowserOperationError)) return undefined;
|
|
const value = (error as BrowserOperationError & { failureEvidence?: unknown }).failureEvidence;
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
const source = value as Record<string, unknown>;
|
|
if (source.status !== "complete" && source.status !== "partial" && source.status !== "unavailable") return undefined;
|
|
return {
|
|
status: source.status,
|
|
...(typeof source.reason === "string" ? { reason: source.reason } : {}),
|
|
...(source.screenshot === undefined ? {} : { screenshot: source.screenshot }),
|
|
...(source.diagnostics === undefined ? {} : { diagnostics: source.diagnostics }),
|
|
};
|
|
}
|
|
|
|
function attachedFailureEvidenceReferences(summary: AttachedFailureEvidence, reason: "failure" | "requested"): ScenarioEvidenceReference[] {
|
|
const references: ScenarioEvidenceReference[] = [];
|
|
for (const metadata of [summary.screenshot, summary.diagnostics]) {
|
|
if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) continue;
|
|
const reference = evidenceReference(metadata as Parameters<typeof evidenceReference>[0]);
|
|
if (reference) references.push(reference);
|
|
}
|
|
if (references.length || summary.status === "complete") return references;
|
|
return [{
|
|
kind: "other",
|
|
artifactId: `evidence-${reason}`,
|
|
status: "failed",
|
|
private: true,
|
|
forwardedToModel: false,
|
|
error: summary.reason ?? (summary.status === "partial" ? "partial failure evidence" : "failure evidence unavailable"),
|
|
}];
|
|
}
|
|
|
|
async function captureOperationFailureEvidence(runtime: BrowserRuntime | undefined, runId: string, operation: ScenarioOperation | undefined, signal: AbortSignal, error: unknown): Promise<ScenarioEvidenceReference[]> {
|
|
const attached = attachedFailureEvidence(error);
|
|
if (attached) return attachedFailureEvidenceReferences(attached, "failure");
|
|
return captureEvidence(runtime, runId, "failure", operation, signal);
|
|
}
|
|
|
|
async function captureEvidence(runtime: BrowserRuntime | undefined, runId: string, reason: "failure" | "requested", operation: ScenarioOperation | undefined, signal: AbortSignal): Promise<ScenarioEvidenceReference[]> {
|
|
if (!runtime) return [];
|
|
const unavailable = (): ScenarioEvidenceReference => ({
|
|
kind: "other",
|
|
artifactId: `evidence-${reason}`,
|
|
status: "failed",
|
|
private: true,
|
|
forwardedToModel: false,
|
|
error: "evidence capture failed",
|
|
});
|
|
try {
|
|
const evidence: FailureEvidence = await runtime.captureEvidence({ runId, reason, ...(operation ? { operation: operationName(operation) } : {}) }, signal);
|
|
const references = [evidence.screenshot, evidence.diagnostics].map((item) => item ? evidenceReference(item) : undefined).filter((item): item is NonNullable<ReturnType<typeof evidenceReference>> => Boolean(item));
|
|
return evidence.status === "unavailable" && references.length === 0 ? [unavailable()] : references;
|
|
} catch {
|
|
return [unavailable()];
|
|
}
|
|
}
|
|
|
|
interface ScenarioCaseResult {
|
|
id: string;
|
|
status: "passed" | "failed";
|
|
elapsedMs: number;
|
|
operations: ScenarioOperationReport[];
|
|
browser?: { product?: string; headless?: boolean; owned?: boolean; profile?: string; sessionId?: string };
|
|
failure?: { phase: string; message: string };
|
|
evidence: ScenarioEvidenceReference[];
|
|
cleanup: { status: "passed" | "failed" | "not_run"; detail: string };
|
|
}
|
|
|
|
async function runOneScenario(entry: LoadedScenarios["scenarios"][number], options: ScenarioCliOptions, manager: BrowserSessionManager, runtimeFactory: BrowserRuntimeFactory, artifactStoreFactory: typeof createPrivateArtifactStore, signal: AbortSignal): Promise<ScenarioCaseResult> {
|
|
const scenario = entry.scenario;
|
|
const started = Date.now();
|
|
const scenarioTimeoutMs = scenario.budgets?.timeoutMs ?? FAST_BROWSER_BUDGETS.runMs.default;
|
|
const scenarioDeadline = started + scenarioTimeoutMs;
|
|
const scenarioController = new AbortController();
|
|
const abortFromParent = (): void => scenarioController.abort(signal.reason ?? new Error("scenario run cancelled"));
|
|
if (signal.aborted) abortFromParent();
|
|
else signal.addEventListener("abort", abortFromParent, { once: true });
|
|
const scenarioTimer = setTimeout(() => scenarioController.abort(new Error("scenario deadline exceeded")), scenarioTimeoutMs);
|
|
scenarioTimer.unref?.();
|
|
const executionSignal = scenarioController.signal;
|
|
const remainingBudget = (): number => {
|
|
executionSignal.throwIfAborted();
|
|
const remaining = scenarioDeadline - Date.now();
|
|
if (remaining < 1) throw new ScenarioExecutionError("budget", "scenario deadline exceeded");
|
|
return Math.min(remaining, FAST_BROWSER_BUDGETS.runMs.hardCeiling);
|
|
};
|
|
const operations: ScenarioOperationReport[] = [];
|
|
const evidence: ScenarioEvidenceReference[] = [];
|
|
let runtime: BrowserRuntime | undefined;
|
|
let session: BrowserSession | undefined;
|
|
let managedServer: OwnedFixtureProcessHandle | undefined;
|
|
let managedServerController: AbortController | undefined;
|
|
let managedServerStopping = false;
|
|
let unexpectedServerExit = false;
|
|
let store: ReturnType<typeof createPrivateArtifactStore> | undefined;
|
|
let artifactRoot: string | undefined;
|
|
let failure: { phase: string; message: string } | undefined;
|
|
let cleanupStatus: "passed" | "failed" | "not_run" = "not_run";
|
|
let cleanupDetail = "cleanup not run";
|
|
let result: ScenarioCaseResult | undefined;
|
|
try {
|
|
remainingBudget();
|
|
if (scenario.hooks?.setup) await runHook(scenario.hooks.setup, options.cwd, executionSignal);
|
|
const overrides = (scenario.settings ?? {}) as ConfigOverrides;
|
|
const config = await loadFastBrowserConfig(options.cwd, overrides, { env: process.env });
|
|
const fixtureServer = scenario.fixture?.server;
|
|
if (fixtureServer) {
|
|
if (fixtureServer.mode === "managed") {
|
|
const shutdownTimeoutMs = fixtureServer.shutdownTimeoutMs ?? FAST_BROWSER_SCENARIO_SERVER_BOUNDS.serverShutdownMs.default;
|
|
const termTimeoutMs = Math.max(1, Math.floor(shutdownTimeoutMs / 2));
|
|
managedServerController = new AbortController();
|
|
managedServer = startOwnedFixtureProcess({
|
|
argv: [fixtureServer.start.command, ...fixtureServer.start.args],
|
|
cwd: resolve(options.cwd, fixtureServer.start.cwd ?? "."),
|
|
env: safeHookEnvironment(fixtureServer.start),
|
|
termTimeoutMs,
|
|
killTimeoutMs: Math.max(1, shutdownTimeoutMs - termTimeoutMs),
|
|
}, managedServerController.signal);
|
|
void managedServer.exit.then(() => {
|
|
if (!managedServerStopping && !executionSignal.aborted) {
|
|
unexpectedServerExit = true;
|
|
scenarioController.abort(new Error("managed fixture server exited before scenario completion"));
|
|
}
|
|
});
|
|
}
|
|
await waitForScenarioHttpReadiness(fixtureServer.readiness, executionSignal, scenarioDeadline);
|
|
}
|
|
if (scenario.hooks?.reset) await runHook(scenario.hooks.reset, options.cwd, executionSignal);
|
|
const effectiveConfig = config.browser?.mode === "launch"
|
|
? { ...config, browser: { ...config.browser, headless: options.headless } }
|
|
: config;
|
|
remainingBudget();
|
|
session = await manager.open(effectiveConfig, executionSignal);
|
|
artifactRoot = await mkdtemp(join(tmpdir(), "fast-browser-scenario-artifacts-"));
|
|
store = artifactStoreFactory(join(artifactRoot, "store"));
|
|
runtime = runtimeFactory.create({
|
|
sessionId: session.id,
|
|
cdpUrl: session.config.cdpUrl,
|
|
allowedOrigins: session.config.allowedOrigins,
|
|
...(session.config.python ? { python: session.config.python } : {}),
|
|
artifacts: store,
|
|
}, executionSignal);
|
|
const browser = session.config.browser;
|
|
const browserMetadata = {
|
|
...(browser?.product ? { product: browser.product } : {}),
|
|
...(browser?.headless === undefined ? {} : { headless: browser.headless }),
|
|
owned: session.owned,
|
|
profile: browser?.profile,
|
|
sessionId: session.id,
|
|
};
|
|
const initialTabs = await runtime.listTabs(executionSignal);
|
|
const initialSelected = initialTabs.filter((tab) => tab.selected);
|
|
if (initialSelected.length !== 1) throw new ScenarioExecutionError("resolve", "scenario initial selected tab is unavailable");
|
|
const referenceContext: ScenarioReferenceContext = { initialSelectedTabId: initialSelected[0]!.tabId };
|
|
const operationStates: ScenarioOperationState[] = [];
|
|
for (const operation of scenario.operations) {
|
|
const opStarted = Date.now();
|
|
try {
|
|
const operationResult = await executeOperation(operation, runtime, scenario.credentials, config, options.cwd, operationStates, referenceContext, executionSignal, { timeoutMs: remainingBudget(), minConfidence: scenario.budgets?.minConfidence ?? 0.85 });
|
|
operations.push({ index: operations.length, op: operation.op, status: "passed", elapsedMs: Date.now() - opStarted, mutationState: operationResult.mutationState });
|
|
operationStates.push({ index: operations.length - 1, ...(operation.id === undefined ? {} : { id: operation.id }), op: operation.op, ...operationResult.state });
|
|
} catch (error) {
|
|
const details = operationFailure(error, operation, operation.op);
|
|
operations.push({ index: operations.length, op: operation.op, status: "failed", elapsedMs: Date.now() - opStarted, mutationState: details.mutationState, error: details });
|
|
failure = { phase: details.phase, message: details.message };
|
|
if (scenario.evidence?.onFailure ?? true) evidence.push(...await captureOperationFailureEvidence(runtime, `${scenario.id}-${Date.now()}`, operation, executionSignal, error));
|
|
break;
|
|
}
|
|
}
|
|
if (!failure && (options.captureOnRequest || scenario.evidence?.onRequest)) {
|
|
evidence.push(...await captureEvidence(runtime, `${scenario.id}-${Date.now()}`, "requested", undefined, executionSignal));
|
|
}
|
|
if (!failure && managedServer && !managedServerStopping && (unexpectedServerExit || !managedServer.isAlive())) {
|
|
failure = { phase: "fixture", message: "managed fixture server exited before scenario completion" };
|
|
if (scenario.evidence?.onFailure ?? true) evidence.push(...await captureEvidence(runtime, `${scenario.id}-${Date.now()}`, "failure", undefined, executionSignal));
|
|
}
|
|
if (!failure && executionSignal.aborted) throw new ScenarioExecutionError("budget", "scenario execution cancelled");
|
|
cleanupStatus = "passed";
|
|
cleanupDetail = "owned runtime, artifacts, and browser session cleaned";
|
|
result = {
|
|
id: scenario.id,
|
|
status: failure ? "failed" : "passed",
|
|
elapsedMs: Date.now() - started,
|
|
operations,
|
|
browser: browserMetadata,
|
|
...(failure ? { failure } : {}),
|
|
evidence,
|
|
cleanup: { status: cleanupStatus, detail: cleanupDetail },
|
|
};
|
|
} catch (error) {
|
|
const phase = error instanceof ScenarioExecutionError ? error.phase : "startup";
|
|
failure = { phase, message: sanitizeReportError(error) };
|
|
if (runtime && (scenario.evidence?.onFailure ?? true)) evidence.push(...await captureOperationFailureEvidence(runtime, `${scenario.id}-${Date.now()}`, undefined, executionSignal, error));
|
|
result = {
|
|
id: scenario.id,
|
|
status: "failed",
|
|
elapsedMs: Date.now() - started,
|
|
operations,
|
|
...(session ? { browser: { owned: session.owned, sessionId: session.id, ...(session.config.browser?.product ? { product: session.config.browser.product } : {}) } } : {}),
|
|
failure,
|
|
evidence,
|
|
cleanup: { status: cleanupStatus, detail: cleanupDetail },
|
|
};
|
|
} finally {
|
|
clearTimeout(scenarioTimer);
|
|
signal.removeEventListener("abort", abortFromParent);
|
|
if (runtime || session || artifactRoot || managedServer || scenario.hooks?.cleanup) {
|
|
cleanupStatus = "passed";
|
|
cleanupDetail = "owned runtime, artifacts, browser session, and fixture resources cleaned";
|
|
}
|
|
try { await runtime?.close(); } catch { cleanupStatus = "failed"; cleanupDetail = "runtime cleanup failed"; }
|
|
if (session) {
|
|
try {
|
|
await manager.close(session.id);
|
|
} catch (error) {
|
|
if (!isCompletedBrowserSessionClose(error)) {
|
|
cleanupStatus = "failed";
|
|
cleanupDetail = "browser session cleanup failed";
|
|
}
|
|
}
|
|
}
|
|
if (scenario.hooks?.cleanup) {
|
|
// Main cancellation stops browser work, but cleanup gets its own finite
|
|
// signal so disposable fixture state is still reset on interruption.
|
|
try { await runHook(scenario.hooks.cleanup, options.cwd, new AbortController().signal); } catch { cleanupStatus = "failed"; cleanupDetail = "fixture cleanup hook failed"; }
|
|
}
|
|
if (managedServer) {
|
|
if (!managedServerStopping && !executionSignal.aborted && !managedServer.isAlive()) unexpectedServerExit = true;
|
|
managedServerStopping = true;
|
|
managedServerController?.abort(new Error("fixture server cleanup"));
|
|
try { await managedServer.stop(); } catch { cleanupStatus = "failed"; cleanupDetail = "fixture server cleanup failed"; }
|
|
}
|
|
if (unexpectedServerExit && result?.status === "passed") {
|
|
result.status = "failed";
|
|
result.failure = { phase: "fixture", message: "managed fixture server exited during scenario cleanup" };
|
|
}
|
|
try { await store?.close(); } catch { cleanupStatus = "failed"; cleanupDetail = "artifact cleanup failed"; }
|
|
const retainedEvidence = result?.evidence.filter((item) => item.status === "captured" && item.retainedPath).map((item) => item.retainedPath!) ?? [];
|
|
if (artifactRoot && retainedEvidence.length === 0) {
|
|
try { await rm(artifactRoot, { recursive: true, force: true }); } catch { cleanupStatus = "failed"; cleanupDetail = "artifact cleanup failed"; }
|
|
} else if (retainedEvidence.length > 0) {
|
|
cleanupDetail = `committed private evidence retained until caller cleanup: ${retainedEvidence.join(", ")}`;
|
|
}
|
|
if (result) result.cleanup = { status: cleanupStatus, detail: cleanupDetail };
|
|
}
|
|
return result!;
|
|
}
|
|
|
|
async function runParallel(entries: LoadedScenarios["scenarios"], options: ScenarioCliOptions, manager: BrowserSessionManager, runtimeFactory: BrowserRuntimeFactory, artifactStoreFactory: typeof createPrivateArtifactStore, signal: AbortSignal): Promise<ScenarioCaseResult[]> {
|
|
const results: ScenarioCaseResult[] = [];
|
|
let next = 0;
|
|
const worker = async (): Promise<void> => {
|
|
while (true) {
|
|
const index = next++;
|
|
if (index >= entries.length) return;
|
|
results[index] = await runOneScenario(entries[index]!, options, manager, runtimeFactory, artifactStoreFactory, signal);
|
|
}
|
|
};
|
|
await Promise.all(Array.from({ length: Math.min(options.parallel, entries.length) }, worker));
|
|
return results;
|
|
}
|
|
|
|
/** Validate or execute selected scenarios through the published BrowserRuntime factory. */
|
|
export async function runScenarioCli(argv: readonly string[] = process.argv.slice(2), dependencies: ScenarioCliDependencies = {}): Promise<ScenarioCliResult> {
|
|
let options: ScenarioCliOptions;
|
|
try {
|
|
options = parseScenarioCliArgs(argv);
|
|
} catch (error) {
|
|
const fallback: ScenarioCliOptions = { scenarioPaths: [], format: "human", headless: true, parallel: 1, captureOnRequest: false, checkOnly: false, cwd: process.cwd(), help: false };
|
|
const report = createScenarioReport({ status: "blocked", failure: { phase: "cli", error: sanitizeReportError(error) }, cleanup: { status: "not_run", detail: "cleanup not run" } });
|
|
const result = { exitCode: 2, options: fallback, report }; printResult(result); return result;
|
|
}
|
|
if (options.help) { process.stdout.write(SCENARIO_CLI_USAGE); return { exitCode: 0, options }; }
|
|
const started = Date.now();
|
|
try {
|
|
const loaded = await loadScenarioFiles(options.scenarioPaths, { cwd: options.cwd });
|
|
assertCliCapabilities(loaded);
|
|
if (options.checkOnly) {
|
|
const report = createScenarioReport({ status: "validated", elapsedMs: Date.now() - started, scenarios: loaded.scenarios.map(({ scenario }) => ({ id: scenario.id, status: "validated", elapsedMs: 0 })), cleanup: { status: "not_run", detail: "browser execution was not started" } });
|
|
if (options.reportPath) writeScenarioReport(resolveReportPath(options.reportPath, options.cwd), report);
|
|
const result = { exitCode: 0, options, loaded, report }; printResult(result); return result;
|
|
}
|
|
const manager = dependencies.sessionManager ?? createBrowserSessionManager();
|
|
const ownsManager = dependencies.sessionManager === undefined;
|
|
const runtimeFactory = dependencies.runtimeFactory ?? createBrowserRuntimeFactory();
|
|
const artifactStoreFactory = dependencies.artifactStoreFactory ?? createPrivateArtifactStore;
|
|
const controller = new AbortController();
|
|
const onSignal = () => controller.abort(new Error("scenario run cancelled"));
|
|
const externalSignal = dependencies.signal;
|
|
const onExternalSignal = () => controller.abort(externalSignal?.reason ?? new Error("scenario run cancelled"));
|
|
if (externalSignal?.aborted) onExternalSignal();
|
|
else externalSignal?.addEventListener("abort", onExternalSignal, { once: true });
|
|
process.once("SIGINT", onSignal);
|
|
process.once("SIGTERM", onSignal);
|
|
try {
|
|
const cases = await runParallel(loaded.scenarios, options, manager, runtimeFactory, artifactStoreFactory, controller.signal);
|
|
const failed = cases.find((item) => item.status === "failed");
|
|
const cleanupFailed = cases.some((item) => item.cleanup.status === "failed");
|
|
const report = createScenarioReport({
|
|
status: failed || cleanupFailed ? "failed" : "passed",
|
|
elapsedMs: Date.now() - started,
|
|
scenarios: cases,
|
|
evidence: cases.flatMap((item) => item.evidence),
|
|
...(failed?.failure ? { failure: { scenarioId: failed.id, phase: failed.failure.phase, error: failed.failure.message } } : {}),
|
|
cleanup: { status: cleanupFailed ? "failed" : "passed", detail: cleanupFailed ? "one or more owned resources failed cleanup" : "owned runtime, artifact, fixture, and browser resources cleaned" },
|
|
});
|
|
if (options.reportPath) writeScenarioReport(resolveReportPath(options.reportPath, options.cwd), report);
|
|
const result = { exitCode: report.status === "passed" ? 0 : 1, options, loaded, report }; printResult(result); return result;
|
|
} finally {
|
|
process.removeListener("SIGINT", onSignal);
|
|
process.removeListener("SIGTERM", onSignal);
|
|
externalSignal?.removeEventListener("abort", onExternalSignal);
|
|
if (ownsManager) await manager.closeAll().catch(() => {});
|
|
}
|
|
} catch (error) {
|
|
const report = createScenarioReport({ status: "failed", elapsedMs: Date.now() - started, failure: { phase: "input", error: sanitizeReportError(error) }, cleanup: { status: "not_run", detail: "browser execution was not started" } });
|
|
if (options.reportPath) { try { writeScenarioReport(resolveReportPath(options.reportPath, options.cwd), report); } catch { /* preserve original failure */ } }
|
|
const result = { exitCode: 2, options, report }; printResult(result); return result;
|
|
}
|
|
}
|
|
|
|
function resolveReportPath(path: string, cwd: string): string { return path.startsWith("/") ? path : `${cwd}/${path}`; }
|
|
|
|
if (import.meta.main) { const result = await runScenarioCli(); process.exitCode = result.exitCode; }
|