273 lines
12 KiB
TypeScript
273 lines
12 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { chmod, mkdtemp, mkdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import test from "node:test";
|
|
import {
|
|
createScenarioReport,
|
|
formatScenarioReport,
|
|
formatScenarioReportJson,
|
|
sanitizeReportError,
|
|
sanitizeRuntimeScenarioReport,
|
|
writeScenarioReport,
|
|
} from "./reports.ts";
|
|
|
|
async function temporary(fn: (directory: string) => Promise<void>): Promise<void> {
|
|
const directory = await mkdtemp(join(tmpdir(), "fast-browser-report-test-"));
|
|
try { await fn(directory); } finally { await rm(directory, { recursive: true, force: true }); }
|
|
}
|
|
|
|
test("redacts secrets, URL paths/query values, headers, and unbounded text", () => {
|
|
const message = sanitizeReportError(
|
|
"request https://fixture.test/private?token=private-token failed Authorization: Bearer provider-secret password=fixture-password",
|
|
["private-token", "provider-secret", "fixture-password"],
|
|
);
|
|
assert.doesNotMatch(message, /private-token|provider-secret|fixture-password/);
|
|
assert.match(message, /https:\/\/fixture\.test/);
|
|
assert.doesNotMatch(sanitizeReportError("token=unknown-private-value"), /unknown-private-value/);
|
|
assert.doesNotMatch(message, /\/private|Bearer/);
|
|
|
|
const report = createScenarioReport({
|
|
status: "failed",
|
|
elapsedMs: 99,
|
|
failure: { scenarioId: "login", phase: "act", error: new Error("password=fixture-password") , secrets: ["fixture-password"] },
|
|
scenarios: [{
|
|
id: "login",
|
|
status: "failed",
|
|
elapsedMs: 99,
|
|
browser: { product: "chromium", profile: "/private/profile", sessionId: "session-1" },
|
|
failure: { phase: "act", message: "https://fixture.test/private?token=fixture-password" },
|
|
evidence: [{ kind: "screenshot", artifactId: "shot-1", status: "captured", file: "/private/shot.png" }],
|
|
}],
|
|
evidence: [{ kind: "request", artifactId: "request-1", status: "failed", error: "Authorization: Bearer fixture-password" }],
|
|
cleanup: { status: "failed", detail: "worker cleanup failed" },
|
|
});
|
|
const serialized = JSON.stringify(report);
|
|
assert.doesNotMatch(serialized, /fixture-password|private\/profile|private\/shot/);
|
|
assert.equal(report.failure?.message, "password=[redacted]");
|
|
assert.equal(report.evidence[0]?.forwardedToModel, false);
|
|
assert.equal(report.evidence[0]?.private, true);
|
|
});
|
|
|
|
test("direct human and JSON formatters sanitize cleanup and metadata strings while retaining safe artifact paths", () => {
|
|
const secret = "fixture-secret";
|
|
const report = {
|
|
version: 1 as const,
|
|
kind: "fast-browser-scenario-report" as const,
|
|
generatedAt: new Date().toISOString(),
|
|
status: "failed" as const,
|
|
elapsedMs: 3,
|
|
scenarios: [{
|
|
id: "scenario",
|
|
status: "failed" as const,
|
|
elapsedMs: 2,
|
|
browser: { sessionId: `session-${secret}`, profile: "safe-profile" },
|
|
cleanup: { status: "failed" as const, detail: `cleanup password=${secret}` },
|
|
evidence: [{ kind: "other" as const, artifactId: "artifact-1", status: "captured" as const, file: "artifact-1.json", retainedPath: "/private/run/artifact-1.json", private: true as const, forwardedToModel: false as const }],
|
|
}],
|
|
evidence: [],
|
|
cleanup: { status: "failed" as const, detail: `session=${secret}` },
|
|
};
|
|
const human = formatScenarioReport(report, [secret]);
|
|
const json = formatScenarioReportJson(report, [secret]);
|
|
assert.doesNotMatch(human, new RegExp(secret));
|
|
assert.doesNotMatch(json, new RegExp(secret));
|
|
assert.match(json, /\/private\/run\/artifact-1\.json/);
|
|
assert.match(human, /cleanup password=\[redacted\]/);
|
|
const runtime = {
|
|
version: 1 as const,
|
|
scenarioId: "scenario",
|
|
status: "failed" as const,
|
|
elapsedMs: 1,
|
|
operations: [],
|
|
evidence: [{ artifactId: `artifact-${secret}`, kind: "diagnostics" as const, contentType: "application/json" as const, path: "/private/run/a-1.json", bytes: 2, owner: { sessionId: secret, runId: "run", operationId: "op" }, permissions: "0600" as const, localOnly: true as const, modelForwarded: false as const }],
|
|
cleanup: "complete" as const,
|
|
};
|
|
assert.doesNotMatch(formatScenarioReport(runtime, [secret]), new RegExp(secret));
|
|
assert.match(formatScenarioReportJson(runtime, [secret]), /a-1\.json/);
|
|
});
|
|
|
|
test("preserves the original failure when evidence capture fails", () => {
|
|
const report = createScenarioReport({
|
|
status: "failed",
|
|
failure: {
|
|
scenarioId: "form",
|
|
phase: "act",
|
|
error: "original action failed",
|
|
},
|
|
evidence: [{ kind: "screenshot", artifactId: "capture", status: "failed", error: "capture unavailable" }],
|
|
cleanup: { status: "failed", detail: "cleanup failed after evidence error" },
|
|
});
|
|
assert.deepEqual(report.failure, { scenarioId: "form", phase: "act", message: "original action failed" });
|
|
assert.equal(report.evidence[0]?.status, "failed");
|
|
assert.equal(report.cleanup.status, "failed");
|
|
const human = formatScenarioReport(report);
|
|
assert.match(human, /original action failed/);
|
|
assert.match(human, /failure evidence unavailable/);
|
|
});
|
|
|
|
test("sanitizes the published runtime report shape and preserves artifact ownership flags", () => {
|
|
const report = sanitizeRuntimeScenarioReport({
|
|
version: 1,
|
|
scenarioId: "local-login",
|
|
status: "failed",
|
|
elapsedMs: 9,
|
|
operations: [{
|
|
index: 0,
|
|
op: "step",
|
|
status: "failed",
|
|
elapsedMs: 4,
|
|
mutationState: "unknown",
|
|
error: {
|
|
code: "worker_failure",
|
|
operation: "act",
|
|
phase: "act",
|
|
message: "https://fixture.test/private?token=secret",
|
|
mutationState: "unknown",
|
|
retry: "never",
|
|
},
|
|
}],
|
|
evidence: [{
|
|
artifactId: "shot-1",
|
|
kind: "screenshot",
|
|
contentType: "image/png",
|
|
path: "/private/session/run/shot-1.png",
|
|
bytes: 12,
|
|
owner: { sessionId: "session-1", runId: "run-1", operationId: "op-1" },
|
|
permissions: "0600",
|
|
localOnly: true,
|
|
modelForwarded: false,
|
|
}],
|
|
cleanup: "failed",
|
|
}, ["secret"]);
|
|
assert.equal(report.version, 1);
|
|
assert.equal(report.operations[0]?.error?.message, "https://fixture.test");
|
|
assert.equal(report.evidence[0]?.path, "shot-1.png");
|
|
assert.equal(report.evidence[0]?.localOnly, true);
|
|
assert.equal(report.evidence[0]?.modelForwarded, false);
|
|
});
|
|
|
|
test("writes versioned reports atomically with private permissions and no arbitrary details", async () => {
|
|
await temporary(async (directory) => {
|
|
const reportPath = join(directory, "reports", "result.json");
|
|
const report = createScenarioReport({
|
|
status: "passed",
|
|
generatedAt: "2026-09-19T21:00:00.000Z",
|
|
scenarios: [{ id: "local-form", status: "passed", elapsedMs: 4, browser: { product: "chromium", headless: true, owned: true } }],
|
|
cleanup: { status: "passed", detail: "owned fixture and browser cleaned" },
|
|
});
|
|
writeScenarioReport(reportPath, report);
|
|
const content = JSON.parse(await readFile(reportPath, "utf8"));
|
|
assert.equal(content.version, 1);
|
|
assert.equal(content.kind, "fast-browser-scenario-report");
|
|
assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
|
|
assert.equal((await stat(join(directory, "reports"))).mode & 0o777, 0o700);
|
|
assert.equal(content.scenarios[0].browser.headless, true);
|
|
|
|
await writeFile(reportPath, "old report");
|
|
assert.throws(() => writeScenarioReport(join(directory, "reports"), report));
|
|
assert.equal(await readFile(reportPath, "utf8"), "old report");
|
|
});
|
|
});
|
|
|
|
test("on-disk report writes redact known secrets for both report branches", async () => {
|
|
await temporary(async (directory) => {
|
|
const secret = "disk-secret";
|
|
const envelopePath = join(directory, "envelope.json");
|
|
writeScenarioReport(envelopePath, createScenarioReport({
|
|
status: "failed",
|
|
failure: { phase: "cleanup", error: `password=${secret}` },
|
|
cleanup: { status: "failed", detail: `cleanup token=${secret}` },
|
|
}), [secret]);
|
|
assert.doesNotMatch(await readFile(envelopePath, "utf8"), new RegExp(secret));
|
|
|
|
const runtimePath = join(directory, "runtime.json");
|
|
writeScenarioReport(runtimePath, {
|
|
version: 1,
|
|
scenarioId: "scenario",
|
|
status: "failed",
|
|
elapsedMs: 1,
|
|
operations: [],
|
|
evidence: [{ artifactId: "a-1", kind: "diagnostics", contentType: "application/json", path: "/private/a-1.json", bytes: 2, owner: { sessionId: secret, runId: "run", operationId: "op" }, permissions: "0600", localOnly: true, modelForwarded: false }],
|
|
cleanup: "complete",
|
|
}, [secret]);
|
|
assert.doesNotMatch(await readFile(runtimePath, "utf8"), new RegExp(secret));
|
|
});
|
|
});
|
|
|
|
test("evidence reasons use fixed producer values and redact transport/cookie forms", async () => {
|
|
await temporary(async (directory) => {
|
|
const report = createScenarioReport({
|
|
status: "failed",
|
|
evidence: [
|
|
{ kind: "other", artifactId: "opaque", status: "failed", error: "opaque-token-123" },
|
|
{ kind: "other", artifactId: "ws", status: "failed", error: "ws://user:password@example.test/private" },
|
|
{ kind: "other", artifactId: "cookie", status: "failed", error: "cookie=session-secret" },
|
|
{ kind: "other", artifactId: "known", status: "failed", error: "browser session is unavailable" },
|
|
],
|
|
});
|
|
assert.deepEqual(report.evidence.map((entry) => entry.error), [
|
|
"failure evidence unavailable",
|
|
"failure evidence unavailable",
|
|
"failure evidence unavailable",
|
|
"browser session is unavailable",
|
|
]);
|
|
const path = join(directory, "reasons.json");
|
|
writeScenarioReport(path, report, ["session-secret"]);
|
|
const saved = await readFile(path, "utf8");
|
|
assert.doesNotMatch(saved, /opaque-token|password@example|session-secret/);
|
|
assert.match(saved, /browser session is unavailable/);
|
|
|
|
const runtimePath = join(directory, "runtime-reasons.json");
|
|
writeScenarioReport(runtimePath, {
|
|
version: 1,
|
|
scenarioId: "reason-runtime",
|
|
status: "failed",
|
|
elapsedMs: 1,
|
|
operations: [{ index: 0, op: "read", status: "failed", elapsedMs: 1, mutationState: "not_attempted", error: { code: "worker_failure", operation: "read", phase: "dispatch", message: "ws://user:password@example.test/private cookie=session-secret", mutationState: "not_attempted", retry: "never" } }],
|
|
evidence: [],
|
|
cleanup: "failed",
|
|
});
|
|
const runtimeSaved = await readFile(runtimePath, "utf8");
|
|
assert.doesNotMatch(runtimeSaved, /password@example|session-secret/);
|
|
});
|
|
});
|
|
|
|
test("accepts an owned 0755 report directory without changing its mode", async () => {
|
|
await temporary(async (directory) => {
|
|
await chmod(directory, 0o755);
|
|
const reportPath = join(directory, "result.json");
|
|
const report = createScenarioReport({ status: "passed", cleanup: { status: "passed", detail: "done" } });
|
|
writeScenarioReport(reportPath, report);
|
|
assert.equal((await stat(directory)).mode & 0o777, 0o755);
|
|
assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
|
|
});
|
|
});
|
|
|
|
test("rejects unsafe or symlink report destinations", async () => {
|
|
await temporary(async (directory) => {
|
|
const report = createScenarioReport({ status: "passed", cleanup: { status: "passed", detail: "done" } });
|
|
const unsafe = join(directory, "unsafe");
|
|
await mkdir(unsafe);
|
|
await chmod(unsafe, 0o777);
|
|
assert.throws(() => writeScenarioReport(join(unsafe, "result.json"), report), /group\/world writable/);
|
|
const linkedDirectory = join(directory, "linked-directory");
|
|
await symlink(directory, linkedDirectory);
|
|
assert.throws(() => writeScenarioReport(join(linkedDirectory, "result.json"), report), /symlink/);
|
|
const linkedFile = join(directory, "linked-file");
|
|
await writeFile(join(directory, "real.json"), "old");
|
|
await symlink(join(directory, "real.json"), linkedFile);
|
|
assert.throws(() => writeScenarioReport(linkedFile, report), /regular file|symlink/);
|
|
});
|
|
});
|
|
|
|
test("rejects remote report destinations and invalid evidence paths", async () => {
|
|
const report = createScenarioReport({
|
|
status: "failed",
|
|
evidence: [{ kind: "screenshot", artifactId: "shot", status: "captured", file: "https://evil.test/shot.png" }],
|
|
cleanup: { status: "not_run", detail: "not run" },
|
|
});
|
|
assert.equal(report.evidence.length, 0);
|
|
await assert.rejects(async () => writeScenarioReport("https://evil.test/report.json", report));
|
|
});
|