69 lines
3.4 KiB
TypeScript
69 lines
3.4 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
import {
|
|
pairedArmOrder,
|
|
safeFailure,
|
|
summarizeBenchmarkAttempts,
|
|
validWorkerCounters,
|
|
summarizeTimings,
|
|
validateBenchmarkOptions,
|
|
type BenchmarkAttempt,
|
|
} from "./performance-benchmark.ts";
|
|
|
|
async function files(): Promise<{ root: string; paths: string[] }> {
|
|
const root = await mkdtemp(join("/tmp", "fast-browser-performance-test-"));
|
|
const paths = await Promise.all(["baseline.py", "optimized.py", "python", "chromium"].map(async (name) => {
|
|
const path = join(root, name);
|
|
await writeFile(path, "# test\n", { mode: 0o700 });
|
|
return path;
|
|
}));
|
|
return { root, paths };
|
|
}
|
|
|
|
test("timing aggregation reports bounded p50 and p95 without wall-clock assumptions", () => {
|
|
assert.deepEqual(summarizeTimings([10, 20, 30, 40]), { count: 4, p50: 25, p95: 38.5 });
|
|
assert.deepEqual(summarizeTimings([]), { count: 0, p50: null, p95: null });
|
|
});
|
|
|
|
test("paired order alternates AB and BA while retaining failed attempts", () => {
|
|
assert.deepEqual(pairedArmOrder(1), ["baseline", "optimized"]);
|
|
assert.deepEqual(pairedArmOrder(2), ["optimized", "baseline"]);
|
|
});
|
|
|
|
test("attempt aggregation excludes failures from timing summaries but retains them for accounting", () => {
|
|
const attempts: BenchmarkAttempt[] = [
|
|
{ arm: "baseline", phase: "warm", sample: 1, status: "passed", elapsedMs: 10, startupMs: null },
|
|
{ arm: "baseline", phase: "warm", sample: 2, status: "failed", elapsedMs: null, startupMs: null, error: "worker failed" },
|
|
{ arm: "baseline", phase: "warm", sample: 3, status: "passed", elapsedMs: 30, startupMs: null },
|
|
];
|
|
assert.deepEqual(summarizeBenchmarkAttempts(attempts, "baseline", "warm"), { count: 2, p50: 20, p95: 29 });
|
|
assert.equal(attempts.filter((attempt) => attempt.status === "failed").length, 1);
|
|
});
|
|
|
|
test("benchmark input bounds reject unbounded samples and candidate sets", async () => {
|
|
const { root, paths } = await files();
|
|
try {
|
|
assert.throws(() => validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, samples: 41 }), /samples/);
|
|
assert.throws(() => validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, candidateCounts: [1] }), /candidate counts/);
|
|
const valid = validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, samples: 1, coldSamples: 1, candidateCounts: [0, 10, 60] });
|
|
assert.equal(valid.samples, 1);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("missing or malformed worker counters are unavailable", () => {
|
|
assert.equal(validWorkerCounters({ cdp: { "Page.enable": 1 }, ipc: { _send: 2 } }), true);
|
|
assert.equal(validWorkerCounters({ cdp: { "Page.enable": -1 }, ipc: { _send: 2 } }), false);
|
|
assert.equal(validWorkerCounters({ cdp: { "Page.enable": 1 }, ipc: { "bad key": 2 } }), false);
|
|
assert.equal(validWorkerCounters(undefined), false);
|
|
});
|
|
|
|
test("report failures use fixed safe categories and never include raw error text", () => {
|
|
const safe = safeFailure("popup-close", "unverified");
|
|
assert.equal(safe, "popup-close:unverified");
|
|
assert.doesNotMatch(safe, /https?:|secret|Ada|tmp|performance/);
|
|
});
|