test(fast-browser): add paired local browser performance benchmark
This commit is contained in:
@@ -0,0 +1,109 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>Fast-browser performance fixture</title>
|
||||||
|
<style>
|
||||||
|
body { font: 16px system-ui, sans-serif; margin: 2rem; }
|
||||||
|
.candidate-grid { display: grid; gap: .35rem; max-width: 42rem; }
|
||||||
|
.status { min-height: 1.5rem; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<main>
|
||||||
|
<h1>Local form validation</h1>
|
||||||
|
<p id="status" class="status">Fixture ready</p>
|
||||||
|
<p id="profile-status" class="status">Profile draft loaded</p>
|
||||||
|
<p id="popup-status" class="status">Popup idle</p>
|
||||||
|
<p id="preview" class="status"></p>
|
||||||
|
<form id="form" novalidate></form>
|
||||||
|
</main>
|
||||||
|
<script>
|
||||||
|
(() => {
|
||||||
|
const params = new URLSearchParams(location.search);
|
||||||
|
const mode = params.get('mode') || 'density';
|
||||||
|
const count = Math.max(0, Math.min(60, Number(params.get('count') || 0)));
|
||||||
|
const form = document.querySelector('#form');
|
||||||
|
const status = document.querySelector('#status');
|
||||||
|
const profileStatus = document.querySelector('#profile-status');
|
||||||
|
const popupStatus = document.querySelector('#popup-status');
|
||||||
|
const preview = document.querySelector('#preview');
|
||||||
|
const label = (text, input) => {
|
||||||
|
const wrapper = document.createElement('div');
|
||||||
|
const caption = document.createElement('label');
|
||||||
|
caption.textContent = text;
|
||||||
|
caption.htmlFor = input.id;
|
||||||
|
wrapper.append(caption, input);
|
||||||
|
return wrapper;
|
||||||
|
};
|
||||||
|
if (mode === 'popup') {
|
||||||
|
status.textContent = 'Popup ready';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (mode === 'density') {
|
||||||
|
const grid = document.createElement('div');
|
||||||
|
grid.className = 'candidate-grid';
|
||||||
|
for (let index = 0; index < count; index += 1) {
|
||||||
|
const input = document.createElement('input');
|
||||||
|
input.id = `candidate-${index}`;
|
||||||
|
input.type = 'text';
|
||||||
|
input.placeholder = `Search filter ${index + 1}`;
|
||||||
|
grid.append(label(`Filter ${index + 1}`, input));
|
||||||
|
}
|
||||||
|
form.append(grid);
|
||||||
|
status.textContent = `Density fixture with ${count} controls`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const display = document.createElement('input');
|
||||||
|
display.id = 'display-name';
|
||||||
|
display.type = 'text';
|
||||||
|
display.value = 'existing value';
|
||||||
|
form.append(label('Display name', display));
|
||||||
|
|
||||||
|
const save = document.createElement('button');
|
||||||
|
save.type = 'button';
|
||||||
|
save.textContent = 'Save profile';
|
||||||
|
save.addEventListener('click', () => {
|
||||||
|
profileStatus.textContent = display.value === 'Ada Lovelace' ? 'Profile saved' : 'Profile validation failed';
|
||||||
|
});
|
||||||
|
form.append(save);
|
||||||
|
|
||||||
|
const previewButton = document.createElement('button');
|
||||||
|
previewButton.type = 'button';
|
||||||
|
previewButton.textContent = 'Preview profile';
|
||||||
|
previewButton.addEventListener('click', () => {
|
||||||
|
preview.textContent = `Preview ready: ${display.value}`;
|
||||||
|
});
|
||||||
|
form.append(previewButton);
|
||||||
|
|
||||||
|
const immediate = document.createElement('button');
|
||||||
|
immediate.type = 'button';
|
||||||
|
immediate.textContent = 'Immediate action';
|
||||||
|
immediate.addEventListener('click', () => { status.textContent = 'Immediate complete'; });
|
||||||
|
form.append(immediate);
|
||||||
|
|
||||||
|
const delayed = document.createElement('button');
|
||||||
|
delayed.type = 'button';
|
||||||
|
delayed.textContent = 'Delayed action';
|
||||||
|
delayed.addEventListener('click', () => setTimeout(() => { status.textContent = 'Delayed complete'; }, 40));
|
||||||
|
form.append(delayed);
|
||||||
|
|
||||||
|
const popupButton = document.createElement('button');
|
||||||
|
popupButton.type = 'button';
|
||||||
|
popupButton.textContent = 'Open popup';
|
||||||
|
popupButton.addEventListener('click', () => {
|
||||||
|
popupStatus.textContent = 'Popup requested';
|
||||||
|
window.open(`${location.origin}${location.pathname}?mode=popup`, '_blank', 'popup,width=420,height=240');
|
||||||
|
});
|
||||||
|
form.append(popupButton);
|
||||||
|
for (let index = 0; index < 10; index += 1) {
|
||||||
|
const input = document.createElement('input');
|
||||||
|
input.id = `extra-${index}`;
|
||||||
|
input.type = 'text';
|
||||||
|
input.placeholder = `Extra field ${index + 1}`;
|
||||||
|
form.append(label(`Extra field ${index + 1}`, input));
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import test from "node:test";
|
||||||
|
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import {
|
||||||
|
pairedArmOrder,
|
||||||
|
safeFailure,
|
||||||
|
summarizeBenchmarkAttempts,
|
||||||
|
validWorkerCounters,
|
||||||
|
summarizeTimings,
|
||||||
|
validateBenchmarkOptions,
|
||||||
|
type BenchmarkAttempt,
|
||||||
|
} from "./performance-benchmark.ts";
|
||||||
|
|
||||||
|
async function files(): Promise<{ root: string; paths: string[] }> {
|
||||||
|
const root = await mkdtemp(join("/tmp", "fast-browser-performance-test-"));
|
||||||
|
const paths = await Promise.all(["baseline.py", "optimized.py", "python", "chromium"].map(async (name) => {
|
||||||
|
const path = join(root, name);
|
||||||
|
await writeFile(path, "# test\n", { mode: 0o700 });
|
||||||
|
return path;
|
||||||
|
}));
|
||||||
|
return { root, paths };
|
||||||
|
}
|
||||||
|
|
||||||
|
test("timing aggregation reports bounded p50 and p95 without wall-clock assumptions", () => {
|
||||||
|
assert.deepEqual(summarizeTimings([10, 20, 30, 40]), { count: 4, p50: 25, p95: 38.5 });
|
||||||
|
assert.deepEqual(summarizeTimings([]), { count: 0, p50: null, p95: null });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("paired order alternates AB and BA while retaining failed attempts", () => {
|
||||||
|
assert.deepEqual(pairedArmOrder(1), ["baseline", "optimized"]);
|
||||||
|
assert.deepEqual(pairedArmOrder(2), ["optimized", "baseline"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("attempt aggregation excludes failures from timing summaries but retains them for accounting", () => {
|
||||||
|
const attempts: BenchmarkAttempt[] = [
|
||||||
|
{ arm: "baseline", phase: "warm", sample: 1, status: "passed", elapsedMs: 10, startupMs: null },
|
||||||
|
{ arm: "baseline", phase: "warm", sample: 2, status: "failed", elapsedMs: null, startupMs: null, error: "worker failed" },
|
||||||
|
{ arm: "baseline", phase: "warm", sample: 3, status: "passed", elapsedMs: 30, startupMs: null },
|
||||||
|
];
|
||||||
|
assert.deepEqual(summarizeBenchmarkAttempts(attempts, "baseline", "warm"), { count: 2, p50: 20, p95: 29 });
|
||||||
|
assert.equal(attempts.filter((attempt) => attempt.status === "failed").length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("benchmark input bounds reject unbounded samples and candidate sets", async () => {
|
||||||
|
const { root, paths } = await files();
|
||||||
|
try {
|
||||||
|
assert.throws(() => validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, samples: 41 }), /samples/);
|
||||||
|
assert.throws(() => validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, candidateCounts: [1] }), /candidate counts/);
|
||||||
|
const valid = validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, samples: 1, coldSamples: 1, candidateCounts: [0, 10, 60] });
|
||||||
|
assert.equal(valid.samples, 1);
|
||||||
|
} finally {
|
||||||
|
await rm(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("missing or malformed worker counters are unavailable", () => {
|
||||||
|
assert.equal(validWorkerCounters({ cdp: { "Page.enable": 1 }, ipc: { _send: 2 } }), true);
|
||||||
|
assert.equal(validWorkerCounters({ cdp: { "Page.enable": -1 }, ipc: { _send: 2 } }), false);
|
||||||
|
assert.equal(validWorkerCounters({ cdp: { "Page.enable": 1 }, ipc: { "bad key": 2 } }), false);
|
||||||
|
assert.equal(validWorkerCounters(undefined), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("report failures use fixed safe categories and never include raw error text", () => {
|
||||||
|
const safe = safeFailure("popup-close", "unverified");
|
||||||
|
assert.equal(safe, "popup-close:unverified");
|
||||||
|
assert.doesNotMatch(safe, /https?:|secret|Ada|tmp|performance/);
|
||||||
|
});
|
||||||
@@ -0,0 +1,601 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { spawn as nodeSpawn, spawnSync } from "node:child_process";
|
||||||
|
import { existsSync, readFileSync, statSync } from "node:fs";
|
||||||
|
import { chmod, mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||||
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { dirname, join } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { createBrowserTransport, type ChildProcessLike, type SpawnLike } from "./bridge.ts";
|
||||||
|
import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts";
|
||||||
|
import type { BrowserRuntime, BrowserTransportFactory, Candidate, PrivateArtifactStore } from "./contracts.ts";
|
||||||
|
import { createBrowserRuntimeFactory } from "./runtime.ts";
|
||||||
|
|
||||||
|
const ROOT = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const CURRENT_WORKER = join(ROOT, "browser_worker.py");
|
||||||
|
const PROBE = join(ROOT, "performance_probe.py");
|
||||||
|
const FIXTURE = join(ROOT, "fixtures", "performance.html");
|
||||||
|
const DEFAULT_PYTHON = join(process.cwd(), ".pi", "fast-browser-demo", "venv", "bin", "python");
|
||||||
|
const DEFAULT_BROWSER = "/usr/bin/chromium";
|
||||||
|
const DEFAULT_COUNTS = [0, 10, 60] as const;
|
||||||
|
const DEFAULT_SAMPLES = 20;
|
||||||
|
const DEFAULT_COLD_SAMPLES = 1;
|
||||||
|
const MAX_SAMPLES = 40;
|
||||||
|
const MAX_COLD_SAMPLES = 5;
|
||||||
|
const MAX_COUNTS = 3;
|
||||||
|
|
||||||
|
type Arm = "baseline" | "optimized";
|
||||||
|
type Phase = "warm" | "cold-start";
|
||||||
|
|
||||||
|
export interface PerformanceBenchmarkOptions {
|
||||||
|
baselineWorkerPath: string;
|
||||||
|
optimizedWorkerPath?: string;
|
||||||
|
pythonPath?: string;
|
||||||
|
browserPath?: string;
|
||||||
|
samples?: number;
|
||||||
|
coldSamples?: number;
|
||||||
|
candidateCounts?: readonly number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TimingSummary {
|
||||||
|
count: number;
|
||||||
|
p50: number | null;
|
||||||
|
p95: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BenchmarkAttempt {
|
||||||
|
arm: Arm;
|
||||||
|
phase: Phase;
|
||||||
|
sample: number;
|
||||||
|
order?: 1 | 2;
|
||||||
|
status: "passed" | "failed";
|
||||||
|
elapsedMs: number | null;
|
||||||
|
startupMs: number | null;
|
||||||
|
error?: string;
|
||||||
|
protocolOperations?: Record<string, number>;
|
||||||
|
densityObserveMs?: Partial<Record<"0" | "10" | "60", number>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerMethodCounts {
|
||||||
|
cdp: Record<string, number>;
|
||||||
|
ipc: Record<string, number>;
|
||||||
|
cdpCalls: number;
|
||||||
|
ipcCalls: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BenchmarkReport {
|
||||||
|
ok: boolean;
|
||||||
|
measurement: { workerMethods: "selected Python worker CDP/IPC helper methods"; protocolOperations: "TypeScript JSONL operation names"; warmPriming: "one density=0 observe per arm, excluded from warm samples"; coldStartup: "browser session open plus first worker observe"; runtimeNarrowingMeasured: false };
|
||||||
|
samples: number;
|
||||||
|
coldSamples: number;
|
||||||
|
candidateCounts: readonly number[];
|
||||||
|
attempts: readonly BenchmarkAttempt[];
|
||||||
|
warm: Record<Arm, TimingSummary>;
|
||||||
|
coldStartup: Record<Arm, TimingSummary>;
|
||||||
|
workerMethods: Partial<Record<Arm, WorkerMethodCounts>>;
|
||||||
|
protocolOperations: Partial<Record<Arm, Record<string, number>>>;
|
||||||
|
densityObserve: Record<Arm, Record<"0" | "10" | "60", TimingSummary>>;
|
||||||
|
pairedOrder: readonly (readonly [Arm, Arm])[];
|
||||||
|
counterStatus: "complete" | "unavailable";
|
||||||
|
cleanup: { status: "passed" | "failed"; error?: string; retainedRoot: boolean };
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FixtureServer {
|
||||||
|
origin: string;
|
||||||
|
close(): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ArmHandle {
|
||||||
|
arm: Arm;
|
||||||
|
session: BrowserSession;
|
||||||
|
runtime: BrowserRuntime;
|
||||||
|
protocol: Record<string, number>;
|
||||||
|
countFile: string;
|
||||||
|
closed: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const EMPTY_ARTIFACT_STORE: PrivateArtifactStore = {
|
||||||
|
async open() {
|
||||||
|
throw new Error("benchmark evidence is disabled");
|
||||||
|
},
|
||||||
|
async close() { /* no-op */ },
|
||||||
|
};
|
||||||
|
|
||||||
|
type FailureStage = "cli" | "cold-start" | "warm-setup" | "warm-flow" | "density-observe" | "popup-close" | "counter" | "cleanup";
|
||||||
|
type FailureCategory = "failed" | "unverified" | "unavailable";
|
||||||
|
|
||||||
|
export function safeFailure(stage: FailureStage, category: FailureCategory = "failed"): string {
|
||||||
|
return `${stage}:${category}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
class BenchmarkFailure extends Error {
|
||||||
|
readonly stage: FailureStage;
|
||||||
|
readonly category: FailureCategory;
|
||||||
|
|
||||||
|
constructor(stage: FailureStage, category: FailureCategory) {
|
||||||
|
super(`${stage}:${category}`);
|
||||||
|
this.stage = stage;
|
||||||
|
this.category = category;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function failureFor(error: unknown, fallback: FailureStage): string {
|
||||||
|
return error instanceof BenchmarkFailure ? safeFailure(error.stage, error.category) : safeFailure(fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
function boundedInteger(value: unknown, label: string, min: number, max: number): number {
|
||||||
|
if (!Number.isSafeInteger(value) || (value as number) < min || (value as number) > max) {
|
||||||
|
throw new Error(`${label} must be an integer between ${min} and ${max}`);
|
||||||
|
}
|
||||||
|
return value as number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requiredFile(value: string, label: string): string {
|
||||||
|
if (typeof value !== "string" || value.length === 0 || value.length > 4_000 || value.includes("\0")) {
|
||||||
|
throw new Error(`${label} is invalid`);
|
||||||
|
}
|
||||||
|
if (!existsSync(value) || !statSync(value).isFile()) throw new Error(`${label} is not a file`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateBenchmarkOptions(options: PerformanceBenchmarkOptions): Required<PerformanceBenchmarkOptions> {
|
||||||
|
if (!options || typeof options !== "object") throw new Error("benchmark options are required");
|
||||||
|
const baselineWorkerPath = requiredFile(options.baselineWorkerPath, "baseline worker path");
|
||||||
|
const optimizedWorkerPath = requiredFile(options.optimizedWorkerPath ?? CURRENT_WORKER, "optimized worker path");
|
||||||
|
const pythonPath = requiredFile(options.pythonPath ?? process.env.FAST_BROWSER_PYTHON ?? DEFAULT_PYTHON, "Python path");
|
||||||
|
const browserPath = requiredFile(options.browserPath ?? DEFAULT_BROWSER, "browser path");
|
||||||
|
const samples = boundedInteger(options.samples ?? DEFAULT_SAMPLES, "samples", 1, MAX_SAMPLES);
|
||||||
|
const coldSamples = boundedInteger(options.coldSamples ?? DEFAULT_COLD_SAMPLES, "cold samples", 1, MAX_COLD_SAMPLES);
|
||||||
|
const candidateCounts = [...(options.candidateCounts ?? DEFAULT_COUNTS)];
|
||||||
|
if (candidateCounts.length < 1 || candidateCounts.length > MAX_COUNTS || candidateCounts.some((value) => ![0, 10, 60].includes(value))) {
|
||||||
|
throw new Error("candidate counts must be a non-empty subset of 0, 10, and 60");
|
||||||
|
}
|
||||||
|
return { baselineWorkerPath, optimizedWorkerPath, pythonPath, browserPath, samples, coldSamples, candidateCounts };
|
||||||
|
}
|
||||||
|
|
||||||
|
function percentile(values: readonly number[], fraction: number): number | null {
|
||||||
|
if (values.length === 0) return null;
|
||||||
|
const sorted = [...values].sort((a, b) => a - b);
|
||||||
|
const position = (sorted.length - 1) * fraction;
|
||||||
|
const lower = Math.floor(position);
|
||||||
|
const upper = Math.ceil(position);
|
||||||
|
if (lower === upper) return sorted[lower]!;
|
||||||
|
return sorted[lower]! + (sorted[upper]! - sorted[lower]!) * (position - lower);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarizeTimings(values: readonly number[]): TimingSummary {
|
||||||
|
return { count: values.length, p50: percentile(values, 0.5), p95: percentile(values, 0.95) };
|
||||||
|
}
|
||||||
|
|
||||||
|
function increment(target: Record<string, number>, key: string): void {
|
||||||
|
target[key] = (target[key] ?? 0) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function countProtocol(child: ReturnType<typeof nodeSpawn>, target: Record<string, number>): void {
|
||||||
|
const stdin = child.stdin;
|
||||||
|
if (!stdin) throw new Error("benchmark worker stdin is unavailable");
|
||||||
|
const write = stdin.write.bind(stdin);
|
||||||
|
stdin.write = ((data: string | Uint8Array, ...args: unknown[]) => {
|
||||||
|
const text = typeof data === "string" ? data : Buffer.from(data).toString("utf8");
|
||||||
|
for (const line of text.split("\n")) {
|
||||||
|
if (!line) continue;
|
||||||
|
try {
|
||||||
|
const wire = JSON.parse(line) as { operation?: { op?: unknown } };
|
||||||
|
const op = wire.operation?.op;
|
||||||
|
if (typeof op === "string" && /^[A-Za-z][A-Za-z0-9]{0,40}$/.test(op)) increment(target, op);
|
||||||
|
} catch { /* bridge owns malformed-request handling */ }
|
||||||
|
}
|
||||||
|
return write(data, ...(args as []));
|
||||||
|
}) as typeof stdin.write;
|
||||||
|
}
|
||||||
|
|
||||||
|
function instrumentedSpawn(
|
||||||
|
workerPath: string,
|
||||||
|
countFile: string,
|
||||||
|
protocol: Record<string, number>,
|
||||||
|
): SpawnLike {
|
||||||
|
return (command, args, options) => {
|
||||||
|
const child = nodeSpawn(command, [
|
||||||
|
...args,
|
||||||
|
"--performance-worker", workerPath,
|
||||||
|
"--performance-count-file", countFile,
|
||||||
|
], options as never);
|
||||||
|
countProtocol(child, protocol);
|
||||||
|
return child as unknown as ChildProcessLike;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createInstrumentedRuntime(
|
||||||
|
armWorkerPath: string,
|
||||||
|
countFile: string,
|
||||||
|
protocol: Record<string, number>,
|
||||||
|
session: BrowserSession,
|
||||||
|
origin: string,
|
||||||
|
pythonPath: string,
|
||||||
|
signal: AbortSignal,
|
||||||
|
): BrowserRuntime {
|
||||||
|
const transportFactory: BrowserTransportFactory = {
|
||||||
|
create(options, parentSignal) {
|
||||||
|
return createBrowserTransport({
|
||||||
|
...options,
|
||||||
|
workerPath: PROBE,
|
||||||
|
spawn: instrumentedSpawn(armWorkerPath, countFile, protocol),
|
||||||
|
}, parentSignal);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return createBrowserRuntimeFactory(transportFactory).create({
|
||||||
|
sessionId: session.id,
|
||||||
|
cdpUrl: session.config.cdpUrl,
|
||||||
|
allowedOrigins: [origin],
|
||||||
|
python: pythonPath,
|
||||||
|
artifacts: EMPTY_ARTIFACT_STORE,
|
||||||
|
}, signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createFixtureServer(): Promise<FixtureServer> {
|
||||||
|
const template = readFileSync(FIXTURE, "utf8");
|
||||||
|
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
|
||||||
|
const path = new URL(request.url ?? "/", "http://127.0.0.1").pathname;
|
||||||
|
if (path !== "/performance.html") {
|
||||||
|
response.writeHead(404);
|
||||||
|
response.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
response.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" });
|
||||||
|
response.end(template);
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
server.once("error", reject);
|
||||||
|
server.listen(0, "127.0.0.1", resolve);
|
||||||
|
});
|
||||||
|
const address = server.address();
|
||||||
|
if (!address || typeof address === "string") throw new Error("performance fixture did not expose a port");
|
||||||
|
return {
|
||||||
|
origin: `http://127.0.0.1:${address.port}`,
|
||||||
|
async close() {
|
||||||
|
server.closeAllConnections?.();
|
||||||
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function launchConfig(origin: string, url: string, browserPath: string): Parameters<BrowserSessionManager["open"]>[0] {
|
||||||
|
return {
|
||||||
|
allowedOrigins: [origin],
|
||||||
|
browser: { mode: "launch", product: "chromium", executablePath: browserPath, headless: true, startUrl: url },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function candidate(observation: Awaited<ReturnType<BrowserRuntime["observe"]>>, name: string, action: "click" | "fill"): Candidate {
|
||||||
|
const matches = observation.candidates.filter((item) => item.name === name && item.actions.includes(action));
|
||||||
|
assert.equal(matches.length, 1, `expected one ${action} candidate`);
|
||||||
|
return matches[0]!;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runWarmFlow(handle: ArmHandle, origin: string, candidateCounts: readonly number[], signal: AbortSignal): Promise<Partial<Record<"0" | "10" | "60", number>>> {
|
||||||
|
const densityObserveMs: Partial<Record<"0" | "10" | "60", number>> = {};
|
||||||
|
for (const count of candidateCounts) {
|
||||||
|
const density = await handle.runtime.navigate({ url: `${origin}/performance.html?mode=density&count=${count}` }, signal);
|
||||||
|
assert.equal(density.status, "committed");
|
||||||
|
const observeStarted = Date.now();
|
||||||
|
try {
|
||||||
|
const observation = await handle.runtime.observe(signal);
|
||||||
|
densityObserveMs[String(count) as "0" | "10" | "60"] = Date.now() - observeStarted;
|
||||||
|
assert.equal(observation.candidates.length, count);
|
||||||
|
} catch {
|
||||||
|
throw new BenchmarkFailure("density-observe", "failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const page = await handle.runtime.navigate({ url: `${origin}/performance.html?mode=flow&count=10` }, signal);
|
||||||
|
assert.equal(page.status, "committed");
|
||||||
|
const tabs = await handle.runtime.listTabs(signal);
|
||||||
|
const main = tabs.find((tab) => tab.selected);
|
||||||
|
assert(main, "main benchmark tab is unavailable");
|
||||||
|
const mainTab = main;
|
||||||
|
|
||||||
|
let observation = await handle.runtime.observe(signal);
|
||||||
|
const field = candidate(observation, "Display name", "fill");
|
||||||
|
const filled = await handle.runtime.act({ observationId: observation.observationId, candidateId: field.id, action: "fill", text: "Ada Lovelace" }, signal);
|
||||||
|
assert.equal(filled.verified, true);
|
||||||
|
|
||||||
|
observation = await handle.runtime.observe(signal);
|
||||||
|
const save = candidate(observation, "Save profile", "click");
|
||||||
|
const saved = await handle.runtime.act({ observationId: observation.observationId, candidateId: save.id, action: "click", postcondition: { kind: "text", value: "Profile saved" } }, signal);
|
||||||
|
assert.equal(saved.verified, true);
|
||||||
|
const saveAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Profile saved" }, wait: "bounded" }, signal);
|
||||||
|
assert.equal(saveAssert.status, "satisfied");
|
||||||
|
|
||||||
|
observation = await handle.runtime.observe(signal);
|
||||||
|
const previewButton = candidate(observation, "Preview profile", "click");
|
||||||
|
const previewRequested = await handle.runtime.act({ observationId: observation.observationId, candidateId: previewButton.id, action: "click", postcondition: { kind: "text", value: "Preview ready" } }, signal);
|
||||||
|
assert.equal(previewRequested.verified, true);
|
||||||
|
|
||||||
|
observation = await handle.runtime.observe(signal);
|
||||||
|
const immediate = candidate(observation, "Immediate action", "click");
|
||||||
|
const clicked = await handle.runtime.act({ observationId: observation.observationId, candidateId: immediate.id, action: "click", postcondition: { kind: "text", value: "Immediate complete" } }, signal);
|
||||||
|
assert.equal(clicked.verified, true);
|
||||||
|
const immediateAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Immediate complete" }, wait: "none" }, signal);
|
||||||
|
assert.equal(immediateAssert.status, "satisfied");
|
||||||
|
|
||||||
|
observation = await handle.runtime.observe(signal);
|
||||||
|
const delayed = candidate(observation, "Delayed action", "click");
|
||||||
|
const delayedResult = await handle.runtime.act({ observationId: observation.observationId, candidateId: delayed.id, action: "click", postcondition: { kind: "text", value: "Delayed complete" } }, signal);
|
||||||
|
assert.equal(delayedResult.verified, true);
|
||||||
|
const delayedAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Delayed complete" }, wait: "bounded" }, signal);
|
||||||
|
assert.equal(delayedAssert.status, "satisfied");
|
||||||
|
|
||||||
|
observation = await handle.runtime.observe(signal);
|
||||||
|
const popupButton = candidate(observation, "Open popup", "click");
|
||||||
|
const popupRequest = await handle.runtime.act({ observationId: observation.observationId, candidateId: popupButton.id, action: "click", postcondition: { kind: "text", value: "Popup requested" } }, signal);
|
||||||
|
assert.equal(popupRequest.verified, true);
|
||||||
|
const popup = await handle.runtime.waitPopup({ openerTabId: mainTab.tabId }, signal);
|
||||||
|
assert.equal(popup.status, "found");
|
||||||
|
assert(popup.tab);
|
||||||
|
const popupTab = popup.tab;
|
||||||
|
const popupRead = await handle.runtime.read({ tabId: popupTab.tabId }, signal);
|
||||||
|
assert.match(popupRead.visibleText, /Popup ready/);
|
||||||
|
const popupReady = await handle.runtime.assert({ tabId: popupTab.tabId, assertion: { kind: "visibleText", match: "contains", value: "Popup ready" }, wait: "bounded" }, signal);
|
||||||
|
assert.equal(popupReady.status, "satisfied");
|
||||||
|
try {
|
||||||
|
await handle.runtime.closeTab({ tabId: popupTab.tabId }, signal);
|
||||||
|
} catch {
|
||||||
|
throw new BenchmarkFailure("popup-close", "unverified");
|
||||||
|
}
|
||||||
|
const preview = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Preview ready: Ada Lovelace" }, wait: "bounded" }, signal);
|
||||||
|
assert.equal(preview.status, "satisfied");
|
||||||
|
return densityObserveMs;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validCounterMap(value: unknown): value is Record<string, number> {
|
||||||
|
return !!value && typeof value === "object" && !Array.isArray(value)
|
||||||
|
&& Object.entries(value).every(([key, count]) => /^[A-Za-z0-9_.-]{1,120}$/.test(key) && Number.isSafeInteger(count) && count >= 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validWorkerCounters(value: unknown): value is { cdp: Record<string, number>; ipc: Record<string, number> } {
|
||||||
|
if (!value || typeof value !== "object") return false;
|
||||||
|
const counters = value as { cdp?: unknown; ipc?: unknown };
|
||||||
|
return validCounterMap(counters.cdp) && validCounterMap(counters.ipc);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readWorkerCounts(path: string): Promise<WorkerMethodCounts | undefined> {
|
||||||
|
try {
|
||||||
|
const value = JSON.parse(await readFile(path, "utf8")) as { cdp?: unknown; ipc?: unknown };
|
||||||
|
if (!validWorkerCounters(value)) return undefined;
|
||||||
|
const cdp = value.cdp;
|
||||||
|
const ipc = value.ipc;
|
||||||
|
return {
|
||||||
|
cdp,
|
||||||
|
ipc,
|
||||||
|
cdpCalls: Object.values(cdp).reduce((sum, count) => sum + count, 0),
|
||||||
|
ipcCalls: Object.values(ipc).reduce((sum, count) => sum + count, 0),
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function ownedProcessLines(root: string): string[] | undefined {
|
||||||
|
const result = spawnSync("ps", ["-eo", "pid=,args="], { encoding: "utf8", timeout: 5_000 });
|
||||||
|
if (result.status !== 0) return undefined;
|
||||||
|
return String(result.stdout).split(/\r?\n/).filter((line) => line.includes(root));
|
||||||
|
}
|
||||||
|
|
||||||
|
function snapshotCounts(value: Record<string, number>): Record<string, number> {
|
||||||
|
return { ...value };
|
||||||
|
}
|
||||||
|
|
||||||
|
function countDelta(before: Record<string, number>, after: Record<string, number>): Record<string, number> {
|
||||||
|
const result: Record<string, number> = {};
|
||||||
|
for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) {
|
||||||
|
const difference = (after[key] ?? 0) - (before[key] ?? 0);
|
||||||
|
if (difference > 0) result[key] = difference;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function closeHandle(handle: ArmHandle | undefined): Promise<string | undefined> {
|
||||||
|
if (!handle || handle.closed) return undefined;
|
||||||
|
handle.closed = true;
|
||||||
|
try {
|
||||||
|
await handle.runtime.close();
|
||||||
|
return undefined;
|
||||||
|
} catch {
|
||||||
|
return safeFailure("cleanup");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function pairedArmOrder(sample: number): readonly [Arm, Arm] {
|
||||||
|
return sample % 2 === 1 ? ["baseline", "optimized"] : ["optimized", "baseline"];
|
||||||
|
}
|
||||||
|
|
||||||
|
function attemptsFor(attempts: readonly BenchmarkAttempt[], arm: Arm, phase: Phase): number[] {
|
||||||
|
return attempts.filter((item) => item.arm === arm && item.phase === phase && item.status === "passed" && item.elapsedMs !== null).map((item) => item.elapsedMs!);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarizeBenchmarkAttempts(attempts: readonly BenchmarkAttempt[], arm: Arm, phase: Phase): TimingSummary {
|
||||||
|
return summarizeTimings(attemptsFor(attempts, arm, phase));
|
||||||
|
}
|
||||||
|
|
||||||
|
function densitySummaries(attempts: readonly BenchmarkAttempt[]): Record<Arm, Record<"0" | "10" | "60", TimingSummary>> {
|
||||||
|
const result = {} as Record<Arm, Record<"0" | "10" | "60", TimingSummary>>;
|
||||||
|
for (const arm of ["baseline", "optimized"] as const) {
|
||||||
|
result[arm] = {} as Record<"0" | "10" | "60", TimingSummary>;
|
||||||
|
for (const count of ["0", "10", "60"] as const) {
|
||||||
|
result[arm][count] = summarizeTimings(attempts
|
||||||
|
.filter((attempt) => attempt.arm === arm && attempt.phase === "warm" && attempt.status === "passed" && attempt.densityObserveMs?.[count] !== undefined)
|
||||||
|
.map((attempt) => attempt.densityObserveMs![count]!));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptions): Promise<BenchmarkReport> {
|
||||||
|
const config = validateBenchmarkOptions(options);
|
||||||
|
const fixture = await createFixtureServer();
|
||||||
|
const root = await mkdtemp(join(tmpdir(), "fast-browser-performance-"));
|
||||||
|
await mkdir(join(root, "leases"), { recursive: true, mode: 0o700 });
|
||||||
|
await chmod(join(root, "leases"), 0o700);
|
||||||
|
const manager = createBrowserSessionManager({ profileDirectory: join(root, "profiles"), leaseDirectory: join(root, "leases"), startupTimeoutMs: 20_000, closeTimeoutMs: 4_000 });
|
||||||
|
const controller = new AbortController();
|
||||||
|
const attempts: BenchmarkAttempt[] = [];
|
||||||
|
const workerMethods: Partial<Record<Arm, WorkerMethodCounts>> = {};
|
||||||
|
const protocolOperations: Partial<Record<Arm, Record<string, number>>> = {};
|
||||||
|
const handles: Partial<Record<Arm, ArmHandle>> = {};
|
||||||
|
const setupErrors: Partial<Record<Arm, string>> = {};
|
||||||
|
const pairedOrder: Array<readonly [Arm, Arm]> = [];
|
||||||
|
let cleanupError: string | undefined;
|
||||||
|
let retainedRoot = false;
|
||||||
|
let counterUnavailable = false;
|
||||||
|
try {
|
||||||
|
const armPaths: Record<Arm, string> = { baseline: config.baselineWorkerPath, optimized: config.optimizedWorkerPath };
|
||||||
|
for (const arm of ["baseline", "optimized"] as const) {
|
||||||
|
for (let sample = 1; sample <= config.coldSamples; sample += 1) {
|
||||||
|
const countFile = join(root, `${arm}-cold-${sample}.json`);
|
||||||
|
const protocol: Record<string, number> = {};
|
||||||
|
let session: BrowserSession | undefined;
|
||||||
|
let runtime: BrowserRuntime | undefined;
|
||||||
|
const started = Date.now();
|
||||||
|
let startupMs: number | null = null;
|
||||||
|
try {
|
||||||
|
const opened = Date.now();
|
||||||
|
session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=10`, config.browserPath), controller.signal);
|
||||||
|
const first = Date.now();
|
||||||
|
runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
|
||||||
|
const observation = await runtime.observe(controller.signal);
|
||||||
|
assert.equal(observation.candidates.length, 10);
|
||||||
|
startupMs = first - opened + (Date.now() - first);
|
||||||
|
attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol } });
|
||||||
|
} catch (error) {
|
||||||
|
attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: null, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol } });
|
||||||
|
} finally {
|
||||||
|
try { await runtime?.close(); } catch { cleanupError ??= safeFailure("cleanup"); }
|
||||||
|
if (session) await manager.close(session.id).catch(() => { cleanupError ??= safeFailure("cleanup"); });
|
||||||
|
const counts = await readWorkerCounts(countFile);
|
||||||
|
if (counts) workerMethods[arm] = counts;
|
||||||
|
else counterUnavailable = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const countFile = join(root, `${arm}-warm.json`);
|
||||||
|
const protocol: Record<string, number> = {};
|
||||||
|
const session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=0`, config.browserPath), controller.signal);
|
||||||
|
const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
|
||||||
|
const handle: ArmHandle = { arm, session, runtime, protocol, countFile, closed: false };
|
||||||
|
handles[arm] = handle;
|
||||||
|
await runtime.observe(controller.signal); // prime the persistent worker; excluded from warm samples
|
||||||
|
} catch (error) {
|
||||||
|
const failedHandle = handles[arm];
|
||||||
|
if (failedHandle) {
|
||||||
|
await closeHandle(failedHandle);
|
||||||
|
await manager.close(failedHandle.session.id).catch(() => {});
|
||||||
|
}
|
||||||
|
delete handles[arm];
|
||||||
|
setupErrors[arm] = safeFailure("warm-setup");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let sample = 1; sample <= config.samples; sample += 1) {
|
||||||
|
const order = pairedArmOrder(sample);
|
||||||
|
pairedOrder.push(order);
|
||||||
|
for (let position = 0; position < order.length; position += 1) {
|
||||||
|
const arm = order[position]!;
|
||||||
|
const handle = handles[arm];
|
||||||
|
const before = handle ? snapshotCounts(handle.protocol) : {};
|
||||||
|
const started = Date.now();
|
||||||
|
try {
|
||||||
|
if (!handle) throw new Error("arm setup failed");
|
||||||
|
const densityObserveMs = await runWarmFlow(handle, fixture.origin, config.candidateCounts, controller.signal);
|
||||||
|
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), densityObserveMs });
|
||||||
|
} catch (error) {
|
||||||
|
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: null, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {} });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
for (const arm of ["baseline", "optimized"] as const) {
|
||||||
|
const handle = handles[arm];
|
||||||
|
const error = await closeHandle(handle);
|
||||||
|
if (error) cleanupError ??= error;
|
||||||
|
if (handle) {
|
||||||
|
const counts = await readWorkerCounts(handle.countFile);
|
||||||
|
if (counts) workerMethods[arm] = counts;
|
||||||
|
else counterUnavailable = true;
|
||||||
|
protocolOperations[arm] = { ...handle.protocol };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await manager.closeAll().catch(() => { cleanupError ??= safeFailure("cleanup"); });
|
||||||
|
await fixture.close().catch(() => { cleanupError ??= safeFailure("cleanup"); });
|
||||||
|
const leftovers = ownedProcessLines(root);
|
||||||
|
if (leftovers === undefined || leftovers.length > 0) {
|
||||||
|
cleanupError ??= safeFailure("cleanup", "unavailable");
|
||||||
|
retainedRoot = true;
|
||||||
|
}
|
||||||
|
if (!cleanupError) {
|
||||||
|
await rm(root, { recursive: true, force: true }).catch(() => {
|
||||||
|
cleanupError ??= safeFailure("cleanup");
|
||||||
|
retainedRoot = true;
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
retainedRoot = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const warm: Record<Arm, TimingSummary> = {
|
||||||
|
baseline: summarizeBenchmarkAttempts(attempts, "baseline", "warm"),
|
||||||
|
optimized: summarizeBenchmarkAttempts(attempts, "optimized", "warm"),
|
||||||
|
};
|
||||||
|
const coldStartup: Record<Arm, TimingSummary> = {
|
||||||
|
baseline: summarizeTimings(attempts.filter((item) => item.arm === "baseline" && item.phase === "cold-start" && item.status === "passed" && item.startupMs !== null).map((item) => item.startupMs!)),
|
||||||
|
optimized: summarizeTimings(attempts.filter((item) => item.arm === "optimized" && item.phase === "cold-start" && item.status === "passed" && item.startupMs !== null).map((item) => item.startupMs!)),
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
ok: !cleanupError && !counterUnavailable && workerMethods.baseline !== undefined && workerMethods.optimized !== undefined && attempts.length === config.samples * 2 + config.coldSamples * 2 && attempts.every((item) => item.status === "passed"),
|
||||||
|
measurement: { workerMethods: "selected Python worker CDP/IPC helper methods", protocolOperations: "TypeScript JSONL operation names", warmPriming: "one density=0 observe per arm, excluded from warm samples", coldStartup: "browser session open plus first worker observe", runtimeNarrowingMeasured: false },
|
||||||
|
samples: config.samples,
|
||||||
|
coldSamples: config.coldSamples,
|
||||||
|
candidateCounts: config.candidateCounts,
|
||||||
|
attempts,
|
||||||
|
warm,
|
||||||
|
coldStartup,
|
||||||
|
workerMethods,
|
||||||
|
protocolOperations,
|
||||||
|
densityObserve: densitySummaries(attempts),
|
||||||
|
pairedOrder,
|
||||||
|
counterStatus: !counterUnavailable && workerMethods.baseline && workerMethods.optimized ? "complete" : "unavailable",
|
||||||
|
cleanup: cleanupError ? { status: "failed", error: cleanupError, retainedRoot } : { status: "passed", retainedRoot: false },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseArgs(argv: readonly string[]): PerformanceBenchmarkOptions & { help?: boolean } {
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
for (let index = 0; index < argv.length; index += 1) {
|
||||||
|
const arg = argv[index]!;
|
||||||
|
if (arg === "--help" || arg === "-h") return { baselineWorkerPath: "", help: true };
|
||||||
|
const values: Record<string, string> = { "--baseline-worker": "baselineWorkerPath", "--optimized-worker": "optimizedWorkerPath", "--python": "pythonPath", "--browser": "browserPath", "--samples": "samples", "--cold-samples": "coldSamples", "--counts": "candidateCounts" };
|
||||||
|
const key = values[arg];
|
||||||
|
if (!key || index + 1 >= argv.length) throw new Error(`unknown or incomplete option: ${arg}`);
|
||||||
|
const value = argv[++index]!;
|
||||||
|
result[key] = key === "samples" || key === "coldSamples" ? Number(value) : key === "candidateCounts" ? value.split(",").map(Number) : value;
|
||||||
|
}
|
||||||
|
if (typeof result.baselineWorkerPath !== "string") throw new Error("--baseline-worker is required");
|
||||||
|
return result as unknown as PerformanceBenchmarkOptions;
|
||||||
|
}
|
||||||
|
|
||||||
|
function printReport(report: BenchmarkReport): void {
|
||||||
|
console.log(JSON.stringify(report, null, 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.main) {
|
||||||
|
try {
|
||||||
|
const parsed = parseArgs(process.argv.slice(2));
|
||||||
|
if (parsed.help) {
|
||||||
|
console.log("usage: node --experimental-strip-types performance-benchmark.ts --baseline-worker PATH [--optimized-worker PATH] [--samples N] [--cold-samples N] [--counts 0,10,60]");
|
||||||
|
} else {
|
||||||
|
const report = await runPerformanceBenchmark(parsed);
|
||||||
|
printReport(report);
|
||||||
|
if (!report.ok) process.exitCode = 1;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(safeFailure("cli"));
|
||||||
|
process.exitCode = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
"""Private benchmark wrapper for counting browser-harness calls.
|
||||||
|
|
||||||
|
The wrapper is deliberately a transport-only probe. It records method names and
|
||||||
|
counts, never CDP arguments, URLs, DOM values, page text, or screenshots. The
|
||||||
|
selected worker is executed unchanged after the helper module is wrapped.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import runpy
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from collections import Counter
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
def _take_flag(argv: list[str], flag: str) -> tuple[str | None, list[str]]:
|
||||||
|
remaining: list[str] = []
|
||||||
|
value: str | None = None
|
||||||
|
index = 0
|
||||||
|
while index < len(argv):
|
||||||
|
item = argv[index]
|
||||||
|
if item == flag:
|
||||||
|
if value is not None or index + 1 >= len(argv):
|
||||||
|
raise ValueError(f"{flag} must be supplied once")
|
||||||
|
value = argv[index + 1]
|
||||||
|
index += 2
|
||||||
|
continue
|
||||||
|
remaining.append(item)
|
||||||
|
index += 1
|
||||||
|
return value, remaining
|
||||||
|
|
||||||
|
|
||||||
|
def _method_name(value: Any) -> str:
|
||||||
|
if isinstance(value, str) and value and len(value) <= 120 and all(
|
||||||
|
character.isalnum() or character in "._-" for character in value
|
||||||
|
):
|
||||||
|
return value
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _wrap_helpers(counts: dict[str, Counter[str]] | None = None) -> dict[str, Counter[str]]:
|
||||||
|
helpers = importlib.import_module("browser_harness.helpers")
|
||||||
|
counts = counts or {"cdp": Counter(), "ipc": Counter()}
|
||||||
|
|
||||||
|
original_cdp = getattr(helpers, "cdp", None)
|
||||||
|
if callable(original_cdp):
|
||||||
|
def counted_cdp(*args: Any, **kwargs: Any) -> Any:
|
||||||
|
method = _method_name(args[0] if args else None)
|
||||||
|
counts["cdp"][method] += 1
|
||||||
|
return original_cdp(*args, **kwargs)
|
||||||
|
helpers.cdp = counted_cdp
|
||||||
|
|
||||||
|
original_send = getattr(helpers, "_send", None)
|
||||||
|
if callable(original_send):
|
||||||
|
def counted_send(*args: Any, **kwargs: Any) -> Any:
|
||||||
|
# Do not inspect the request. The IPC method is intentionally the
|
||||||
|
# fixed helper method name rather than an untrusted operation value.
|
||||||
|
counts["ipc"]["_send"] += 1
|
||||||
|
return original_send(*args, **kwargs)
|
||||||
|
helpers._send = counted_send
|
||||||
|
return counts
|
||||||
|
|
||||||
|
|
||||||
|
def _write_counts(path: str, counts: dict[str, Counter[str]]) -> None:
|
||||||
|
payload = {
|
||||||
|
"cdp": dict(sorted(counts["cdp"].items())),
|
||||||
|
"ipc": dict(sorted(counts["ipc"].items())),
|
||||||
|
}
|
||||||
|
destination = Path(path)
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix="performance-count-", dir=str(destination.parent))
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
||||||
|
json.dump(payload, stream, separators=(",", ":"), sort_keys=True)
|
||||||
|
stream.write("\n")
|
||||||
|
os.chmod(temporary, 0o600)
|
||||||
|
os.replace(temporary, destination)
|
||||||
|
os.chmod(destination, 0o600)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
os.unlink(temporary)
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
raw = list(sys.argv[1:] if argv is None else argv)
|
||||||
|
worker_path, raw = _take_flag(raw, "--performance-worker")
|
||||||
|
count_path, worker_argv = _take_flag(raw, "--performance-count-file")
|
||||||
|
if not worker_path or not count_path:
|
||||||
|
raise SystemExit("performance_probe requires --performance-worker and --performance-count-file")
|
||||||
|
if not Path(worker_path).is_file():
|
||||||
|
raise SystemExit("performance worker path is not a file")
|
||||||
|
counts: dict[str, Counter[str]] = {"cdp": Counter(), "ipc": Counter()}
|
||||||
|
try:
|
||||||
|
sys.argv = [worker_path, *worker_argv]
|
||||||
|
namespace = runpy.run_path(worker_path, run_name="performance_worker")
|
||||||
|
backend = namespace.get("RealBrowserBackend")
|
||||||
|
main_function = namespace.get("main")
|
||||||
|
if backend is None or not callable(main_function):
|
||||||
|
raise SystemExit("performance worker does not expose its expected entry points")
|
||||||
|
original_load = backend._load_harness
|
||||||
|
def load_and_probe(instance: Any) -> Any:
|
||||||
|
result = original_load(instance)
|
||||||
|
_wrap_helpers(counts)
|
||||||
|
return result
|
||||||
|
backend._load_harness = load_and_probe
|
||||||
|
main_function(worker_argv)
|
||||||
|
finally:
|
||||||
|
_write_counts(count_path, counts)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Reference in New Issue
Block a user