|
|
|
@@ -0,0 +1,601 @@
|
|
|
|
|
import assert from "node:assert/strict";
|
|
|
|
|
import { spawn as nodeSpawn, spawnSync } from "node:child_process";
|
|
|
|
|
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
|
|
|
import { chmod, mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
|
|
|
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
|
|
|
|
import { tmpdir } from "node:os";
|
|
|
|
|
import { dirname, join } from "node:path";
|
|
|
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
|
import { createBrowserTransport, type ChildProcessLike, type SpawnLike } from "./bridge.ts";
|
|
|
|
|
import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts";
|
|
|
|
|
import type { BrowserRuntime, BrowserTransportFactory, Candidate, PrivateArtifactStore } from "./contracts.ts";
|
|
|
|
|
import { createBrowserRuntimeFactory } from "./runtime.ts";
|
|
|
|
|
|
|
|
|
|
const ROOT = dirname(fileURLToPath(import.meta.url));
|
|
|
|
|
const CURRENT_WORKER = join(ROOT, "browser_worker.py");
|
|
|
|
|
const PROBE = join(ROOT, "performance_probe.py");
|
|
|
|
|
const FIXTURE = join(ROOT, "fixtures", "performance.html");
|
|
|
|
|
const DEFAULT_PYTHON = join(process.cwd(), ".pi", "fast-browser-demo", "venv", "bin", "python");
|
|
|
|
|
const DEFAULT_BROWSER = "/usr/bin/chromium";
|
|
|
|
|
const DEFAULT_COUNTS = [0, 10, 60] as const;
|
|
|
|
|
const DEFAULT_SAMPLES = 20;
|
|
|
|
|
const DEFAULT_COLD_SAMPLES = 1;
|
|
|
|
|
const MAX_SAMPLES = 40;
|
|
|
|
|
const MAX_COLD_SAMPLES = 5;
|
|
|
|
|
const MAX_COUNTS = 3;
|
|
|
|
|
|
|
|
|
|
type Arm = "baseline" | "optimized";
|
|
|
|
|
type Phase = "warm" | "cold-start";
|
|
|
|
|
|
|
|
|
|
export interface PerformanceBenchmarkOptions {
|
|
|
|
|
baselineWorkerPath: string;
|
|
|
|
|
optimizedWorkerPath?: string;
|
|
|
|
|
pythonPath?: string;
|
|
|
|
|
browserPath?: string;
|
|
|
|
|
samples?: number;
|
|
|
|
|
coldSamples?: number;
|
|
|
|
|
candidateCounts?: readonly number[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface TimingSummary {
|
|
|
|
|
count: number;
|
|
|
|
|
p50: number | null;
|
|
|
|
|
p95: number | null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface BenchmarkAttempt {
|
|
|
|
|
arm: Arm;
|
|
|
|
|
phase: Phase;
|
|
|
|
|
sample: number;
|
|
|
|
|
order?: 1 | 2;
|
|
|
|
|
status: "passed" | "failed";
|
|
|
|
|
elapsedMs: number | null;
|
|
|
|
|
startupMs: number | null;
|
|
|
|
|
error?: string;
|
|
|
|
|
protocolOperations?: Record<string, number>;
|
|
|
|
|
densityObserveMs?: Partial<Record<"0" | "10" | "60", number>>;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface WorkerMethodCounts {
|
|
|
|
|
cdp: Record<string, number>;
|
|
|
|
|
ipc: Record<string, number>;
|
|
|
|
|
cdpCalls: number;
|
|
|
|
|
ipcCalls: number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface BenchmarkReport {
|
|
|
|
|
ok: boolean;
|
|
|
|
|
measurement: { workerMethods: "selected Python worker CDP/IPC helper methods"; protocolOperations: "TypeScript JSONL operation names"; warmPriming: "one density=0 observe per arm, excluded from warm samples"; coldStartup: "browser session open plus first worker observe"; runtimeNarrowingMeasured: false };
|
|
|
|
|
samples: number;
|
|
|
|
|
coldSamples: number;
|
|
|
|
|
candidateCounts: readonly number[];
|
|
|
|
|
attempts: readonly BenchmarkAttempt[];
|
|
|
|
|
warm: Record<Arm, TimingSummary>;
|
|
|
|
|
coldStartup: Record<Arm, TimingSummary>;
|
|
|
|
|
workerMethods: Partial<Record<Arm, WorkerMethodCounts>>;
|
|
|
|
|
protocolOperations: Partial<Record<Arm, Record<string, number>>>;
|
|
|
|
|
densityObserve: Record<Arm, Record<"0" | "10" | "60", TimingSummary>>;
|
|
|
|
|
pairedOrder: readonly (readonly [Arm, Arm])[];
|
|
|
|
|
counterStatus: "complete" | "unavailable";
|
|
|
|
|
cleanup: { status: "passed" | "failed"; error?: string; retainedRoot: boolean };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface FixtureServer {
|
|
|
|
|
origin: string;
|
|
|
|
|
close(): Promise<void>;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface ArmHandle {
|
|
|
|
|
arm: Arm;
|
|
|
|
|
session: BrowserSession;
|
|
|
|
|
runtime: BrowserRuntime;
|
|
|
|
|
protocol: Record<string, number>;
|
|
|
|
|
countFile: string;
|
|
|
|
|
closed: boolean;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const EMPTY_ARTIFACT_STORE: PrivateArtifactStore = {
|
|
|
|
|
async open() {
|
|
|
|
|
throw new Error("benchmark evidence is disabled");
|
|
|
|
|
},
|
|
|
|
|
async close() { /* no-op */ },
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
type FailureStage = "cli" | "cold-start" | "warm-setup" | "warm-flow" | "density-observe" | "popup-close" | "counter" | "cleanup";
|
|
|
|
|
type FailureCategory = "failed" | "unverified" | "unavailable";
|
|
|
|
|
|
|
|
|
|
export function safeFailure(stage: FailureStage, category: FailureCategory = "failed"): string {
|
|
|
|
|
return `${stage}:${category}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
class BenchmarkFailure extends Error {
|
|
|
|
|
readonly stage: FailureStage;
|
|
|
|
|
readonly category: FailureCategory;
|
|
|
|
|
|
|
|
|
|
constructor(stage: FailureStage, category: FailureCategory) {
|
|
|
|
|
super(`${stage}:${category}`);
|
|
|
|
|
this.stage = stage;
|
|
|
|
|
this.category = category;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function failureFor(error: unknown, fallback: FailureStage): string {
|
|
|
|
|
return error instanceof BenchmarkFailure ? safeFailure(error.stage, error.category) : safeFailure(fallback);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function boundedInteger(value: unknown, label: string, min: number, max: number): number {
|
|
|
|
|
if (!Number.isSafeInteger(value) || (value as number) < min || (value as number) > max) {
|
|
|
|
|
throw new Error(`${label} must be an integer between ${min} and ${max}`);
|
|
|
|
|
}
|
|
|
|
|
return value as number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function requiredFile(value: string, label: string): string {
|
|
|
|
|
if (typeof value !== "string" || value.length === 0 || value.length > 4_000 || value.includes("\0")) {
|
|
|
|
|
throw new Error(`${label} is invalid`);
|
|
|
|
|
}
|
|
|
|
|
if (!existsSync(value) || !statSync(value).isFile()) throw new Error(`${label} is not a file`);
|
|
|
|
|
return value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function validateBenchmarkOptions(options: PerformanceBenchmarkOptions): Required<PerformanceBenchmarkOptions> {
|
|
|
|
|
if (!options || typeof options !== "object") throw new Error("benchmark options are required");
|
|
|
|
|
const baselineWorkerPath = requiredFile(options.baselineWorkerPath, "baseline worker path");
|
|
|
|
|
const optimizedWorkerPath = requiredFile(options.optimizedWorkerPath ?? CURRENT_WORKER, "optimized worker path");
|
|
|
|
|
const pythonPath = requiredFile(options.pythonPath ?? process.env.FAST_BROWSER_PYTHON ?? DEFAULT_PYTHON, "Python path");
|
|
|
|
|
const browserPath = requiredFile(options.browserPath ?? DEFAULT_BROWSER, "browser path");
|
|
|
|
|
const samples = boundedInteger(options.samples ?? DEFAULT_SAMPLES, "samples", 1, MAX_SAMPLES);
|
|
|
|
|
const coldSamples = boundedInteger(options.coldSamples ?? DEFAULT_COLD_SAMPLES, "cold samples", 1, MAX_COLD_SAMPLES);
|
|
|
|
|
const candidateCounts = [...(options.candidateCounts ?? DEFAULT_COUNTS)];
|
|
|
|
|
if (candidateCounts.length < 1 || candidateCounts.length > MAX_COUNTS || candidateCounts.some((value) => ![0, 10, 60].includes(value))) {
|
|
|
|
|
throw new Error("candidate counts must be a non-empty subset of 0, 10, and 60");
|
|
|
|
|
}
|
|
|
|
|
return { baselineWorkerPath, optimizedWorkerPath, pythonPath, browserPath, samples, coldSamples, candidateCounts };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function percentile(values: readonly number[], fraction: number): number | null {
|
|
|
|
|
if (values.length === 0) return null;
|
|
|
|
|
const sorted = [...values].sort((a, b) => a - b);
|
|
|
|
|
const position = (sorted.length - 1) * fraction;
|
|
|
|
|
const lower = Math.floor(position);
|
|
|
|
|
const upper = Math.ceil(position);
|
|
|
|
|
if (lower === upper) return sorted[lower]!;
|
|
|
|
|
return sorted[lower]! + (sorted[upper]! - sorted[lower]!) * (position - lower);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function summarizeTimings(values: readonly number[]): TimingSummary {
|
|
|
|
|
return { count: values.length, p50: percentile(values, 0.5), p95: percentile(values, 0.95) };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function increment(target: Record<string, number>, key: string): void {
|
|
|
|
|
target[key] = (target[key] ?? 0) + 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function countProtocol(child: ReturnType<typeof nodeSpawn>, target: Record<string, number>): void {
|
|
|
|
|
const stdin = child.stdin;
|
|
|
|
|
if (!stdin) throw new Error("benchmark worker stdin is unavailable");
|
|
|
|
|
const write = stdin.write.bind(stdin);
|
|
|
|
|
stdin.write = ((data: string | Uint8Array, ...args: unknown[]) => {
|
|
|
|
|
const text = typeof data === "string" ? data : Buffer.from(data).toString("utf8");
|
|
|
|
|
for (const line of text.split("\n")) {
|
|
|
|
|
if (!line) continue;
|
|
|
|
|
try {
|
|
|
|
|
const wire = JSON.parse(line) as { operation?: { op?: unknown } };
|
|
|
|
|
const op = wire.operation?.op;
|
|
|
|
|
if (typeof op === "string" && /^[A-Za-z][A-Za-z0-9]{0,40}$/.test(op)) increment(target, op);
|
|
|
|
|
} catch { /* bridge owns malformed-request handling */ }
|
|
|
|
|
}
|
|
|
|
|
return write(data, ...(args as []));
|
|
|
|
|
}) as typeof stdin.write;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function instrumentedSpawn(
|
|
|
|
|
workerPath: string,
|
|
|
|
|
countFile: string,
|
|
|
|
|
protocol: Record<string, number>,
|
|
|
|
|
): SpawnLike {
|
|
|
|
|
return (command, args, options) => {
|
|
|
|
|
const child = nodeSpawn(command, [
|
|
|
|
|
...args,
|
|
|
|
|
"--performance-worker", workerPath,
|
|
|
|
|
"--performance-count-file", countFile,
|
|
|
|
|
], options as never);
|
|
|
|
|
countProtocol(child, protocol);
|
|
|
|
|
return child as unknown as ChildProcessLike;
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createInstrumentedRuntime(
|
|
|
|
|
armWorkerPath: string,
|
|
|
|
|
countFile: string,
|
|
|
|
|
protocol: Record<string, number>,
|
|
|
|
|
session: BrowserSession,
|
|
|
|
|
origin: string,
|
|
|
|
|
pythonPath: string,
|
|
|
|
|
signal: AbortSignal,
|
|
|
|
|
): BrowserRuntime {
|
|
|
|
|
const transportFactory: BrowserTransportFactory = {
|
|
|
|
|
create(options, parentSignal) {
|
|
|
|
|
return createBrowserTransport({
|
|
|
|
|
...options,
|
|
|
|
|
workerPath: PROBE,
|
|
|
|
|
spawn: instrumentedSpawn(armWorkerPath, countFile, protocol),
|
|
|
|
|
}, parentSignal);
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
return createBrowserRuntimeFactory(transportFactory).create({
|
|
|
|
|
sessionId: session.id,
|
|
|
|
|
cdpUrl: session.config.cdpUrl,
|
|
|
|
|
allowedOrigins: [origin],
|
|
|
|
|
python: pythonPath,
|
|
|
|
|
artifacts: EMPTY_ARTIFACT_STORE,
|
|
|
|
|
}, signal);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function createFixtureServer(): Promise<FixtureServer> {
|
|
|
|
|
const template = readFileSync(FIXTURE, "utf8");
|
|
|
|
|
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
|
|
|
|
|
const path = new URL(request.url ?? "/", "http://127.0.0.1").pathname;
|
|
|
|
|
if (path !== "/performance.html") {
|
|
|
|
|
response.writeHead(404);
|
|
|
|
|
response.end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
response.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" });
|
|
|
|
|
response.end(template);
|
|
|
|
|
});
|
|
|
|
|
await new Promise<void>((resolve, reject) => {
|
|
|
|
|
server.once("error", reject);
|
|
|
|
|
server.listen(0, "127.0.0.1", resolve);
|
|
|
|
|
});
|
|
|
|
|
const address = server.address();
|
|
|
|
|
if (!address || typeof address === "string") throw new Error("performance fixture did not expose a port");
|
|
|
|
|
return {
|
|
|
|
|
origin: `http://127.0.0.1:${address.port}`,
|
|
|
|
|
async close() {
|
|
|
|
|
server.closeAllConnections?.();
|
|
|
|
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function launchConfig(origin: string, url: string, browserPath: string): Parameters<BrowserSessionManager["open"]>[0] {
|
|
|
|
|
return {
|
|
|
|
|
allowedOrigins: [origin],
|
|
|
|
|
browser: { mode: "launch", product: "chromium", executablePath: browserPath, headless: true, startUrl: url },
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function candidate(observation: Awaited<ReturnType<BrowserRuntime["observe"]>>, name: string, action: "click" | "fill"): Candidate {
|
|
|
|
|
const matches = observation.candidates.filter((item) => item.name === name && item.actions.includes(action));
|
|
|
|
|
assert.equal(matches.length, 1, `expected one ${action} candidate`);
|
|
|
|
|
return matches[0]!;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function runWarmFlow(handle: ArmHandle, origin: string, candidateCounts: readonly number[], signal: AbortSignal): Promise<Partial<Record<"0" | "10" | "60", number>>> {
|
|
|
|
|
const densityObserveMs: Partial<Record<"0" | "10" | "60", number>> = {};
|
|
|
|
|
for (const count of candidateCounts) {
|
|
|
|
|
const density = await handle.runtime.navigate({ url: `${origin}/performance.html?mode=density&count=${count}` }, signal);
|
|
|
|
|
assert.equal(density.status, "committed");
|
|
|
|
|
const observeStarted = Date.now();
|
|
|
|
|
try {
|
|
|
|
|
const observation = await handle.runtime.observe(signal);
|
|
|
|
|
densityObserveMs[String(count) as "0" | "10" | "60"] = Date.now() - observeStarted;
|
|
|
|
|
assert.equal(observation.candidates.length, count);
|
|
|
|
|
} catch {
|
|
|
|
|
throw new BenchmarkFailure("density-observe", "failed");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const page = await handle.runtime.navigate({ url: `${origin}/performance.html?mode=flow&count=10` }, signal);
|
|
|
|
|
assert.equal(page.status, "committed");
|
|
|
|
|
const tabs = await handle.runtime.listTabs(signal);
|
|
|
|
|
const main = tabs.find((tab) => tab.selected);
|
|
|
|
|
assert(main, "main benchmark tab is unavailable");
|
|
|
|
|
const mainTab = main;
|
|
|
|
|
|
|
|
|
|
let observation = await handle.runtime.observe(signal);
|
|
|
|
|
const field = candidate(observation, "Display name", "fill");
|
|
|
|
|
const filled = await handle.runtime.act({ observationId: observation.observationId, candidateId: field.id, action: "fill", text: "Ada Lovelace" }, signal);
|
|
|
|
|
assert.equal(filled.verified, true);
|
|
|
|
|
|
|
|
|
|
observation = await handle.runtime.observe(signal);
|
|
|
|
|
const save = candidate(observation, "Save profile", "click");
|
|
|
|
|
const saved = await handle.runtime.act({ observationId: observation.observationId, candidateId: save.id, action: "click", postcondition: { kind: "text", value: "Profile saved" } }, signal);
|
|
|
|
|
assert.equal(saved.verified, true);
|
|
|
|
|
const saveAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Profile saved" }, wait: "bounded" }, signal);
|
|
|
|
|
assert.equal(saveAssert.status, "satisfied");
|
|
|
|
|
|
|
|
|
|
observation = await handle.runtime.observe(signal);
|
|
|
|
|
const previewButton = candidate(observation, "Preview profile", "click");
|
|
|
|
|
const previewRequested = await handle.runtime.act({ observationId: observation.observationId, candidateId: previewButton.id, action: "click", postcondition: { kind: "text", value: "Preview ready" } }, signal);
|
|
|
|
|
assert.equal(previewRequested.verified, true);
|
|
|
|
|
|
|
|
|
|
observation = await handle.runtime.observe(signal);
|
|
|
|
|
const immediate = candidate(observation, "Immediate action", "click");
|
|
|
|
|
const clicked = await handle.runtime.act({ observationId: observation.observationId, candidateId: immediate.id, action: "click", postcondition: { kind: "text", value: "Immediate complete" } }, signal);
|
|
|
|
|
assert.equal(clicked.verified, true);
|
|
|
|
|
const immediateAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Immediate complete" }, wait: "none" }, signal);
|
|
|
|
|
assert.equal(immediateAssert.status, "satisfied");
|
|
|
|
|
|
|
|
|
|
observation = await handle.runtime.observe(signal);
|
|
|
|
|
const delayed = candidate(observation, "Delayed action", "click");
|
|
|
|
|
const delayedResult = await handle.runtime.act({ observationId: observation.observationId, candidateId: delayed.id, action: "click", postcondition: { kind: "text", value: "Delayed complete" } }, signal);
|
|
|
|
|
assert.equal(delayedResult.verified, true);
|
|
|
|
|
const delayedAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Delayed complete" }, wait: "bounded" }, signal);
|
|
|
|
|
assert.equal(delayedAssert.status, "satisfied");
|
|
|
|
|
|
|
|
|
|
observation = await handle.runtime.observe(signal);
|
|
|
|
|
const popupButton = candidate(observation, "Open popup", "click");
|
|
|
|
|
const popupRequest = await handle.runtime.act({ observationId: observation.observationId, candidateId: popupButton.id, action: "click", postcondition: { kind: "text", value: "Popup requested" } }, signal);
|
|
|
|
|
assert.equal(popupRequest.verified, true);
|
|
|
|
|
const popup = await handle.runtime.waitPopup({ openerTabId: mainTab.tabId }, signal);
|
|
|
|
|
assert.equal(popup.status, "found");
|
|
|
|
|
assert(popup.tab);
|
|
|
|
|
const popupTab = popup.tab;
|
|
|
|
|
const popupRead = await handle.runtime.read({ tabId: popupTab.tabId }, signal);
|
|
|
|
|
assert.match(popupRead.visibleText, /Popup ready/);
|
|
|
|
|
const popupReady = await handle.runtime.assert({ tabId: popupTab.tabId, assertion: { kind: "visibleText", match: "contains", value: "Popup ready" }, wait: "bounded" }, signal);
|
|
|
|
|
assert.equal(popupReady.status, "satisfied");
|
|
|
|
|
try {
|
|
|
|
|
await handle.runtime.closeTab({ tabId: popupTab.tabId }, signal);
|
|
|
|
|
} catch {
|
|
|
|
|
throw new BenchmarkFailure("popup-close", "unverified");
|
|
|
|
|
}
|
|
|
|
|
const preview = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Preview ready: Ada Lovelace" }, wait: "bounded" }, signal);
|
|
|
|
|
assert.equal(preview.status, "satisfied");
|
|
|
|
|
return densityObserveMs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function validCounterMap(value: unknown): value is Record<string, number> {
|
|
|
|
|
return !!value && typeof value === "object" && !Array.isArray(value)
|
|
|
|
|
&& Object.entries(value).every(([key, count]) => /^[A-Za-z0-9_.-]{1,120}$/.test(key) && Number.isSafeInteger(count) && count >= 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function validWorkerCounters(value: unknown): value is { cdp: Record<string, number>; ipc: Record<string, number> } {
|
|
|
|
|
if (!value || typeof value !== "object") return false;
|
|
|
|
|
const counters = value as { cdp?: unknown; ipc?: unknown };
|
|
|
|
|
return validCounterMap(counters.cdp) && validCounterMap(counters.ipc);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function readWorkerCounts(path: string): Promise<WorkerMethodCounts | undefined> {
|
|
|
|
|
try {
|
|
|
|
|
const value = JSON.parse(await readFile(path, "utf8")) as { cdp?: unknown; ipc?: unknown };
|
|
|
|
|
if (!validWorkerCounters(value)) return undefined;
|
|
|
|
|
const cdp = value.cdp;
|
|
|
|
|
const ipc = value.ipc;
|
|
|
|
|
return {
|
|
|
|
|
cdp,
|
|
|
|
|
ipc,
|
|
|
|
|
cdpCalls: Object.values(cdp).reduce((sum, count) => sum + count, 0),
|
|
|
|
|
ipcCalls: Object.values(ipc).reduce((sum, count) => sum + count, 0),
|
|
|
|
|
};
|
|
|
|
|
} catch {
|
|
|
|
|
return undefined;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ownedProcessLines(root: string): string[] | undefined {
|
|
|
|
|
const result = spawnSync("ps", ["-eo", "pid=,args="], { encoding: "utf8", timeout: 5_000 });
|
|
|
|
|
if (result.status !== 0) return undefined;
|
|
|
|
|
return String(result.stdout).split(/\r?\n/).filter((line) => line.includes(root));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function snapshotCounts(value: Record<string, number>): Record<string, number> {
|
|
|
|
|
return { ...value };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function countDelta(before: Record<string, number>, after: Record<string, number>): Record<string, number> {
|
|
|
|
|
const result: Record<string, number> = {};
|
|
|
|
|
for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) {
|
|
|
|
|
const difference = (after[key] ?? 0) - (before[key] ?? 0);
|
|
|
|
|
if (difference > 0) result[key] = difference;
|
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function closeHandle(handle: ArmHandle | undefined): Promise<string | undefined> {
|
|
|
|
|
if (!handle || handle.closed) return undefined;
|
|
|
|
|
handle.closed = true;
|
|
|
|
|
try {
|
|
|
|
|
await handle.runtime.close();
|
|
|
|
|
return undefined;
|
|
|
|
|
} catch {
|
|
|
|
|
return safeFailure("cleanup");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function pairedArmOrder(sample: number): readonly [Arm, Arm] {
|
|
|
|
|
return sample % 2 === 1 ? ["baseline", "optimized"] : ["optimized", "baseline"];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function attemptsFor(attempts: readonly BenchmarkAttempt[], arm: Arm, phase: Phase): number[] {
|
|
|
|
|
return attempts.filter((item) => item.arm === arm && item.phase === phase && item.status === "passed" && item.elapsedMs !== null).map((item) => item.elapsedMs!);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function summarizeBenchmarkAttempts(attempts: readonly BenchmarkAttempt[], arm: Arm, phase: Phase): TimingSummary {
|
|
|
|
|
return summarizeTimings(attemptsFor(attempts, arm, phase));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function densitySummaries(attempts: readonly BenchmarkAttempt[]): Record<Arm, Record<"0" | "10" | "60", TimingSummary>> {
|
|
|
|
|
const result = {} as Record<Arm, Record<"0" | "10" | "60", TimingSummary>>;
|
|
|
|
|
for (const arm of ["baseline", "optimized"] as const) {
|
|
|
|
|
result[arm] = {} as Record<"0" | "10" | "60", TimingSummary>;
|
|
|
|
|
for (const count of ["0", "10", "60"] as const) {
|
|
|
|
|
result[arm][count] = summarizeTimings(attempts
|
|
|
|
|
.filter((attempt) => attempt.arm === arm && attempt.phase === "warm" && attempt.status === "passed" && attempt.densityObserveMs?.[count] !== undefined)
|
|
|
|
|
.map((attempt) => attempt.densityObserveMs![count]!));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptions): Promise<BenchmarkReport> {
|
|
|
|
|
const config = validateBenchmarkOptions(options);
|
|
|
|
|
const fixture = await createFixtureServer();
|
|
|
|
|
const root = await mkdtemp(join(tmpdir(), "fast-browser-performance-"));
|
|
|
|
|
await mkdir(join(root, "leases"), { recursive: true, mode: 0o700 });
|
|
|
|
|
await chmod(join(root, "leases"), 0o700);
|
|
|
|
|
const manager = createBrowserSessionManager({ profileDirectory: join(root, "profiles"), leaseDirectory: join(root, "leases"), startupTimeoutMs: 20_000, closeTimeoutMs: 4_000 });
|
|
|
|
|
const controller = new AbortController();
|
|
|
|
|
const attempts: BenchmarkAttempt[] = [];
|
|
|
|
|
const workerMethods: Partial<Record<Arm, WorkerMethodCounts>> = {};
|
|
|
|
|
const protocolOperations: Partial<Record<Arm, Record<string, number>>> = {};
|
|
|
|
|
const handles: Partial<Record<Arm, ArmHandle>> = {};
|
|
|
|
|
const setupErrors: Partial<Record<Arm, string>> = {};
|
|
|
|
|
const pairedOrder: Array<readonly [Arm, Arm]> = [];
|
|
|
|
|
let cleanupError: string | undefined;
|
|
|
|
|
let retainedRoot = false;
|
|
|
|
|
let counterUnavailable = false;
|
|
|
|
|
try {
|
|
|
|
|
const armPaths: Record<Arm, string> = { baseline: config.baselineWorkerPath, optimized: config.optimizedWorkerPath };
|
|
|
|
|
for (const arm of ["baseline", "optimized"] as const) {
|
|
|
|
|
for (let sample = 1; sample <= config.coldSamples; sample += 1) {
|
|
|
|
|
const countFile = join(root, `${arm}-cold-${sample}.json`);
|
|
|
|
|
const protocol: Record<string, number> = {};
|
|
|
|
|
let session: BrowserSession | undefined;
|
|
|
|
|
let runtime: BrowserRuntime | undefined;
|
|
|
|
|
const started = Date.now();
|
|
|
|
|
let startupMs: number | null = null;
|
|
|
|
|
try {
|
|
|
|
|
const opened = Date.now();
|
|
|
|
|
session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=10`, config.browserPath), controller.signal);
|
|
|
|
|
const first = Date.now();
|
|
|
|
|
runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
|
|
|
|
|
const observation = await runtime.observe(controller.signal);
|
|
|
|
|
assert.equal(observation.candidates.length, 10);
|
|
|
|
|
startupMs = first - opened + (Date.now() - first);
|
|
|
|
|
attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol } });
|
|
|
|
|
} catch (error) {
|
|
|
|
|
attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: null, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol } });
|
|
|
|
|
} finally {
|
|
|
|
|
try { await runtime?.close(); } catch { cleanupError ??= safeFailure("cleanup"); }
|
|
|
|
|
if (session) await manager.close(session.id).catch(() => { cleanupError ??= safeFailure("cleanup"); });
|
|
|
|
|
const counts = await readWorkerCounts(countFile);
|
|
|
|
|
if (counts) workerMethods[arm] = counts;
|
|
|
|
|
else counterUnavailable = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
const countFile = join(root, `${arm}-warm.json`);
|
|
|
|
|
const protocol: Record<string, number> = {};
|
|
|
|
|
const session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=0`, config.browserPath), controller.signal);
|
|
|
|
|
const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
|
|
|
|
|
const handle: ArmHandle = { arm, session, runtime, protocol, countFile, closed: false };
|
|
|
|
|
handles[arm] = handle;
|
|
|
|
|
await runtime.observe(controller.signal); // prime the persistent worker; excluded from warm samples
|
|
|
|
|
} catch (error) {
|
|
|
|
|
const failedHandle = handles[arm];
|
|
|
|
|
if (failedHandle) {
|
|
|
|
|
await closeHandle(failedHandle);
|
|
|
|
|
await manager.close(failedHandle.session.id).catch(() => {});
|
|
|
|
|
}
|
|
|
|
|
delete handles[arm];
|
|
|
|
|
setupErrors[arm] = safeFailure("warm-setup");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (let sample = 1; sample <= config.samples; sample += 1) {
|
|
|
|
|
const order = pairedArmOrder(sample);
|
|
|
|
|
pairedOrder.push(order);
|
|
|
|
|
for (let position = 0; position < order.length; position += 1) {
|
|
|
|
|
const arm = order[position]!;
|
|
|
|
|
const handle = handles[arm];
|
|
|
|
|
const before = handle ? snapshotCounts(handle.protocol) : {};
|
|
|
|
|
const started = Date.now();
|
|
|
|
|
try {
|
|
|
|
|
if (!handle) throw new Error("arm setup failed");
|
|
|
|
|
const densityObserveMs = await runWarmFlow(handle, fixture.origin, config.candidateCounts, controller.signal);
|
|
|
|
|
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), densityObserveMs });
|
|
|
|
|
} catch (error) {
|
|
|
|
|
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: null, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {} });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
for (const arm of ["baseline", "optimized"] as const) {
|
|
|
|
|
const handle = handles[arm];
|
|
|
|
|
const error = await closeHandle(handle);
|
|
|
|
|
if (error) cleanupError ??= error;
|
|
|
|
|
if (handle) {
|
|
|
|
|
const counts = await readWorkerCounts(handle.countFile);
|
|
|
|
|
if (counts) workerMethods[arm] = counts;
|
|
|
|
|
else counterUnavailable = true;
|
|
|
|
|
protocolOperations[arm] = { ...handle.protocol };
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
await manager.closeAll().catch(() => { cleanupError ??= safeFailure("cleanup"); });
|
|
|
|
|
await fixture.close().catch(() => { cleanupError ??= safeFailure("cleanup"); });
|
|
|
|
|
const leftovers = ownedProcessLines(root);
|
|
|
|
|
if (leftovers === undefined || leftovers.length > 0) {
|
|
|
|
|
cleanupError ??= safeFailure("cleanup", "unavailable");
|
|
|
|
|
retainedRoot = true;
|
|
|
|
|
}
|
|
|
|
|
if (!cleanupError) {
|
|
|
|
|
await rm(root, { recursive: true, force: true }).catch(() => {
|
|
|
|
|
cleanupError ??= safeFailure("cleanup");
|
|
|
|
|
retainedRoot = true;
|
|
|
|
|
});
|
|
|
|
|
} else {
|
|
|
|
|
retainedRoot = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const warm: Record<Arm, TimingSummary> = {
|
|
|
|
|
baseline: summarizeBenchmarkAttempts(attempts, "baseline", "warm"),
|
|
|
|
|
optimized: summarizeBenchmarkAttempts(attempts, "optimized", "warm"),
|
|
|
|
|
};
|
|
|
|
|
const coldStartup: Record<Arm, TimingSummary> = {
|
|
|
|
|
baseline: summarizeTimings(attempts.filter((item) => item.arm === "baseline" && item.phase === "cold-start" && item.status === "passed" && item.startupMs !== null).map((item) => item.startupMs!)),
|
|
|
|
|
optimized: summarizeTimings(attempts.filter((item) => item.arm === "optimized" && item.phase === "cold-start" && item.status === "passed" && item.startupMs !== null).map((item) => item.startupMs!)),
|
|
|
|
|
};
|
|
|
|
|
return {
|
|
|
|
|
ok: !cleanupError && !counterUnavailable && workerMethods.baseline !== undefined && workerMethods.optimized !== undefined && attempts.length === config.samples * 2 + config.coldSamples * 2 && attempts.every((item) => item.status === "passed"),
|
|
|
|
|
measurement: { workerMethods: "selected Python worker CDP/IPC helper methods", protocolOperations: "TypeScript JSONL operation names", warmPriming: "one density=0 observe per arm, excluded from warm samples", coldStartup: "browser session open plus first worker observe", runtimeNarrowingMeasured: false },
|
|
|
|
|
samples: config.samples,
|
|
|
|
|
coldSamples: config.coldSamples,
|
|
|
|
|
candidateCounts: config.candidateCounts,
|
|
|
|
|
attempts,
|
|
|
|
|
warm,
|
|
|
|
|
coldStartup,
|
|
|
|
|
workerMethods,
|
|
|
|
|
protocolOperations,
|
|
|
|
|
densityObserve: densitySummaries(attempts),
|
|
|
|
|
pairedOrder,
|
|
|
|
|
counterStatus: !counterUnavailable && workerMethods.baseline && workerMethods.optimized ? "complete" : "unavailable",
|
|
|
|
|
cleanup: cleanupError ? { status: "failed", error: cleanupError, retainedRoot } : { status: "passed", retainedRoot: false },
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function parseArgs(argv: readonly string[]): PerformanceBenchmarkOptions & { help?: boolean } {
|
|
|
|
|
const result: Record<string, unknown> = {};
|
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
|
|
|
const arg = argv[index]!;
|
|
|
|
|
if (arg === "--help" || arg === "-h") return { baselineWorkerPath: "", help: true };
|
|
|
|
|
const values: Record<string, string> = { "--baseline-worker": "baselineWorkerPath", "--optimized-worker": "optimizedWorkerPath", "--python": "pythonPath", "--browser": "browserPath", "--samples": "samples", "--cold-samples": "coldSamples", "--counts": "candidateCounts" };
|
|
|
|
|
const key = values[arg];
|
|
|
|
|
if (!key || index + 1 >= argv.length) throw new Error(`unknown or incomplete option: ${arg}`);
|
|
|
|
|
const value = argv[++index]!;
|
|
|
|
|
result[key] = key === "samples" || key === "coldSamples" ? Number(value) : key === "candidateCounts" ? value.split(",").map(Number) : value;
|
|
|
|
|
}
|
|
|
|
|
if (typeof result.baselineWorkerPath !== "string") throw new Error("--baseline-worker is required");
|
|
|
|
|
return result as unknown as PerformanceBenchmarkOptions;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function printReport(report: BenchmarkReport): void {
|
|
|
|
|
console.log(JSON.stringify(report, null, 2));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (import.meta.main) {
|
|
|
|
|
try {
|
|
|
|
|
const parsed = parseArgs(process.argv.slice(2));
|
|
|
|
|
if (parsed.help) {
|
|
|
|
|
console.log("usage: node --experimental-strip-types performance-benchmark.ts --baseline-worker PATH [--optimized-worker PATH] [--samples N] [--cold-samples N] [--counts 0,10,60]");
|
|
|
|
|
} else {
|
|
|
|
|
const report = await runPerformanceBenchmark(parsed);
|
|
|
|
|
printReport(report);
|
|
|
|
|
if (!report.ok) process.exitCode = 1;
|
|
|
|
|
}
|
|
|
|
|
} catch (error) {
|
|
|
|
|
console.error(safeFailure("cli"));
|
|
|
|
|
process.exitCode = 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
