diff --git a/extensions/fast-browser/fixtures/performance.html b/extensions/fast-browser/fixtures/performance.html new file mode 100644 index 0000000..d9e977e --- /dev/null +++ b/extensions/fast-browser/fixtures/performance.html @@ -0,0 +1,109 @@ + + + + + Fast-browser performance fixture + + + +
+

Local form validation

+

Fixture ready

+

Profile draft loaded

+ +

+
+
+ + + diff --git a/extensions/fast-browser/performance-benchmark.test.ts b/extensions/fast-browser/performance-benchmark.test.ts new file mode 100644 index 0000000..2982532 --- /dev/null +++ b/extensions/fast-browser/performance-benchmark.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { + pairedArmOrder, + safeFailure, + summarizeBenchmarkAttempts, + validWorkerCounters, + summarizeTimings, + validateBenchmarkOptions, + type BenchmarkAttempt, +} from "./performance-benchmark.ts"; + +async function files(): Promise<{ root: string; paths: string[] }> { + const root = await mkdtemp(join("/tmp", "fast-browser-performance-test-")); + const paths = await Promise.all(["baseline.py", "optimized.py", "python", "chromium"].map(async (name) => { + const path = join(root, name); + await writeFile(path, "# test\n", { mode: 0o700 }); + return path; + })); + return { root, paths }; +} + +test("timing aggregation reports bounded p50 and p95 without wall-clock assumptions", () => { + assert.deepEqual(summarizeTimings([10, 20, 30, 40]), { count: 4, p50: 25, p95: 38.5 }); + assert.deepEqual(summarizeTimings([]), { count: 0, p50: null, p95: null }); +}); + +test("paired order alternates AB and BA while retaining failed attempts", () => { + assert.deepEqual(pairedArmOrder(1), ["baseline", "optimized"]); + assert.deepEqual(pairedArmOrder(2), ["optimized", "baseline"]); +}); + +test("attempt aggregation excludes failures from timing summaries but retains them for accounting", () => { + const attempts: BenchmarkAttempt[] = [ + { arm: "baseline", phase: "warm", sample: 1, status: "passed", elapsedMs: 10, startupMs: null }, + { arm: "baseline", phase: "warm", sample: 2, status: "failed", elapsedMs: null, startupMs: null, error: "worker failed" }, + { arm: "baseline", phase: "warm", sample: 3, status: "passed", elapsedMs: 30, startupMs: null }, + ]; + assert.deepEqual(summarizeBenchmarkAttempts(attempts, "baseline", "warm"), { count: 2, p50: 20, p95: 29 }); + assert.equal(attempts.filter((attempt) => attempt.status === "failed").length, 1); +}); + +test("benchmark input bounds reject unbounded samples and candidate sets", async () => { + const { root, paths } = await files(); + try { + assert.throws(() => validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, samples: 41 }), /samples/); + assert.throws(() => validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, candidateCounts: [1] }), /candidate counts/); + const valid = validateBenchmarkOptions({ baselineWorkerPath: paths[0]!, optimizedWorkerPath: paths[1]!, pythonPath: paths[2]!, browserPath: paths[3]!, samples: 1, coldSamples: 1, candidateCounts: [0, 10, 60] }); + assert.equal(valid.samples, 1); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("missing or malformed worker counters are unavailable", () => { + assert.equal(validWorkerCounters({ cdp: { "Page.enable": 1 }, ipc: { _send: 2 } }), true); + assert.equal(validWorkerCounters({ cdp: { "Page.enable": -1 }, ipc: { _send: 2 } }), false); + assert.equal(validWorkerCounters({ cdp: { "Page.enable": 1 }, ipc: { "bad key": 2 } }), false); + assert.equal(validWorkerCounters(undefined), false); +}); + +test("report failures use fixed safe categories and never include raw error text", () => { + const safe = safeFailure("popup-close", "unverified"); + assert.equal(safe, "popup-close:unverified"); + assert.doesNotMatch(safe, /https?:|secret|Ada|tmp|performance/); +}); diff --git a/extensions/fast-browser/performance-benchmark.ts b/extensions/fast-browser/performance-benchmark.ts new file mode 100644 index 0000000..4d6d5db --- /dev/null +++ b/extensions/fast-browser/performance-benchmark.ts @@ -0,0 +1,601 @@ +import assert from "node:assert/strict"; +import { spawn as nodeSpawn, spawnSync } from "node:child_process"; +import { existsSync, readFileSync, statSync } from "node:fs"; +import { chmod, mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createBrowserTransport, type ChildProcessLike, type SpawnLike } from "./bridge.ts"; +import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts"; +import type { BrowserRuntime, BrowserTransportFactory, Candidate, PrivateArtifactStore } from "./contracts.ts"; +import { createBrowserRuntimeFactory } from "./runtime.ts"; + +const ROOT = dirname(fileURLToPath(import.meta.url)); +const CURRENT_WORKER = join(ROOT, "browser_worker.py"); +const PROBE = join(ROOT, "performance_probe.py"); +const FIXTURE = join(ROOT, "fixtures", "performance.html"); +const DEFAULT_PYTHON = join(process.cwd(), ".pi", "fast-browser-demo", "venv", "bin", "python"); +const DEFAULT_BROWSER = "/usr/bin/chromium"; +const DEFAULT_COUNTS = [0, 10, 60] as const; +const DEFAULT_SAMPLES = 20; +const DEFAULT_COLD_SAMPLES = 1; +const MAX_SAMPLES = 40; +const MAX_COLD_SAMPLES = 5; +const MAX_COUNTS = 3; + +type Arm = "baseline" | "optimized"; +type Phase = "warm" | "cold-start"; + +export interface PerformanceBenchmarkOptions { + baselineWorkerPath: string; + optimizedWorkerPath?: string; + pythonPath?: string; + browserPath?: string; + samples?: number; + coldSamples?: number; + candidateCounts?: readonly number[]; +} + +export interface TimingSummary { + count: number; + p50: number | null; + p95: number | null; +} + +export interface BenchmarkAttempt { + arm: Arm; + phase: Phase; + sample: number; + order?: 1 | 2; + status: "passed" | "failed"; + elapsedMs: number | null; + startupMs: number | null; + error?: string; + protocolOperations?: Record; + densityObserveMs?: Partial>; +} + +export interface WorkerMethodCounts { + cdp: Record; + ipc: Record; + cdpCalls: number; + ipcCalls: number; +} + +export interface BenchmarkReport { + ok: boolean; + measurement: { workerMethods: "selected Python worker CDP/IPC helper methods"; protocolOperations: "TypeScript JSONL operation names"; warmPriming: "one density=0 observe per arm, excluded from warm samples"; coldStartup: "browser session open plus first worker observe"; runtimeNarrowingMeasured: false }; + samples: number; + coldSamples: number; + candidateCounts: readonly number[]; + attempts: readonly BenchmarkAttempt[]; + warm: Record; + coldStartup: Record; + workerMethods: Partial>; + protocolOperations: Partial>>; + densityObserve: Record>; + pairedOrder: readonly (readonly [Arm, Arm])[]; + counterStatus: "complete" | "unavailable"; + cleanup: { status: "passed" | "failed"; error?: string; retainedRoot: boolean }; +} + +interface FixtureServer { + origin: string; + close(): Promise; +} + +interface ArmHandle { + arm: Arm; + session: BrowserSession; + runtime: BrowserRuntime; + protocol: Record; + countFile: string; + closed: boolean; +} + +const EMPTY_ARTIFACT_STORE: PrivateArtifactStore = { + async open() { + throw new Error("benchmark evidence is disabled"); + }, + async close() { /* no-op */ }, +}; + +type FailureStage = "cli" | "cold-start" | "warm-setup" | "warm-flow" | "density-observe" | "popup-close" | "counter" | "cleanup"; +type FailureCategory = "failed" | "unverified" | "unavailable"; + +export function safeFailure(stage: FailureStage, category: FailureCategory = "failed"): string { + return `${stage}:${category}`; +} + +class BenchmarkFailure extends Error { + readonly stage: FailureStage; + readonly category: FailureCategory; + + constructor(stage: FailureStage, category: FailureCategory) { + super(`${stage}:${category}`); + this.stage = stage; + this.category = category; + } +} + +function failureFor(error: unknown, fallback: FailureStage): string { + return error instanceof BenchmarkFailure ? safeFailure(error.stage, error.category) : safeFailure(fallback); +} + +function boundedInteger(value: unknown, label: string, min: number, max: number): number { + if (!Number.isSafeInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer between ${min} and ${max}`); + } + return value as number; +} + +function requiredFile(value: string, label: string): string { + if (typeof value !== "string" || value.length === 0 || value.length > 4_000 || value.includes("\0")) { + throw new Error(`${label} is invalid`); + } + if (!existsSync(value) || !statSync(value).isFile()) throw new Error(`${label} is not a file`); + return value; +} + +export function validateBenchmarkOptions(options: PerformanceBenchmarkOptions): Required { + if (!options || typeof options !== "object") throw new Error("benchmark options are required"); + const baselineWorkerPath = requiredFile(options.baselineWorkerPath, "baseline worker path"); + const optimizedWorkerPath = requiredFile(options.optimizedWorkerPath ?? CURRENT_WORKER, "optimized worker path"); + const pythonPath = requiredFile(options.pythonPath ?? process.env.FAST_BROWSER_PYTHON ?? DEFAULT_PYTHON, "Python path"); + const browserPath = requiredFile(options.browserPath ?? DEFAULT_BROWSER, "browser path"); + const samples = boundedInteger(options.samples ?? DEFAULT_SAMPLES, "samples", 1, MAX_SAMPLES); + const coldSamples = boundedInteger(options.coldSamples ?? DEFAULT_COLD_SAMPLES, "cold samples", 1, MAX_COLD_SAMPLES); + const candidateCounts = [...(options.candidateCounts ?? DEFAULT_COUNTS)]; + if (candidateCounts.length < 1 || candidateCounts.length > MAX_COUNTS || candidateCounts.some((value) => ![0, 10, 60].includes(value))) { + throw new Error("candidate counts must be a non-empty subset of 0, 10, and 60"); + } + return { baselineWorkerPath, optimizedWorkerPath, pythonPath, browserPath, samples, coldSamples, candidateCounts }; +} + +function percentile(values: readonly number[], fraction: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const position = (sorted.length - 1) * fraction; + const lower = Math.floor(position); + const upper = Math.ceil(position); + if (lower === upper) return sorted[lower]!; + return sorted[lower]! + (sorted[upper]! - sorted[lower]!) * (position - lower); +} + +export function summarizeTimings(values: readonly number[]): TimingSummary { + return { count: values.length, p50: percentile(values, 0.5), p95: percentile(values, 0.95) }; +} + +function increment(target: Record, key: string): void { + target[key] = (target[key] ?? 0) + 1; +} + +function countProtocol(child: ReturnType, target: Record): void { + const stdin = child.stdin; + if (!stdin) throw new Error("benchmark worker stdin is unavailable"); + const write = stdin.write.bind(stdin); + stdin.write = ((data: string | Uint8Array, ...args: unknown[]) => { + const text = typeof data === "string" ? data : Buffer.from(data).toString("utf8"); + for (const line of text.split("\n")) { + if (!line) continue; + try { + const wire = JSON.parse(line) as { operation?: { op?: unknown } }; + const op = wire.operation?.op; + if (typeof op === "string" && /^[A-Za-z][A-Za-z0-9]{0,40}$/.test(op)) increment(target, op); + } catch { /* bridge owns malformed-request handling */ } + } + return write(data, ...(args as [])); + }) as typeof stdin.write; +} + +function instrumentedSpawn( + workerPath: string, + countFile: string, + protocol: Record, +): SpawnLike { + return (command, args, options) => { + const child = nodeSpawn(command, [ + ...args, + "--performance-worker", workerPath, + "--performance-count-file", countFile, + ], options as never); + countProtocol(child, protocol); + return child as unknown as ChildProcessLike; + }; +} + +function createInstrumentedRuntime( + armWorkerPath: string, + countFile: string, + protocol: Record, + session: BrowserSession, + origin: string, + pythonPath: string, + signal: AbortSignal, +): BrowserRuntime { + const transportFactory: BrowserTransportFactory = { + create(options, parentSignal) { + return createBrowserTransport({ + ...options, + workerPath: PROBE, + spawn: instrumentedSpawn(armWorkerPath, countFile, protocol), + }, parentSignal); + }, + }; + return createBrowserRuntimeFactory(transportFactory).create({ + sessionId: session.id, + cdpUrl: session.config.cdpUrl, + allowedOrigins: [origin], + python: pythonPath, + artifacts: EMPTY_ARTIFACT_STORE, + }, signal); +} + +async function createFixtureServer(): Promise { + const template = readFileSync(FIXTURE, "utf8"); + const server = createServer((request: IncomingMessage, response: ServerResponse) => { + const path = new URL(request.url ?? "/", "http://127.0.0.1").pathname; + if (path !== "/performance.html") { + response.writeHead(404); + response.end(); + return; + } + response.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" }); + response.end(template); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("performance fixture did not expose a port"); + return { + origin: `http://127.0.0.1:${address.port}`, + async close() { + server.closeAllConnections?.(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} + +function launchConfig(origin: string, url: string, browserPath: string): Parameters[0] { + return { + allowedOrigins: [origin], + browser: { mode: "launch", product: "chromium", executablePath: browserPath, headless: true, startUrl: url }, + }; +} + +function candidate(observation: Awaited>, name: string, action: "click" | "fill"): Candidate { + const matches = observation.candidates.filter((item) => item.name === name && item.actions.includes(action)); + assert.equal(matches.length, 1, `expected one ${action} candidate`); + return matches[0]!; +} + +async function runWarmFlow(handle: ArmHandle, origin: string, candidateCounts: readonly number[], signal: AbortSignal): Promise>> { + const densityObserveMs: Partial> = {}; + for (const count of candidateCounts) { + const density = await handle.runtime.navigate({ url: `${origin}/performance.html?mode=density&count=${count}` }, signal); + assert.equal(density.status, "committed"); + const observeStarted = Date.now(); + try { + const observation = await handle.runtime.observe(signal); + densityObserveMs[String(count) as "0" | "10" | "60"] = Date.now() - observeStarted; + assert.equal(observation.candidates.length, count); + } catch { + throw new BenchmarkFailure("density-observe", "failed"); + } + } + const page = await handle.runtime.navigate({ url: `${origin}/performance.html?mode=flow&count=10` }, signal); + assert.equal(page.status, "committed"); + const tabs = await handle.runtime.listTabs(signal); + const main = tabs.find((tab) => tab.selected); + assert(main, "main benchmark tab is unavailable"); + const mainTab = main; + + let observation = await handle.runtime.observe(signal); + const field = candidate(observation, "Display name", "fill"); + const filled = await handle.runtime.act({ observationId: observation.observationId, candidateId: field.id, action: "fill", text: "Ada Lovelace" }, signal); + assert.equal(filled.verified, true); + + observation = await handle.runtime.observe(signal); + const save = candidate(observation, "Save profile", "click"); + const saved = await handle.runtime.act({ observationId: observation.observationId, candidateId: save.id, action: "click", postcondition: { kind: "text", value: "Profile saved" } }, signal); + assert.equal(saved.verified, true); + const saveAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Profile saved" }, wait: "bounded" }, signal); + assert.equal(saveAssert.status, "satisfied"); + + observation = await handle.runtime.observe(signal); + const previewButton = candidate(observation, "Preview profile", "click"); + const previewRequested = await handle.runtime.act({ observationId: observation.observationId, candidateId: previewButton.id, action: "click", postcondition: { kind: "text", value: "Preview ready" } }, signal); + assert.equal(previewRequested.verified, true); + + observation = await handle.runtime.observe(signal); + const immediate = candidate(observation, "Immediate action", "click"); + const clicked = await handle.runtime.act({ observationId: observation.observationId, candidateId: immediate.id, action: "click", postcondition: { kind: "text", value: "Immediate complete" } }, signal); + assert.equal(clicked.verified, true); + const immediateAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Immediate complete" }, wait: "none" }, signal); + assert.equal(immediateAssert.status, "satisfied"); + + observation = await handle.runtime.observe(signal); + const delayed = candidate(observation, "Delayed action", "click"); + const delayedResult = await handle.runtime.act({ observationId: observation.observationId, candidateId: delayed.id, action: "click", postcondition: { kind: "text", value: "Delayed complete" } }, signal); + assert.equal(delayedResult.verified, true); + const delayedAssert = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Delayed complete" }, wait: "bounded" }, signal); + assert.equal(delayedAssert.status, "satisfied"); + + observation = await handle.runtime.observe(signal); + const popupButton = candidate(observation, "Open popup", "click"); + const popupRequest = await handle.runtime.act({ observationId: observation.observationId, candidateId: popupButton.id, action: "click", postcondition: { kind: "text", value: "Popup requested" } }, signal); + assert.equal(popupRequest.verified, true); + const popup = await handle.runtime.waitPopup({ openerTabId: mainTab.tabId }, signal); + assert.equal(popup.status, "found"); + assert(popup.tab); + const popupTab = popup.tab; + const popupRead = await handle.runtime.read({ tabId: popupTab.tabId }, signal); + assert.match(popupRead.visibleText, /Popup ready/); + const popupReady = await handle.runtime.assert({ tabId: popupTab.tabId, assertion: { kind: "visibleText", match: "contains", value: "Popup ready" }, wait: "bounded" }, signal); + assert.equal(popupReady.status, "satisfied"); + try { + await handle.runtime.closeTab({ tabId: popupTab.tabId }, signal); + } catch { + throw new BenchmarkFailure("popup-close", "unverified"); + } + const preview = await handle.runtime.assert({ assertion: { kind: "visibleText", match: "contains", value: "Preview ready: Ada Lovelace" }, wait: "bounded" }, signal); + assert.equal(preview.status, "satisfied"); + return densityObserveMs; +} + +function validCounterMap(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) + && Object.entries(value).every(([key, count]) => /^[A-Za-z0-9_.-]{1,120}$/.test(key) && Number.isSafeInteger(count) && count >= 0); +} + +export function validWorkerCounters(value: unknown): value is { cdp: Record; ipc: Record } { + if (!value || typeof value !== "object") return false; + const counters = value as { cdp?: unknown; ipc?: unknown }; + return validCounterMap(counters.cdp) && validCounterMap(counters.ipc); +} + +async function readWorkerCounts(path: string): Promise { + try { + const value = JSON.parse(await readFile(path, "utf8")) as { cdp?: unknown; ipc?: unknown }; + if (!validWorkerCounters(value)) return undefined; + const cdp = value.cdp; + const ipc = value.ipc; + return { + cdp, + ipc, + cdpCalls: Object.values(cdp).reduce((sum, count) => sum + count, 0), + ipcCalls: Object.values(ipc).reduce((sum, count) => sum + count, 0), + }; + } catch { + return undefined; + } +} + +function ownedProcessLines(root: string): string[] | undefined { + const result = spawnSync("ps", ["-eo", "pid=,args="], { encoding: "utf8", timeout: 5_000 }); + if (result.status !== 0) return undefined; + return String(result.stdout).split(/\r?\n/).filter((line) => line.includes(root)); +} + +function snapshotCounts(value: Record): Record { + return { ...value }; +} + +function countDelta(before: Record, after: Record): Record { + const result: Record = {}; + for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) { + const difference = (after[key] ?? 0) - (before[key] ?? 0); + if (difference > 0) result[key] = difference; + } + return result; +} + +async function closeHandle(handle: ArmHandle | undefined): Promise { + if (!handle || handle.closed) return undefined; + handle.closed = true; + try { + await handle.runtime.close(); + return undefined; + } catch { + return safeFailure("cleanup"); + } +} + +export function pairedArmOrder(sample: number): readonly [Arm, Arm] { + return sample % 2 === 1 ? ["baseline", "optimized"] : ["optimized", "baseline"]; +} + +function attemptsFor(attempts: readonly BenchmarkAttempt[], arm: Arm, phase: Phase): number[] { + return attempts.filter((item) => item.arm === arm && item.phase === phase && item.status === "passed" && item.elapsedMs !== null).map((item) => item.elapsedMs!); +} + +export function summarizeBenchmarkAttempts(attempts: readonly BenchmarkAttempt[], arm: Arm, phase: Phase): TimingSummary { + return summarizeTimings(attemptsFor(attempts, arm, phase)); +} + +function densitySummaries(attempts: readonly BenchmarkAttempt[]): Record> { + const result = {} as Record>; + for (const arm of ["baseline", "optimized"] as const) { + result[arm] = {} as Record<"0" | "10" | "60", TimingSummary>; + for (const count of ["0", "10", "60"] as const) { + result[arm][count] = summarizeTimings(attempts + .filter((attempt) => attempt.arm === arm && attempt.phase === "warm" && attempt.status === "passed" && attempt.densityObserveMs?.[count] !== undefined) + .map((attempt) => attempt.densityObserveMs![count]!)); + } + } + return result; +} + +export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptions): Promise { + const config = validateBenchmarkOptions(options); + const fixture = await createFixtureServer(); + const root = await mkdtemp(join(tmpdir(), "fast-browser-performance-")); + await mkdir(join(root, "leases"), { recursive: true, mode: 0o700 }); + await chmod(join(root, "leases"), 0o700); + const manager = createBrowserSessionManager({ profileDirectory: join(root, "profiles"), leaseDirectory: join(root, "leases"), startupTimeoutMs: 20_000, closeTimeoutMs: 4_000 }); + const controller = new AbortController(); + const attempts: BenchmarkAttempt[] = []; + const workerMethods: Partial> = {}; + const protocolOperations: Partial>> = {}; + const handles: Partial> = {}; + const setupErrors: Partial> = {}; + const pairedOrder: Array = []; + let cleanupError: string | undefined; + let retainedRoot = false; + let counterUnavailable = false; + try { + const armPaths: Record = { baseline: config.baselineWorkerPath, optimized: config.optimizedWorkerPath }; + for (const arm of ["baseline", "optimized"] as const) { + for (let sample = 1; sample <= config.coldSamples; sample += 1) { + const countFile = join(root, `${arm}-cold-${sample}.json`); + const protocol: Record = {}; + let session: BrowserSession | undefined; + let runtime: BrowserRuntime | undefined; + const started = Date.now(); + let startupMs: number | null = null; + try { + const opened = Date.now(); + session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=10`, config.browserPath), controller.signal); + const first = Date.now(); + runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal); + const observation = await runtime.observe(controller.signal); + assert.equal(observation.candidates.length, 10); + startupMs = first - opened + (Date.now() - first); + attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol } }); + } catch (error) { + attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: null, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol } }); + } finally { + try { await runtime?.close(); } catch { cleanupError ??= safeFailure("cleanup"); } + if (session) await manager.close(session.id).catch(() => { cleanupError ??= safeFailure("cleanup"); }); + const counts = await readWorkerCounts(countFile); + if (counts) workerMethods[arm] = counts; + else counterUnavailable = true; + } + } + try { + const countFile = join(root, `${arm}-warm.json`); + const protocol: Record = {}; + const session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=0`, config.browserPath), controller.signal); + const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal); + const handle: ArmHandle = { arm, session, runtime, protocol, countFile, closed: false }; + handles[arm] = handle; + await runtime.observe(controller.signal); // prime the persistent worker; excluded from warm samples + } catch (error) { + const failedHandle = handles[arm]; + if (failedHandle) { + await closeHandle(failedHandle); + await manager.close(failedHandle.session.id).catch(() => {}); + } + delete handles[arm]; + setupErrors[arm] = safeFailure("warm-setup"); + } + } + + for (let sample = 1; sample <= config.samples; sample += 1) { + const order = pairedArmOrder(sample); + pairedOrder.push(order); + for (let position = 0; position < order.length; position += 1) { + const arm = order[position]!; + const handle = handles[arm]; + const before = handle ? snapshotCounts(handle.protocol) : {}; + const started = Date.now(); + try { + if (!handle) throw new Error("arm setup failed"); + const densityObserveMs = await runWarmFlow(handle, fixture.origin, config.candidateCounts, controller.signal); + attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), densityObserveMs }); + } catch (error) { + attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: null, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {} }); + } + } + } + } finally { + for (const arm of ["baseline", "optimized"] as const) { + const handle = handles[arm]; + const error = await closeHandle(handle); + if (error) cleanupError ??= error; + if (handle) { + const counts = await readWorkerCounts(handle.countFile); + if (counts) workerMethods[arm] = counts; + else counterUnavailable = true; + protocolOperations[arm] = { ...handle.protocol }; + } + } + await manager.closeAll().catch(() => { cleanupError ??= safeFailure("cleanup"); }); + await fixture.close().catch(() => { cleanupError ??= safeFailure("cleanup"); }); + const leftovers = ownedProcessLines(root); + if (leftovers === undefined || leftovers.length > 0) { + cleanupError ??= safeFailure("cleanup", "unavailable"); + retainedRoot = true; + } + if (!cleanupError) { + await rm(root, { recursive: true, force: true }).catch(() => { + cleanupError ??= safeFailure("cleanup"); + retainedRoot = true; + }); + } else { + retainedRoot = true; + } + } + const warm: Record = { + baseline: summarizeBenchmarkAttempts(attempts, "baseline", "warm"), + optimized: summarizeBenchmarkAttempts(attempts, "optimized", "warm"), + }; + const coldStartup: Record = { + baseline: summarizeTimings(attempts.filter((item) => item.arm === "baseline" && item.phase === "cold-start" && item.status === "passed" && item.startupMs !== null).map((item) => item.startupMs!)), + optimized: summarizeTimings(attempts.filter((item) => item.arm === "optimized" && item.phase === "cold-start" && item.status === "passed" && item.startupMs !== null).map((item) => item.startupMs!)), + }; + return { + ok: !cleanupError && !counterUnavailable && workerMethods.baseline !== undefined && workerMethods.optimized !== undefined && attempts.length === config.samples * 2 + config.coldSamples * 2 && attempts.every((item) => item.status === "passed"), + measurement: { workerMethods: "selected Python worker CDP/IPC helper methods", protocolOperations: "TypeScript JSONL operation names", warmPriming: "one density=0 observe per arm, excluded from warm samples", coldStartup: "browser session open plus first worker observe", runtimeNarrowingMeasured: false }, + samples: config.samples, + coldSamples: config.coldSamples, + candidateCounts: config.candidateCounts, + attempts, + warm, + coldStartup, + workerMethods, + protocolOperations, + densityObserve: densitySummaries(attempts), + pairedOrder, + counterStatus: !counterUnavailable && workerMethods.baseline && workerMethods.optimized ? "complete" : "unavailable", + cleanup: cleanupError ? { status: "failed", error: cleanupError, retainedRoot } : { status: "passed", retainedRoot: false }, + }; +} + +function parseArgs(argv: readonly string[]): PerformanceBenchmarkOptions & { help?: boolean } { + const result: Record = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]!; + if (arg === "--help" || arg === "-h") return { baselineWorkerPath: "", help: true }; + const values: Record = { "--baseline-worker": "baselineWorkerPath", "--optimized-worker": "optimizedWorkerPath", "--python": "pythonPath", "--browser": "browserPath", "--samples": "samples", "--cold-samples": "coldSamples", "--counts": "candidateCounts" }; + const key = values[arg]; + if (!key || index + 1 >= argv.length) throw new Error(`unknown or incomplete option: ${arg}`); + const value = argv[++index]!; + result[key] = key === "samples" || key === "coldSamples" ? Number(value) : key === "candidateCounts" ? value.split(",").map(Number) : value; + } + if (typeof result.baselineWorkerPath !== "string") throw new Error("--baseline-worker is required"); + return result as unknown as PerformanceBenchmarkOptions; +} + +function printReport(report: BenchmarkReport): void { + console.log(JSON.stringify(report, null, 2)); +} + +if (import.meta.main) { + try { + const parsed = parseArgs(process.argv.slice(2)); + if (parsed.help) { + console.log("usage: node --experimental-strip-types performance-benchmark.ts --baseline-worker PATH [--optimized-worker PATH] [--samples N] [--cold-samples N] [--counts 0,10,60]"); + } else { + const report = await runPerformanceBenchmark(parsed); + printReport(report); + if (!report.ok) process.exitCode = 1; + } + } catch (error) { + console.error(safeFailure("cli")); + process.exitCode = 1; + } +} diff --git a/extensions/fast-browser/performance_probe.py b/extensions/fast-browser/performance_probe.py new file mode 100644 index 0000000..fe465fb --- /dev/null +++ b/extensions/fast-browser/performance_probe.py @@ -0,0 +1,119 @@ +"""Private benchmark wrapper for counting browser-harness calls. + +The wrapper is deliberately a transport-only probe. It records method names and +counts, never CDP arguments, URLs, DOM values, page text, or screenshots. The +selected worker is executed unchanged after the helper module is wrapped. +""" +from __future__ import annotations + +import importlib +import json +import os +from pathlib import Path +import runpy +import sys +import tempfile +from collections import Counter +from typing import Any + + +def _take_flag(argv: list[str], flag: str) -> tuple[str | None, list[str]]: + remaining: list[str] = [] + value: str | None = None + index = 0 + while index < len(argv): + item = argv[index] + if item == flag: + if value is not None or index + 1 >= len(argv): + raise ValueError(f"{flag} must be supplied once") + value = argv[index + 1] + index += 2 + continue + remaining.append(item) + index += 1 + return value, remaining + + +def _method_name(value: Any) -> str: + if isinstance(value, str) and value and len(value) <= 120 and all( + character.isalnum() or character in "._-" for character in value + ): + return value + return "unknown" + + +def _wrap_helpers(counts: dict[str, Counter[str]] | None = None) -> dict[str, Counter[str]]: + helpers = importlib.import_module("browser_harness.helpers") + counts = counts or {"cdp": Counter(), "ipc": Counter()} + + original_cdp = getattr(helpers, "cdp", None) + if callable(original_cdp): + def counted_cdp(*args: Any, **kwargs: Any) -> Any: + method = _method_name(args[0] if args else None) + counts["cdp"][method] += 1 + return original_cdp(*args, **kwargs) + helpers.cdp = counted_cdp + + original_send = getattr(helpers, "_send", None) + if callable(original_send): + def counted_send(*args: Any, **kwargs: Any) -> Any: + # Do not inspect the request. The IPC method is intentionally the + # fixed helper method name rather than an untrusted operation value. + counts["ipc"]["_send"] += 1 + return original_send(*args, **kwargs) + helpers._send = counted_send + return counts + + +def _write_counts(path: str, counts: dict[str, Counter[str]]) -> None: + payload = { + "cdp": dict(sorted(counts["cdp"].items())), + "ipc": dict(sorted(counts["ipc"].items())), + } + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix="performance-count-", dir=str(destination.parent)) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(payload, stream, separators=(",", ":"), sort_keys=True) + stream.write("\n") + os.chmod(temporary, 0o600) + os.replace(temporary, destination) + os.chmod(destination, 0o600) + finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + + +def main(argv: list[str] | None = None) -> int: + raw = list(sys.argv[1:] if argv is None else argv) + worker_path, raw = _take_flag(raw, "--performance-worker") + count_path, worker_argv = _take_flag(raw, "--performance-count-file") + if not worker_path or not count_path: + raise SystemExit("performance_probe requires --performance-worker and --performance-count-file") + if not Path(worker_path).is_file(): + raise SystemExit("performance worker path is not a file") + counts: dict[str, Counter[str]] = {"cdp": Counter(), "ipc": Counter()} + try: + sys.argv = [worker_path, *worker_argv] + namespace = runpy.run_path(worker_path, run_name="performance_worker") + backend = namespace.get("RealBrowserBackend") + main_function = namespace.get("main") + if backend is None or not callable(main_function): + raise SystemExit("performance worker does not expose its expected entry points") + original_load = backend._load_harness + def load_and_probe(instance: Any) -> Any: + result = original_load(instance) + _wrap_helpers(counts) + return result + backend._load_harness = load_and_probe + main_function(worker_argv) + finally: + _write_counts(count_path, counts) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())