From 12a6bb6afbc512445264700bc30393f900ffa9bd Mon Sep 17 00:00:00 2001 From: JB Date: Mon, 21 Sep 2026 13:33:51 -0400 Subject: [PATCH] test(fast-browser): retain per-operation timing and failure costs --- .../performance-benchmark.test.ts | 6 +++ .../fast-browser/performance-benchmark.ts | 51 +++++++++++++++---- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/extensions/fast-browser/performance-benchmark.test.ts b/extensions/fast-browser/performance-benchmark.test.ts index 2982532..9624cd5 100644 --- a/extensions/fast-browser/performance-benchmark.test.ts +++ b/extensions/fast-browser/performance-benchmark.test.ts @@ -6,6 +6,7 @@ import { pairedArmOrder, safeFailure, summarizeBenchmarkAttempts, + timingDelta, validWorkerCounters, summarizeTimings, validateBenchmarkOptions, @@ -27,6 +28,11 @@ test("timing aggregation reports bounded p50 and p95 without wall-clock assumpti assert.deepEqual(summarizeTimings([]), { count: 0, p50: null, p95: null }); }); +test("transport timer deltas preserve bounded operation names and failed costs", () => { + assert.deepEqual(timingDelta({ observe: 12, closeTab: 4 }, { observe: 19, closeTab: 4, waitPopup: 8 }), { observe: 7, waitPopup: 8 }); + assert.deepEqual(timingDelta({ observe: 10 }, { observe: 9 }), {}); +}); + test("paired order alternates AB and BA while retaining failed attempts", () => { assert.deepEqual(pairedArmOrder(1), ["baseline", "optimized"]); assert.deepEqual(pairedArmOrder(2), ["optimized", "baseline"]); diff --git a/extensions/fast-browser/performance-benchmark.ts b/extensions/fast-browser/performance-benchmark.ts index 4d6d5db..99749cf 100644 --- a/extensions/fast-browser/performance-benchmark.ts +++ b/extensions/fast-browser/performance-benchmark.ts @@ -8,7 +8,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { createBrowserTransport, type ChildProcessLike, type SpawnLike } from "./bridge.ts"; import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts"; -import type { BrowserRuntime, BrowserTransportFactory, Candidate, PrivateArtifactStore } from "./contracts.ts"; +import type { BrowserOperation, BrowserRuntime, BrowserTransport, BrowserTransportFactory, BrowserTransportRequest, BrowserTransportResultFor, Candidate, PrivateArtifactStore } from "./contracts.ts"; import { createBrowserRuntimeFactory } from "./runtime.ts"; const ROOT = dirname(fileURLToPath(import.meta.url)); @@ -26,6 +26,8 @@ const MAX_COUNTS = 3; type Arm = "baseline" | "optimized"; type Phase = "warm" | "cold-start"; +export type TimedOperation = "observe" | "act" | "read" | "assert" | "navigate" | "reload" | "listTabs" | "selectTab" | "waitPopup" | "closeTab" | "captureEvidence" | "narrowOriginPolicy" | "close"; +export type OperationTimings = Partial>; export interface PerformanceBenchmarkOptions { baselineWorkerPath: string; @@ -54,6 +56,7 @@ export interface BenchmarkAttempt { error?: string; protocolOperations?: Record; densityObserveMs?: Partial>; + transportMs?: OperationTimings; } export interface WorkerMethodCounts { @@ -90,6 +93,7 @@ interface ArmHandle { session: BrowserSession; runtime: BrowserRuntime; protocol: Record; + transportMs: OperationTimings; countFile: string; closed: boolean; } @@ -205,10 +209,26 @@ function instrumentedSpawn( }; } +function timedTransport(base: BrowserTransport, timings: OperationTimings): BrowserTransport { + return { + async dispatch(request: BrowserTransportRequest, signal: AbortSignal): Promise> { + const operation = request.operation.op as TimedOperation; + const started = performance.now(); + try { + return await base.dispatch(request, signal); + } finally { + timings[operation] = (timings[operation] ?? 0) + performance.now() - started; + } + }, + close: () => base.close(), + }; +} + function createInstrumentedRuntime( armWorkerPath: string, countFile: string, protocol: Record, + transportMs: OperationTimings, session: BrowserSession, origin: string, pythonPath: string, @@ -216,11 +236,12 @@ function createInstrumentedRuntime( ): BrowserRuntime { const transportFactory: BrowserTransportFactory = { create(options, parentSignal) { - return createBrowserTransport({ + const transport = createBrowserTransport({ ...options, workerPath: PROBE, spawn: instrumentedSpawn(armWorkerPath, countFile, protocol), }, parentSignal); + return timedTransport(transport, transportMs); }, }; return createBrowserRuntimeFactory(transportFactory).create({ @@ -384,6 +405,15 @@ function snapshotCounts(value: Record): Record { return { ...value }; } +export function timingDelta(before: OperationTimings, after: OperationTimings): OperationTimings { + const result: OperationTimings = {}; + for (const key of new Set([...Object.keys(before), ...Object.keys(after)] as TimedOperation[])) { + const difference = (after[key] ?? 0) - (before[key] ?? 0); + if (difference > 0) result[key] = difference; + } + return result; +} + function countDelta(before: Record, after: Record): Record { const result: Record = {}; for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) { @@ -452,6 +482,7 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio for (let sample = 1; sample <= config.coldSamples; sample += 1) { const countFile = join(root, `${arm}-cold-${sample}.json`); const protocol: Record = {}; + const transportMs: OperationTimings = {}; let session: BrowserSession | undefined; let runtime: BrowserRuntime | undefined; const started = Date.now(); @@ -460,13 +491,13 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio const opened = Date.now(); session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=10`, config.browserPath), controller.signal); const first = Date.now(); - runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal); + runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, transportMs, session, fixture.origin, config.pythonPath, controller.signal); const observation = await runtime.observe(controller.signal); assert.equal(observation.candidates.length, 10); startupMs = first - opened + (Date.now() - first); - attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol } }); + attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol }, transportMs: { ...transportMs } }); } catch (error) { - attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: null, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol } }); + attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: Date.now() - started, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol }, transportMs: { ...transportMs } }); } finally { try { await runtime?.close(); } catch { cleanupError ??= safeFailure("cleanup"); } if (session) await manager.close(session.id).catch(() => { cleanupError ??= safeFailure("cleanup"); }); @@ -478,9 +509,10 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio try { const countFile = join(root, `${arm}-warm.json`); const protocol: Record = {}; + const transportMs: OperationTimings = {}; const session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=0`, config.browserPath), controller.signal); - const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal); - const handle: ArmHandle = { arm, session, runtime, protocol, countFile, closed: false }; + const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, transportMs, session, fixture.origin, config.pythonPath, controller.signal); + const handle: ArmHandle = { arm, session, runtime, protocol, transportMs, countFile, closed: false }; handles[arm] = handle; await runtime.observe(controller.signal); // prime the persistent worker; excluded from warm samples } catch (error) { @@ -501,13 +533,14 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio const arm = order[position]!; const handle = handles[arm]; const before = handle ? snapshotCounts(handle.protocol) : {}; + const beforeTransport = handle ? { ...handle.transportMs } : {}; const started = Date.now(); try { if (!handle) throw new Error("arm setup failed"); const densityObserveMs = await runWarmFlow(handle, fixture.origin, config.candidateCounts, controller.signal); - attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), densityObserveMs }); + attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), transportMs: timingDelta(beforeTransport, handle.transportMs), densityObserveMs }); } catch (error) { - attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: null, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {} }); + attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: Date.now() - started, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {}, transportMs: handle ? timingDelta(beforeTransport, handle.transportMs) : {} }); } } }