jb/harness-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(fast-browser): retain per-operation timing and failure costs

Browse Source
This commit is contained in:
jb
2026-09-21 13:33:51 -04:00
parent 6a725c1f47
commit 12a6bb6afb
2 changed files with 48 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/fast-browser/performance-benchmark.test.ts
+6
View File
@@ -6,6 +6,7 @@ import {
pairedArmOrder,
safeFailure,
summarizeBenchmarkAttempts,
timingDelta,
validWorkerCounters,
summarizeTimings,
validateBenchmarkOptions,
@@ -27,6 +28,11 @@ test("timing aggregation reports bounded p50 and p95 without wall-clock assumpti
assert.deepEqual(summarizeTimings([]), { count: 0, p50: null, p95: null });
});
test("transport timer deltas preserve bounded operation names and failed costs", () => {
assert.deepEqual(timingDelta({ observe: 12, closeTab: 4 }, { observe: 19, closeTab: 4, waitPopup: 8 }), { observe: 7, waitPopup: 8 });
assert.deepEqual(timingDelta({ observe: 10 }, { observe: 9 }), {});
});
test("paired order alternates AB and BA while retaining failed attempts", () => {
assert.deepEqual(pairedArmOrder(1), ["baseline", "optimized"]);
assert.deepEqual(pairedArmOrder(2), ["optimized", "baseline"]);
extensions/fast-browser/performance-benchmark.ts
+42 -9
View File
@@ -8,7 +8,7 @@ import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { createBrowserTransport, type ChildProcessLike, type SpawnLike } from "./bridge.ts";
import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts";
import type { BrowserRuntime, BrowserTransportFactory, Candidate, PrivateArtifactStore } from "./contracts.ts";
import type { BrowserOperation, BrowserRuntime, BrowserTransport, BrowserTransportFactory, BrowserTransportRequest, BrowserTransportResultFor, Candidate, PrivateArtifactStore } from "./contracts.ts";
import { createBrowserRuntimeFactory } from "./runtime.ts";
const ROOT = dirname(fileURLToPath(import.meta.url));
@@ -26,6 +26,8 @@ const MAX_COUNTS = 3;
type Arm = "baseline" | "optimized";
type Phase = "warm" | "cold-start";
export type TimedOperation = "observe" | "act" | "read" | "assert" | "navigate" | "reload" | "listTabs" | "selectTab" | "waitPopup" | "closeTab" | "captureEvidence" | "narrowOriginPolicy" | "close";
export type OperationTimings = Partial<Record<TimedOperation, number>>;
export interface PerformanceBenchmarkOptions {
baselineWorkerPath: string;
@@ -54,6 +56,7 @@ export interface BenchmarkAttempt {
error?: string;
protocolOperations?: Record<string, number>;
densityObserveMs?: Partial<Record<"0" | "10" | "60", number>>;
transportMs?: OperationTimings;
}
export interface WorkerMethodCounts {
@@ -90,6 +93,7 @@ interface ArmHandle {
session: BrowserSession;
runtime: BrowserRuntime;
protocol: Record<string, number>;
transportMs: OperationTimings;
countFile: string;
closed: boolean;
}
@@ -205,10 +209,26 @@ function instrumentedSpawn(
};
}
function timedTransport(base: BrowserTransport, timings: OperationTimings): BrowserTransport {
return {
async dispatch<Operation extends BrowserOperation>(request: BrowserTransportRequest<Operation>, signal: AbortSignal): Promise<BrowserTransportResultFor<Operation>> {
const operation = request.operation.op as TimedOperation;
const started = performance.now();
try {
return await base.dispatch(request, signal);
} finally {
timings[operation] = (timings[operation] ?? 0) + performance.now() - started;
}
},
close: () => base.close(),
};
}
function createInstrumentedRuntime(
armWorkerPath: string,
countFile: string,
protocol: Record<string, number>,
transportMs: OperationTimings,
session: BrowserSession,
origin: string,
pythonPath: string,
@@ -216,11 +236,12 @@ function createInstrumentedRuntime(
): BrowserRuntime {
const transportFactory: BrowserTransportFactory = {
create(options, parentSignal) {
return createBrowserTransport({
const transport = createBrowserTransport({
...options,
workerPath: PROBE,
spawn: instrumentedSpawn(armWorkerPath, countFile, protocol),
}, parentSignal);
return timedTransport(transport, transportMs);
},
};
return createBrowserRuntimeFactory(transportFactory).create({
@@ -384,6 +405,15 @@ function snapshotCounts(value: Record<string, number>): Record<string, number> {
return { ...value };
}
export function timingDelta(before: OperationTimings, after: OperationTimings): OperationTimings {
const result: OperationTimings = {};
for (const key of new Set([...Object.keys(before), ...Object.keys(after)] as TimedOperation[])) {
const difference = (after[key] ?? 0) - (before[key] ?? 0);
if (difference > 0) result[key] = difference;
}
return result;
}
function countDelta(before: Record<string, number>, after: Record<string, number>): Record<string, number> {
const result: Record<string, number> = {};
for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) {
@@ -452,6 +482,7 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
for (let sample = 1; sample <= config.coldSamples; sample += 1) {
const countFile = join(root, `${arm}-cold-${sample}.json`);
const protocol: Record<string, number> = {};
const transportMs: OperationTimings = {};
let session: BrowserSession | undefined;
let runtime: BrowserRuntime | undefined;
const started = Date.now();
@@ -460,13 +491,13 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
const opened = Date.now();
session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=10`, config.browserPath), controller.signal);
const first = Date.now();
runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, transportMs, session, fixture.origin, config.pythonPath, controller.signal);
const observation = await runtime.observe(controller.signal);
assert.equal(observation.candidates.length, 10);
startupMs = first - opened + (Date.now() - first);
attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol } });
attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol }, transportMs: { ...transportMs } });
} catch (error) {
attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: null, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol } });
attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: Date.now() - started, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol }, transportMs: { ...transportMs } });
} finally {
try { await runtime?.close(); } catch { cleanupError ??= safeFailure("cleanup"); }
if (session) await manager.close(session.id).catch(() => { cleanupError ??= safeFailure("cleanup"); });
@@ -478,9 +509,10 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
try {
const countFile = join(root, `${arm}-warm.json`);
const protocol: Record<string, number> = {};
const transportMs: OperationTimings = {};
const session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=0`, config.browserPath), controller.signal);
const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
const handle: ArmHandle = { arm, session, runtime, protocol, countFile, closed: false };
const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, transportMs, session, fixture.origin, config.pythonPath, controller.signal);
const handle: ArmHandle = { arm, session, runtime, protocol, transportMs, countFile, closed: false };
handles[arm] = handle;
await runtime.observe(controller.signal); // prime the persistent worker; excluded from warm samples
} catch (error) {
@@ -501,13 +533,14 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
const arm = order[position]!;
const handle = handles[arm];
const before = handle ? snapshotCounts(handle.protocol) : {};
const beforeTransport = handle ? { ...handle.transportMs } : {};
const started = Date.now();
try {
if (!handle) throw new Error("arm setup failed");
const densityObserveMs = await runWarmFlow(handle, fixture.origin, config.candidateCounts, controller.signal);
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), densityObserveMs });
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), transportMs: timingDelta(beforeTransport, handle.transportMs), densityObserveMs });
} catch (error) {
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: null, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {} });
attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: Date.now() - started, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {}, transportMs: handle ? timingDelta(beforeTransport, handle.transportMs) : {} });
}
}
}