test(fast-browser): retain per-operation timing and failure costs

extensions/fast-browser/performance-benchmark.test.ts

@@ -6,6 +6,7 @@ import {
   pairedArmOrder,
   safeFailure,
   summarizeBenchmarkAttempts,
+  timingDelta,
   validWorkerCounters,
   summarizeTimings,
   validateBenchmarkOptions,
@@ -27,6 +28,11 @@ test("timing aggregation reports bounded p50 and p95 without wall-clock assumpti
   assert.deepEqual(summarizeTimings([]), { count: 0, p50: null, p95: null });
 });
 
+test("transport timer deltas preserve bounded operation names and failed costs", () => {
+  assert.deepEqual(timingDelta({ observe: 12, closeTab: 4 }, { observe: 19, closeTab: 4, waitPopup: 8 }), { observe: 7, waitPopup: 8 });
+  assert.deepEqual(timingDelta({ observe: 10 }, { observe: 9 }), {});
+});
+
 test("paired order alternates AB and BA while retaining failed attempts", () => {
   assert.deepEqual(pairedArmOrder(1), ["baseline", "optimized"]);
   assert.deepEqual(pairedArmOrder(2), ["optimized", "baseline"]);

extensions/fast-browser/performance-benchmark.ts

@@ -8,7 +8,7 @@ import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { createBrowserTransport, type ChildProcessLike, type SpawnLike } from "./bridge.ts";
 import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts";
-import type { BrowserRuntime, BrowserTransportFactory, Candidate, PrivateArtifactStore } from "./contracts.ts";
+import type { BrowserOperation, BrowserRuntime, BrowserTransport, BrowserTransportFactory, BrowserTransportRequest, BrowserTransportResultFor, Candidate, PrivateArtifactStore } from "./contracts.ts";
 import { createBrowserRuntimeFactory } from "./runtime.ts";
 
 const ROOT = dirname(fileURLToPath(import.meta.url));
@@ -26,6 +26,8 @@ const MAX_COUNTS = 3;
 
 type Arm = "baseline" | "optimized";
 type Phase = "warm" | "cold-start";
+export type TimedOperation = "observe" | "act" | "read" | "assert" | "navigate" | "reload" | "listTabs" | "selectTab" | "waitPopup" | "closeTab" | "captureEvidence" | "narrowOriginPolicy" | "close";
+export type OperationTimings = Partial<Record<TimedOperation, number>>;
 
 export interface PerformanceBenchmarkOptions {
   baselineWorkerPath: string;
@@ -54,6 +56,7 @@ export interface BenchmarkAttempt {
   error?: string;
   protocolOperations?: Record<string, number>;
   densityObserveMs?: Partial<Record<"0" | "10" | "60", number>>;
+  transportMs?: OperationTimings;
 }
 
 export interface WorkerMethodCounts {
@@ -90,6 +93,7 @@ interface ArmHandle {
   session: BrowserSession;
   runtime: BrowserRuntime;
   protocol: Record<string, number>;
+  transportMs: OperationTimings;
   countFile: string;
   closed: boolean;
 }
@@ -205,10 +209,26 @@ function instrumentedSpawn(
   };
 }
 
+function timedTransport(base: BrowserTransport, timings: OperationTimings): BrowserTransport {
+  return {
+    async dispatch<Operation extends BrowserOperation>(request: BrowserTransportRequest<Operation>, signal: AbortSignal): Promise<BrowserTransportResultFor<Operation>> {
+      const operation = request.operation.op as TimedOperation;
+      const started = performance.now();
+      try {
+        return await base.dispatch(request, signal);
+      } finally {
+        timings[operation] = (timings[operation] ?? 0) + performance.now() - started;
+      }
+    },
+    close: () => base.close(),
+  };
+}
+
 function createInstrumentedRuntime(
   armWorkerPath: string,
   countFile: string,
   protocol: Record<string, number>,
+  transportMs: OperationTimings,
   session: BrowserSession,
   origin: string,
   pythonPath: string,
@@ -216,11 +236,12 @@ function createInstrumentedRuntime(
 ): BrowserRuntime {
   const transportFactory: BrowserTransportFactory = {
     create(options, parentSignal) {
-      return createBrowserTransport({
+      const transport = createBrowserTransport({
         ...options,
         workerPath: PROBE,
         spawn: instrumentedSpawn(armWorkerPath, countFile, protocol),
       }, parentSignal);
+      return timedTransport(transport, transportMs);
     },
   };
   return createBrowserRuntimeFactory(transportFactory).create({
@@ -384,6 +405,15 @@ function snapshotCounts(value: Record<string, number>): Record<string, number> {
   return { ...value };
 }
 
+export function timingDelta(before: OperationTimings, after: OperationTimings): OperationTimings {
+  const result: OperationTimings = {};
+  for (const key of new Set([...Object.keys(before), ...Object.keys(after)] as TimedOperation[])) {
+    const difference = (after[key] ?? 0) - (before[key] ?? 0);
+    if (difference > 0) result[key] = difference;
+  }
+  return result;
+}
+
 function countDelta(before: Record<string, number>, after: Record<string, number>): Record<string, number> {
   const result: Record<string, number> = {};
   for (const key of new Set([...Object.keys(before), ...Object.keys(after)])) {
@@ -452,6 +482,7 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
       for (let sample = 1; sample <= config.coldSamples; sample += 1) {
         const countFile = join(root, `${arm}-cold-${sample}.json`);
         const protocol: Record<string, number> = {};
+        const transportMs: OperationTimings = {};
         let session: BrowserSession | undefined;
         let runtime: BrowserRuntime | undefined;
         const started = Date.now();
@@ -460,13 +491,13 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
           const opened = Date.now();
           session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=10`, config.browserPath), controller.signal);
           const first = Date.now();
-          runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
+          runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, transportMs, session, fixture.origin, config.pythonPath, controller.signal);
           const observation = await runtime.observe(controller.signal);
           assert.equal(observation.candidates.length, 10);
           startupMs = first - opened + (Date.now() - first);
-          attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol } });
+          attempts.push({ arm, phase: "cold-start", sample, status: "passed", elapsedMs: Date.now() - started, startupMs, protocolOperations: { ...protocol }, transportMs: { ...transportMs } });
         } catch (error) {
-          attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: null, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol } });
+          attempts.push({ arm, phase: "cold-start", sample, status: "failed", elapsedMs: Date.now() - started, startupMs, error: safeFailure("cold-start"), protocolOperations: { ...protocol }, transportMs: { ...transportMs } });
         } finally {
           try { await runtime?.close(); } catch { cleanupError ??= safeFailure("cleanup"); }
           if (session) await manager.close(session.id).catch(() => { cleanupError ??= safeFailure("cleanup"); });
@@ -478,9 +509,10 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
       try {
         const countFile = join(root, `${arm}-warm.json`);
         const protocol: Record<string, number> = {};
+        const transportMs: OperationTimings = {};
         const session = await manager.open(launchConfig(fixture.origin, `${fixture.origin}/performance.html?mode=density&count=0`, config.browserPath), controller.signal);
-        const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, session, fixture.origin, config.pythonPath, controller.signal);
+        const runtime = createInstrumentedRuntime(armPaths[arm], countFile, protocol, transportMs, session, fixture.origin, config.pythonPath, controller.signal);
-        const handle: ArmHandle = { arm, session, runtime, protocol, countFile, closed: false };
+        const handle: ArmHandle = { arm, session, runtime, protocol, transportMs, countFile, closed: false };
         handles[arm] = handle;
         await runtime.observe(controller.signal); // prime the persistent worker; excluded from warm samples
       } catch (error) {
@@ -501,13 +533,14 @@ export async function runPerformanceBenchmark(options: PerformanceBenchmarkOptio
         const arm = order[position]!;
         const handle = handles[arm];
         const before = handle ? snapshotCounts(handle.protocol) : {};
+        const beforeTransport = handle ? { ...handle.transportMs } : {};
         const started = Date.now();
         try {
           if (!handle) throw new Error("arm setup failed");
           const densityObserveMs = await runWarmFlow(handle, fixture.origin, config.candidateCounts, controller.signal);
-          attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), densityObserveMs });
+          attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "passed", elapsedMs: Date.now() - started, startupMs: null, protocolOperations: countDelta(before, handle.protocol), transportMs: timingDelta(beforeTransport, handle.transportMs), densityObserveMs });
         } catch (error) {
-          attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: null, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {} });
+          attempts.push({ arm, phase: "warm", sample, order: (position + 1) as 1 | 2, status: "failed", elapsedMs: Date.now() - started, startupMs: null, error: handle ? failureFor(error, "warm-flow") : (setupErrors[arm] ?? safeFailure("warm-setup")), protocolOperations: handle ? countDelta(before, handle.protocol) : {}, transportMs: handle ? timingDelta(beforeTransport, handle.transportMs) : {} });
         }
       }
     }