Add metrics collection and terminal performance tests

packages/server/src/server/agent/agent-manager.ts

@@ -231,6 +231,16 @@ export type ManagedAgent =
   | ManagedAgentError
   | ManagedAgentClosed;
 
+export interface AgentMetricsSnapshot {
+  total: number;
+  byLifecycle: Record<string, number>;
+  withActiveForegroundTurn: number;
+  timelineStats: {
+    totalItems: number;
+    maxItemsPerAgent: number;
+  };
+}
+
 type ActiveManagedAgent =
   | ManagedAgentInitializing
   | ManagedAgentIdle
@@ -345,6 +355,37 @@ export class AgentManager {
     this.onAgentAttention = callback;
   }
 
+  public getMetricsSnapshot(): AgentMetricsSnapshot {
+    const byLifecycle: Record<string, number> = {};
+    let withActiveForegroundTurn = 0;
+    let totalItems = 0;
+    let maxItemsPerAgent = 0;
+
+    for (const agent of this.agents.values()) {
+      byLifecycle[agent.lifecycle] = (byLifecycle[agent.lifecycle] ?? 0) + 1;
+
+      if (agent.activeForegroundTurnId !== null) {
+        withActiveForegroundTurn++;
+      }
+
+      const len = agent.timeline.length;
+      totalItems += len;
+      if (len > maxItemsPerAgent) {
+        maxItemsPerAgent = len;
+      }
+    }
+
+    return {
+      total: this.agents.size,
+      byLifecycle,
+      withActiveForegroundTurn,
+      timelineStats: {
+        totalItems,
+        maxItemsPerAgent,
+      },
+    };
+  }
+
   private touchUpdatedAt(agent: ManagedAgent): Date {
     const nowMs = Date.now();
     const previousMs = agent.updatedAt.getTime();

packages/server/src/server/session.ts

@@ -291,6 +291,8 @@ export type SessionRuntimeMetrics = {
   checkoutDiffFallbackRefreshTargetCount: number;
   terminalDirectorySubscriptionCount: number;
   terminalSubscriptionCount: number;
+  inflightRequests: number;
+  peakInflightRequests: number;
 };
 
 type FetchAgentsRequestMessage = Extract<SessionInboundMessage, { type: "fetch_agents_request" }>;
@@ -610,6 +612,8 @@ export class Session {
   private readonly activeTerminalStreams = new Map<number, ActiveTerminalStream>();
   private readonly terminalIdToSlot = new Map<string, number>();
   private nextTerminalSlot = 0;
+  private inflightRequests = 0;
+  private peakInflightRequests = 0;
   private readonly checkoutDiffSubscriptions = new Map<string, { targetKey: string }>();
   private readonly checkoutDiffTargets = new Map<string, CheckoutDiffWatchTarget>();
   private readonly workspaceGitWatchTargets = new Map<string, WorkspaceGitWatchTarget>();
@@ -754,6 +758,8 @@ export class Session {
       checkoutDiffFallbackRefreshTargetCount,
       terminalDirectorySubscriptionCount: this.subscribedTerminalDirectories.size,
       terminalSubscriptionCount: this.activeTerminalStreams.size,
+      inflightRequests: this.inflightRequests,
+      peakInflightRequests: this.peakInflightRequests,
     };
   }
 
@@ -1454,6 +1460,11 @@ export class Session {
    * Main entry point for processing session messages
    */
   public async handleMessage(msg: SessionInboundMessage): Promise<void> {
+    this.inflightRequests++;
+    if (this.inflightRequests > this.peakInflightRequests) {
+      this.peakInflightRequests = this.inflightRequests;
+    }
+    try {
     this.sessionLogger.trace({ inbound: msg }, "inbound message");
     try {
       switch (msg.type) {
@@ -1779,6 +1790,13 @@ export class Session {
         },
       });
     }
+    } finally {
+      this.inflightRequests--;
+    }
+  }
+
+  public resetPeakInflight(): void {
+    this.peakInflightRequests = this.inflightRequests;
   }
 
   public handleBinaryFrame(frame: TerminalStreamFrame): void {

packages/server/src/server/websocket-server.ts

@@ -197,6 +197,7 @@ type WebSocketRuntimeCounters = {
   hostRejected: number;
 };
 
+const SLOW_REQUEST_THRESHOLD_MS = 500;
 const EXTERNAL_SESSION_DISCONNECT_GRACE_MS = 90_000;
 const HELLO_TIMEOUT_MS = 15_000;
 const WS_CLOSE_HELLO_TIMEOUT = 4001;
@@ -275,6 +276,7 @@ export class VoiceAssistantWebSocketServer {
   };
   private readonly inboundMessageCounts = new Map<string, number>();
   private readonly inboundSessionRequestCounts = new Map<string, number>();
+  private readonly requestLatencies = new Map<string, number[]>();
   private runtimeMetricsInterval: ReturnType<typeof setInterval> | null = null;
 
   constructor(
@@ -1056,7 +1058,21 @@ export class VoiceAssistantWebSocketServer {
 
       if (message.type === "session") {
         this.recordInboundSessionRequestType(message.message.type);
+        const startMs = performance.now();
         await activeConnection.session.handleMessage(message.message);
+        const durationMs = performance.now() - startMs;
+        this.recordRequestLatency(message.message.type, durationMs);
+
+        if (durationMs >= SLOW_REQUEST_THRESHOLD_MS) {
+          activeConnection.connectionLogger.warn(
+            {
+              requestType: message.message.type,
+              durationMs: Math.round(durationMs),
+              inflightRequests: activeConnection.session.getRuntimeMetrics().inflightRequests,
+            },
+            "ws_slow_request",
+          );
+        }
       }
     } catch (error) {
       const err = error instanceof Error ? error : new Error(String(error));
@@ -1147,10 +1163,49 @@ export class VoiceAssistantWebSocketServer {
     this.incrementCount(this.inboundSessionRequestCounts, type);
   }
 
+  private recordRequestLatency(type: string, durationMs: number): void {
+    let latencies = this.requestLatencies.get(type);
+    if (!latencies) {
+      latencies = [];
+      this.requestLatencies.set(type, latencies);
+    }
+    latencies.push(durationMs);
+  }
+
   private getTopCounts(map: Map<string, number>, limit: number): Array<[string, number]> {
     return [...map.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit);
   }
 
+  private computeLatencyStats(): Array<{
+    type: string;
+    count: number;
+    minMs: number;
+    maxMs: number;
+    p50Ms: number;
+    totalMs: number;
+  }> {
+    const stats: Array<{
+      type: string;
+      count: number;
+      minMs: number;
+      maxMs: number;
+      p50Ms: number;
+      totalMs: number;
+    }> = [];
+    for (const [type, latencies] of this.requestLatencies) {
+      if (latencies.length === 0) continue;
+      latencies.sort((a, b) => a - b);
+      const count = latencies.length;
+      const minMs = Math.round(latencies[0]!);
+      const maxMs = Math.round(latencies[count - 1]!);
+      const p50Ms = Math.round(latencies[Math.floor(count / 2)]!);
+      const totalMs = Math.round(latencies.reduce((sum, v) => sum + v, 0));
+      stats.push({ type, count, minMs, maxMs, p50Ms, totalMs });
+    }
+    stats.sort((a, b) => b.totalMs - a.totalMs);
+    return stats.slice(0, 15);
+  }
+
   private collectSessionRuntimeMetrics(): SessionRuntimeMetrics {
     const uniqueConnections = new Set<SessionConnection>(this.externalSessionsByKey.values());
     let checkoutDiffTargetCount = 0;
@@ -1159,6 +1214,8 @@ export class VoiceAssistantWebSocketServer {
     let checkoutDiffFallbackRefreshTargetCount = 0;
     let terminalDirectorySubscriptionCount = 0;
     let terminalSubscriptionCount = 0;
+    let inflightRequests = 0;
+    let peakInflightRequests = 0;
 
     for (const connection of uniqueConnections) {
       const sessionMetrics = connection.session.getRuntimeMetrics();
@@ -1169,6 +1226,9 @@ export class VoiceAssistantWebSocketServer {
         sessionMetrics.checkoutDiffFallbackRefreshTargetCount;
       terminalDirectorySubscriptionCount += sessionMetrics.terminalDirectorySubscriptionCount;
       terminalSubscriptionCount += sessionMetrics.terminalSubscriptionCount;
+      inflightRequests += sessionMetrics.inflightRequests;
+      peakInflightRequests = Math.max(peakInflightRequests, sessionMetrics.peakInflightRequests);
+      connection.session.resetPeakInflight();
     }
 
     return {
@@ -1178,6 +1238,8 @@ export class VoiceAssistantWebSocketServer {
       checkoutDiffFallbackRefreshTargetCount,
       terminalDirectorySubscriptionCount,
       terminalSubscriptionCount,
+      inflightRequests,
+      peakInflightRequests,
     };
   }
 
@@ -1192,6 +1254,8 @@ export class VoiceAssistantWebSocketServer {
         connection.sockets.size === 0 && connection.externalDisconnectCleanupTimeout !== null,
     ).length;
     const sessionMetrics = this.collectSessionRuntimeMetrics();
+    const latencyStats = this.computeLatencyStats();
+    const agentSnapshot = this.agentManager.getMetricsSnapshot();
 
     this.logger.info(
       {
@@ -1210,6 +1274,8 @@ export class VoiceAssistantWebSocketServer {
         inboundMessageTypesTop: this.getTopCounts(this.inboundMessageCounts, 12),
         inboundSessionRequestTypesTop: this.getTopCounts(this.inboundSessionRequestCounts, 20),
         runtime: sessionMetrics,
+        latency: latencyStats,
+        agents: agentSnapshot,
       },
       "ws_runtime_metrics",
     );
@@ -1221,6 +1287,7 @@ export class VoiceAssistantWebSocketServer {
     }
     this.inboundMessageCounts.clear();
     this.inboundSessionRequestCounts.clear();
+    this.requestLatencies.clear();
     this.runtimeWindowStartedAt = now;
   }