feat(daemon-client): add opt-in runtime metrics collector
Aggregates inbound message counts, bytes, handler timings, and agent_stream breakdowns over a rolling window and flushes via logger. Gated on runtimeMetricsIntervalMs config; disabled by default.
This commit is contained in:
@@ -0,0 +1,258 @@
|
|||||||
|
import type { SessionOutboundMessage } from "../shared/messages.js";
|
||||||
|
|
||||||
|
type RuntimeMetricsLogger = {
|
||||||
|
info(obj: object, msg?: string): void;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RuntimeMetricsHandlerTiming = {
|
||||||
|
count: number;
|
||||||
|
totalMs: number;
|
||||||
|
maxMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RuntimeMetricsBucket = {
|
||||||
|
inboundMessageCounts: Map<string, number>;
|
||||||
|
inboundMessageBytes: Map<string, number>;
|
||||||
|
inboundMessageHandlerMs: Map<string, RuntimeMetricsHandlerTiming>;
|
||||||
|
inboundAgentStreamCounts: Map<string, number>;
|
||||||
|
inboundAgentStreamByAgentCounts: Map<string, number>;
|
||||||
|
inboundBinaryFrameCounts: Map<string, number>;
|
||||||
|
endedAt: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RuntimeMetricsContext = {
|
||||||
|
connectionPath: "direct" | "relay";
|
||||||
|
serverId: string | null;
|
||||||
|
getConnectionStatus: () => string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RuntimeMetricsOptions = {
|
||||||
|
windowMs?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_ROLLING_WINDOW_MS = 60_000;
|
||||||
|
|
||||||
|
export class DaemonClientRuntimeMetrics {
|
||||||
|
private readonly startedAt = Date.now();
|
||||||
|
private readonly windowMs: number;
|
||||||
|
private readonly buckets: RuntimeMetricsBucket[] = [];
|
||||||
|
private readonly inboundMessageCounts = new Map<string, number>();
|
||||||
|
private readonly inboundMessageBytes = new Map<string, number>();
|
||||||
|
private readonly inboundMessageHandlerMs = new Map<string, RuntimeMetricsHandlerTiming>();
|
||||||
|
private readonly inboundAgentStreamCounts = new Map<string, number>();
|
||||||
|
private readonly inboundAgentStreamByAgentCounts = new Map<string, number>();
|
||||||
|
private readonly inboundBinaryFrameCounts = new Map<string, number>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly logger: RuntimeMetricsLogger,
|
||||||
|
private readonly context: RuntimeMetricsContext,
|
||||||
|
options?: RuntimeMetricsOptions,
|
||||||
|
) {
|
||||||
|
this.windowMs =
|
||||||
|
typeof options?.windowMs === "number" && options.windowMs > 0
|
||||||
|
? options.windowMs
|
||||||
|
: DEFAULT_ROLLING_WINDOW_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
recordMessage(type: string, bytes: number, handlerMs: number): void {
|
||||||
|
incrementCount(this.inboundMessageCounts, type, 1);
|
||||||
|
incrementCount(this.inboundMessageBytes, type, bytes);
|
||||||
|
incrementHandlerTiming(this.inboundMessageHandlerMs, type, handlerMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
recordAgentStream(
|
||||||
|
payload: Extract<SessionOutboundMessage, { type: "agent_stream" }>["payload"],
|
||||||
|
): void {
|
||||||
|
const { agentId, event } = payload;
|
||||||
|
const eventType = event.type === "timeline" ? `timeline:${event.item.type}` : event.type;
|
||||||
|
incrementCount(this.inboundAgentStreamCounts, eventType, 1);
|
||||||
|
incrementCount(this.inboundAgentStreamByAgentCounts, agentId, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
recordBinaryFrame(kind: string, bytes: number, handlerMs: number): void {
|
||||||
|
incrementCount(this.inboundBinaryFrameCounts, kind, 1);
|
||||||
|
incrementCount(this.inboundMessageBytes, `binary:${kind}`, bytes);
|
||||||
|
incrementHandlerTiming(this.inboundMessageHandlerMs, `binary:${kind}`, handlerMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
flush(options?: { final?: boolean }): void {
|
||||||
|
const now = Date.now();
|
||||||
|
const bucket = this.consumeCurrentBucket(now);
|
||||||
|
if (bucket) {
|
||||||
|
this.buckets.push(bucket);
|
||||||
|
}
|
||||||
|
this.pruneBuckets(now);
|
||||||
|
|
||||||
|
const aggregate = this.aggregateBuckets();
|
||||||
|
const hasActivity =
|
||||||
|
aggregate.inboundMessageCounts.size > 0 || aggregate.inboundBinaryFrameCounts.size > 0;
|
||||||
|
if (!hasActivity && !options?.final) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
{
|
||||||
|
windowMs: Math.min(this.windowMs, Math.max(0, now - this.startedAt)),
|
||||||
|
rollingWindowMs: this.windowMs,
|
||||||
|
bucketCount: this.buckets.length,
|
||||||
|
final: Boolean(options?.final),
|
||||||
|
connectionPath: this.context.connectionPath,
|
||||||
|
serverId: this.context.serverId,
|
||||||
|
connectionStatus: this.context.getConnectionStatus(),
|
||||||
|
inboundMessageTypesTop: getTopCounts(aggregate.inboundMessageCounts, 20),
|
||||||
|
inboundMessageBytesTop: getTopCounts(aggregate.inboundMessageBytes, 20),
|
||||||
|
inboundAgentStreamTypesTop: getTopCounts(aggregate.inboundAgentStreamCounts, 20),
|
||||||
|
inboundAgentStreamAgentsTop: getTopCounts(aggregate.inboundAgentStreamByAgentCounts, 20),
|
||||||
|
inboundBinaryFrameTypesTop: getTopCounts(aggregate.inboundBinaryFrameCounts, 12),
|
||||||
|
handlerTimingTop: getTopHandlerTimings(aggregate.inboundMessageHandlerMs, 20),
|
||||||
|
},
|
||||||
|
"ws_runtime_metrics_client",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private consumeCurrentBucket(now: number): RuntimeMetricsBucket | null {
|
||||||
|
const hasActivity =
|
||||||
|
this.inboundMessageCounts.size > 0 || this.inboundBinaryFrameCounts.size > 0;
|
||||||
|
if (!hasActivity) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bucket = {
|
||||||
|
inboundMessageCounts: new Map(this.inboundMessageCounts),
|
||||||
|
inboundMessageBytes: new Map(this.inboundMessageBytes),
|
||||||
|
inboundMessageHandlerMs: cloneHandlerTimingMap(this.inboundMessageHandlerMs),
|
||||||
|
inboundAgentStreamCounts: new Map(this.inboundAgentStreamCounts),
|
||||||
|
inboundAgentStreamByAgentCounts: new Map(this.inboundAgentStreamByAgentCounts),
|
||||||
|
inboundBinaryFrameCounts: new Map(this.inboundBinaryFrameCounts),
|
||||||
|
endedAt: now,
|
||||||
|
};
|
||||||
|
|
||||||
|
this.inboundMessageCounts.clear();
|
||||||
|
this.inboundMessageBytes.clear();
|
||||||
|
this.inboundMessageHandlerMs.clear();
|
||||||
|
this.inboundAgentStreamCounts.clear();
|
||||||
|
this.inboundAgentStreamByAgentCounts.clear();
|
||||||
|
this.inboundBinaryFrameCounts.clear();
|
||||||
|
return bucket;
|
||||||
|
}
|
||||||
|
|
||||||
|
private pruneBuckets(now: number): void {
|
||||||
|
const cutoff = now - this.windowMs;
|
||||||
|
while (this.buckets.length > 0 && this.buckets[0]!.endedAt < cutoff) {
|
||||||
|
this.buckets.shift();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private aggregateBuckets(): RuntimeMetricsBucket {
|
||||||
|
const aggregate = createEmptyBucket(Date.now());
|
||||||
|
for (const bucket of this.buckets) {
|
||||||
|
mergeCountMap(aggregate.inboundMessageCounts, bucket.inboundMessageCounts);
|
||||||
|
mergeCountMap(aggregate.inboundMessageBytes, bucket.inboundMessageBytes);
|
||||||
|
mergeHandlerTimingMap(aggregate.inboundMessageHandlerMs, bucket.inboundMessageHandlerMs);
|
||||||
|
mergeCountMap(aggregate.inboundAgentStreamCounts, bucket.inboundAgentStreamCounts);
|
||||||
|
mergeCountMap(
|
||||||
|
aggregate.inboundAgentStreamByAgentCounts,
|
||||||
|
bucket.inboundAgentStreamByAgentCounts,
|
||||||
|
);
|
||||||
|
mergeCountMap(aggregate.inboundBinaryFrameCounts, bucket.inboundBinaryFrameCounts);
|
||||||
|
}
|
||||||
|
return aggregate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createEmptyBucket(endedAt: number): RuntimeMetricsBucket {
|
||||||
|
return {
|
||||||
|
inboundMessageCounts: new Map(),
|
||||||
|
inboundMessageBytes: new Map(),
|
||||||
|
inboundMessageHandlerMs: new Map(),
|
||||||
|
inboundAgentStreamCounts: new Map(),
|
||||||
|
inboundAgentStreamByAgentCounts: new Map(),
|
||||||
|
inboundBinaryFrameCounts: new Map(),
|
||||||
|
endedAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function incrementCount(map: Map<string, number>, key: string, amount: number): void {
|
||||||
|
map.set(key, (map.get(key) ?? 0) + amount);
|
||||||
|
}
|
||||||
|
|
||||||
|
function incrementHandlerTiming(
|
||||||
|
map: Map<string, RuntimeMetricsHandlerTiming>,
|
||||||
|
key: string,
|
||||||
|
handlerMs: number,
|
||||||
|
): void {
|
||||||
|
const existing = map.get(key);
|
||||||
|
if (existing) {
|
||||||
|
existing.count += 1;
|
||||||
|
existing.totalMs += handlerMs;
|
||||||
|
existing.maxMs = Math.max(existing.maxMs, handlerMs);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
map.set(key, {
|
||||||
|
count: 1,
|
||||||
|
totalMs: handlerMs,
|
||||||
|
maxMs: handlerMs,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function cloneHandlerTimingMap(
|
||||||
|
map: Map<string, RuntimeMetricsHandlerTiming>,
|
||||||
|
): Map<string, RuntimeMetricsHandlerTiming> {
|
||||||
|
return new Map(
|
||||||
|
[...map.entries()].map(([key, value]) => [
|
||||||
|
key,
|
||||||
|
{ count: value.count, totalMs: value.totalMs, maxMs: value.maxMs },
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeCountMap(target: Map<string, number>, source: Map<string, number>): void {
|
||||||
|
for (const [key, value] of source) {
|
||||||
|
incrementCount(target, key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeHandlerTimingMap(
|
||||||
|
target: Map<string, RuntimeMetricsHandlerTiming>,
|
||||||
|
source: Map<string, RuntimeMetricsHandlerTiming>,
|
||||||
|
): void {
|
||||||
|
for (const [key, value] of source) {
|
||||||
|
const existing = target.get(key);
|
||||||
|
if (existing) {
|
||||||
|
existing.count += value.count;
|
||||||
|
existing.totalMs += value.totalMs;
|
||||||
|
existing.maxMs = Math.max(existing.maxMs, value.maxMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
target.set(key, {
|
||||||
|
count: value.count,
|
||||||
|
totalMs: value.totalMs,
|
||||||
|
maxMs: value.maxMs,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTopCounts(map: Map<string, number>, limit: number): Array<[string, number]> {
|
||||||
|
return [...map.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTopHandlerTimings(
|
||||||
|
map: Map<string, RuntimeMetricsHandlerTiming>,
|
||||||
|
limit: number,
|
||||||
|
): Array<{
|
||||||
|
type: string;
|
||||||
|
count: number;
|
||||||
|
totalMs: number;
|
||||||
|
avgMs: number;
|
||||||
|
maxMs: number;
|
||||||
|
}> {
|
||||||
|
const rows = [...map.entries()].map(([type, value]) => ({
|
||||||
|
type,
|
||||||
|
count: value.count,
|
||||||
|
totalMs: Math.round(value.totalMs),
|
||||||
|
avgMs: Math.round((value.totalMs / value.count) * 100) / 100,
|
||||||
|
maxMs: Math.round(value.maxMs * 100) / 100,
|
||||||
|
}));
|
||||||
|
rows.sort((a, b) => b.totalMs - a.totalMs);
|
||||||
|
return rows.slice(0, limit);
|
||||||
|
}
|
||||||
@@ -99,6 +99,7 @@ import {
|
|||||||
type DaemonTransportFactory,
|
type DaemonTransportFactory,
|
||||||
type WebSocketFactory,
|
type WebSocketFactory,
|
||||||
} from "./daemon-client-transport.js";
|
} from "./daemon-client-transport.js";
|
||||||
|
import { DaemonClientRuntimeMetrics } from "./daemon-client-runtime-metrics.js";
|
||||||
|
|
||||||
export interface Logger {
|
export interface Logger {
|
||||||
debug(obj: object, msg?: string): void;
|
debug(obj: object, msg?: string): void;
|
||||||
@@ -109,11 +110,16 @@ export interface Logger {
|
|||||||
|
|
||||||
const consoleLogger: Logger = {
|
const consoleLogger: Logger = {
|
||||||
debug: () => {},
|
debug: () => {},
|
||||||
info: (obj, msg) => console.info(msg, obj),
|
info: (obj, msg) => console.log(msg, obj),
|
||||||
warn: (obj, msg) => console.warn(msg, obj),
|
warn: (obj, msg) => console.warn(msg, obj),
|
||||||
error: (obj, msg) => console.error(msg, obj),
|
error: (obj, msg) => console.error(msg, obj),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const perfNow: () => number =
|
||||||
|
typeof performance !== "undefined" && typeof performance.now === "function"
|
||||||
|
? () => performance.now()
|
||||||
|
: () => Date.now();
|
||||||
|
|
||||||
export type {
|
export type {
|
||||||
DaemonTransport,
|
DaemonTransport,
|
||||||
DaemonTransportFactory,
|
DaemonTransportFactory,
|
||||||
@@ -198,6 +204,8 @@ export type DaemonClientConfig = {
|
|||||||
baseDelayMs?: number;
|
baseDelayMs?: number;
|
||||||
maxDelayMs?: number;
|
maxDelayMs?: number;
|
||||||
};
|
};
|
||||||
|
runtimeMetricsIntervalMs?: number;
|
||||||
|
runtimeMetricsWindowMs?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type SendMessageOptions = {
|
export type SendMessageOptions = {
|
||||||
@@ -650,6 +658,8 @@ export class DaemonClient {
|
|||||||
private readonly logClientIdHash: string;
|
private readonly logClientIdHash: string;
|
||||||
private readonly logGeneration: number | null;
|
private readonly logGeneration: number | null;
|
||||||
private lastServerInfoMessage: ServerInfoStatusPayload | null = null;
|
private lastServerInfoMessage: ServerInfoStatusPayload | null = null;
|
||||||
|
private runtimeMetricsInterval: ReturnType<typeof setInterval> | null = null;
|
||||||
|
private runtimeMetrics: DaemonClientRuntimeMetrics | null = null;
|
||||||
|
|
||||||
constructor(private config: DaemonClientConfig) {
|
constructor(private config: DaemonClientConfig) {
|
||||||
this.logger = config.logger ?? consoleLogger;
|
this.logger = config.logger ?? consoleLogger;
|
||||||
@@ -673,6 +683,28 @@ export class DaemonClient {
|
|||||||
Number.isFinite(this.config.runtimeGeneration)
|
Number.isFinite(this.config.runtimeGeneration)
|
||||||
? this.config.runtimeGeneration
|
? this.config.runtimeGeneration
|
||||||
: null;
|
: null;
|
||||||
|
const runtimeMetricsIntervalMs =
|
||||||
|
typeof config.runtimeMetricsIntervalMs === "number" && config.runtimeMetricsIntervalMs > 0
|
||||||
|
? config.runtimeMetricsIntervalMs
|
||||||
|
: 0;
|
||||||
|
if (runtimeMetricsIntervalMs > 0) {
|
||||||
|
const runtimeMetricsWindowMs =
|
||||||
|
typeof config.runtimeMetricsWindowMs === "number" && config.runtimeMetricsWindowMs > 0
|
||||||
|
? Math.max(config.runtimeMetricsWindowMs, runtimeMetricsIntervalMs)
|
||||||
|
: undefined;
|
||||||
|
this.runtimeMetrics = new DaemonClientRuntimeMetrics(
|
||||||
|
this.logger,
|
||||||
|
{
|
||||||
|
connectionPath: this.logConnectionPath,
|
||||||
|
serverId: this.logServerId,
|
||||||
|
getConnectionStatus: () => this.connectionState.status,
|
||||||
|
},
|
||||||
|
runtimeMetricsWindowMs ? { windowMs: runtimeMetricsWindowMs } : undefined,
|
||||||
|
);
|
||||||
|
this.runtimeMetricsInterval = setInterval(() => {
|
||||||
|
this.runtimeMetrics?.flush();
|
||||||
|
}, runtimeMetricsIntervalMs);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
@@ -883,6 +915,12 @@ export class DaemonClient {
|
|||||||
this.rejectPendingSendQueue(new Error("Daemon client closed"));
|
this.rejectPendingSendQueue(new Error("Daemon client closed"));
|
||||||
this.clearTerminalSlots();
|
this.clearTerminalSlots();
|
||||||
this.lastServerInfoMessage = null;
|
this.lastServerInfoMessage = null;
|
||||||
|
if (this.runtimeMetricsInterval) {
|
||||||
|
clearInterval(this.runtimeMetricsInterval);
|
||||||
|
this.runtimeMetricsInterval = null;
|
||||||
|
this.runtimeMetrics?.flush({ final: true });
|
||||||
|
this.runtimeMetrics = null;
|
||||||
|
}
|
||||||
this.updateConnectionState(
|
this.updateConnectionState(
|
||||||
{ status: "disposed" },
|
{ status: "disposed" },
|
||||||
{ event: "DISPOSE", reason: "Client closed", reasonCode: "disposed" },
|
{ event: "DISPOSE", reason: "Client closed", reasonCode: "disposed" },
|
||||||
@@ -3703,7 +3741,17 @@ export class DaemonClient {
|
|||||||
if (rawBytes) {
|
if (rawBytes) {
|
||||||
const frame = decodeTerminalStreamFrame(rawBytes);
|
const frame = decodeTerminalStreamFrame(rawBytes);
|
||||||
if (frame) {
|
if (frame) {
|
||||||
|
const binaryStartMs = perfNow();
|
||||||
this.handleBinaryFrame(frame);
|
this.handleBinaryFrame(frame);
|
||||||
|
this.runtimeMetrics?.recordBinaryFrame(
|
||||||
|
frame.opcode === TerminalStreamOpcode.Output
|
||||||
|
? "output"
|
||||||
|
: frame.opcode === TerminalStreamOpcode.Snapshot
|
||||||
|
? "snapshot"
|
||||||
|
: "other",
|
||||||
|
rawBytes.byteLength,
|
||||||
|
perfNow() - binaryStartMs,
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3712,6 +3760,8 @@ export class DaemonClient {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const bytes = rawBytes?.byteLength ?? payload.length;
|
||||||
|
const startMs = perfNow();
|
||||||
let parsedJson: unknown;
|
let parsedJson: unknown;
|
||||||
try {
|
try {
|
||||||
parsedJson = JSON.parse(payload);
|
parsedJson = JSON.parse(payload);
|
||||||
@@ -3727,10 +3777,16 @@ export class DaemonClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (parsed.data.type === "pong") {
|
if (parsed.data.type === "pong") {
|
||||||
|
this.runtimeMetrics?.recordMessage("pong", bytes, perfNow() - startMs);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.handleSessionMessage(parsed.data.message);
|
this.handleSessionMessage(parsed.data.message);
|
||||||
|
const msgType = parsed.data.message.type;
|
||||||
|
this.runtimeMetrics?.recordMessage(msgType, bytes, perfNow() - startMs);
|
||||||
|
if (parsed.data.message.type === "agent_stream") {
|
||||||
|
this.runtimeMetrics?.recordAgentStream(parsed.data.message.payload);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private handleBinaryFrame(frame: TerminalStreamFrame): void {
|
private handleBinaryFrame(frame: TerminalStreamFrame): void {
|
||||||
|
|||||||
Reference in New Issue
Block a user