perf(agent-tools): shrink MCP tool catalog context cost (#1939)
Drop advertised MCP output schemas (no supported harness forwards them to the model) and remove the redundant wait_for_agent tool. Core agent tool catalog drops ~20.5k -> ~5.6k estimated tokens; ~24.9k -> ~9k with browser tools. Runtime structuredContent is preserved via server-side text serialization. Also corrects the agent-wait timeout guidance for blocking callers.
This commit is contained in:
Mohamed Boudra
@@ -0,0 +1,384 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
||||
import type { Tool } from "@modelcontextprotocol/sdk/types.js";
|
||||
import pino from "pino";
|
||||
|
||||
import { createAgentMcpServer } from "../src/server/agent/mcp-server.js";
|
||||
import type { AgentManager, ManagedAgent } from "../src/server/agent/agent-manager.js";
|
||||
import type { AgentStorage, StoredAgentRecord } from "../src/server/agent/agent-storage.js";
|
||||
import type { AgentProvider, ProviderSnapshotEntry } from "../src/server/agent/agent-sdk-types.js";
|
||||
import type { ProviderSnapshotManager } from "../src/server/agent/provider-snapshot-manager.js";
|
||||
import type { BrowserToolsBroker } from "../src/server/browser-tools/broker.js";
|
||||
import type { BrowserToolsResponsePayload } from "../src/server/browser-tools/errors.js";
|
||||
|
||||
type CatalogScope = "agent" | "top-level";
|
||||
|
||||
interface CliOptions {
|
||||
format: "markdown" | "json";
|
||||
scope: CatalogScope;
|
||||
top: number;
|
||||
}
|
||||
|
||||
interface FieldSize {
|
||||
field: string;
|
||||
bytes: number;
|
||||
estimatedTokens: number;
|
||||
}
|
||||
|
||||
interface ToolSize {
|
||||
name: string;
|
||||
bytes: number;
|
||||
estimatedTokens: number;
|
||||
fields: FieldSize[];
|
||||
}
|
||||
|
||||
interface CatalogMeasurement {
|
||||
label: string;
|
||||
browserToolsEnabled: boolean;
|
||||
toolCount: number;
|
||||
bytes: number;
|
||||
estimatedTokens: number;
|
||||
withoutOutputSchemas: {
|
||||
bytes: number;
|
||||
estimatedTokens: number;
|
||||
savedBytes: number;
|
||||
savedEstimatedTokens: number;
|
||||
};
|
||||
fields: FieldSize[];
|
||||
tools: ToolSize[];
|
||||
}
|
||||
|
||||
const DEFAULT_TOP_COUNT = 15;
|
||||
const MEASUREMENT_AGENT_ID = "agent-tools-measurement";
|
||||
const MEASUREMENT_CWD = "/tmp/paseo-agent-tools-measurement";
|
||||
const MEASUREMENT_WORKSPACE_ID = "workspace_agent_tools_measurement";
|
||||
const FIELD_NAMES = [
|
||||
"name",
|
||||
"title",
|
||||
"description",
|
||||
"inputSchema",
|
||||
"outputSchema",
|
||||
"annotations",
|
||||
"execution",
|
||||
"_meta",
|
||||
] as const;
|
||||
|
||||
function parseCliOptions(args: string[]): CliOptions {
|
||||
let format: CliOptions["format"] = "markdown";
|
||||
let scope: CatalogScope = "agent";
|
||||
let top = DEFAULT_TOP_COUNT;
|
||||
|
||||
for (const arg of args) {
|
||||
if (arg === "--json") {
|
||||
format = "json";
|
||||
continue;
|
||||
}
|
||||
if (arg === "--markdown") {
|
||||
format = "markdown";
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--scope=")) {
|
||||
const value = arg.slice("--scope=".length);
|
||||
if (value !== "agent" && value !== "top-level") {
|
||||
throw new Error("--scope must be agent or top-level");
|
||||
}
|
||||
scope = value;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--top=")) {
|
||||
const value = Number.parseInt(arg.slice("--top=".length), 10);
|
||||
if (!Number.isInteger(value) || value < 1) {
|
||||
throw new Error("--top must be a positive integer");
|
||||
}
|
||||
top = value;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help") {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
return { format, scope, top };
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
process.stdout.write(`Usage: npm run measure:agent-tools -- [options]
|
||||
|
||||
Measures the MCP tools/list payload for Paseo agent tools.
|
||||
|
||||
Options:
|
||||
--scope=agent|top-level Catalog shape to measure. Defaults to agent.
|
||||
--top=N Number of largest tools to show. Defaults to ${DEFAULT_TOP_COUNT}.
|
||||
--json Emit machine-readable JSON.
|
||||
--markdown Emit Markdown. This is the default.
|
||||
`);
|
||||
}
|
||||
|
||||
function estimateTokens(text: string): number {
|
||||
return Math.ceil(Buffer.byteLength(text, "utf8") / 4);
|
||||
}
|
||||
|
||||
function measureJson(value: unknown): { bytes: number; estimatedTokens: number } {
|
||||
const text = JSON.stringify(value);
|
||||
return {
|
||||
bytes: Buffer.byteLength(text, "utf8"),
|
||||
estimatedTokens: estimateTokens(text),
|
||||
};
|
||||
}
|
||||
|
||||
function measureField(field: string, value: unknown): FieldSize {
|
||||
return {
|
||||
field,
|
||||
...measureJson({ [field]: value }),
|
||||
};
|
||||
}
|
||||
|
||||
function stripOutputSchemas(tools: Tool[]): Tool[] {
|
||||
return tools.map((tool) => {
|
||||
const { outputSchema: _outputSchema, ...rest } = tool;
|
||||
return rest;
|
||||
});
|
||||
}
|
||||
|
||||
function measureTool(tool: Tool): ToolSize {
|
||||
const fields = FIELD_NAMES.flatMap((field) =>
|
||||
Object.hasOwn(tool, field) ? [measureField(field, tool[field])] : [],
|
||||
);
|
||||
return {
|
||||
name: tool.name,
|
||||
...measureJson(tool),
|
||||
fields,
|
||||
};
|
||||
}
|
||||
|
||||
function sumFieldSizes(tools: ToolSize[]): FieldSize[] {
|
||||
const totals = new Map<string, { bytes: number; estimatedTokens: number }>();
|
||||
for (const tool of tools) {
|
||||
for (const field of tool.fields) {
|
||||
const current = totals.get(field.field) ?? { bytes: 0, estimatedTokens: 0 };
|
||||
totals.set(field.field, {
|
||||
bytes: current.bytes + field.bytes,
|
||||
estimatedTokens: current.estimatedTokens + field.estimatedTokens,
|
||||
});
|
||||
}
|
||||
}
|
||||
return [...totals.entries()]
|
||||
.map(([field, size]) => ({
|
||||
field,
|
||||
bytes: size.bytes,
|
||||
estimatedTokens: size.estimatedTokens,
|
||||
}))
|
||||
.sort((a, b) => b.bytes - a.bytes);
|
||||
}
|
||||
|
||||
async function measureCatalog(params: {
|
||||
label: string;
|
||||
browserToolsEnabled: boolean;
|
||||
scope: CatalogScope;
|
||||
}): Promise<CatalogMeasurement> {
|
||||
const server = await createAgentMcpServer({
|
||||
agentManager: new MeasurementAgentManager() as unknown as AgentManager,
|
||||
agentStorage: new MeasurementAgentStorage() as unknown as AgentStorage,
|
||||
providerSnapshotManager:
|
||||
new MeasurementProviderSnapshotManager() as unknown as ProviderSnapshotManager,
|
||||
browserToolsEnabled: params.browserToolsEnabled,
|
||||
browserToolsBroker: new MeasurementBrowserToolsBroker() as unknown as BrowserToolsBroker,
|
||||
callerAgentId: params.scope === "agent" ? MEASUREMENT_AGENT_ID : undefined,
|
||||
logger: pino({ level: "silent" }),
|
||||
});
|
||||
const client = new Client({ name: "paseo-agent-tools-measurement", version: "0.0.0" });
|
||||
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
|
||||
|
||||
await server.connect(serverTransport);
|
||||
await client.connect(clientTransport);
|
||||
try {
|
||||
const result = await client.listTools();
|
||||
const size = measureJson(result.tools);
|
||||
const strippedSize = measureJson(stripOutputSchemas(result.tools));
|
||||
const tools = result.tools.map(measureTool).sort((a, b) => b.bytes - a.bytes);
|
||||
return {
|
||||
label: params.label,
|
||||
browserToolsEnabled: params.browserToolsEnabled,
|
||||
toolCount: result.tools.length,
|
||||
...size,
|
||||
withoutOutputSchemas: {
|
||||
...strippedSize,
|
||||
savedBytes: size.bytes - strippedSize.bytes,
|
||||
savedEstimatedTokens: size.estimatedTokens - strippedSize.estimatedTokens,
|
||||
},
|
||||
fields: sumFieldSizes(tools),
|
||||
tools,
|
||||
};
|
||||
} finally {
|
||||
await client.close();
|
||||
await server.close();
|
||||
}
|
||||
}
|
||||
|
||||
function formatInteger(value: number): string {
|
||||
return value.toLocaleString("en-US");
|
||||
}
|
||||
|
||||
function formatMarkdown(measurements: CatalogMeasurement[], top: number): string {
|
||||
const [withoutBrowser, withBrowser] = measurements;
|
||||
const delta =
|
||||
withoutBrowser && withBrowser
|
||||
? {
|
||||
tools: withBrowser.toolCount - withoutBrowser.toolCount,
|
||||
bytes: withBrowser.bytes - withoutBrowser.bytes,
|
||||
estimatedTokens: withBrowser.estimatedTokens - withoutBrowser.estimatedTokens,
|
||||
}
|
||||
: null;
|
||||
const lines: string[] = [];
|
||||
lines.push("# Paseo Agent Tool Catalog Context");
|
||||
lines.push("");
|
||||
lines.push("Token counts are estimates from compact JSON bytes / 4.");
|
||||
lines.push("");
|
||||
lines.push("| Catalog | Tools | Bytes | Estimated tokens | Output-schema savings |");
|
||||
lines.push("| --- | ---: | ---: | ---: | ---: |");
|
||||
for (const measurement of measurements) {
|
||||
lines.push(
|
||||
`| ${measurement.label} | ${formatInteger(measurement.toolCount)} | ${formatInteger(
|
||||
measurement.bytes,
|
||||
)} | ${formatInteger(measurement.estimatedTokens)} | ${formatInteger(
|
||||
measurement.withoutOutputSchemas.savedEstimatedTokens,
|
||||
)} est. tokens |`,
|
||||
);
|
||||
}
|
||||
if (delta) {
|
||||
lines.push(
|
||||
`| Browser-tool delta | +${formatInteger(delta.tools)} | +${formatInteger(
|
||||
delta.bytes,
|
||||
)} | +${formatInteger(delta.estimatedTokens)} | |`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const measurement of measurements) {
|
||||
lines.push("");
|
||||
lines.push(`## ${measurement.label}`);
|
||||
lines.push("");
|
||||
lines.push("### Field totals");
|
||||
lines.push("");
|
||||
lines.push("| Field | Bytes | Estimated tokens |");
|
||||
lines.push("| --- | ---: | ---: |");
|
||||
for (const field of measurement.fields) {
|
||||
lines.push(
|
||||
`| ${field.field} | ${formatInteger(field.bytes)} | ${formatInteger(
|
||||
field.estimatedTokens,
|
||||
)} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push(`### Largest ${Math.min(top, measurement.tools.length)} tools`);
|
||||
lines.push("");
|
||||
lines.push("| Tool | Bytes | Estimated tokens | Description | Input schema | Output schema |");
|
||||
lines.push("| --- | ---: | ---: | ---: | ---: | ---: |");
|
||||
for (const tool of measurement.tools.slice(0, top)) {
|
||||
const fields = new Map(tool.fields.map((field) => [field.field, field.estimatedTokens]));
|
||||
lines.push(
|
||||
`| ${tool.name} | ${formatInteger(tool.bytes)} | ${formatInteger(
|
||||
tool.estimatedTokens,
|
||||
)} | ${formatInteger(fields.get("description") ?? 0)} | ${formatInteger(
|
||||
fields.get("inputSchema") ?? 0,
|
||||
)} | ${formatInteger(fields.get("outputSchema") ?? 0)} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
class MeasurementAgentManager {
|
||||
private readonly agent = {
|
||||
id: MEASUREMENT_AGENT_ID,
|
||||
cwd: MEASUREMENT_CWD,
|
||||
workspaceId: MEASUREMENT_WORKSPACE_ID,
|
||||
provider: "codex",
|
||||
currentModeId: "default",
|
||||
config: {
|
||||
provider: "codex",
|
||||
cwd: MEASUREMENT_CWD,
|
||||
},
|
||||
};
|
||||
|
||||
public getAgent(agentId: string): ManagedAgent | null {
|
||||
return agentId === MEASUREMENT_AGENT_ID ? (this.agent as unknown as ManagedAgent) : null;
|
||||
}
|
||||
|
||||
public listAgents(): ManagedAgent[] {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
class MeasurementAgentStorage {
|
||||
public async list(): Promise<StoredAgentRecord[]> {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
class MeasurementProviderSnapshotManager {
|
||||
public listRegisteredProviderIds(): AgentProvider[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
public hasProvider(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public getProviderLabel(provider: AgentProvider): string {
|
||||
return provider;
|
||||
}
|
||||
|
||||
public async listProviders(): Promise<ProviderSnapshotEntry[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
public async getProvider(): Promise<ProviderSnapshotEntry> {
|
||||
throw new Error("Provider catalog is not used by the measurement script");
|
||||
}
|
||||
|
||||
public async listModels(): Promise<[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
public async listModes(): Promise<[]> {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
class MeasurementBrowserToolsBroker {
|
||||
public async execute(): Promise<BrowserToolsResponsePayload> {
|
||||
throw new Error("Browser tools are not executed by the measurement script");
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseCliOptions(process.argv.slice(2));
|
||||
const measurements = await Promise.all([
|
||||
measureCatalog({
|
||||
label: "Core tools",
|
||||
browserToolsEnabled: false,
|
||||
scope: options.scope,
|
||||
}),
|
||||
measureCatalog({
|
||||
label: "Core + browser tools",
|
||||
browserToolsEnabled: true,
|
||||
scope: options.scope,
|
||||
}),
|
||||
]);
|
||||
|
||||
if (options.format === "json") {
|
||||
process.stdout.write(`${JSON.stringify({ scope: options.scope, measurements }, null, 2)}\n`);
|
||||
return;
|
||||
}
|
||||
process.stdout.write(formatMarkdown(measurements, options.top));
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
process.stderr.write(`${message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Reference in New Issue
Block a user