test: add manual terminal perf harness

This commit is contained in:
Mohamed Boudra
2026-04-21 20:47:28 +07:00
parent 6a03873d81
commit c402db8b1c
6 changed files with 728 additions and 2 deletions
@@ -89,6 +89,16 @@ type ActiveTurn = {
completed: Promise<AgentRunResult>;
};
type LargeAgentStreamPayloadRequest = {
bytes: number;
kind: "diff" | "file" | "image";
};
type AgentStreamStressRequest = {
count: number;
coalesced: boolean;
};
function resolveModelProfile(modelId: string | null | undefined): {
modelId: string;
durationMs: number;
@@ -115,6 +125,52 @@ function promptToText(prompt: AgentPromptInput): string {
.trim();
}
function parseLargeAgentStreamPayloadPrompt(
prompt: AgentPromptInput,
): LargeAgentStreamPayloadRequest | null {
const text = promptToText(prompt);
const match =
/emit\s+(\d+)\s+(?:byte\s+)?(?:large\s+)?(diff|file|image)\s+agent stream (?:update|payload)/i.exec(
text,
);
if (!match) {
return null;
}
const bytes = Number(match[1]);
if (!Number.isFinite(bytes) || bytes <= 0) {
return null;
}
return {
bytes: Math.min(bytes, 1_000_000),
kind: match[2]?.toLowerCase() as LargeAgentStreamPayloadRequest["kind"],
};
}
function parseAgentStreamStressPrompt(prompt: AgentPromptInput): AgentStreamStressRequest | null {
const text = promptToText(prompt);
const match = /emit\s+(\d+)\s+(coalesced\s+)?agent stream updates/i.exec(text);
if (!match) {
return null;
}
const count = Number(match[1]);
if (!Number.isFinite(count) || count <= 0) {
return null;
}
return {
count: Math.min(count, 5_000),
coalesced: Boolean(match[2]),
};
}
function buildRepeatedPayload(bytes: number, prefix: string): string {
const line = `${prefix} ${"x".repeat(96)}\n`;
let output = "";
while (output.length < bytes) {
output += line;
}
return output.slice(0, bytes);
}
function buildMarkdownDocument(iteration: number, prompt: AgentPromptInput): string {
const promptPreview = promptToText(prompt).slice(0, 160) || "No prompt text supplied.";
return [
@@ -282,7 +338,15 @@ export class MockLoadTestAgentSession implements AgentSession {
};
this.activeTurn = turn;
this.schedule(turn, 0);
const largePayload = parseLargeAgentStreamPayloadPrompt(prompt);
const stress = parseAgentStreamStressPrompt(prompt);
if (largePayload) {
this.scheduleLargePayloadTurn(turn, largePayload);
} else if (stress) {
this.scheduleStressTurn(turn, stress);
} else {
this.schedule(turn, 0);
}
return { turnId };
}
@@ -380,6 +444,147 @@ export class MockLoadTestAgentSession implements AgentSession {
turn.timer.unref?.();
}
private scheduleLargePayloadTurn(
turn: ActiveTurn,
largePayload: LargeAgentStreamPayloadRequest,
): void {
turn.timer = setTimeout(() => {
this.emitLargePayloadTurn(turn, largePayload);
}, 0);
turn.timer.unref?.();
}
private scheduleStressTurn(turn: ActiveTurn, stress: AgentStreamStressRequest): void {
turn.timer = setTimeout(() => {
this.emitStressTurn(turn, stress);
}, 0);
turn.timer.unref?.();
}
private emitStressTurn(turn: ActiveTurn, stress: AgentStreamStressRequest): void {
if (this.activeTurn !== turn) {
return;
}
this.clearTurnTimer(turn);
this.emit({
type: "turn_started",
provider: this.provider,
turnId: turn.turnId,
});
for (let index = 0; index < stress.count; index += 1) {
this.emitTimeline(
turn.turnId,
stress.coalesced
? {
type: "assistant_message",
text: `stress-update-${index}`,
}
: {
type: "todo",
items: [{ text: `stress-update-${index}`, completed: index % 2 === 0 }],
},
);
}
this.activeTurn = null;
const usage = {
inputTokens: 1,
outputTokens: stress.count,
contextWindowUsedTokens: stress.count,
contextWindowMaxTokens: 128_000,
};
this.emit({
type: "turn_completed",
provider: this.provider,
turnId: turn.turnId,
usage,
});
turn.resolve({
sessionId: this.id,
finalText: "Synthetic agent stream stress complete",
usage,
timeline: [],
canceled: false,
});
}
private emitLargePayloadTurn(
turn: ActiveTurn,
largePayload: LargeAgentStreamPayloadRequest,
): void {
if (this.activeTurn !== turn) {
return;
}
this.clearTurnTimer(turn);
this.emit({
type: "turn_started",
provider: this.provider,
turnId: turn.turnId,
});
const payload = buildRepeatedPayload(largePayload.bytes, largePayload.kind);
if (largePayload.kind === "diff") {
this.emitTimeline(
turn.turnId,
createToolCall({
turnId: turn.turnId,
iteration: 0,
name: "edit",
status: "completed",
detail: {
type: "edit",
filePath: "src/large-diff.ts",
unifiedDiff: `diff --git a/src/large-diff.ts b/src/large-diff.ts\n${payload}`,
},
}),
);
} else if (largePayload.kind === "file") {
this.emitTimeline(
turn.turnId,
createToolCall({
turnId: turn.turnId,
iteration: 0,
name: "read",
status: "completed",
detail: {
type: "read",
filePath: "src/large-file.txt",
content: payload,
},
}),
);
} else {
this.emitTimeline(turn.turnId, {
type: "assistant_message",
text: `data:image/png;base64,${payload}`,
});
}
this.activeTurn = null;
const usage = {
inputTokens: 1,
outputTokens: largePayload.bytes,
contextWindowUsedTokens: largePayload.bytes,
contextWindowMaxTokens: 128_000,
};
this.emit({
type: "turn_completed",
provider: this.provider,
turnId: turn.turnId,
usage,
});
turn.resolve({
sessionId: this.id,
finalText: "Synthetic large payload complete",
usage,
timeline: [],
canceled: false,
});
}
private tick(turn: ActiveTurn): void {
if (this.activeTurn !== turn) {
return;
@@ -376,6 +376,16 @@ function getFrameText(frame: TerminalStreamFrame): string {
return "";
}
function sendRawTerminalInput(input: { ws: WebSocket; slot: number; text: string }): void {
input.ws.send(
encodeTerminalStreamFrame({
opcode: TerminalStreamOpcode.Input,
slot: input.slot,
payload: new Uint8Array(Buffer.from(input.text, "utf8")),
}),
);
}
async function subscribeRawTerminal(
ws: WebSocket,
terminalId: string,
@@ -181,6 +181,107 @@ function buildToolCallForPrompt(provider: string, prompt: string) {
return null;
}
function parseAgentStreamStressPrompt(prompt: string): {
count: number;
coalesced: boolean;
} | null {
const match = /emit\s+(\d+)\s+(coalesced\s+)?agent stream updates/i.exec(prompt);
if (!match) {
return null;
}
const count = Number(match[1]);
if (!Number.isFinite(count) || count <= 0) {
return null;
}
return {
count: Math.min(count, 5000),
coalesced: Boolean(match[2]),
};
}
function parseLargeAgentStreamPayloadPrompt(prompt: string): {
bytes: number;
kind: "diff" | "file" | "image";
} | null {
const match =
/emit\s+(\d+)\s+(?:byte\s+)?(?:large\s+)?(diff|file|image)\s+agent stream (?:update|payload)/i.exec(
prompt,
);
if (!match) {
return null;
}
const bytes = Number(match[1]);
if (!Number.isFinite(bytes) || bytes <= 0) {
return null;
}
return {
bytes: Math.min(bytes, 1_000_000),
kind: match[2]?.toLowerCase() as "diff" | "file" | "image",
};
}
function buildRepeatedPayload(bytes: number, prefix: string): string {
const line = `${prefix} ${"x".repeat(96)}\n`;
let output = "";
while (output.length < bytes) {
output += line;
}
return output.slice(0, bytes);
}
function buildLargeTimelineItem(input: {
bytes: number;
kind: "diff" | "file" | "image";
callId: string;
provider: AgentStreamEvent["provider"];
}): AgentStreamEvent {
const payload = buildRepeatedPayload(input.bytes, input.kind);
if (input.kind === "diff") {
return {
type: "timeline",
provider: input.provider,
item: {
type: "tool_call",
name: "apply_patch",
callId: input.callId,
status: "completed",
detail: {
type: "edit",
filePath: "src/large-diff.ts",
unifiedDiff: `diff --git a/src/large-diff.ts b/src/large-diff.ts\n${payload}`,
},
error: null,
},
};
}
if (input.kind === "file") {
return {
type: "timeline",
provider: input.provider,
item: {
type: "tool_call",
name: "read_file",
callId: input.callId,
status: "completed",
detail: {
type: "read",
filePath: "src/large-file.txt",
content: payload,
},
error: null,
},
};
}
return {
type: "timeline",
provider: input.provider,
item: {
type: "assistant_message",
text: `data:image/png;base64,${payload}`,
},
};
}
class FakeAgentSession implements AgentSession {
readonly capabilities = TEST_CAPABILITIES;
readonly id: string;
@@ -372,6 +473,57 @@ class FakeAgentSession implements AgentSession {
await this.appendHistoryEvent(turnStarted);
this.notifySubscribers(turnStarted);
const stress = parseAgentStreamStressPrompt(textPrompt);
if (stress !== null) {
for (let index = 0; index < stress.count; index += 1) {
const stressUpdate: AgentStreamEvent = {
type: "timeline",
provider: this.providerName,
item: stress.coalesced
? {
type: "assistant_message",
text: `stress-update-${index}`,
}
: {
type: "todo",
items: [{ text: `stress-update-${index}`, completed: index % 2 === 0 }],
},
};
await this.appendHistoryEvent(stressUpdate);
this.notifySubscribers(stressUpdate);
await new Promise((resolve) => setImmediate(resolve));
}
const completed: AgentStreamEvent = {
type: "turn_completed",
provider: this.providerName,
usage: { inputTokens: 1, outputTokens: stress.count },
};
await this.appendHistoryEvent(completed);
this.notifySubscribers(completed);
return;
}
const largePayload = parseLargeAgentStreamPayloadPrompt(textPrompt);
if (largePayload !== null) {
const largeUpdate = buildLargeTimelineItem({
...largePayload,
callId: randomUUID(),
provider: this.providerName,
});
await this.appendHistoryEvent(largeUpdate);
this.notifySubscribers(largeUpdate);
const completed: AgentStreamEvent = {
type: "turn_completed",
provider: this.providerName,
usage: { inputTokens: 1, outputTokens: largePayload.bytes },
};
await this.appendHistoryEvent(completed);
this.notifySubscribers(completed);
return;
}
const tool = buildToolCallForPrompt(this.providerName, textPrompt);
if (tool) {
const needsPermission = this.needsPermissionForTool(tool.name, tool.input ?? {});