test: add manual terminal perf harness
This commit is contained in:
@@ -89,6 +89,16 @@ type ActiveTurn = {
|
||||
completed: Promise<AgentRunResult>;
|
||||
};
|
||||
|
||||
type LargeAgentStreamPayloadRequest = {
|
||||
bytes: number;
|
||||
kind: "diff" | "file" | "image";
|
||||
};
|
||||
|
||||
type AgentStreamStressRequest = {
|
||||
count: number;
|
||||
coalesced: boolean;
|
||||
};
|
||||
|
||||
function resolveModelProfile(modelId: string | null | undefined): {
|
||||
modelId: string;
|
||||
durationMs: number;
|
||||
@@ -115,6 +125,52 @@ function promptToText(prompt: AgentPromptInput): string {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function parseLargeAgentStreamPayloadPrompt(
|
||||
prompt: AgentPromptInput,
|
||||
): LargeAgentStreamPayloadRequest | null {
|
||||
const text = promptToText(prompt);
|
||||
const match =
|
||||
/emit\s+(\d+)\s+(?:byte\s+)?(?:large\s+)?(diff|file|image)\s+agent stream (?:update|payload)/i.exec(
|
||||
text,
|
||||
);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const bytes = Number(match[1]);
|
||||
if (!Number.isFinite(bytes) || bytes <= 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
bytes: Math.min(bytes, 1_000_000),
|
||||
kind: match[2]?.toLowerCase() as LargeAgentStreamPayloadRequest["kind"],
|
||||
};
|
||||
}
|
||||
|
||||
function parseAgentStreamStressPrompt(prompt: AgentPromptInput): AgentStreamStressRequest | null {
|
||||
const text = promptToText(prompt);
|
||||
const match = /emit\s+(\d+)\s+(coalesced\s+)?agent stream updates/i.exec(text);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const count = Number(match[1]);
|
||||
if (!Number.isFinite(count) || count <= 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
count: Math.min(count, 5_000),
|
||||
coalesced: Boolean(match[2]),
|
||||
};
|
||||
}
|
||||
|
||||
function buildRepeatedPayload(bytes: number, prefix: string): string {
|
||||
const line = `${prefix} ${"x".repeat(96)}\n`;
|
||||
let output = "";
|
||||
while (output.length < bytes) {
|
||||
output += line;
|
||||
}
|
||||
return output.slice(0, bytes);
|
||||
}
|
||||
|
||||
function buildMarkdownDocument(iteration: number, prompt: AgentPromptInput): string {
|
||||
const promptPreview = promptToText(prompt).slice(0, 160) || "No prompt text supplied.";
|
||||
return [
|
||||
@@ -282,7 +338,15 @@ export class MockLoadTestAgentSession implements AgentSession {
|
||||
};
|
||||
this.activeTurn = turn;
|
||||
|
||||
this.schedule(turn, 0);
|
||||
const largePayload = parseLargeAgentStreamPayloadPrompt(prompt);
|
||||
const stress = parseAgentStreamStressPrompt(prompt);
|
||||
if (largePayload) {
|
||||
this.scheduleLargePayloadTurn(turn, largePayload);
|
||||
} else if (stress) {
|
||||
this.scheduleStressTurn(turn, stress);
|
||||
} else {
|
||||
this.schedule(turn, 0);
|
||||
}
|
||||
return { turnId };
|
||||
}
|
||||
|
||||
@@ -380,6 +444,147 @@ export class MockLoadTestAgentSession implements AgentSession {
|
||||
turn.timer.unref?.();
|
||||
}
|
||||
|
||||
private scheduleLargePayloadTurn(
|
||||
turn: ActiveTurn,
|
||||
largePayload: LargeAgentStreamPayloadRequest,
|
||||
): void {
|
||||
turn.timer = setTimeout(() => {
|
||||
this.emitLargePayloadTurn(turn, largePayload);
|
||||
}, 0);
|
||||
turn.timer.unref?.();
|
||||
}
|
||||
|
||||
private scheduleStressTurn(turn: ActiveTurn, stress: AgentStreamStressRequest): void {
|
||||
turn.timer = setTimeout(() => {
|
||||
this.emitStressTurn(turn, stress);
|
||||
}, 0);
|
||||
turn.timer.unref?.();
|
||||
}
|
||||
|
||||
private emitStressTurn(turn: ActiveTurn, stress: AgentStreamStressRequest): void {
|
||||
if (this.activeTurn !== turn) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.clearTurnTimer(turn);
|
||||
this.emit({
|
||||
type: "turn_started",
|
||||
provider: this.provider,
|
||||
turnId: turn.turnId,
|
||||
});
|
||||
|
||||
for (let index = 0; index < stress.count; index += 1) {
|
||||
this.emitTimeline(
|
||||
turn.turnId,
|
||||
stress.coalesced
|
||||
? {
|
||||
type: "assistant_message",
|
||||
text: `stress-update-${index}`,
|
||||
}
|
||||
: {
|
||||
type: "todo",
|
||||
items: [{ text: `stress-update-${index}`, completed: index % 2 === 0 }],
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
this.activeTurn = null;
|
||||
const usage = {
|
||||
inputTokens: 1,
|
||||
outputTokens: stress.count,
|
||||
contextWindowUsedTokens: stress.count,
|
||||
contextWindowMaxTokens: 128_000,
|
||||
};
|
||||
this.emit({
|
||||
type: "turn_completed",
|
||||
provider: this.provider,
|
||||
turnId: turn.turnId,
|
||||
usage,
|
||||
});
|
||||
turn.resolve({
|
||||
sessionId: this.id,
|
||||
finalText: "Synthetic agent stream stress complete",
|
||||
usage,
|
||||
timeline: [],
|
||||
canceled: false,
|
||||
});
|
||||
}
|
||||
|
||||
private emitLargePayloadTurn(
|
||||
turn: ActiveTurn,
|
||||
largePayload: LargeAgentStreamPayloadRequest,
|
||||
): void {
|
||||
if (this.activeTurn !== turn) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.clearTurnTimer(turn);
|
||||
this.emit({
|
||||
type: "turn_started",
|
||||
provider: this.provider,
|
||||
turnId: turn.turnId,
|
||||
});
|
||||
|
||||
const payload = buildRepeatedPayload(largePayload.bytes, largePayload.kind);
|
||||
if (largePayload.kind === "diff") {
|
||||
this.emitTimeline(
|
||||
turn.turnId,
|
||||
createToolCall({
|
||||
turnId: turn.turnId,
|
||||
iteration: 0,
|
||||
name: "edit",
|
||||
status: "completed",
|
||||
detail: {
|
||||
type: "edit",
|
||||
filePath: "src/large-diff.ts",
|
||||
unifiedDiff: `diff --git a/src/large-diff.ts b/src/large-diff.ts\n${payload}`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else if (largePayload.kind === "file") {
|
||||
this.emitTimeline(
|
||||
turn.turnId,
|
||||
createToolCall({
|
||||
turnId: turn.turnId,
|
||||
iteration: 0,
|
||||
name: "read",
|
||||
status: "completed",
|
||||
detail: {
|
||||
type: "read",
|
||||
filePath: "src/large-file.txt",
|
||||
content: payload,
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
this.emitTimeline(turn.turnId, {
|
||||
type: "assistant_message",
|
||||
text: `data:image/png;base64,${payload}`,
|
||||
});
|
||||
}
|
||||
|
||||
this.activeTurn = null;
|
||||
const usage = {
|
||||
inputTokens: 1,
|
||||
outputTokens: largePayload.bytes,
|
||||
contextWindowUsedTokens: largePayload.bytes,
|
||||
contextWindowMaxTokens: 128_000,
|
||||
};
|
||||
this.emit({
|
||||
type: "turn_completed",
|
||||
provider: this.provider,
|
||||
turnId: turn.turnId,
|
||||
usage,
|
||||
});
|
||||
turn.resolve({
|
||||
sessionId: this.id,
|
||||
finalText: "Synthetic large payload complete",
|
||||
usage,
|
||||
timeline: [],
|
||||
canceled: false,
|
||||
});
|
||||
}
|
||||
|
||||
private tick(turn: ActiveTurn): void {
|
||||
if (this.activeTurn !== turn) {
|
||||
return;
|
||||
|
||||
@@ -376,6 +376,16 @@ function getFrameText(frame: TerminalStreamFrame): string {
|
||||
return "";
|
||||
}
|
||||
|
||||
function sendRawTerminalInput(input: { ws: WebSocket; slot: number; text: string }): void {
|
||||
input.ws.send(
|
||||
encodeTerminalStreamFrame({
|
||||
opcode: TerminalStreamOpcode.Input,
|
||||
slot: input.slot,
|
||||
payload: new Uint8Array(Buffer.from(input.text, "utf8")),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function subscribeRawTerminal(
|
||||
ws: WebSocket,
|
||||
terminalId: string,
|
||||
|
||||
@@ -181,6 +181,107 @@ function buildToolCallForPrompt(provider: string, prompt: string) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseAgentStreamStressPrompt(prompt: string): {
|
||||
count: number;
|
||||
coalesced: boolean;
|
||||
} | null {
|
||||
const match = /emit\s+(\d+)\s+(coalesced\s+)?agent stream updates/i.exec(prompt);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const count = Number(match[1]);
|
||||
if (!Number.isFinite(count) || count <= 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
count: Math.min(count, 5000),
|
||||
coalesced: Boolean(match[2]),
|
||||
};
|
||||
}
|
||||
|
||||
function parseLargeAgentStreamPayloadPrompt(prompt: string): {
|
||||
bytes: number;
|
||||
kind: "diff" | "file" | "image";
|
||||
} | null {
|
||||
const match =
|
||||
/emit\s+(\d+)\s+(?:byte\s+)?(?:large\s+)?(diff|file|image)\s+agent stream (?:update|payload)/i.exec(
|
||||
prompt,
|
||||
);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const bytes = Number(match[1]);
|
||||
if (!Number.isFinite(bytes) || bytes <= 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
bytes: Math.min(bytes, 1_000_000),
|
||||
kind: match[2]?.toLowerCase() as "diff" | "file" | "image",
|
||||
};
|
||||
}
|
||||
|
||||
function buildRepeatedPayload(bytes: number, prefix: string): string {
|
||||
const line = `${prefix} ${"x".repeat(96)}\n`;
|
||||
let output = "";
|
||||
while (output.length < bytes) {
|
||||
output += line;
|
||||
}
|
||||
return output.slice(0, bytes);
|
||||
}
|
||||
|
||||
function buildLargeTimelineItem(input: {
|
||||
bytes: number;
|
||||
kind: "diff" | "file" | "image";
|
||||
callId: string;
|
||||
provider: AgentStreamEvent["provider"];
|
||||
}): AgentStreamEvent {
|
||||
const payload = buildRepeatedPayload(input.bytes, input.kind);
|
||||
if (input.kind === "diff") {
|
||||
return {
|
||||
type: "timeline",
|
||||
provider: input.provider,
|
||||
item: {
|
||||
type: "tool_call",
|
||||
name: "apply_patch",
|
||||
callId: input.callId,
|
||||
status: "completed",
|
||||
detail: {
|
||||
type: "edit",
|
||||
filePath: "src/large-diff.ts",
|
||||
unifiedDiff: `diff --git a/src/large-diff.ts b/src/large-diff.ts\n${payload}`,
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (input.kind === "file") {
|
||||
return {
|
||||
type: "timeline",
|
||||
provider: input.provider,
|
||||
item: {
|
||||
type: "tool_call",
|
||||
name: "read_file",
|
||||
callId: input.callId,
|
||||
status: "completed",
|
||||
detail: {
|
||||
type: "read",
|
||||
filePath: "src/large-file.txt",
|
||||
content: payload,
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: "timeline",
|
||||
provider: input.provider,
|
||||
item: {
|
||||
type: "assistant_message",
|
||||
text: `data:image/png;base64,${payload}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
class FakeAgentSession implements AgentSession {
|
||||
readonly capabilities = TEST_CAPABILITIES;
|
||||
readonly id: string;
|
||||
@@ -372,6 +473,57 @@ class FakeAgentSession implements AgentSession {
|
||||
await this.appendHistoryEvent(turnStarted);
|
||||
this.notifySubscribers(turnStarted);
|
||||
|
||||
const stress = parseAgentStreamStressPrompt(textPrompt);
|
||||
if (stress !== null) {
|
||||
for (let index = 0; index < stress.count; index += 1) {
|
||||
const stressUpdate: AgentStreamEvent = {
|
||||
type: "timeline",
|
||||
provider: this.providerName,
|
||||
item: stress.coalesced
|
||||
? {
|
||||
type: "assistant_message",
|
||||
text: `stress-update-${index}`,
|
||||
}
|
||||
: {
|
||||
type: "todo",
|
||||
items: [{ text: `stress-update-${index}`, completed: index % 2 === 0 }],
|
||||
},
|
||||
};
|
||||
await this.appendHistoryEvent(stressUpdate);
|
||||
this.notifySubscribers(stressUpdate);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
|
||||
const completed: AgentStreamEvent = {
|
||||
type: "turn_completed",
|
||||
provider: this.providerName,
|
||||
usage: { inputTokens: 1, outputTokens: stress.count },
|
||||
};
|
||||
await this.appendHistoryEvent(completed);
|
||||
this.notifySubscribers(completed);
|
||||
return;
|
||||
}
|
||||
|
||||
const largePayload = parseLargeAgentStreamPayloadPrompt(textPrompt);
|
||||
if (largePayload !== null) {
|
||||
const largeUpdate = buildLargeTimelineItem({
|
||||
...largePayload,
|
||||
callId: randomUUID(),
|
||||
provider: this.providerName,
|
||||
});
|
||||
await this.appendHistoryEvent(largeUpdate);
|
||||
this.notifySubscribers(largeUpdate);
|
||||
|
||||
const completed: AgentStreamEvent = {
|
||||
type: "turn_completed",
|
||||
provider: this.providerName,
|
||||
usage: { inputTokens: 1, outputTokens: largePayload.bytes },
|
||||
};
|
||||
await this.appendHistoryEvent(completed);
|
||||
this.notifySubscribers(completed);
|
||||
return;
|
||||
}
|
||||
|
||||
const tool = buildToolCallForPrompt(this.providerName, textPrompt);
|
||||
if (tool) {
|
||||
const needsPermission = this.needsPermissionForTool(tool.name, tool.input ?? {});
|
||||
|
||||
Reference in New Issue
Block a user