From 5c86956fefd84fe63c30f07cc67df4293289b0a1 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 7 May 2026 13:19:31 +0800 Subject: [PATCH] Render Codex image thread items as path markdown (#785) --- .../providers/codex-app-server-agent.test.ts | 128 ++++++++++++++++++ .../agent/providers/codex-app-server-agent.ts | 107 ++++++++++++++- .../agent/providers/provider-image-output.ts | 77 +++++++++++ 3 files changed, 306 insertions(+), 6 deletions(-) create mode 100644 packages/server/src/server/agent/providers/provider-image-output.ts diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 1ea62f356..199b476c1 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -83,6 +83,14 @@ function asInternals(session: CodexTestSession): CodexSessionTestAccess { return castInternals(session); } +function markdownImageSource(markdown: string): string { + const match = markdown.match(/^!\[[^\]]*]\((.*)\)$/); + if (!match) { + throw new Error(`Expected markdown image, got: ${markdown}`); + } + return match[1].replace(/\\\)/g, ")"); +} + describe("Codex app-server provider", () => { test("passes ephemeral: true to thread/start when constructed as ephemeral", async () => { const requests: Array<{ method: string; params: unknown }> = []; @@ -1065,6 +1073,126 @@ describe("Codex app-server provider", () => { }); }); + test("emits imageView thread items as assistant markdown images using the path", () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("item/completed", { + item: { + id: "image-view-1", + type: "imageView", + path: "/tmp/paseo image.png", + }, + }); + + expect(events).toEqual([ + { + type: "timeline", + provider: "codex", + turnId: "test-turn", + item: { + type: "assistant_message", + text: "![Image](/tmp/paseo image.png)", + }, + }, + ]); + }); + + test.each([ + ["savedPath", { savedPath: "/tmp/generated-camel.png" }, "/tmp/generated-camel.png"], + ["saved_path", { saved_path: "/tmp/generated-snake.png" }, "/tmp/generated-snake.png"], + ])( + "emits imageGeneration thread items with %s as assistant markdown images", + (_fieldName, imageFields, expectedPath) => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("item/completed", { + item: { + id: `image-generation-${_fieldName}`, + type: "imageGeneration", + status: "completed", + ...imageFields, + }, + }); + + expect(events).toEqual([ + { + type: "timeline", + provider: "codex", + turnId: "test-turn", + item: { + type: "assistant_message", + text: `![Image](${expectedPath})`, + }, + }, + ]); + }, + ); + + test("materializes imageGeneration base64 results before rendering markdown", () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("item/completed", { + item: { + id: "image-generation-base64", + type: "imageGeneration", + status: "completed", + result: `data:image/png;base64,${ONE_BY_ONE_PNG_BASE64}`, + }, + }); + + expect(events).toHaveLength(1); + const event = events[0]; + expect(event).toMatchObject({ + type: "timeline", + provider: "codex", + turnId: "test-turn", + item: { type: "assistant_message" }, + }); + if (event?.type !== "timeline" || event.item.type !== "assistant_message") { + throw new Error("Expected assistant timeline event"); + } + expect(event.item.text).not.toContain("data:image"); + expect(event.item.text).not.toContain(ONE_BY_ONE_PNG_BASE64); + const source = markdownImageSource(event.item.text); + expect(source).toMatch(/paseo-attachments\/.+\.png$/); + expect(existsSync(source)).toBe(true); + rmSync(source, { force: true }); + }); + + test("ignores incomplete imageGeneration thread items without failing the turn", () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + expect(() => + asInternals(session).handleNotification("item/completed", { + item: { + id: "image-generation-incomplete", + type: "imageGeneration", + status: "in_progress", + }, + }), + ).not.toThrow(); + asInternals(session).handleNotification("turn/completed", { + turn: { status: "completed", error: null }, + }); + + expect(events).toEqual([ + { + type: "turn_completed", + provider: "codex", + turnId: "test-turn", + usage: undefined, + }, + ]); + }); + test("emits usage_updated on token usage updates and keeps usage on turn completion", () => { const session = createSession(); const events: AgentStreamEvent[] = []; diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index 44fdd8a3a..9fa49eea2 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -33,6 +33,7 @@ import { homedir } from "node:os"; import type { ChildProcess, ChildProcessWithoutNullStreams } from "node:child_process"; import { randomUUID } from "node:crypto"; import { Dirent } from "node:fs"; +import * as fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -55,6 +56,10 @@ import { terminateWithTreeKill } from "../../../utils/tree-kill.js"; import { spawnProcess } from "../../../utils/spawn.js"; import { extractCodexTerminalSessionId, nonEmptyString } from "./tool-call-mapper-utils.js"; import { buildCodexFeatures, codexModelSupportsFastMode } from "./codex-feature-definitions.js"; +import { + renderProviderImageOutputAsAssistantMarkdown, + type ProviderImageOutput, +} from "./provider-image-output.js"; import { formatDiagnosticStatus, formatProviderDiagnostic, @@ -85,6 +90,13 @@ const APP_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS = 1_000; const CODEX_PROVIDER = "codex" as const; const CODEX_IMAGE_ATTACHMENT_DIR = "paseo-attachments"; const ASSISTANT_MESSAGE_BOUNDARY_MARKDOWN = "\n\n---\n\n"; +const CODEX_TOOL_THREAD_ITEM_TYPES = new Set([ + "commandExecution", + "fileChange", + "mcpToolCall", + "webSearch", + "collabAgentToolCall", +]); const CODEX_PLAN_IMPLEMENTATION_PROMPT_PREFIX = "The user approved the plan. Implement it now. Do not restate or revise the plan unless blocked."; @@ -1202,6 +1214,10 @@ function normalizeCodexThreadItemType(rawType: string | undefined): string | und return "webSearch"; case "CollabAgentToolCall": return "collabAgentToolCall"; + case "ImageView": + return "imageView"; + case "ImageGeneration": + return "imageGeneration"; default: return rawType; } @@ -1544,6 +1560,84 @@ function mapCodexThreadUserMessageItem( return { type: "user_message", text }; } +function firstStringField( + record: Record, + fields: readonly string[], +): string | null { + for (const field of fields) { + const value = record[field]; + if (typeof value === "string") { + return value; + } + } + return null; +} + +function codexImageOutputFromResult(result: unknown): ProviderImageOutput | null { + if (typeof result === "string") { + const trimmed = result.trim(); + if ( + trimmed.toLowerCase().startsWith("data:image/") || + (/^[A-Za-z0-9+/]+={0,2}$/.test(trimmed) && trimmed.length > 64) + ) { + return { data: trimmed }; + } + return { url: trimmed }; + } + const resultRecord = toObjectRecord(result); + if (!resultRecord) { + return null; + } + return { + path: firstStringField(resultRecord, ["path", "savedPath", "saved_path"]), + url: firstStringField(resultRecord, ["url"]), + data: firstStringField(resultRecord, ["data"]), + mimeType: firstStringField(resultRecord, ["mimeType", "mime_type"]), + }; +} + +function writeImageAttachmentSync(mimeType: string, data: string): string { + const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR); + fsSync.mkdirSync(attachmentsDir, { recursive: true }); + const normalized = normalizeImageData(mimeType, data); + const extension = getImageExtension(normalized.mimeType); + const filename = `${randomUUID()}.${extension}`; + const filePath = path.join(attachmentsDir, filename); + fsSync.writeFileSync(filePath, Buffer.from(normalized.data, "base64")); + return filePath; +} + +function materializeCodexImageOutput(image: { data: string; mimeType: string | null }): { + path: string; +} { + return { + path: writeImageAttachmentSync(image.mimeType ?? "image/png", image.data), + }; +} + +function mapCodexThreadImageItem( + normalizedType: string, + normalizedItem: Record, +): AgentTimelineItem | null { + if (normalizedType === "imageView") { + return renderProviderImageOutputAsAssistantMarkdown({ + path: firstStringField(normalizedItem, ["path"]), + }); + } + + const savedPath = firstStringField(normalizedItem, ["savedPath", "saved_path"]); + const result = codexImageOutputFromResult(normalizedItem.result); + return renderProviderImageOutputAsAssistantMarkdown( + { + path: savedPath ?? result?.path ?? null, + url: result?.url ?? null, + data: result?.data ?? null, + mimeType: result?.mimeType ?? null, + }, + { materialize: materializeCodexImageOutput }, + ); +} + function threadItemToTimeline( item: unknown, options?: { includeUserMessage?: boolean; cwd?: string | null }, @@ -1560,6 +1654,13 @@ function threadItemToTimeline( ? { ...itemRecord, type: normalizedType } : itemRecord; + if (normalizedType === "imageView" || normalizedType === "imageGeneration") { + return mapCodexThreadImageItem(normalizedType, normalizedItem); + } + if (normalizedType && CODEX_TOOL_THREAD_ITEM_TYPES.has(normalizedType)) { + return mapCodexToolCallFromThreadItem(normalizedItem, { cwd }); + } + switch (normalizedType) { case "userMessage": return mapCodexThreadUserMessageItem(normalizedItem, includeUserMessage); @@ -1572,12 +1673,6 @@ function threadItemToTimeline( return mapCodexThreadPlanItem(normalizedItem); case "reasoning": return mapCodexThreadReasoningItem(normalizedItem); - case "commandExecution": - case "fileChange": - case "mcpToolCall": - case "webSearch": - case "collabAgentToolCall": - return mapCodexToolCallFromThreadItem(normalizedItem, { cwd }); default: return null; } diff --git a/packages/server/src/server/agent/providers/provider-image-output.ts b/packages/server/src/server/agent/providers/provider-image-output.ts new file mode 100644 index 000000000..bc726081c --- /dev/null +++ b/packages/server/src/server/agent/providers/provider-image-output.ts @@ -0,0 +1,77 @@ +import type { AgentTimelineItem } from "../agent-sdk-types.js"; + +export interface ProviderImageOutput { + path?: string | null; + url?: string | null; + data?: string | null; + mimeType?: string | null; + altText?: string | null; +} + +export interface MaterializedProviderImage { + path: string; +} + +interface RenderProviderImageOutputOptions { + materialize?: (image: { data: string; mimeType: string | null }) => MaterializedProviderImage; +} + +function nonEmptyString(value: string | null | undefined): string | null { + const trimmed = value?.trim(); + return trimmed ? trimmed : null; +} + +function isDataImageSource(source: string): boolean { + return source.trim().toLowerCase().startsWith("data:image/"); +} + +function escapeMarkdownImageAlt(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/\]/g, "\\]"); +} + +function escapeMarkdownImageSource(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/\)/g, "\\)"); +} + +export function renderProviderImageOutputAsAssistantMarkdown( + image: ProviderImageOutput, + options: RenderProviderImageOutputOptions = {}, +): AgentTimelineItem | null { + const source = nonEmptyString(image.path) ?? nonEmptyString(image.url); + if (source && !isDataImageSource(source)) { + const altText = escapeMarkdownImageAlt(nonEmptyString(image.altText) ?? "Image"); + return { + type: "assistant_message", + text: `![${altText}](${escapeMarkdownImageSource(source)})`, + }; + } + + const data = nonEmptyString(image.data) ?? (source && isDataImageSource(source) ? source : null); + if (!data) { + return null; + } + + let materialized: MaterializedProviderImage | null = null; + try { + materialized = options.materialize + ? options.materialize({ + data, + mimeType: nonEmptyString(image.mimeType), + }) + : null; + } catch { + materialized = null; + } + if (!materialized?.path || isDataImageSource(materialized.path)) { + return { + type: "assistant_message", + text: "Image output was omitted because it was not available as a file path or URL.", + }; + } + + const altText = escapeMarkdownImageAlt(nonEmptyString(image.altText) ?? "Image"); + return { + type: "assistant_message", + text: `![${altText}](${escapeMarkdownImageSource(materialized.path)})`, + }; +}