Render Codex image thread items as path markdown (#785)

2026-05-07 13:19:31 +08:00
parent 80bdb4d45b
commit 5c86956fef
3 changed files with 306 additions and 6 deletions
@@ -83,6 +83,14 @@ function asInternals(session: CodexTestSession): CodexSessionTestAccess {
  return castInternals<CodexSessionTestAccess>(session);
 }

+function markdownImageSource(markdown: string): string {
+  const match = markdown.match(/^!\[[^\]]*]\((.*)\)$/);
+  if (!match) {
+    throw new Error(`Expected markdown image, got: ${markdown}`);
+  }
+  return match[1].replace(/\\\)/g, ")");
+}
+
 describe("Codex app-server provider", () => {
  test("passes ephemeral: true to thread/start when constructed as ephemeral", async () => {
    const requests: Array<{ method: string; params: unknown }> = [];
@@ -1065,6 +1073,126 @@ describe("Codex app-server provider", () => {
    });
  });

+  test("emits imageView thread items as assistant markdown images using the path", () => {
+    const session = createSession();
+    const events: AgentStreamEvent[] = [];
+    session.subscribe((event) => events.push(event));
+
+    asInternals(session).handleNotification("item/completed", {
+      item: {
+        id: "image-view-1",
+        type: "imageView",
+        path: "/tmp/paseo image.png",
+      },
+    });
+
+    expect(events).toEqual([
+      {
+        type: "timeline",
+        provider: "codex",
+        turnId: "test-turn",
+        item: {
+          type: "assistant_message",
+          text: "![Image](/tmp/paseo image.png)",
+        },
+      },
+    ]);
+  });
+
+  test.each([
+    ["savedPath", { savedPath: "/tmp/generated-camel.png" }, "/tmp/generated-camel.png"],
+    ["saved_path", { saved_path: "/tmp/generated-snake.png" }, "/tmp/generated-snake.png"],
+  ])(
+    "emits imageGeneration thread items with %s as assistant markdown images",
+    (_fieldName, imageFields, expectedPath) => {
+      const session = createSession();
+      const events: AgentStreamEvent[] = [];
+      session.subscribe((event) => events.push(event));
+
+      asInternals(session).handleNotification("item/completed", {
+        item: {
+          id: `image-generation-${_fieldName}`,
+          type: "imageGeneration",
+          status: "completed",
+          ...imageFields,
+        },
+      });
+
+      expect(events).toEqual([
+        {
+          type: "timeline",
+          provider: "codex",
+          turnId: "test-turn",
+          item: {
+            type: "assistant_message",
+            text: `![Image](${expectedPath})`,
+          },
+        },
+      ]);
+    },
+  );
+
+  test("materializes imageGeneration base64 results before rendering markdown", () => {
+    const session = createSession();
+    const events: AgentStreamEvent[] = [];
+    session.subscribe((event) => events.push(event));
+
+    asInternals(session).handleNotification("item/completed", {
+      item: {
+        id: "image-generation-base64",
+        type: "imageGeneration",
+        status: "completed",
+        result: `data:image/png;base64,${ONE_BY_ONE_PNG_BASE64}`,
+      },
+    });
+
+    expect(events).toHaveLength(1);
+    const event = events[0];
+    expect(event).toMatchObject({
+      type: "timeline",
+      provider: "codex",
+      turnId: "test-turn",
+      item: { type: "assistant_message" },
+    });
+    if (event?.type !== "timeline" || event.item.type !== "assistant_message") {
+      throw new Error("Expected assistant timeline event");
+    }
+    expect(event.item.text).not.toContain("data:image");
+    expect(event.item.text).not.toContain(ONE_BY_ONE_PNG_BASE64);
+    const source = markdownImageSource(event.item.text);
+    expect(source).toMatch(/paseo-attachments\/.+\.png$/);
+    expect(existsSync(source)).toBe(true);
+    rmSync(source, { force: true });
+  });
+
+  test("ignores incomplete imageGeneration thread items without failing the turn", () => {
+    const session = createSession();
+    const events: AgentStreamEvent[] = [];
+    session.subscribe((event) => events.push(event));
+
+    expect(() =>
+      asInternals(session).handleNotification("item/completed", {
+        item: {
+          id: "image-generation-incomplete",
+          type: "imageGeneration",
+          status: "in_progress",
+        },
+      }),
+    ).not.toThrow();
+    asInternals(session).handleNotification("turn/completed", {
+      turn: { status: "completed", error: null },
+    });
+
+    expect(events).toEqual([
+      {
+        type: "turn_completed",
+        provider: "codex",
+        turnId: "test-turn",
+        usage: undefined,
+      },
+    ]);
+  });
+
  test("emits usage_updated on token usage updates and keeps usage on turn completion", () => {
    const session = createSession();
    const events: AgentStreamEvent[] = [];
@@ -33,6 +33,7 @@ import { homedir } from "node:os";
 import type { ChildProcess, ChildProcessWithoutNullStreams } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { Dirent } from "node:fs";
+import * as fsSync from "node:fs";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
@@ -55,6 +56,10 @@ import { terminateWithTreeKill } from "../../../utils/tree-kill.js";
 import { spawnProcess } from "../../../utils/spawn.js";
 import { extractCodexTerminalSessionId, nonEmptyString } from "./tool-call-mapper-utils.js";
 import { buildCodexFeatures, codexModelSupportsFastMode } from "./codex-feature-definitions.js";
+import {
+  renderProviderImageOutputAsAssistantMarkdown,
+  type ProviderImageOutput,
+} from "./provider-image-output.js";
 import {
  formatDiagnosticStatus,
  formatProviderDiagnostic,
@@ -85,6 +90,13 @@ const APP_SERVER_FORCE_SHUTDOWN_TIMEOUT_MS = 1_000;
 const CODEX_PROVIDER = "codex" as const;
 const CODEX_IMAGE_ATTACHMENT_DIR = "paseo-attachments";
 const ASSISTANT_MESSAGE_BOUNDARY_MARKDOWN = "\n\n---\n\n";
+const CODEX_TOOL_THREAD_ITEM_TYPES = new Set([
+  "commandExecution",
+  "fileChange",
+  "mcpToolCall",
+  "webSearch",
+  "collabAgentToolCall",
+]);
 const CODEX_PLAN_IMPLEMENTATION_PROMPT_PREFIX =
  "The user approved the plan. Implement it now. Do not restate or revise the plan unless blocked.";

@@ -1202,6 +1214,10 @@ function normalizeCodexThreadItemType(rawType: string | undefined): string | und
      return "webSearch";
    case "CollabAgentToolCall":
      return "collabAgentToolCall";
+    case "ImageView":
+      return "imageView";
+    case "ImageGeneration":
+      return "imageGeneration";
    default:
      return rawType;
  }
@@ -1544,6 +1560,84 @@ function mapCodexThreadUserMessageItem(
  return { type: "user_message", text };
 }

+function firstStringField(
+  record: Record<string, unknown>,
+  fields: readonly string[],
+): string | null {
+  for (const field of fields) {
+    const value = record[field];
+    if (typeof value === "string") {
+      return value;
+    }
+  }
+  return null;
+}
+
+function codexImageOutputFromResult(result: unknown): ProviderImageOutput | null {
+  if (typeof result === "string") {
+    const trimmed = result.trim();
+    if (
+      trimmed.toLowerCase().startsWith("data:image/") ||
+      (/^[A-Za-z0-9+/]+={0,2}$/.test(trimmed) && trimmed.length > 64)
+    ) {
+      return { data: trimmed };
+    }
+    return { url: trimmed };
+  }
+  const resultRecord = toObjectRecord(result);
+  if (!resultRecord) {
+    return null;
+  }
+  return {
+    path: firstStringField(resultRecord, ["path", "savedPath", "saved_path"]),
+    url: firstStringField(resultRecord, ["url"]),
+    data: firstStringField(resultRecord, ["data"]),
+    mimeType: firstStringField(resultRecord, ["mimeType", "mime_type"]),
+  };
+}
+
+function writeImageAttachmentSync(mimeType: string, data: string): string {
+  const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR);
+  fsSync.mkdirSync(attachmentsDir, { recursive: true });
+  const normalized = normalizeImageData(mimeType, data);
+  const extension = getImageExtension(normalized.mimeType);
+  const filename = `${randomUUID()}.${extension}`;
+  const filePath = path.join(attachmentsDir, filename);
+  fsSync.writeFileSync(filePath, Buffer.from(normalized.data, "base64"));
+  return filePath;
+}
+
+function materializeCodexImageOutput(image: { data: string; mimeType: string | null }): {
+  path: string;
+} {
+  return {
+    path: writeImageAttachmentSync(image.mimeType ?? "image/png", image.data),
+  };
+}
+
+function mapCodexThreadImageItem(
+  normalizedType: string,
+  normalizedItem: Record<string, unknown>,
+): AgentTimelineItem | null {
+  if (normalizedType === "imageView") {
+    return renderProviderImageOutputAsAssistantMarkdown({
+      path: firstStringField(normalizedItem, ["path"]),
+    });
+  }
+
+  const savedPath = firstStringField(normalizedItem, ["savedPath", "saved_path"]);
+  const result = codexImageOutputFromResult(normalizedItem.result);
+  return renderProviderImageOutputAsAssistantMarkdown(
+    {
+      path: savedPath ?? result?.path ?? null,
+      url: result?.url ?? null,
+      data: result?.data ?? null,
+      mimeType: result?.mimeType ?? null,
+    },
+    { materialize: materializeCodexImageOutput },
+  );
+}
+
 function threadItemToTimeline(
  item: unknown,
  options?: { includeUserMessage?: boolean; cwd?: string | null },
@@ -1560,6 +1654,13 @@ function threadItemToTimeline(
      ? { ...itemRecord, type: normalizedType }
      : itemRecord;

+  if (normalizedType === "imageView" || normalizedType === "imageGeneration") {
+    return mapCodexThreadImageItem(normalizedType, normalizedItem);
+  }
+  if (normalizedType && CODEX_TOOL_THREAD_ITEM_TYPES.has(normalizedType)) {
+    return mapCodexToolCallFromThreadItem(normalizedItem, { cwd });
+  }
+
  switch (normalizedType) {
    case "userMessage":
      return mapCodexThreadUserMessageItem(normalizedItem, includeUserMessage);
@@ -1572,12 +1673,6 @@ function threadItemToTimeline(
      return mapCodexThreadPlanItem(normalizedItem);
    case "reasoning":
      return mapCodexThreadReasoningItem(normalizedItem);
-    case "commandExecution":
-    case "fileChange":
-    case "mcpToolCall":
-    case "webSearch":
-    case "collabAgentToolCall":
-      return mapCodexToolCallFromThreadItem(normalizedItem, { cwd });
    default:
      return null;
  }
@@ -0,0 +1,77 @@
+import type { AgentTimelineItem } from "../agent-sdk-types.js";
+
+export interface ProviderImageOutput {
+  path?: string | null;
+  url?: string | null;
+  data?: string | null;
+  mimeType?: string | null;
+  altText?: string | null;
+}
+
+export interface MaterializedProviderImage {
+  path: string;
+}
+
+interface RenderProviderImageOutputOptions {
+  materialize?: (image: { data: string; mimeType: string | null }) => MaterializedProviderImage;
+}
+
+function nonEmptyString(value: string | null | undefined): string | null {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed : null;
+}
+
+function isDataImageSource(source: string): boolean {
+  return source.trim().toLowerCase().startsWith("data:image/");
+}
+
+function escapeMarkdownImageAlt(value: string): string {
+  return value.replace(/\\/g, "\\\\").replace(/\]/g, "\\]");
+}
+
+function escapeMarkdownImageSource(value: string): string {
+  return value.replace(/\\/g, "\\\\").replace(/\)/g, "\\)");
+}
+
+export function renderProviderImageOutputAsAssistantMarkdown(
+  image: ProviderImageOutput,
+  options: RenderProviderImageOutputOptions = {},
+): AgentTimelineItem | null {
+  const source = nonEmptyString(image.path) ?? nonEmptyString(image.url);
+  if (source && !isDataImageSource(source)) {
+    const altText = escapeMarkdownImageAlt(nonEmptyString(image.altText) ?? "Image");
+    return {
+      type: "assistant_message",
+      text: `![${altText}](${escapeMarkdownImageSource(source)})`,
+    };
+  }
+
+  const data = nonEmptyString(image.data) ?? (source && isDataImageSource(source) ? source : null);
+  if (!data) {
+    return null;
+  }
+
+  let materialized: MaterializedProviderImage | null = null;
+  try {
+    materialized = options.materialize
+      ? options.materialize({
+          data,
+          mimeType: nonEmptyString(image.mimeType),
+        })
+      : null;
+  } catch {
+    materialized = null;
+  }
+  if (!materialized?.path || isDataImageSource(materialized.path)) {
+    return {
+      type: "assistant_message",
+      text: "Image output was omitted because it was not available as a file path or URL.",
+    };
+  }
+
+  const altText = escapeMarkdownImageAlt(nonEmptyString(image.altText) ?? "Image");
+  return {
+    type: "assistant_message",
+    text: `![${altText}](${escapeMarkdownImageSource(materialized.path)})`,
+  };
+}