Render images from Claude Code tool results in chat (#1717)
* feat(providers): render images Claude Code emits instead of base64 Claude returns images as base64 blocks inside tool_result content; they were dumped into the tool output as a wall of raw base64 text. Materialize them to a temp file and emit an assistant_message markdown image — the path Codex already uses — so the client fetches the bytes over the existing WebSocket binary-frame channel. No protocol or client change; assistant_message is an existing type. Also strips the base64 out of the failed-tool-call error field, and widens the Claude history-replay filter so the image survives reload. Collapses the image writer Codex had duplicated twice into one shared materializeProviderImage. * fix(providers): match only materialized image paths in the history filter The history-replay filter recognized a provider image by a leading "![". Review flagged that user-authored text starting with an image-markdown pointing at the attachments dir could then replay as assistant output. Tighten the predicate to the content-hashed <sha256>.<ext> shape the writer actually emits, so only materialized images pass. * fix(providers): handle Windows paths in the image-markdown predicate On Windows the materialized image path uses backslash separators, and escapeMarkdownImageSource doubles each backslash in the markdown source. The history-replay predicate matched a single separator, so on a Windows daemon the image was dropped on reload; the test path assertions failed for the same reason. Allow one-or-more separators in the predicate, unescape the source in the test helper, and add a Windows-path case to the predicate test.
This commit is contained in:
@@ -0,0 +1,225 @@
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { createTestLogger } from "../../../../test-utils/test-logger.js";
|
||||
import type { AgentStreamEvent, AgentTimelineItem } from "../../agent-sdk-types.js";
|
||||
import { ClaudeAgentClient } from "./agent.js";
|
||||
|
||||
const ONE_BY_ONE_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+X1r0AAAAASUVORK5CYII=";
|
||||
|
||||
interface ClaudeImageTestSession {
|
||||
translateMessageToEvents(message: SDKMessage): AgentStreamEvent[];
|
||||
convertHistoryEntry(entry: unknown): AgentTimelineItem[];
|
||||
}
|
||||
|
||||
async function createSession(): Promise<ClaudeImageTestSession> {
|
||||
const client = new ClaudeAgentClient({
|
||||
logger: createTestLogger(),
|
||||
resolveBinary: async () => "/test/claude/bin",
|
||||
});
|
||||
const session = await client.createSession({ provider: "claude", cwd: process.cwd() });
|
||||
return session as unknown as ClaudeImageTestSession;
|
||||
}
|
||||
|
||||
function imageToolResultUserMessage(): SDKMessage {
|
||||
return {
|
||||
type: "user",
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "toolu_read_png",
|
||||
tool_name: "Read",
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: ONE_BY_ONE_PNG_BASE64,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
uuid: "user-image-result-1",
|
||||
session_id: "session-1",
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
function imageToolResultHistoryEntry(): unknown {
|
||||
return {
|
||||
type: "user",
|
||||
uuid: "user-image-result-1",
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "toolu_read_png",
|
||||
tool_name: "Read",
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: ONE_BY_ONE_PNG_BASE64,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function erroredImageToolResultUserMessage(): SDKMessage {
|
||||
return {
|
||||
type: "user",
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "toolu_read_png",
|
||||
tool_name: "Read",
|
||||
is_error: true,
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
source: {
|
||||
type: "base64",
|
||||
media_type: "image/png",
|
||||
data: ONE_BY_ONE_PNG_BASE64,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
uuid: "user-image-error-1",
|
||||
session_id: "session-1",
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
function multiImageToolResultUserMessage(): SDKMessage {
|
||||
const imageBlock = {
|
||||
type: "image",
|
||||
source: { type: "base64", media_type: "image/png", data: ONE_BY_ONE_PNG_BASE64 },
|
||||
};
|
||||
return {
|
||||
type: "user",
|
||||
parent_tool_use_id: null,
|
||||
message: {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "toolu_read_png",
|
||||
tool_name: "Read",
|
||||
content: [imageBlock, imageBlock],
|
||||
},
|
||||
],
|
||||
},
|
||||
uuid: "user-image-result-multi",
|
||||
session_id: "session-1",
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
function imageMessages(items: AgentTimelineItem[]): string[] {
|
||||
return items
|
||||
.filter((item) => item.type === "assistant_message")
|
||||
.map((item) => (item as { text: string }).text)
|
||||
.filter((text) => text.startsWith("!["));
|
||||
}
|
||||
|
||||
function markdownImageSource(markdown: string): string {
|
||||
const match = markdown.match(/^!\[[^\]]*]\((.*)\)$/);
|
||||
if (!match) {
|
||||
throw new Error(`Expected markdown image, got: ${markdown}`);
|
||||
}
|
||||
// Reverse escapeMarkdownImageSource: "\\" -> "\" and "\)" -> ")" (Windows paths are escaped).
|
||||
return match[1].replace(/\\(.)/g, "$1");
|
||||
}
|
||||
|
||||
describe("Claude tool_result image rendering", () => {
|
||||
test("emits the image as assistant markdown and keeps base64 out of the live tool output", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const events = session.translateMessageToEvents(imageToolResultUserMessage());
|
||||
|
||||
const timelineItems = events
|
||||
.filter((event) => event.type === "timeline")
|
||||
.map((event) => (event as { item: AgentTimelineItem }).item);
|
||||
const [imageMessage, ...extraImages] = imageMessages(timelineItems);
|
||||
expect(extraImages).toEqual([]);
|
||||
|
||||
const source = markdownImageSource(imageMessage);
|
||||
expect(source).toMatch(/paseo-attachments[\\/][0-9a-f]{64}\.png$/);
|
||||
expect(existsSync(source)).toBe(true);
|
||||
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
|
||||
|
||||
rmSync(source, { force: true });
|
||||
});
|
||||
|
||||
test("replays the image as assistant markdown through history conversion", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const items = session.convertHistoryEntry(imageToolResultHistoryEntry());
|
||||
|
||||
const [imageMessage, ...extraImages] = imageMessages(items);
|
||||
expect(extraImages).toEqual([]);
|
||||
|
||||
const source = markdownImageSource(imageMessage);
|
||||
expect(source).toMatch(/paseo-attachments[\\/][0-9a-f]{64}\.png$/);
|
||||
expect(existsSync(source)).toBe(true);
|
||||
expect(JSON.stringify(items)).not.toContain(ONE_BY_ONE_PNG_BASE64);
|
||||
|
||||
rmSync(source, { force: true });
|
||||
});
|
||||
|
||||
test("keeps base64 out of an errored tool_result that carries an image", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const events = session.translateMessageToEvents(erroredImageToolResultUserMessage());
|
||||
|
||||
const timelineItems = events
|
||||
.filter((event) => event.type === "timeline")
|
||||
.map((event) => (event as { item: AgentTimelineItem }).item);
|
||||
const [imageMessage, ...extraImages] = imageMessages(timelineItems);
|
||||
expect(extraImages).toEqual([]);
|
||||
|
||||
const source = markdownImageSource(imageMessage);
|
||||
expect(existsSync(source)).toBe(true);
|
||||
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
|
||||
expect(JSON.stringify(events)).toContain("[image]");
|
||||
|
||||
rmSync(source, { force: true });
|
||||
});
|
||||
|
||||
test("emits one image message per image block in a multi-image tool_result", async () => {
|
||||
const session = await createSession();
|
||||
|
||||
const events = session.translateMessageToEvents(multiImageToolResultUserMessage());
|
||||
|
||||
const timelineItems = events
|
||||
.filter((event) => event.type === "timeline")
|
||||
.map((event) => (event as { item: AgentTimelineItem }).item);
|
||||
const sources = imageMessages(timelineItems).map(markdownImageSource);
|
||||
|
||||
expect(sources).toHaveLength(2);
|
||||
// Identical bytes materialize to one content-hashed file (idempotent), one message per block.
|
||||
expect(new Set(sources).size).toBe(1);
|
||||
expect(existsSync(sources[0])).toBe(true);
|
||||
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
|
||||
|
||||
rmSync(sources[0], { force: true });
|
||||
});
|
||||
});
|
||||
@@ -677,6 +677,7 @@ test("maps tool_result content shapes into deterministic string output", async (
|
||||
const session = await createSession();
|
||||
const internal: {
|
||||
buildToolOutput: (
|
||||
content: unknown,
|
||||
block: Record<string, unknown>,
|
||||
entry: Record<string, unknown> | undefined,
|
||||
) => Record<string, unknown> | undefined;
|
||||
@@ -724,6 +725,7 @@ test("maps tool_result content shapes into deterministic string output", async (
|
||||
try {
|
||||
for (const fixture of fixtures) {
|
||||
const output = internal.buildToolOutput(
|
||||
fixture.content,
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "tool-1",
|
||||
@@ -750,6 +752,7 @@ test("Grep tool_result string content flows to a search detail with content", as
|
||||
const session = await createSession();
|
||||
const internal: {
|
||||
buildToolOutput: (
|
||||
content: unknown,
|
||||
block: Record<string, unknown>,
|
||||
entry: Record<string, unknown> | undefined,
|
||||
) => Record<string, unknown> | undefined;
|
||||
@@ -765,12 +768,14 @@ test("Grep tool_result string content flows to a search detail with content", as
|
||||
};
|
||||
|
||||
try {
|
||||
const grepContent = "Found 2 files\nsrc/foo.tsx\nsrc/bar.tsx";
|
||||
const output = internal.buildToolOutput(
|
||||
grepContent,
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: "tool-grep-1",
|
||||
tool_name: "Grep",
|
||||
content: "Found 2 files\nsrc/foo.tsx\nsrc/bar.tsx",
|
||||
content: grepContent,
|
||||
is_error: false,
|
||||
},
|
||||
grepEntry,
|
||||
|
||||
@@ -45,6 +45,12 @@ import { realClaudeRewindSdk, revertClaudeConversation, revertClaudeFiles } from
|
||||
import { normalizeProviderReplayTimestamp } from "../../provider-history-timestamps.js";
|
||||
import { claudeProjectDirSync } from "./project-dir.js";
|
||||
import { SETTING_APPLIES_NEXT_TURN_NOTICE } from "../../provider-notices.js";
|
||||
import {
|
||||
isProviderImageMarkdown,
|
||||
materializeProviderImage,
|
||||
renderProviderImageOutputAsAssistantMarkdown,
|
||||
type ProviderImageOutput,
|
||||
} from "../provider-image-output.js";
|
||||
|
||||
import {
|
||||
getAgentStreamEventTurnId,
|
||||
@@ -590,6 +596,44 @@ function coerceToolResultContentToString(content: unknown): string {
|
||||
return deterministicStringify(content);
|
||||
}
|
||||
|
||||
function toBase64ImageOutput(block: unknown): ProviderImageOutput | null {
|
||||
const record = toObjectRecord(block);
|
||||
if (!record || record.type !== "image") {
|
||||
return null;
|
||||
}
|
||||
const source = toObjectRecord(record.source);
|
||||
if (!source || source.type !== "base64" || typeof source.data !== "string") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
data: source.data,
|
||||
mimeType: typeof source.media_type === "string" ? source.media_type : null,
|
||||
};
|
||||
}
|
||||
|
||||
// Claude returns images inside tool_result content as base64 Anthropic blocks. Left in place they
|
||||
// reach coerceToolResultContentToString, which JSON.stringifies the whole array — dumping base64
|
||||
// into the tool output. We pull those blocks out to render them as image markdown and leave a
|
||||
// "[image]" placeholder so image-only results still produce non-empty output.
|
||||
function splitClaudeToolResultImages(content: unknown): {
|
||||
images: ProviderImageOutput[];
|
||||
text: unknown;
|
||||
} {
|
||||
if (!Array.isArray(content)) {
|
||||
return { images: [], text: content };
|
||||
}
|
||||
const images: ProviderImageOutput[] = [];
|
||||
const text = content.map((block) => {
|
||||
const image = toBase64ImageOutput(block);
|
||||
if (image) {
|
||||
images.push(image);
|
||||
return { type: "text", text: "[image]" };
|
||||
}
|
||||
return block;
|
||||
});
|
||||
return { images, text };
|
||||
}
|
||||
|
||||
function normalizeClaudeTranscriptText(value: unknown): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
@@ -4353,8 +4397,10 @@ class ClaudeAgentSession implements AgentSession {
|
||||
? block.tool_use_id
|
||||
: (entry?.id ?? null);
|
||||
|
||||
// Extract output from block.content (SDK always returns content in string form)
|
||||
const output = this.buildToolOutput(block, entry);
|
||||
// Pull image blocks out of the result so base64 never reaches the tool output, and render each
|
||||
// one as an assistant_message markdown image after the tool_call (matching how Codex emits).
|
||||
const { images, text } = splitClaudeToolResultImages(block.content);
|
||||
const output = this.buildToolOutput(text, block, entry);
|
||||
|
||||
if (block.is_error) {
|
||||
this.pushToolCall(
|
||||
@@ -4363,7 +4409,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
callId,
|
||||
input: entry?.input ?? null,
|
||||
output: output ?? null,
|
||||
error: block,
|
||||
error: { ...block, content: text },
|
||||
}),
|
||||
items,
|
||||
);
|
||||
@@ -4379,6 +4425,15 @@ class ClaudeAgentSession implements AgentSession {
|
||||
);
|
||||
}
|
||||
|
||||
for (const image of images) {
|
||||
const imageItem = renderProviderImageOutputAsAssistantMarkdown(image, {
|
||||
materialize: materializeProviderImage,
|
||||
});
|
||||
if (imageItem) {
|
||||
items.push(imageItem);
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof block.tool_use_id === "string") {
|
||||
this.toolUseCache.delete(block.tool_use_id);
|
||||
this.sidechainTracker.delete(block.tool_use_id);
|
||||
@@ -4386,6 +4441,7 @@ class ClaudeAgentSession implements AgentSession {
|
||||
}
|
||||
|
||||
private buildToolOutput(
|
||||
content: unknown,
|
||||
block: ClaudeContentChunk,
|
||||
entry: ToolUseCacheEntry | undefined,
|
||||
): AgentMetadata | undefined {
|
||||
@@ -4397,11 +4453,11 @@ class ClaudeAgentSession implements AgentSession {
|
||||
const blockToolName = typeof block.tool_name === "string" ? block.tool_name : undefined;
|
||||
const server = entry?.server ?? blockServer ?? "tool";
|
||||
const tool = entry?.name ?? blockToolName ?? "tool";
|
||||
const content = coerceToolResultContentToString(block.content);
|
||||
const coercedContent = coerceToolResultContentToString(content);
|
||||
const input = entry?.input;
|
||||
|
||||
// Build structured result based on tool type
|
||||
const structured = this.buildStructuredToolResult(server, tool, content, input);
|
||||
const structured = this.buildStructuredToolResult(server, tool, coercedContent, input);
|
||||
|
||||
if (structured) {
|
||||
return structured;
|
||||
@@ -4410,13 +4466,13 @@ class ClaudeAgentSession implements AgentSession {
|
||||
// Fallback format - try to parse JSON first
|
||||
const result: AgentMetadata = {};
|
||||
|
||||
if (content.length > 0) {
|
||||
if (coercedContent.length > 0) {
|
||||
try {
|
||||
// If content is a JSON string, parse it
|
||||
result.output = JSON.parse(content);
|
||||
result.output = JSON.parse(coercedContent);
|
||||
} catch {
|
||||
// If not JSON, return unchanged (no extra wrapping)
|
||||
result.output = content;
|
||||
result.output = coercedContent;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4936,6 +4992,10 @@ function convertClaudeHistoryEntryPreamble(
|
||||
return { proceed: { content } };
|
||||
}
|
||||
|
||||
function isProviderImageMessage(item: AgentTimelineItem): boolean {
|
||||
return item.type === "assistant_message" && isProviderImageMarkdown(item.text);
|
||||
}
|
||||
|
||||
export function convertClaudeHistoryEntry(
|
||||
entry: ClaudeHistoryEntry,
|
||||
mapBlocks: (content: string | ClaudeContentChunk[]) => AgentTimelineItem[],
|
||||
@@ -4979,7 +5039,12 @@ export function convertClaudeHistoryEntry(
|
||||
if (hasToolBlock && normalizedBlocks) {
|
||||
const mapped = mapBlocks(normalizedBlocks);
|
||||
if (entry.type === "user") {
|
||||
const toolItems = mapped.filter((item) => item.type === "tool_call");
|
||||
// tool_result handling (handleToolResult) emits image markdown as an assistant_message
|
||||
// alongside the tool_call. User-entry text blocks also map to assistant_message in this path
|
||||
// and must stay suppressed, so keep tool_calls plus only the image assistant_messages.
|
||||
const toolItems = mapped.filter(
|
||||
(item) => item.type === "tool_call" || isProviderImageMessage(item),
|
||||
);
|
||||
return timeline.length ? [...timeline, ...toolItems] : toolItems;
|
||||
}
|
||||
return mapped;
|
||||
|
||||
@@ -39,7 +39,6 @@ import type { Logger } from "pino";
|
||||
import type { ChildProcess, ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { Dirent } from "node:fs";
|
||||
import * as fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -79,6 +78,7 @@ import {
|
||||
} from "./codex/app-server-transport.js";
|
||||
import { type CodexUserMessageTurnIndex, revertCodexConversation } from "./codex/rewind.js";
|
||||
import {
|
||||
materializeProviderImage,
|
||||
renderProviderImageOutputAsAssistantMarkdown,
|
||||
type ProviderImageOutput,
|
||||
} from "./provider-image-output.js";
|
||||
@@ -114,7 +114,6 @@ function isCodexAlreadyUnarchivedError(error: unknown, threadId: string): boolea
|
||||
const TURN_START_TIMEOUT_MS = 90 * 1000;
|
||||
const INTERRUPT_TIMEOUT_MS = 2_000;
|
||||
const CODEX_PROVIDER = "codex" as const;
|
||||
const CODEX_IMAGE_ATTACHMENT_DIR = "paseo-attachments";
|
||||
// Codex treats most app-server client names as the model-request originator.
|
||||
// This reserved Codex name is non-originating, so requests keep Codex's default
|
||||
// CLI identity instead of showing up as Paseo in provider usage logs.
|
||||
@@ -1626,25 +1625,6 @@ function codexImageOutputFromResult(result: unknown): ProviderImageOutput | null
|
||||
};
|
||||
}
|
||||
|
||||
function writeImageAttachmentSync(mimeType: string, data: string): string {
|
||||
const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR);
|
||||
fsSync.mkdirSync(attachmentsDir, { recursive: true });
|
||||
const normalized = normalizeImageData(mimeType, data);
|
||||
const extension = getImageExtension(normalized.mimeType);
|
||||
const filename = `${randomUUID()}.${extension}`;
|
||||
const filePath = path.join(attachmentsDir, filename);
|
||||
fsSync.writeFileSync(filePath, Buffer.from(normalized.data, "base64"));
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function materializeCodexImageOutput(image: { data: string; mimeType: string | null }): {
|
||||
path: string;
|
||||
} {
|
||||
return {
|
||||
path: writeImageAttachmentSync(image.mimeType ?? "image/png", image.data),
|
||||
};
|
||||
}
|
||||
|
||||
function mapCodexThreadImageItem(
|
||||
normalizedType: string,
|
||||
normalizedItem: Record<string, unknown>,
|
||||
@@ -1664,7 +1644,7 @@ function mapCodexThreadImageItem(
|
||||
data: result?.data ?? null,
|
||||
mimeType: result?.mimeType ?? null,
|
||||
},
|
||||
{ materialize: materializeCodexImageOutput },
|
||||
{ materialize: materializeProviderImage },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1809,40 +1789,6 @@ function toSandboxPolicy(type: string, networkAccess?: boolean): Record<string,
|
||||
}
|
||||
}
|
||||
|
||||
function getImageExtension(mimeType: string): string {
|
||||
switch (mimeType) {
|
||||
case "image/jpeg":
|
||||
return "jpg";
|
||||
case "image/png":
|
||||
return "png";
|
||||
case "image/webp":
|
||||
return "webp";
|
||||
case "image/gif":
|
||||
return "gif";
|
||||
case "image/bmp":
|
||||
return "bmp";
|
||||
case "image/tiff":
|
||||
return "tiff";
|
||||
default:
|
||||
return "bin";
|
||||
}
|
||||
}
|
||||
|
||||
interface ImageDataPayload {
|
||||
mimeType: string;
|
||||
data: string;
|
||||
}
|
||||
|
||||
function normalizeImageData(mimeType: string, data: string): ImageDataPayload {
|
||||
if (data.startsWith("data:")) {
|
||||
const match = data.match(/^data:([^;]+);base64,(.*)$/);
|
||||
if (match) {
|
||||
return { mimeType: match[1], data: match[2] };
|
||||
}
|
||||
}
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
const ThreadStartedNotificationSchema = z
|
||||
.object({
|
||||
thread: z.object({ id: z.string() }).passthrough(),
|
||||
@@ -2691,17 +2637,6 @@ const CodexNotificationSchema = z.union([
|
||||
),
|
||||
]);
|
||||
|
||||
async function writeImageAttachment(mimeType: string, data: string): Promise<string> {
|
||||
const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR);
|
||||
await fs.mkdir(attachmentsDir, { recursive: true });
|
||||
const normalized = normalizeImageData(mimeType, data);
|
||||
const extension = getImageExtension(normalized.mimeType);
|
||||
const filename = `${randomUUID()}.${extension}`;
|
||||
const filePath = path.join(attachmentsDir, filename);
|
||||
await fs.writeFile(filePath, Buffer.from(normalized.data, "base64"));
|
||||
return filePath;
|
||||
}
|
||||
|
||||
async function readCodexConfiguredDefaults(
|
||||
client: CodexAppServerClient,
|
||||
logger: Logger,
|
||||
@@ -2794,7 +2729,10 @@ export async function codexAppServerTurnInputFromPrompt(
|
||||
}
|
||||
if (block.type === "image") {
|
||||
try {
|
||||
const filePath = await writeImageAttachment(block.mimeType, block.data);
|
||||
const filePath = materializeProviderImage({
|
||||
data: block.data,
|
||||
mimeType: block.mimeType,
|
||||
}).path;
|
||||
output.push({ type: "localImage", path: filePath });
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { isProviderImageMarkdown } from "./provider-image-output.js";
|
||||
|
||||
const HASH = "a".repeat(64);
|
||||
|
||||
describe("isProviderImageMarkdown", () => {
|
||||
test("matches the markdown emitted for a materialized attachment", () => {
|
||||
expect(isProviderImageMarkdown(``)).toBe(true);
|
||||
expect(isProviderImageMarkdown(``)).toBe(
|
||||
true,
|
||||
);
|
||||
// Windows: backslash path separators are doubled by escapeMarkdownImageSource.
|
||||
expect(
|
||||
isProviderImageMarkdown(
|
||||
``,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects user-authored markdown that is not a materialized attachment", () => {
|
||||
// No content hash — a hand-written path, not something the writer produced.
|
||||
expect(isProviderImageMarkdown("")).toBe(false);
|
||||
expect(isProviderImageMarkdown("")).toBe(false);
|
||||
// Image markdown that does not start the text.
|
||||
expect(isProviderImageMarkdown("see the chart: ")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,8 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import * as fsSync from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import type { AgentTimelineItem } from "../agent-sdk-types.js";
|
||||
|
||||
export interface ProviderImageOutput {
|
||||
@@ -12,6 +17,68 @@ export interface MaterializedProviderImage {
|
||||
path: string;
|
||||
}
|
||||
|
||||
const PROVIDER_IMAGE_ATTACHMENT_DIR = "paseo-attachments";
|
||||
|
||||
function getImageExtension(mimeType: string): string {
|
||||
switch (mimeType) {
|
||||
case "image/jpeg":
|
||||
return "jpg";
|
||||
case "image/png":
|
||||
return "png";
|
||||
case "image/webp":
|
||||
return "webp";
|
||||
case "image/gif":
|
||||
return "gif";
|
||||
case "image/bmp":
|
||||
return "bmp";
|
||||
case "image/tiff":
|
||||
return "tiff";
|
||||
default:
|
||||
return "bin";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeImageData(mimeType: string, data: string): { mimeType: string; data: string } {
|
||||
if (data.startsWith("data:")) {
|
||||
const match = data.match(/^data:([^;]+);base64,(.*)$/);
|
||||
if (match) {
|
||||
return { mimeType: match[1], data: match[2] };
|
||||
}
|
||||
}
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
// Filenames are a content hash of the bytes so re-materializing the same image
|
||||
// is idempotent: history replay reuses the existing temp file instead of leaking
|
||||
// a fresh one on every load.
|
||||
export function materializeProviderImage(image: {
|
||||
data: string;
|
||||
mimeType: string | null;
|
||||
}): MaterializedProviderImage {
|
||||
const attachmentsDir = path.join(os.tmpdir(), PROVIDER_IMAGE_ATTACHMENT_DIR);
|
||||
fsSync.mkdirSync(attachmentsDir, { recursive: true });
|
||||
const normalized = normalizeImageData(image.mimeType ?? "image/png", image.data);
|
||||
const bytes = Buffer.from(normalized.data, "base64");
|
||||
const extension = getImageExtension(normalized.mimeType);
|
||||
const hash = createHash("sha256").update(bytes).digest("hex");
|
||||
const filePath = path.join(attachmentsDir, `${hash}.${extension}`);
|
||||
fsSync.writeFileSync(filePath, bytes);
|
||||
return { path: filePath };
|
||||
}
|
||||
|
||||
// Recognizes the markdown renderProviderImageOutputAsAssistantMarkdown emits for a materialized
|
||||
// provider image: its source is a content-hashed file in the attachments dir. Matching the full
|
||||
// <hash>.<ext> shape (not just a leading "![") keeps user-authored text from being mistaken for a
|
||||
// provider image when it reaches the history-replay filter. The separator class allows one-or-more
|
||||
// because on Windows the path uses "\\" and escapeMarkdownImageSource doubles each backslash.
|
||||
const PROVIDER_IMAGE_MARKDOWN = new RegExp(
|
||||
`^!\\[[^\\]]*\\]\\([^)]*${PROVIDER_IMAGE_ATTACHMENT_DIR}[/\\\\]+[0-9a-f]{64}\\.[a-z0-9]+\\)`,
|
||||
);
|
||||
|
||||
export function isProviderImageMarkdown(text: string): boolean {
|
||||
return PROVIDER_IMAGE_MARKDOWN.test(text);
|
||||
}
|
||||
|
||||
interface RenderProviderImageOutputOptions {
|
||||
materialize?: (image: { data: string; mimeType: string | null }) => MaterializedProviderImage;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user