Render images from Claude Code tool results in chat (#1717)

* feat(providers): render images Claude Code emits instead of base64

Claude returns images as base64 blocks inside tool_result content; they were
dumped into the tool output as a wall of raw base64 text. Materialize them to a
temp file and emit an assistant_message markdown image — the path Codex already
uses — so the client fetches the bytes over the existing WebSocket binary-frame
channel. No protocol or client change; assistant_message is an existing type.

Also strips the base64 out of the failed-tool-call error field, and widens the
Claude history-replay filter so the image survives reload. Collapses the image
writer Codex had duplicated twice into one shared materializeProviderImage.

* fix(providers): match only materialized image paths in the history filter

The history-replay filter recognized a provider image by a leading "![". Review
flagged that user-authored text starting with an image-markdown pointing at the
attachments dir could then replay as assistant output. Tighten the predicate to
the content-hashed <sha256>.<ext> shape the writer actually emits, so only
materialized images pass.

* fix(providers): handle Windows paths in the image-markdown predicate

On Windows the materialized image path uses backslash separators, and
escapeMarkdownImageSource doubles each backslash in the markdown source. The
history-replay predicate matched a single separator, so on a Windows daemon the
image was dropped on reload; the test path assertions failed for the same
reason. Allow one-or-more separators in the predicate, unescape the source in
the test helper, and add a Windows-path case to the predicate test.
This commit is contained in:
Mohamed Boudra
2026-06-25 04:11:46 +00:00
committed by GitHub
parent 1b9543023e
commit b625b69302
6 changed files with 406 additions and 78 deletions
@@ -0,0 +1,225 @@
import { existsSync, rmSync } from "node:fs";
import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
import { describe, expect, test } from "vitest";
import { createTestLogger } from "../../../../test-utils/test-logger.js";
import type { AgentStreamEvent, AgentTimelineItem } from "../../agent-sdk-types.js";
import { ClaudeAgentClient } from "./agent.js";
const ONE_BY_ONE_PNG_BASE64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+X1r0AAAAASUVORK5CYII=";
interface ClaudeImageTestSession {
translateMessageToEvents(message: SDKMessage): AgentStreamEvent[];
convertHistoryEntry(entry: unknown): AgentTimelineItem[];
}
async function createSession(): Promise<ClaudeImageTestSession> {
const client = new ClaudeAgentClient({
logger: createTestLogger(),
resolveBinary: async () => "/test/claude/bin",
});
const session = await client.createSession({ provider: "claude", cwd: process.cwd() });
return session as unknown as ClaudeImageTestSession;
}
function imageToolResultUserMessage(): SDKMessage {
return {
type: "user",
parent_tool_use_id: null,
message: {
role: "user",
content: [
{
type: "tool_result",
tool_use_id: "toolu_read_png",
tool_name: "Read",
content: [
{
type: "image",
source: {
type: "base64",
media_type: "image/png",
data: ONE_BY_ONE_PNG_BASE64,
},
},
],
},
],
},
uuid: "user-image-result-1",
session_id: "session-1",
} as unknown as SDKMessage;
}
function imageToolResultHistoryEntry(): unknown {
return {
type: "user",
uuid: "user-image-result-1",
message: {
role: "user",
content: [
{
type: "tool_result",
tool_use_id: "toolu_read_png",
tool_name: "Read",
content: [
{
type: "image",
source: {
type: "base64",
media_type: "image/png",
data: ONE_BY_ONE_PNG_BASE64,
},
},
],
},
],
},
};
}
function erroredImageToolResultUserMessage(): SDKMessage {
return {
type: "user",
parent_tool_use_id: null,
message: {
role: "user",
content: [
{
type: "tool_result",
tool_use_id: "toolu_read_png",
tool_name: "Read",
is_error: true,
content: [
{
type: "image",
source: {
type: "base64",
media_type: "image/png",
data: ONE_BY_ONE_PNG_BASE64,
},
},
],
},
],
},
uuid: "user-image-error-1",
session_id: "session-1",
} as unknown as SDKMessage;
}
function multiImageToolResultUserMessage(): SDKMessage {
const imageBlock = {
type: "image",
source: { type: "base64", media_type: "image/png", data: ONE_BY_ONE_PNG_BASE64 },
};
return {
type: "user",
parent_tool_use_id: null,
message: {
role: "user",
content: [
{
type: "tool_result",
tool_use_id: "toolu_read_png",
tool_name: "Read",
content: [imageBlock, imageBlock],
},
],
},
uuid: "user-image-result-multi",
session_id: "session-1",
} as unknown as SDKMessage;
}
function imageMessages(items: AgentTimelineItem[]): string[] {
return items
.filter((item) => item.type === "assistant_message")
.map((item) => (item as { text: string }).text)
.filter((text) => text.startsWith("!["));
}
function markdownImageSource(markdown: string): string {
const match = markdown.match(/^!\[[^\]]*]\((.*)\)$/);
if (!match) {
throw new Error(`Expected markdown image, got: ${markdown}`);
}
// Reverse escapeMarkdownImageSource: "\\" -> "\" and "\)" -> ")" (Windows paths are escaped).
return match[1].replace(/\\(.)/g, "$1");
}
describe("Claude tool_result image rendering", () => {
test("emits the image as assistant markdown and keeps base64 out of the live tool output", async () => {
const session = await createSession();
const events = session.translateMessageToEvents(imageToolResultUserMessage());
const timelineItems = events
.filter((event) => event.type === "timeline")
.map((event) => (event as { item: AgentTimelineItem }).item);
const [imageMessage, ...extraImages] = imageMessages(timelineItems);
expect(extraImages).toEqual([]);
const source = markdownImageSource(imageMessage);
expect(source).toMatch(/paseo-attachments[\\/][0-9a-f]{64}\.png$/);
expect(existsSync(source)).toBe(true);
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
rmSync(source, { force: true });
});
test("replays the image as assistant markdown through history conversion", async () => {
const session = await createSession();
const items = session.convertHistoryEntry(imageToolResultHistoryEntry());
const [imageMessage, ...extraImages] = imageMessages(items);
expect(extraImages).toEqual([]);
const source = markdownImageSource(imageMessage);
expect(source).toMatch(/paseo-attachments[\\/][0-9a-f]{64}\.png$/);
expect(existsSync(source)).toBe(true);
expect(JSON.stringify(items)).not.toContain(ONE_BY_ONE_PNG_BASE64);
rmSync(source, { force: true });
});
test("keeps base64 out of an errored tool_result that carries an image", async () => {
const session = await createSession();
const events = session.translateMessageToEvents(erroredImageToolResultUserMessage());
const timelineItems = events
.filter((event) => event.type === "timeline")
.map((event) => (event as { item: AgentTimelineItem }).item);
const [imageMessage, ...extraImages] = imageMessages(timelineItems);
expect(extraImages).toEqual([]);
const source = markdownImageSource(imageMessage);
expect(existsSync(source)).toBe(true);
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
expect(JSON.stringify(events)).toContain("[image]");
rmSync(source, { force: true });
});
test("emits one image message per image block in a multi-image tool_result", async () => {
const session = await createSession();
const events = session.translateMessageToEvents(multiImageToolResultUserMessage());
const timelineItems = events
.filter((event) => event.type === "timeline")
.map((event) => (event as { item: AgentTimelineItem }).item);
const sources = imageMessages(timelineItems).map(markdownImageSource);
expect(sources).toHaveLength(2);
// Identical bytes materialize to one content-hashed file (idempotent), one message per block.
expect(new Set(sources).size).toBe(1);
expect(existsSync(sources[0])).toBe(true);
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
rmSync(sources[0], { force: true });
});
});
@@ -677,6 +677,7 @@ test("maps tool_result content shapes into deterministic string output", async (
const session = await createSession();
const internal: {
buildToolOutput: (
content: unknown,
block: Record<string, unknown>,
entry: Record<string, unknown> | undefined,
) => Record<string, unknown> | undefined;
@@ -724,6 +725,7 @@ test("maps tool_result content shapes into deterministic string output", async (
try {
for (const fixture of fixtures) {
const output = internal.buildToolOutput(
fixture.content,
{
type: "tool_result",
tool_use_id: "tool-1",
@@ -750,6 +752,7 @@ test("Grep tool_result string content flows to a search detail with content", as
const session = await createSession();
const internal: {
buildToolOutput: (
content: unknown,
block: Record<string, unknown>,
entry: Record<string, unknown> | undefined,
) => Record<string, unknown> | undefined;
@@ -765,12 +768,14 @@ test("Grep tool_result string content flows to a search detail with content", as
};
try {
const grepContent = "Found 2 files\nsrc/foo.tsx\nsrc/bar.tsx";
const output = internal.buildToolOutput(
grepContent,
{
type: "tool_result",
tool_use_id: "tool-grep-1",
tool_name: "Grep",
content: "Found 2 files\nsrc/foo.tsx\nsrc/bar.tsx",
content: grepContent,
is_error: false,
},
grepEntry,
@@ -45,6 +45,12 @@ import { realClaudeRewindSdk, revertClaudeConversation, revertClaudeFiles } from
import { normalizeProviderReplayTimestamp } from "../../provider-history-timestamps.js";
import { claudeProjectDirSync } from "./project-dir.js";
import { SETTING_APPLIES_NEXT_TURN_NOTICE } from "../../provider-notices.js";
import {
isProviderImageMarkdown,
materializeProviderImage,
renderProviderImageOutputAsAssistantMarkdown,
type ProviderImageOutput,
} from "../provider-image-output.js";
import {
getAgentStreamEventTurnId,
@@ -590,6 +596,44 @@ function coerceToolResultContentToString(content: unknown): string {
return deterministicStringify(content);
}
function toBase64ImageOutput(block: unknown): ProviderImageOutput | null {
const record = toObjectRecord(block);
if (!record || record.type !== "image") {
return null;
}
const source = toObjectRecord(record.source);
if (!source || source.type !== "base64" || typeof source.data !== "string") {
return null;
}
return {
data: source.data,
mimeType: typeof source.media_type === "string" ? source.media_type : null,
};
}
// Claude returns images inside tool_result content as base64 Anthropic blocks. Left in place they
// reach coerceToolResultContentToString, which JSON.stringifies the whole array — dumping base64
// into the tool output. We pull those blocks out to render them as image markdown and leave a
// "[image]" placeholder so image-only results still produce non-empty output.
function splitClaudeToolResultImages(content: unknown): {
images: ProviderImageOutput[];
text: unknown;
} {
if (!Array.isArray(content)) {
return { images: [], text: content };
}
const images: ProviderImageOutput[] = [];
const text = content.map((block) => {
const image = toBase64ImageOutput(block);
if (image) {
images.push(image);
return { type: "text", text: "[image]" };
}
return block;
});
return { images, text };
}
function normalizeClaudeTranscriptText(value: unknown): string | null {
if (typeof value !== "string") {
return null;
@@ -4353,8 +4397,10 @@ class ClaudeAgentSession implements AgentSession {
? block.tool_use_id
: (entry?.id ?? null);
// Extract output from block.content (SDK always returns content in string form)
const output = this.buildToolOutput(block, entry);
// Pull image blocks out of the result so base64 never reaches the tool output, and render each
// one as an assistant_message markdown image after the tool_call (matching how Codex emits).
const { images, text } = splitClaudeToolResultImages(block.content);
const output = this.buildToolOutput(text, block, entry);
if (block.is_error) {
this.pushToolCall(
@@ -4363,7 +4409,7 @@ class ClaudeAgentSession implements AgentSession {
callId,
input: entry?.input ?? null,
output: output ?? null,
error: block,
error: { ...block, content: text },
}),
items,
);
@@ -4379,6 +4425,15 @@ class ClaudeAgentSession implements AgentSession {
);
}
for (const image of images) {
const imageItem = renderProviderImageOutputAsAssistantMarkdown(image, {
materialize: materializeProviderImage,
});
if (imageItem) {
items.push(imageItem);
}
}
if (typeof block.tool_use_id === "string") {
this.toolUseCache.delete(block.tool_use_id);
this.sidechainTracker.delete(block.tool_use_id);
@@ -4386,6 +4441,7 @@ class ClaudeAgentSession implements AgentSession {
}
private buildToolOutput(
content: unknown,
block: ClaudeContentChunk,
entry: ToolUseCacheEntry | undefined,
): AgentMetadata | undefined {
@@ -4397,11 +4453,11 @@ class ClaudeAgentSession implements AgentSession {
const blockToolName = typeof block.tool_name === "string" ? block.tool_name : undefined;
const server = entry?.server ?? blockServer ?? "tool";
const tool = entry?.name ?? blockToolName ?? "tool";
const content = coerceToolResultContentToString(block.content);
const coercedContent = coerceToolResultContentToString(content);
const input = entry?.input;
// Build structured result based on tool type
const structured = this.buildStructuredToolResult(server, tool, content, input);
const structured = this.buildStructuredToolResult(server, tool, coercedContent, input);
if (structured) {
return structured;
@@ -4410,13 +4466,13 @@ class ClaudeAgentSession implements AgentSession {
// Fallback format - try to parse JSON first
const result: AgentMetadata = {};
if (content.length > 0) {
if (coercedContent.length > 0) {
try {
// If content is a JSON string, parse it
result.output = JSON.parse(content);
result.output = JSON.parse(coercedContent);
} catch {
// If not JSON, return unchanged (no extra wrapping)
result.output = content;
result.output = coercedContent;
}
}
@@ -4936,6 +4992,10 @@ function convertClaudeHistoryEntryPreamble(
return { proceed: { content } };
}
function isProviderImageMessage(item: AgentTimelineItem): boolean {
return item.type === "assistant_message" && isProviderImageMarkdown(item.text);
}
export function convertClaudeHistoryEntry(
entry: ClaudeHistoryEntry,
mapBlocks: (content: string | ClaudeContentChunk[]) => AgentTimelineItem[],
@@ -4979,7 +5039,12 @@ export function convertClaudeHistoryEntry(
if (hasToolBlock && normalizedBlocks) {
const mapped = mapBlocks(normalizedBlocks);
if (entry.type === "user") {
const toolItems = mapped.filter((item) => item.type === "tool_call");
// tool_result handling (handleToolResult) emits image markdown as an assistant_message
// alongside the tool_call. User-entry text blocks also map to assistant_message in this path
// and must stay suppressed, so keep tool_calls plus only the image assistant_messages.
const toolItems = mapped.filter(
(item) => item.type === "tool_call" || isProviderImageMessage(item),
);
return timeline.length ? [...timeline, ...toolItems] : toolItems;
}
return mapped;
@@ -39,7 +39,6 @@ import type { Logger } from "pino";
import type { ChildProcess, ChildProcessWithoutNullStreams } from "node:child_process";
import { randomUUID } from "node:crypto";
import { Dirent } from "node:fs";
import * as fsSync from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
@@ -79,6 +78,7 @@ import {
} from "./codex/app-server-transport.js";
import { type CodexUserMessageTurnIndex, revertCodexConversation } from "./codex/rewind.js";
import {
materializeProviderImage,
renderProviderImageOutputAsAssistantMarkdown,
type ProviderImageOutput,
} from "./provider-image-output.js";
@@ -114,7 +114,6 @@ function isCodexAlreadyUnarchivedError(error: unknown, threadId: string): boolea
const TURN_START_TIMEOUT_MS = 90 * 1000;
const INTERRUPT_TIMEOUT_MS = 2_000;
const CODEX_PROVIDER = "codex" as const;
const CODEX_IMAGE_ATTACHMENT_DIR = "paseo-attachments";
// Codex treats most app-server client names as the model-request originator.
// This reserved Codex name is non-originating, so requests keep Codex's default
// CLI identity instead of showing up as Paseo in provider usage logs.
@@ -1626,25 +1625,6 @@ function codexImageOutputFromResult(result: unknown): ProviderImageOutput | null
};
}
function writeImageAttachmentSync(mimeType: string, data: string): string {
const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR);
fsSync.mkdirSync(attachmentsDir, { recursive: true });
const normalized = normalizeImageData(mimeType, data);
const extension = getImageExtension(normalized.mimeType);
const filename = `${randomUUID()}.${extension}`;
const filePath = path.join(attachmentsDir, filename);
fsSync.writeFileSync(filePath, Buffer.from(normalized.data, "base64"));
return filePath;
}
function materializeCodexImageOutput(image: { data: string; mimeType: string | null }): {
path: string;
} {
return {
path: writeImageAttachmentSync(image.mimeType ?? "image/png", image.data),
};
}
function mapCodexThreadImageItem(
normalizedType: string,
normalizedItem: Record<string, unknown>,
@@ -1664,7 +1644,7 @@ function mapCodexThreadImageItem(
data: result?.data ?? null,
mimeType: result?.mimeType ?? null,
},
{ materialize: materializeCodexImageOutput },
{ materialize: materializeProviderImage },
);
}
@@ -1809,40 +1789,6 @@ function toSandboxPolicy(type: string, networkAccess?: boolean): Record<string,
}
}
function getImageExtension(mimeType: string): string {
switch (mimeType) {
case "image/jpeg":
return "jpg";
case "image/png":
return "png";
case "image/webp":
return "webp";
case "image/gif":
return "gif";
case "image/bmp":
return "bmp";
case "image/tiff":
return "tiff";
default:
return "bin";
}
}
interface ImageDataPayload {
mimeType: string;
data: string;
}
function normalizeImageData(mimeType: string, data: string): ImageDataPayload {
if (data.startsWith("data:")) {
const match = data.match(/^data:([^;]+);base64,(.*)$/);
if (match) {
return { mimeType: match[1], data: match[2] };
}
}
return { mimeType, data };
}
const ThreadStartedNotificationSchema = z
.object({
thread: z.object({ id: z.string() }).passthrough(),
@@ -2691,17 +2637,6 @@ const CodexNotificationSchema = z.union([
),
]);
async function writeImageAttachment(mimeType: string, data: string): Promise<string> {
const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR);
await fs.mkdir(attachmentsDir, { recursive: true });
const normalized = normalizeImageData(mimeType, data);
const extension = getImageExtension(normalized.mimeType);
const filename = `${randomUUID()}.${extension}`;
const filePath = path.join(attachmentsDir, filename);
await fs.writeFile(filePath, Buffer.from(normalized.data, "base64"));
return filePath;
}
async function readCodexConfiguredDefaults(
client: CodexAppServerClient,
logger: Logger,
@@ -2794,7 +2729,10 @@ export async function codexAppServerTurnInputFromPrompt(
}
if (block.type === "image") {
try {
const filePath = await writeImageAttachment(block.mimeType, block.data);
const filePath = materializeProviderImage({
data: block.data,
mimeType: block.mimeType,
}).path;
output.push({ type: "localImage", path: filePath });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
@@ -0,0 +1,28 @@
import { describe, expect, test } from "vitest";
import { isProviderImageMarkdown } from "./provider-image-output.js";
const HASH = "a".repeat(64);
describe("isProviderImageMarkdown", () => {
test("matches the markdown emitted for a materialized attachment", () => {
expect(isProviderImageMarkdown(`![Image](/tmp/paseo-attachments/${HASH}.png)`)).toBe(true);
expect(isProviderImageMarkdown(`![shot](/var/folders/x/paseo-attachments/${HASH}.webp)`)).toBe(
true,
);
// Windows: backslash path separators are doubled by escapeMarkdownImageSource.
expect(
isProviderImageMarkdown(
`![Image](C:\\\\Users\\\\me\\\\AppData\\\\Local\\\\Temp\\\\paseo-attachments\\\\${HASH}.png)`,
),
).toBe(true);
});
test("rejects user-authored markdown that is not a materialized attachment", () => {
// No content hash — a hand-written path, not something the writer produced.
expect(isProviderImageMarkdown("![diagram](./paseo-attachments/notes.png)")).toBe(false);
expect(isProviderImageMarkdown("![logo](https://example.com/logo.png)")).toBe(false);
// Image markdown that does not start the text.
expect(isProviderImageMarkdown("see the chart: ![chart](x.png)")).toBe(false);
});
});
@@ -1,3 +1,8 @@
import { createHash } from "node:crypto";
import * as fsSync from "node:fs";
import os from "node:os";
import path from "node:path";
import type { AgentTimelineItem } from "../agent-sdk-types.js";
export interface ProviderImageOutput {
@@ -12,6 +17,68 @@ export interface MaterializedProviderImage {
path: string;
}
const PROVIDER_IMAGE_ATTACHMENT_DIR = "paseo-attachments";
function getImageExtension(mimeType: string): string {
switch (mimeType) {
case "image/jpeg":
return "jpg";
case "image/png":
return "png";
case "image/webp":
return "webp";
case "image/gif":
return "gif";
case "image/bmp":
return "bmp";
case "image/tiff":
return "tiff";
default:
return "bin";
}
}
function normalizeImageData(mimeType: string, data: string): { mimeType: string; data: string } {
if (data.startsWith("data:")) {
const match = data.match(/^data:([^;]+);base64,(.*)$/);
if (match) {
return { mimeType: match[1], data: match[2] };
}
}
return { mimeType, data };
}
// Filenames are a content hash of the bytes so re-materializing the same image
// is idempotent: history replay reuses the existing temp file instead of leaking
// a fresh one on every load.
export function materializeProviderImage(image: {
data: string;
mimeType: string | null;
}): MaterializedProviderImage {
const attachmentsDir = path.join(os.tmpdir(), PROVIDER_IMAGE_ATTACHMENT_DIR);
fsSync.mkdirSync(attachmentsDir, { recursive: true });
const normalized = normalizeImageData(image.mimeType ?? "image/png", image.data);
const bytes = Buffer.from(normalized.data, "base64");
const extension = getImageExtension(normalized.mimeType);
const hash = createHash("sha256").update(bytes).digest("hex");
const filePath = path.join(attachmentsDir, `${hash}.${extension}`);
fsSync.writeFileSync(filePath, bytes);
return { path: filePath };
}
// Recognizes the markdown renderProviderImageOutputAsAssistantMarkdown emits for a materialized
// provider image: its source is a content-hashed file in the attachments dir. Matching the full
// <hash>.<ext> shape (not just a leading "![") keeps user-authored text from being mistaken for a
// provider image when it reaches the history-replay filter. The separator class allows one-or-more
// because on Windows the path uses "\\" and escapeMarkdownImageSource doubles each backslash.
const PROVIDER_IMAGE_MARKDOWN = new RegExp(
`^!\\[[^\\]]*\\]\\([^)]*${PROVIDER_IMAGE_ATTACHMENT_DIR}[/\\\\]+[0-9a-f]{64}\\.[a-z0-9]+\\)`,
);
export function isProviderImageMarkdown(text: string): boolean {
return PROVIDER_IMAGE_MARKDOWN.test(text);
}
interface RenderProviderImageOutputOptions {
materialize?: (image: { data: string; mimeType: string | null }) => MaterializedProviderImage;
}