diff --git a/packages/server/src/server/agent/providers/claude/agent.image-rendering.test.ts b/packages/server/src/server/agent/providers/claude/agent.image-rendering.test.ts new file mode 100644 index 000000000..194b51e5c --- /dev/null +++ b/packages/server/src/server/agent/providers/claude/agent.image-rendering.test.ts @@ -0,0 +1,225 @@ +import { existsSync, rmSync } from "node:fs"; +import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk"; +import { describe, expect, test } from "vitest"; + +import { createTestLogger } from "../../../../test-utils/test-logger.js"; +import type { AgentStreamEvent, AgentTimelineItem } from "../../agent-sdk-types.js"; +import { ClaudeAgentClient } from "./agent.js"; + +const ONE_BY_ONE_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+X1r0AAAAASUVORK5CYII="; + +interface ClaudeImageTestSession { + translateMessageToEvents(message: SDKMessage): AgentStreamEvent[]; + convertHistoryEntry(entry: unknown): AgentTimelineItem[]; +} + +async function createSession(): Promise { + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + resolveBinary: async () => "/test/claude/bin", + }); + const session = await client.createSession({ provider: "claude", cwd: process.cwd() }); + return session as unknown as ClaudeImageTestSession; +} + +function imageToolResultUserMessage(): SDKMessage { + return { + type: "user", + parent_tool_use_id: null, + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_read_png", + tool_name: "Read", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: ONE_BY_ONE_PNG_BASE64, + }, + }, + ], + }, + ], + }, + uuid: "user-image-result-1", + session_id: "session-1", + } as unknown as SDKMessage; +} + +function imageToolResultHistoryEntry(): unknown { + return { + type: "user", + uuid: "user-image-result-1", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_read_png", + tool_name: "Read", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: ONE_BY_ONE_PNG_BASE64, + }, + }, + ], + }, + ], + }, + }; +} + +function erroredImageToolResultUserMessage(): SDKMessage { + return { + type: "user", + parent_tool_use_id: null, + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_read_png", + tool_name: "Read", + is_error: true, + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: ONE_BY_ONE_PNG_BASE64, + }, + }, + ], + }, + ], + }, + uuid: "user-image-error-1", + session_id: "session-1", + } as unknown as SDKMessage; +} + +function multiImageToolResultUserMessage(): SDKMessage { + const imageBlock = { + type: "image", + source: { type: "base64", media_type: "image/png", data: ONE_BY_ONE_PNG_BASE64 }, + }; + return { + type: "user", + parent_tool_use_id: null, + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_read_png", + tool_name: "Read", + content: [imageBlock, imageBlock], + }, + ], + }, + uuid: "user-image-result-multi", + session_id: "session-1", + } as unknown as SDKMessage; +} + +function imageMessages(items: AgentTimelineItem[]): string[] { + return items + .filter((item) => item.type === "assistant_message") + .map((item) => (item as { text: string }).text) + .filter((text) => text.startsWith("![")); +} + +function markdownImageSource(markdown: string): string { + const match = markdown.match(/^!\[[^\]]*]\((.*)\)$/); + if (!match) { + throw new Error(`Expected markdown image, got: ${markdown}`); + } + // Reverse escapeMarkdownImageSource: "\\" -> "\" and "\)" -> ")" (Windows paths are escaped). + return match[1].replace(/\\(.)/g, "$1"); +} + +describe("Claude tool_result image rendering", () => { + test("emits the image as assistant markdown and keeps base64 out of the live tool output", async () => { + const session = await createSession(); + + const events = session.translateMessageToEvents(imageToolResultUserMessage()); + + const timelineItems = events + .filter((event) => event.type === "timeline") + .map((event) => (event as { item: AgentTimelineItem }).item); + const [imageMessage, ...extraImages] = imageMessages(timelineItems); + expect(extraImages).toEqual([]); + + const source = markdownImageSource(imageMessage); + expect(source).toMatch(/paseo-attachments[\\/][0-9a-f]{64}\.png$/); + expect(existsSync(source)).toBe(true); + expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64); + + rmSync(source, { force: true }); + }); + + test("replays the image as assistant markdown through history conversion", async () => { + const session = await createSession(); + + const items = session.convertHistoryEntry(imageToolResultHistoryEntry()); + + const [imageMessage, ...extraImages] = imageMessages(items); + expect(extraImages).toEqual([]); + + const source = markdownImageSource(imageMessage); + expect(source).toMatch(/paseo-attachments[\\/][0-9a-f]{64}\.png$/); + expect(existsSync(source)).toBe(true); + expect(JSON.stringify(items)).not.toContain(ONE_BY_ONE_PNG_BASE64); + + rmSync(source, { force: true }); + }); + + test("keeps base64 out of an errored tool_result that carries an image", async () => { + const session = await createSession(); + + const events = session.translateMessageToEvents(erroredImageToolResultUserMessage()); + + const timelineItems = events + .filter((event) => event.type === "timeline") + .map((event) => (event as { item: AgentTimelineItem }).item); + const [imageMessage, ...extraImages] = imageMessages(timelineItems); + expect(extraImages).toEqual([]); + + const source = markdownImageSource(imageMessage); + expect(existsSync(source)).toBe(true); + expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64); + expect(JSON.stringify(events)).toContain("[image]"); + + rmSync(source, { force: true }); + }); + + test("emits one image message per image block in a multi-image tool_result", async () => { + const session = await createSession(); + + const events = session.translateMessageToEvents(multiImageToolResultUserMessage()); + + const timelineItems = events + .filter((event) => event.type === "timeline") + .map((event) => (event as { item: AgentTimelineItem }).item); + const sources = imageMessages(timelineItems).map(markdownImageSource); + + expect(sources).toHaveLength(2); + // Identical bytes materialize to one content-hashed file (idempotent), one message per block. + expect(new Set(sources).size).toBe(1); + expect(existsSync(sources[0])).toBe(true); + expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64); + + rmSync(sources[0], { force: true }); + }); +}); diff --git a/packages/server/src/server/agent/providers/claude/agent.redesign.test.ts b/packages/server/src/server/agent/providers/claude/agent.redesign.test.ts index 83be9f5ba..f43bbacf7 100644 --- a/packages/server/src/server/agent/providers/claude/agent.redesign.test.ts +++ b/packages/server/src/server/agent/providers/claude/agent.redesign.test.ts @@ -677,6 +677,7 @@ test("maps tool_result content shapes into deterministic string output", async ( const session = await createSession(); const internal: { buildToolOutput: ( + content: unknown, block: Record, entry: Record | undefined, ) => Record | undefined; @@ -724,6 +725,7 @@ test("maps tool_result content shapes into deterministic string output", async ( try { for (const fixture of fixtures) { const output = internal.buildToolOutput( + fixture.content, { type: "tool_result", tool_use_id: "tool-1", @@ -750,6 +752,7 @@ test("Grep tool_result string content flows to a search detail with content", as const session = await createSession(); const internal: { buildToolOutput: ( + content: unknown, block: Record, entry: Record | undefined, ) => Record | undefined; @@ -765,12 +768,14 @@ test("Grep tool_result string content flows to a search detail with content", as }; try { + const grepContent = "Found 2 files\nsrc/foo.tsx\nsrc/bar.tsx"; const output = internal.buildToolOutput( + grepContent, { type: "tool_result", tool_use_id: "tool-grep-1", tool_name: "Grep", - content: "Found 2 files\nsrc/foo.tsx\nsrc/bar.tsx", + content: grepContent, is_error: false, }, grepEntry, diff --git a/packages/server/src/server/agent/providers/claude/agent.ts b/packages/server/src/server/agent/providers/claude/agent.ts index dbfeb9186..62813bcec 100644 --- a/packages/server/src/server/agent/providers/claude/agent.ts +++ b/packages/server/src/server/agent/providers/claude/agent.ts @@ -45,6 +45,12 @@ import { realClaudeRewindSdk, revertClaudeConversation, revertClaudeFiles } from import { normalizeProviderReplayTimestamp } from "../../provider-history-timestamps.js"; import { claudeProjectDirSync } from "./project-dir.js"; import { SETTING_APPLIES_NEXT_TURN_NOTICE } from "../../provider-notices.js"; +import { + isProviderImageMarkdown, + materializeProviderImage, + renderProviderImageOutputAsAssistantMarkdown, + type ProviderImageOutput, +} from "../provider-image-output.js"; import { getAgentStreamEventTurnId, @@ -590,6 +596,44 @@ function coerceToolResultContentToString(content: unknown): string { return deterministicStringify(content); } +function toBase64ImageOutput(block: unknown): ProviderImageOutput | null { + const record = toObjectRecord(block); + if (!record || record.type !== "image") { + return null; + } + const source = toObjectRecord(record.source); + if (!source || source.type !== "base64" || typeof source.data !== "string") { + return null; + } + return { + data: source.data, + mimeType: typeof source.media_type === "string" ? source.media_type : null, + }; +} + +// Claude returns images inside tool_result content as base64 Anthropic blocks. Left in place they +// reach coerceToolResultContentToString, which JSON.stringifies the whole array — dumping base64 +// into the tool output. We pull those blocks out to render them as image markdown and leave a +// "[image]" placeholder so image-only results still produce non-empty output. +function splitClaudeToolResultImages(content: unknown): { + images: ProviderImageOutput[]; + text: unknown; +} { + if (!Array.isArray(content)) { + return { images: [], text: content }; + } + const images: ProviderImageOutput[] = []; + const text = content.map((block) => { + const image = toBase64ImageOutput(block); + if (image) { + images.push(image); + return { type: "text", text: "[image]" }; + } + return block; + }); + return { images, text }; +} + function normalizeClaudeTranscriptText(value: unknown): string | null { if (typeof value !== "string") { return null; @@ -4353,8 +4397,10 @@ class ClaudeAgentSession implements AgentSession { ? block.tool_use_id : (entry?.id ?? null); - // Extract output from block.content (SDK always returns content in string form) - const output = this.buildToolOutput(block, entry); + // Pull image blocks out of the result so base64 never reaches the tool output, and render each + // one as an assistant_message markdown image after the tool_call (matching how Codex emits). + const { images, text } = splitClaudeToolResultImages(block.content); + const output = this.buildToolOutput(text, block, entry); if (block.is_error) { this.pushToolCall( @@ -4363,7 +4409,7 @@ class ClaudeAgentSession implements AgentSession { callId, input: entry?.input ?? null, output: output ?? null, - error: block, + error: { ...block, content: text }, }), items, ); @@ -4379,6 +4425,15 @@ class ClaudeAgentSession implements AgentSession { ); } + for (const image of images) { + const imageItem = renderProviderImageOutputAsAssistantMarkdown(image, { + materialize: materializeProviderImage, + }); + if (imageItem) { + items.push(imageItem); + } + } + if (typeof block.tool_use_id === "string") { this.toolUseCache.delete(block.tool_use_id); this.sidechainTracker.delete(block.tool_use_id); @@ -4386,6 +4441,7 @@ class ClaudeAgentSession implements AgentSession { } private buildToolOutput( + content: unknown, block: ClaudeContentChunk, entry: ToolUseCacheEntry | undefined, ): AgentMetadata | undefined { @@ -4397,11 +4453,11 @@ class ClaudeAgentSession implements AgentSession { const blockToolName = typeof block.tool_name === "string" ? block.tool_name : undefined; const server = entry?.server ?? blockServer ?? "tool"; const tool = entry?.name ?? blockToolName ?? "tool"; - const content = coerceToolResultContentToString(block.content); + const coercedContent = coerceToolResultContentToString(content); const input = entry?.input; // Build structured result based on tool type - const structured = this.buildStructuredToolResult(server, tool, content, input); + const structured = this.buildStructuredToolResult(server, tool, coercedContent, input); if (structured) { return structured; @@ -4410,13 +4466,13 @@ class ClaudeAgentSession implements AgentSession { // Fallback format - try to parse JSON first const result: AgentMetadata = {}; - if (content.length > 0) { + if (coercedContent.length > 0) { try { // If content is a JSON string, parse it - result.output = JSON.parse(content); + result.output = JSON.parse(coercedContent); } catch { // If not JSON, return unchanged (no extra wrapping) - result.output = content; + result.output = coercedContent; } } @@ -4936,6 +4992,10 @@ function convertClaudeHistoryEntryPreamble( return { proceed: { content } }; } +function isProviderImageMessage(item: AgentTimelineItem): boolean { + return item.type === "assistant_message" && isProviderImageMarkdown(item.text); +} + export function convertClaudeHistoryEntry( entry: ClaudeHistoryEntry, mapBlocks: (content: string | ClaudeContentChunk[]) => AgentTimelineItem[], @@ -4979,7 +5039,12 @@ export function convertClaudeHistoryEntry( if (hasToolBlock && normalizedBlocks) { const mapped = mapBlocks(normalizedBlocks); if (entry.type === "user") { - const toolItems = mapped.filter((item) => item.type === "tool_call"); + // tool_result handling (handleToolResult) emits image markdown as an assistant_message + // alongside the tool_call. User-entry text blocks also map to assistant_message in this path + // and must stay suppressed, so keep tool_calls plus only the image assistant_messages. + const toolItems = mapped.filter( + (item) => item.type === "tool_call" || isProviderImageMessage(item), + ); return timeline.length ? [...timeline, ...toolItems] : toolItems; } return mapped; diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index 634044b30..41d91f7d2 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -39,7 +39,6 @@ import type { Logger } from "pino"; import type { ChildProcess, ChildProcessWithoutNullStreams } from "node:child_process"; import { randomUUID } from "node:crypto"; import { Dirent } from "node:fs"; -import * as fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -79,6 +78,7 @@ import { } from "./codex/app-server-transport.js"; import { type CodexUserMessageTurnIndex, revertCodexConversation } from "./codex/rewind.js"; import { + materializeProviderImage, renderProviderImageOutputAsAssistantMarkdown, type ProviderImageOutput, } from "./provider-image-output.js"; @@ -114,7 +114,6 @@ function isCodexAlreadyUnarchivedError(error: unknown, threadId: string): boolea const TURN_START_TIMEOUT_MS = 90 * 1000; const INTERRUPT_TIMEOUT_MS = 2_000; const CODEX_PROVIDER = "codex" as const; -const CODEX_IMAGE_ATTACHMENT_DIR = "paseo-attachments"; // Codex treats most app-server client names as the model-request originator. // This reserved Codex name is non-originating, so requests keep Codex's default // CLI identity instead of showing up as Paseo in provider usage logs. @@ -1626,25 +1625,6 @@ function codexImageOutputFromResult(result: unknown): ProviderImageOutput | null }; } -function writeImageAttachmentSync(mimeType: string, data: string): string { - const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR); - fsSync.mkdirSync(attachmentsDir, { recursive: true }); - const normalized = normalizeImageData(mimeType, data); - const extension = getImageExtension(normalized.mimeType); - const filename = `${randomUUID()}.${extension}`; - const filePath = path.join(attachmentsDir, filename); - fsSync.writeFileSync(filePath, Buffer.from(normalized.data, "base64")); - return filePath; -} - -function materializeCodexImageOutput(image: { data: string; mimeType: string | null }): { - path: string; -} { - return { - path: writeImageAttachmentSync(image.mimeType ?? "image/png", image.data), - }; -} - function mapCodexThreadImageItem( normalizedType: string, normalizedItem: Record, @@ -1664,7 +1644,7 @@ function mapCodexThreadImageItem( data: result?.data ?? null, mimeType: result?.mimeType ?? null, }, - { materialize: materializeCodexImageOutput }, + { materialize: materializeProviderImage }, ); } @@ -1809,40 +1789,6 @@ function toSandboxPolicy(type: string, networkAccess?: boolean): Record { - const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR); - await fs.mkdir(attachmentsDir, { recursive: true }); - const normalized = normalizeImageData(mimeType, data); - const extension = getImageExtension(normalized.mimeType); - const filename = `${randomUUID()}.${extension}`; - const filePath = path.join(attachmentsDir, filename); - await fs.writeFile(filePath, Buffer.from(normalized.data, "base64")); - return filePath; -} - async function readCodexConfiguredDefaults( client: CodexAppServerClient, logger: Logger, @@ -2794,7 +2729,10 @@ export async function codexAppServerTurnInputFromPrompt( } if (block.type === "image") { try { - const filePath = await writeImageAttachment(block.mimeType, block.data); + const filePath = materializeProviderImage({ + data: block.data, + mimeType: block.mimeType, + }).path; output.push({ type: "localImage", path: filePath }); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/packages/server/src/server/agent/providers/provider-image-output.test.ts b/packages/server/src/server/agent/providers/provider-image-output.test.ts new file mode 100644 index 000000000..8f5410cb1 --- /dev/null +++ b/packages/server/src/server/agent/providers/provider-image-output.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from "vitest"; + +import { isProviderImageMarkdown } from "./provider-image-output.js"; + +const HASH = "a".repeat(64); + +describe("isProviderImageMarkdown", () => { + test("matches the markdown emitted for a materialized attachment", () => { + expect(isProviderImageMarkdown(`![Image](/tmp/paseo-attachments/${HASH}.png)`)).toBe(true); + expect(isProviderImageMarkdown(`![shot](/var/folders/x/paseo-attachments/${HASH}.webp)`)).toBe( + true, + ); + // Windows: backslash path separators are doubled by escapeMarkdownImageSource. + expect( + isProviderImageMarkdown( + `![Image](C:\\\\Users\\\\me\\\\AppData\\\\Local\\\\Temp\\\\paseo-attachments\\\\${HASH}.png)`, + ), + ).toBe(true); + }); + + test("rejects user-authored markdown that is not a materialized attachment", () => { + // No content hash — a hand-written path, not something the writer produced. + expect(isProviderImageMarkdown("![diagram](./paseo-attachments/notes.png)")).toBe(false); + expect(isProviderImageMarkdown("![logo](https://example.com/logo.png)")).toBe(false); + // Image markdown that does not start the text. + expect(isProviderImageMarkdown("see the chart: ![chart](x.png)")).toBe(false); + }); +}); diff --git a/packages/server/src/server/agent/providers/provider-image-output.ts b/packages/server/src/server/agent/providers/provider-image-output.ts index bc726081c..0a4a154f7 100644 --- a/packages/server/src/server/agent/providers/provider-image-output.ts +++ b/packages/server/src/server/agent/providers/provider-image-output.ts @@ -1,3 +1,8 @@ +import { createHash } from "node:crypto"; +import * as fsSync from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import type { AgentTimelineItem } from "../agent-sdk-types.js"; export interface ProviderImageOutput { @@ -12,6 +17,68 @@ export interface MaterializedProviderImage { path: string; } +const PROVIDER_IMAGE_ATTACHMENT_DIR = "paseo-attachments"; + +function getImageExtension(mimeType: string): string { + switch (mimeType) { + case "image/jpeg": + return "jpg"; + case "image/png": + return "png"; + case "image/webp": + return "webp"; + case "image/gif": + return "gif"; + case "image/bmp": + return "bmp"; + case "image/tiff": + return "tiff"; + default: + return "bin"; + } +} + +function normalizeImageData(mimeType: string, data: string): { mimeType: string; data: string } { + if (data.startsWith("data:")) { + const match = data.match(/^data:([^;]+);base64,(.*)$/); + if (match) { + return { mimeType: match[1], data: match[2] }; + } + } + return { mimeType, data }; +} + +// Filenames are a content hash of the bytes so re-materializing the same image +// is idempotent: history replay reuses the existing temp file instead of leaking +// a fresh one on every load. +export function materializeProviderImage(image: { + data: string; + mimeType: string | null; +}): MaterializedProviderImage { + const attachmentsDir = path.join(os.tmpdir(), PROVIDER_IMAGE_ATTACHMENT_DIR); + fsSync.mkdirSync(attachmentsDir, { recursive: true }); + const normalized = normalizeImageData(image.mimeType ?? "image/png", image.data); + const bytes = Buffer.from(normalized.data, "base64"); + const extension = getImageExtension(normalized.mimeType); + const hash = createHash("sha256").update(bytes).digest("hex"); + const filePath = path.join(attachmentsDir, `${hash}.${extension}`); + fsSync.writeFileSync(filePath, bytes); + return { path: filePath }; +} + +// Recognizes the markdown renderProviderImageOutputAsAssistantMarkdown emits for a materialized +// provider image: its source is a content-hashed file in the attachments dir. Matching the full +// . shape (not just a leading "![") keeps user-authored text from being mistaken for a +// provider image when it reaches the history-replay filter. The separator class allows one-or-more +// because on Windows the path uses "\\" and escapeMarkdownImageSource doubles each backslash. +const PROVIDER_IMAGE_MARKDOWN = new RegExp( + `^!\\[[^\\]]*\\]\\([^)]*${PROVIDER_IMAGE_ATTACHMENT_DIR}[/\\\\]+[0-9a-f]{64}\\.[a-z0-9]+\\)`, +); + +export function isProviderImageMarkdown(text: string): boolean { + return PROVIDER_IMAGE_MARKDOWN.test(text); +} + interface RenderProviderImageOutputOptions { materialize?: (image: { data: string; mimeType: string | null }) => MaterializedProviderImage; }