Add multimodal prompt blocks for agent prompts

packages/server/src/server/agent/agent-sdk-types.ts

@@ -36,7 +36,11 @@ export type AgentPersistenceHandle = {
   metadata?: AgentMetadata;
 };
 
-export type AgentPromptInput = string | { type: "text"; text: string }[];
+export type AgentPromptContentBlock =
+  | { type: "text"; text: string }
+  | { type: "image"; data: string; mimeType: string };
+
+export type AgentPromptInput = string | AgentPromptContentBlock[];
 
 export type AgentRunOptions = {
   outputSchema?: unknown;

packages/server/src/server/agent/providers/claude-agent.ts

@@ -29,6 +29,7 @@ import type {
   AgentPermissionResponse,
   AgentPermissionUpdate,
   AgentPersistenceHandle,
+  AgentPromptContentBlock,
   AgentPromptInput,
   AgentRunOptions,
   AgentRunResult,
@@ -787,15 +788,32 @@ class ClaudeAgentSession implements AgentSession {
   }
 
   private toSdkUserMessage(prompt: AgentPromptInput): SDKUserMessage {
-    const text = Array.isArray(prompt)
-      ? prompt.map((chunk) => chunk.text).join("\n\n")
-      : prompt;
+    const content = Array.isArray(prompt)
+      ? prompt.flatMap((chunk: AgentPromptContentBlock) => {
+          if (chunk.type === "text") {
+            return [{ type: "text", text: chunk.text }];
+          }
+          if (chunk.type === "image") {
+            return [
+              {
+                type: "image",
+                source: {
+                  type: "base64",
+                  media_type: chunk.mimeType,
+                  data: chunk.data,
+                },
+              },
+            ];
+          }
+          return [];
+        })
+      : [{ type: "text", text: prompt }];
 
     return {
       type: "user",
       message: {
         role: "user",
-        content: [{ type: "text", text }],
+        content,
       },
       parent_tool_use_id: null,
       session_id: this.claudeSessionId ?? this.pendingLocalId,

packages/server/src/server/agent/providers/codex-mcp-agent.ts

@@ -2314,7 +2314,18 @@ function toPromptText(prompt: AgentPromptInput): string {
   if (typeof prompt === "string") {
     return prompt;
   }
-  return prompt.map((chunk) => chunk.text).join("");
+  const parts: string[] = [];
+  for (const chunk of prompt) {
+    if (chunk.type === "text") {
+      parts.push(chunk.text);
+      continue;
+    }
+    if (chunk.type === "image") {
+      const dataUrl = `data:${chunk.mimeType};base64,${chunk.data}`;
+      parts.push(`![user image](${dataUrl})`);
+    }
+  }
+  return parts.join("\n\n");
 }
 
 function getCodexMcpCommand(): string {

packages/server/src/server/session.ts

@@ -42,6 +42,7 @@ import type { ManagedAgent } from "./agent/agent-manager.js";
 import { toAgentPayload } from "./agent/agent-projections.js";
 import type {
   AgentPermissionResponse,
+  AgentPromptContentBlock,
   AgentPromptInput,
   AgentSessionConfig,
   AgentStreamEvent,
@@ -321,16 +322,14 @@ export class Session {
     if (!images || images.length === 0) {
       return normalized;
     }
-
-    const attachmentSummary = images
-      .map((image, index) => {
-        const sizeKb = Math.round((image.data.length * 0.75) / 1024);
-        return `Attachment ${index + 1}: ${image.mimeType}, ~${sizeKb}KB base64`;
-      })
-      .join("\n");
-
-    const base = normalized.length > 0 ? normalized : "User shared image attachment(s).";
-    return `${base}\n\n[Image attachments]\n${attachmentSummary}\n(Actual image bytes omitted; request a screenshot or file if needed.)`;
+    const blocks: AgentPromptContentBlock[] = [];
+    if (normalized.length > 0) {
+      blocks.push({ type: "text", text: normalized });
+    }
+    for (const image of images) {
+      blocks.push({ type: "image", data: image.data, mimeType: image.mimeType });
+    }
+    return blocks;
   }
 
   /**