From 69e009990c77a21f97a5832904099a2e6688181d Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 18 Feb 2026 17:55:13 +0700 Subject: [PATCH] feat(app): support pasting images into prompt attachments --- .../app/e2e/paste-image-attachment.spec.ts | 53 +++++++++ .../app/src/components/agent-input-area.tsx | 1 + packages/app/src/components/message-input.tsx | 105 ++++++++++++++---- .../image-attachments-from-files.test.ts | 79 +++++++++++++ .../src/utils/image-attachments-from-files.ts | 67 +++++++++++ 5 files changed, 284 insertions(+), 21 deletions(-) create mode 100644 packages/app/e2e/paste-image-attachment.spec.ts create mode 100644 packages/app/src/utils/image-attachments-from-files.test.ts create mode 100644 packages/app/src/utils/image-attachments-from-files.ts diff --git a/packages/app/e2e/paste-image-attachment.spec.ts b/packages/app/e2e/paste-image-attachment.spec.ts new file mode 100644 index 000000000..b675b3669 --- /dev/null +++ b/packages/app/e2e/paste-image-attachment.spec.ts @@ -0,0 +1,53 @@ +import { test, expect } from "./fixtures"; +import { ensureHostSelected, gotoHome, setWorkingDirectory } from "./helpers/app"; +import { createTempGitRepo } from "./helpers/workspace"; + +test("pastes clipboard image into prompt attachments", async ({ page }) => { + const repo = await createTempGitRepo("paseo-e2e-paste-image-"); + + try { + await gotoHome(page); + await setWorkingDirectory(page, repo.path); + await ensureHostSelected(page); + + const input = page.getByRole("textbox", { name: "Message agent..." }); + await expect(input).toBeEditable(); + await input.focus(); + + const result = await page.evaluate(() => { + const active = document.activeElement; + if (!(active instanceof HTMLTextAreaElement)) { + return { + pasted: false, + elementTag: active ? active.tagName : null, + defaultPrevented: false, + }; + } + + const file = new File([new Uint8Array([0, 1, 2, 3])], "paste.png", { + type: "image/png", + }); + const dataTransfer = new DataTransfer(); + dataTransfer.items.add(file); + + const event = new ClipboardEvent("paste", { + clipboardData: dataTransfer, + bubbles: true, + cancelable: true, + }); + active.dispatchEvent(event); + + return { + pasted: true, + elementTag: active.tagName, + defaultPrevented: event.defaultPrevented, + }; + }); + + expect(result.pasted).toBe(true); + expect(result.defaultPrevented).toBe(true); + await expect(page.getByTestId("message-input-image-pill")).toHaveCount(1); + } finally { + await repo.cleanup(); + } +}); diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index 49aa47c82..672d485f2 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -711,6 +711,7 @@ export function AgentInputArea({ isSubmitLoading={isProcessing || isSubmitLoading} images={selectedImages} onPickImages={handlePickImage} + onAddImages={addImages} onRemoveImage={handleRemoveImage} client={client} placeholder="Message agent..." diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index 66fd4fcba..26dd3ee5b 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -34,6 +34,10 @@ import { useSessionStore } from "@/stores/session-store"; import { useVoiceOptional } from "@/contexts/voice-context"; import { useToast } from "@/contexts/toast-context"; import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities"; +import { + collectImageFilesFromClipboardData, + filesToImageAttachments, +} from "@/utils/image-attachments-from-files"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { Shortcut } from "@/components/ui/shortcut"; import type { MessageInputKeyboardActionKind } from "@/keyboard/actions"; @@ -58,6 +62,7 @@ export interface MessageInputProps { isSubmitLoading?: boolean; images?: ImageAttachment[]; onPickImages?: () => void; + onAddImages?: (images: ImageAttachment[]) => void; onRemoveImage?: (index: number) => void; client: DaemonClient | null; placeholder?: string; @@ -122,6 +127,7 @@ export const MessageInput = forwardRef( isSubmitLoading = false, images = [], onPickImages, + onAddImages, onRemoveImage, client, placeholder = "Message...", @@ -142,11 +148,11 @@ export const MessageInput = forwardRef( const { theme } = useUnistyles(); const toast = useToast(); const voice = useVoiceOptional(); - const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT); - const textInputRef = useRef< - TextInput | (TextInput & { getNativeRef?: () => unknown }) | null - >(null); - const isInputFocusedRef = useRef(false); + const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT); + const textInputRef = useRef< + TextInput | (TextInput & { getNativeRef?: () => unknown }) | null + >(null); + const isInputFocusedRef = useRef(false); useImperativeHandle(ref, () => ({ focus: () => { @@ -218,19 +224,19 @@ export const MessageInput = forwardRef( ) ); - useEffect(() => { - valueRef.current = value; - }, [value]); + useEffect(() => { + valueRef.current = value; + }, [value]); - // Autofocus on web when autoFocus prop is true - useEffect(() => { - if (!IS_WEB || !autoFocus) return; - // Use requestAnimationFrame to ensure DOM is ready - const rafId = requestAnimationFrame(() => { - textInputRef.current?.focus(); - }); - return () => cancelAnimationFrame(rafId); - }, [autoFocus]); + // Autofocus on web when autoFocus prop is true + useEffect(() => { + if (!IS_WEB || !autoFocus) return; + // Use requestAnimationFrame to ensure DOM is ready + const rafId = requestAnimationFrame(() => { + textInputRef.current?.focus(); + }); + return () => cancelAnimationFrame(rafId); + }, [autoFocus]); const handleDictationTranscript = useCallback( (text: string, _meta: { requestId: string }) => { @@ -472,7 +478,7 @@ export const MessageInput = forwardRef( return typeof v === "object" && v !== null && "scrollHeight" in v; } - function getWebTextArea(): TextAreaHandle | null { + const getWebTextArea = useCallback((): TextAreaHandle | null => { const ref = textInputRef.current; if (!ref) return null; if (typeof (ref as any).getNativeRef === "function") { @@ -481,9 +487,62 @@ export const MessageInput = forwardRef( } if (isTextAreaLike(ref)) return ref; return null; - } + }, []); - function measureWebInputHeight(source: string): boolean { + useEffect(() => { + if (!IS_WEB || !onAddImages) { + return; + } + + const textarea = getWebTextArea(); + if ( + !textarea || + typeof (textarea as any).addEventListener !== "function" || + typeof (textarea as any).removeEventListener !== "function" + ) { + return; + } + + let disposed = false; + const handlePaste = (event: ClipboardEvent) => { + if (!isConnected || disabled || isDictating || isRealtimeVoiceForCurrentAgent) { + return; + } + + const imageFiles = collectImageFilesFromClipboardData(event.clipboardData); + if (imageFiles.length === 0) { + return; + } + + event.preventDefault(); + + void filesToImageAttachments(imageFiles) + .then((attachments) => { + if (disposed || attachments.length === 0) { + return; + } + onAddImages(attachments); + }) + .catch((error) => { + console.error("[MessageInput] Failed to process pasted images:", error); + }); + }; + + (textarea as any).addEventListener("paste", handlePaste); + return () => { + disposed = true; + (textarea as any).removeEventListener("paste", handlePaste); + }; + }, [ + disabled, + isConnected, + isDictating, + isRealtimeVoiceForCurrentAgent, + getWebTextArea, + onAddImages, + ]); + + function measureWebInputHeight(_source: string): boolean { if (!IS_WEB) return false; const textarea = getWebTextArea(); if (!textarea || typeof textarea.scrollHeight !== "number") return false; @@ -596,10 +655,14 @@ export const MessageInput = forwardRef( {/* Image preview pills */} {hasImages && ( - + {images.map((image, index) => ( onRemoveImage(index) : undefined} > diff --git a/packages/app/src/utils/image-attachments-from-files.test.ts b/packages/app/src/utils/image-attachments-from-files.test.ts new file mode 100644 index 000000000..70664c989 --- /dev/null +++ b/packages/app/src/utils/image-attachments-from-files.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import { + collectImageFilesFromClipboardData, + filesToImageAttachments, +} from "./image-attachments-from-files"; + +function createClipboardItem(params: { + kind: string; + type: string; + file?: File | null; +}) { + return { + kind: params.kind, + type: params.type, + getAsFile: () => params.file ?? null, + }; +} + +describe("collectImageFilesFromClipboardData", () => { + it("returns only image files from clipboard items", () => { + const imagePng = new File([new Uint8Array([0, 1, 2, 3])], "paste.png", { + type: "image/png", + }); + const textFile = new File(["not image"], "notes.txt", { + type: "text/plain", + }); + + const files = collectImageFilesFromClipboardData({ + items: [ + createClipboardItem({ kind: "string", type: "text/plain" }), + createClipboardItem({ + kind: "file", + type: "text/plain", + file: textFile, + }), + createClipboardItem({ + kind: "file", + type: "image/png", + file: imagePng, + }), + createClipboardItem({ + kind: "file", + type: "image/jpeg", + file: null, + }), + ], + }); + + expect(files).toEqual([imagePng]); + }); + + it("returns an empty array when clipboard data is missing", () => { + expect(collectImageFilesFromClipboardData(undefined)).toEqual([]); + }); +}); + +describe("filesToImageAttachments", () => { + it("converts files into data URI image attachments and keeps order", async () => { + const pngFile = new File([new Uint8Array([0, 1, 2, 3])], "first.png", { + type: "image/png", + }); + const typeLessFile = new File([new Uint8Array([4, 5, 6, 7])], "second", { + type: "", + }); + + const attachments = await filesToImageAttachments([pngFile, typeLessFile]); + + expect(attachments).toEqual([ + { + uri: "data:image/png;base64,AAECAw==", + mimeType: "image/png", + }, + { + uri: "data:image/jpeg;base64,BAUGBw==", + mimeType: "image/jpeg", + }, + ]); + }); +}); diff --git a/packages/app/src/utils/image-attachments-from-files.ts b/packages/app/src/utils/image-attachments-from-files.ts new file mode 100644 index 000000000..cfff51c6a --- /dev/null +++ b/packages/app/src/utils/image-attachments-from-files.ts @@ -0,0 +1,67 @@ +export interface ClipboardItemLike { + kind?: string; + type?: string; + getAsFile?: () => File | null; +} + +export interface ClipboardDataLike { + items?: ArrayLike | null; +} + +export interface ImageAttachmentFromFile { + uri: string; + mimeType: string; +} + +function toBase64(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer); + const chunkSize = 0x8000; + let binary = ""; + + for (let index = 0; index < bytes.length; index += chunkSize) { + const chunk = bytes.subarray(index, index + chunkSize); + binary += String.fromCharCode(...chunk); + } + + return btoa(binary); +} + +export function collectImageFilesFromClipboardData( + clipboardData?: ClipboardDataLike | null +): File[] { + if (!clipboardData?.items) { + return []; + } + + const files: File[] = []; + for (const item of Array.from(clipboardData.items)) { + if (item?.kind !== "file") { + continue; + } + if (!item.type?.startsWith("image/")) { + continue; + } + const file = item.getAsFile?.(); + if (!file) { + continue; + } + files.push(file); + } + + return files; +} + +export async function filesToImageAttachments( + files: readonly File[] +): Promise { + return Promise.all( + files.map(async (file) => { + const mimeType = file.type || "image/jpeg"; + const base64 = toBase64(await file.arrayBuffer()); + return { + uri: `data:${mimeType};base64,${base64}`, + mimeType, + }; + }) + ); +}