Merge pull request #60 from getpaseo/feature/image-paste-attachment

feat(app): paste images into prompt attachments

packages/app/e2e/paste-image-attachment.spec.ts

@@ -0,0 +1,53 @@
+import { test, expect } from "./fixtures";
+import { ensureHostSelected, gotoHome, setWorkingDirectory } from "./helpers/app";
+import { createTempGitRepo } from "./helpers/workspace";
+
+test("pastes clipboard image into prompt attachments", async ({ page }) => {
+  const repo = await createTempGitRepo("paseo-e2e-paste-image-");
+
+  try {
+    await gotoHome(page);
+    await setWorkingDirectory(page, repo.path);
+    await ensureHostSelected(page);
+
+    const input = page.getByRole("textbox", { name: "Message agent..." });
+    await expect(input).toBeEditable();
+    await input.focus();
+
+    const result = await page.evaluate(() => {
+      const active = document.activeElement;
+      if (!(active instanceof HTMLTextAreaElement)) {
+        return {
+          pasted: false,
+          elementTag: active ? active.tagName : null,
+          defaultPrevented: false,
+        };
+      }
+
+      const file = new File([new Uint8Array([0, 1, 2, 3])], "paste.png", {
+        type: "image/png",
+      });
+      const dataTransfer = new DataTransfer();
+      dataTransfer.items.add(file);
+
+      const event = new ClipboardEvent("paste", {
+        clipboardData: dataTransfer,
+        bubbles: true,
+        cancelable: true,
+      });
+      active.dispatchEvent(event);
+
+      return {
+        pasted: true,
+        elementTag: active.tagName,
+        defaultPrevented: event.defaultPrevented,
+      };
+    });
+
+    expect(result.pasted).toBe(true);
+    expect(result.defaultPrevented).toBe(true);
+    await expect(page.getByTestId("message-input-image-pill")).toHaveCount(1);
+  } finally {
+    await repo.cleanup();
+  }
+});

packages/app/src/components/agent-input-area.tsx

@@ -667,6 +667,7 @@ export function AgentInputArea({
             isSubmitLoading={isProcessing || isSubmitLoading}
             images={selectedImages}
             onPickImages={handlePickImage}
+            onAddImages={addImages}
             onRemoveImage={handleRemoveImage}
             client={client}
             placeholder="Message agent..."

packages/app/src/components/message-input.tsx

@@ -24,6 +24,10 @@ import { useSessionStore } from '@/stores/session-store'
 import { useVoiceOptional } from '@/contexts/voice-context'
 import { useToast } from '@/contexts/toast-context'
 import { resolveVoiceUnavailableMessage } from '@/utils/server-info-capabilities'
+import {
+  collectImageFilesFromClipboardData,
+  filesToImageAttachments,
+} from '@/utils/image-attachments-from-files'
 import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
 import { Shortcut } from '@/components/ui/shortcut'
 import type { MessageInputKeyboardActionKind } from '@/keyboard/actions'
@@ -48,6 +52,7 @@ export interface MessageInputProps {
   isSubmitLoading?: boolean
   images?: ImageAttachment[]
   onPickImages?: () => void
+  onAddImages?: (images: ImageAttachment[]) => void
   onRemoveImage?: (index: number) => void
   client: DaemonClient | null
   placeholder?: string
@@ -113,6 +118,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
     isSubmitLoading = false,
     images = [],
     onPickImages,
+    onAddImages,
     onRemoveImage,
     client,
     placeholder = 'Message...',
@@ -446,7 +452,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
     return typeof v === 'object' && v !== null && 'scrollHeight' in v
   }
 
-  function getWebTextArea(): TextAreaHandle | null {
+  const getWebTextArea = useCallback((): TextAreaHandle | null => {
     const ref = textInputRef.current
     if (!ref) return null
     if (typeof (ref as any).getNativeRef === 'function') {
@@ -455,7 +461,60 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
     }
     if (isTextAreaLike(ref)) return ref
     return null
-  }
+  }, [])
+
+  useEffect(() => {
+    if (!IS_WEB || !onAddImages) {
+      return
+    }
+
+    const textarea = getWebTextArea()
+    if (
+      !textarea ||
+      typeof (textarea as any).addEventListener !== 'function' ||
+      typeof (textarea as any).removeEventListener !== 'function'
+    ) {
+      return
+    }
+
+    let disposed = false
+    const handlePaste = (event: ClipboardEvent) => {
+      if (!isConnected || disabled || isDictating || isRealtimeVoiceForCurrentAgent) {
+        return
+      }
+
+      const imageFiles = collectImageFilesFromClipboardData(event.clipboardData)
+      if (imageFiles.length === 0) {
+        return
+      }
+
+      event.preventDefault()
+
+      void filesToImageAttachments(imageFiles)
+        .then((attachments) => {
+          if (disposed || attachments.length === 0) {
+            return
+          }
+          onAddImages(attachments)
+        })
+        .catch((error) => {
+          console.error('[MessageInput] Failed to process pasted images:', error)
+        })
+    }
+
+    ;(textarea as any).addEventListener('paste', handlePaste)
+    return () => {
+      disposed = true
+      ;(textarea as any).removeEventListener('paste', handlePaste)
+    }
+  }, [
+    disabled,
+    getWebTextArea,
+    isConnected,
+    isDictating,
+    isRealtimeVoiceForCurrentAgent,
+    onAddImages,
+  ])
 
   function measureWebInputHeight(source: string): boolean {
     if (!IS_WEB) return false
@@ -568,10 +627,11 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
       <Animated.View style={[styles.inputWrapper, inputAnimatedStyle]}>
         {/* Image preview pills */}
         {hasImages && (
-          <View style={styles.imagePreviewContainer}>
+          <View style={styles.imagePreviewContainer} testID="message-input-image-preview">
             {images.map((image, index) => (
               <Pressable
                 key={`${image.uri}-${index}`}
+                testID="message-input-image-pill"
                 style={styles.imagePill}
                 onPress={onRemoveImage ? () => onRemoveImage(index) : undefined}
               >

packages/app/src/utils/image-attachments-from-files.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+import {
+  collectImageFilesFromClipboardData,
+  filesToImageAttachments,
+} from "./image-attachments-from-files";
+
+function createClipboardItem(params: {
+  kind: string;
+  type: string;
+  file?: File | null;
+}) {
+  return {
+    kind: params.kind,
+    type: params.type,
+    getAsFile: () => params.file ?? null,
+  };
+}
+
+describe("collectImageFilesFromClipboardData", () => {
+  it("returns only image files from clipboard items", () => {
+    const imagePng = new File([new Uint8Array([0, 1, 2, 3])], "paste.png", {
+      type: "image/png",
+    });
+    const textFile = new File(["not image"], "notes.txt", {
+      type: "text/plain",
+    });
+
+    const files = collectImageFilesFromClipboardData({
+      items: [
+        createClipboardItem({ kind: "string", type: "text/plain" }),
+        createClipboardItem({
+          kind: "file",
+          type: "text/plain",
+          file: textFile,
+        }),
+        createClipboardItem({
+          kind: "file",
+          type: "image/png",
+          file: imagePng,
+        }),
+        createClipboardItem({
+          kind: "file",
+          type: "image/jpeg",
+          file: null,
+        }),
+      ],
+    });
+
+    expect(files).toEqual([imagePng]);
+  });
+
+  it("returns an empty array when clipboard data is missing", () => {
+    expect(collectImageFilesFromClipboardData(undefined)).toEqual([]);
+  });
+});
+
+describe("filesToImageAttachments", () => {
+  it("converts files into data URI image attachments and keeps order", async () => {
+    const pngFile = new File([new Uint8Array([0, 1, 2, 3])], "first.png", {
+      type: "image/png",
+    });
+    const typeLessFile = new File([new Uint8Array([4, 5, 6, 7])], "second", {
+      type: "",
+    });
+
+    const attachments = await filesToImageAttachments([pngFile, typeLessFile]);
+
+    expect(attachments).toEqual([
+      {
+        uri: "data:image/png;base64,AAECAw==",
+        mimeType: "image/png",
+      },
+      {
+        uri: "data:image/jpeg;base64,BAUGBw==",
+        mimeType: "image/jpeg",
+      },
+    ]);
+  });
+});

packages/app/src/utils/image-attachments-from-files.ts

@@ -0,0 +1,67 @@
+export interface ClipboardItemLike {
+  kind?: string;
+  type?: string;
+  getAsFile?: () => File | null;
+}
+
+export interface ClipboardDataLike {
+  items?: ArrayLike<ClipboardItemLike> | null;
+}
+
+export interface ImageAttachmentFromFile {
+  uri: string;
+  mimeType: string;
+}
+
+function toBase64(buffer: ArrayBuffer): string {
+  const bytes = new Uint8Array(buffer);
+  const chunkSize = 0x8000;
+  let binary = "";
+
+  for (let index = 0; index < bytes.length; index += chunkSize) {
+    const chunk = bytes.subarray(index, index + chunkSize);
+    binary += String.fromCharCode(...chunk);
+  }
+
+  return btoa(binary);
+}
+
+export function collectImageFilesFromClipboardData(
+  clipboardData?: ClipboardDataLike | null
+): File[] {
+  if (!clipboardData?.items) {
+    return [];
+  }
+
+  const files: File[] = [];
+  for (const item of Array.from(clipboardData.items)) {
+    if (item?.kind !== "file") {
+      continue;
+    }
+    if (!item.type?.startsWith("image/")) {
+      continue;
+    }
+    const file = item.getAsFile?.();
+    if (!file) {
+      continue;
+    }
+    files.push(file);
+  }
+
+  return files;
+}
+
+export async function filesToImageAttachments(
+  files: readonly File[]
+): Promise<ImageAttachmentFromFile[]> {
+  return Promise.all(
+    files.map(async (file) => {
+      const mimeType = file.type || "image/jpeg";
+      const base64 = toBase64(await file.arrayBuffer());
+      return {
+        uri: `data:${mimeType};base64,${base64}`,
+        mimeType,
+      };
+    })
+  );
+}