feat: add image upload support to agent chat with attachment preview

2025-10-26 07:35:06 +01:00
parent faf087b3a6
commit a3224175ee
12 changed files with 372 additions and 791 deletions
@@ -4,10 +4,13 @@ import {
  Pressable,
  NativeSyntheticEvent,
  TextInputContentSizeChangeEventData,
+  Image,
+  Alert,
 } from "react-native";
 import { useState } from "react";
 import { StyleSheet, useUnistyles } from "react-native-unistyles";
-import { Mic, ArrowUp, AudioLines, Square } from "lucide-react-native";
+import { Mic, ArrowUp, AudioLines, Square, Paperclip, X } from "lucide-react-native";
+import * as ImagePicker from "expo-image-picker";
 import { useSession } from "@/contexts/session-context";
 import { useRealtime } from "@/contexts/realtime-context";
 import { useAudioRecorder } from "@/hooks/use-audio-recorder";
@@ -31,17 +34,21 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
  const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT);
  const [isRecording, setIsRecording] = useState(false);
  const [isProcessing, setIsProcessing] = useState(false);
+  const [selectedImages, setSelectedImages] = useState<Array<{ uri: string; mimeType: string }>>([]);

  async function handleSendMessage() {
    if (!userInput.trim() || !ws.isConnected) return;

    const message = userInput.trim();
+    const imageUris = selectedImages.length > 0 ? selectedImages.map(img => img.uri) : undefined;
+    
    setUserInput("");
+    setSelectedImages([]);
    setInputHeight(MIN_INPUT_HEIGHT);
    setIsProcessing(true);

    try {
-      sendAgentMessage(agentId, message);
+      await sendAgentMessage(agentId, message, imageUris);
    } catch (error) {
      console.error("[AgentInput] Failed to send message:", error);
    } finally {
@@ -49,6 +56,38 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
    }
  }

+  async function handlePickImage() {
+    try {
+      const permissionResult = await ImagePicker.requestMediaLibraryPermissionsAsync();
+      
+      if (!permissionResult.granted) {
+        Alert.alert("Permission required", "Please allow access to your photo library to attach images.");
+        return;
+      }
+
+      const result = await ImagePicker.launchImageLibraryAsync({
+        mediaTypes: "images",
+        allowsMultipleSelection: true,
+        quality: 0.8,
+      });
+
+      if (!result.canceled && result.assets.length > 0) {
+        const newImages = result.assets.map(asset => ({
+          uri: asset.uri,
+          mimeType: asset.mimeType || "image/jpeg",
+        }));
+        setSelectedImages(prev => [...prev, ...newImages]);
+      }
+    } catch (error) {
+      console.error("[AgentInput] Failed to pick image:", error);
+      Alert.alert("Error", "Failed to select image");
+    }
+  }
+
+  function handleRemoveImage(index: number) {
+    setSelectedImages(prev => prev.filter((_, i) => i !== index));
+  }
+
  async function handleVoicePress() {
    if (isRecording) {
      // Stop recording
@@ -105,72 +144,104 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
  }

  const hasText = userInput.trim().length > 0;
+  const hasImages = selectedImages.length > 0;

  return (
    <View style={styles.container}>
-      {/* Text input */}
-      <TextInput
-        value={userInput}
-        onChangeText={setUserInput}
-        placeholder="Message agent..."
-        placeholderTextColor={theme.colors.mutedForeground}
-        style={[
-          styles.textInput,
-          { height: inputHeight, minHeight: MIN_INPUT_HEIGHT, maxHeight: MAX_INPUT_HEIGHT },
-        ]}
-        multiline
-        scrollEnabled={inputHeight >= MAX_INPUT_HEIGHT}
-        onContentSizeChange={handleContentSizeChange}
-        editable={!isRecording && ws.isConnected}
-      />
+      {/* Image preview pills */}
+      {hasImages && (
+        <View style={styles.imagePreviewContainer}>
+          {selectedImages.map((image, index) => (
+            <View key={`${image.uri}-${index}`} style={styles.imagePill}>
+              <Image source={{ uri: image.uri }} style={styles.imageThumbnail} />
+              <Pressable onPress={() => handleRemoveImage(index)} style={styles.removeImageButton}>
+                <X size={16} color={theme.colors.foreground} />
+              </Pressable>
+            </View>
+          ))}
+        </View>
+      )}

-      {/* Buttons */}
-      <View style={styles.buttonRow}>
-          {hasText ? (
-            // Send button when text is entered
-            <Pressable
-              onPress={handleSendMessage}
-              disabled={!ws.isConnected || isProcessing}
-              style={[
-                styles.sendButton,
-                (!ws.isConnected || isProcessing) && styles.buttonDisabled,
-              ]}
-            >
-              <ArrowUp size={20} color="white" />
-            </Pressable>
-          ) : (
-            // Voice and Realtime buttons when no text
-            <>
-              {/* Voice recording button */}
+      {/* Input row */}
+      <View style={styles.inputRow}>
+        {/* Attachment button */}
+        {!isRecording && (
+          <Pressable
+            onPress={handlePickImage}
+            disabled={!ws.isConnected}
+            style={[
+              styles.attachButton,
+              !ws.isConnected && styles.buttonDisabled,
+            ]}
+          >
+            <Paperclip size={20} color={theme.colors.foreground} />
+          </Pressable>
+        )}
+
+        {/* Text input */}
+        <TextInput
+          value={userInput}
+          onChangeText={setUserInput}
+          placeholder="Message agent..."
+          placeholderTextColor={theme.colors.mutedForeground}
+          style={[
+            styles.textInput,
+            { height: inputHeight, minHeight: MIN_INPUT_HEIGHT, maxHeight: MAX_INPUT_HEIGHT },
+          ]}
+          multiline
+          scrollEnabled={inputHeight >= MAX_INPUT_HEIGHT}
+          onContentSizeChange={handleContentSizeChange}
+          editable={!isRecording && ws.isConnected}
+        />
+
+        {/* Buttons */}
+        <View style={styles.buttonRow}>
+            {hasText || hasImages ? (
+              // Send button when text is entered or images are selected
              <Pressable
-                onPress={handleVoicePress}
-                disabled={!ws.isConnected}
+                onPress={handleSendMessage}
+                disabled={!ws.isConnected || isProcessing}
                style={[
-                  styles.voiceButton,
-                  !ws.isConnected && styles.buttonDisabled,
-                  isRecording && styles.voiceButtonRecording,
+                  styles.sendButton,
+                  (!ws.isConnected || isProcessing) && styles.buttonDisabled,
                ]}
              >
-                {isRecording ? (
-                  <Square size={14} color="white" fill="white" />
-                ) : (
-                  <Mic size={20} color={theme.colors.foreground} />
-                )}
+                <ArrowUp size={20} color="white" />
              </Pressable>
+            ) : (
+              // Voice and Realtime buttons when no text
+              <>
+                {/* Voice recording button */}
+                <Pressable
+                  onPress={handleVoicePress}
+                  disabled={!ws.isConnected}
+                  style={[
+                    styles.voiceButton,
+                    !ws.isConnected && styles.buttonDisabled,
+                    isRecording && styles.voiceButtonRecording,
+                  ]}
+                >
+                  {isRecording ? (
+                    <Square size={14} color="white" fill="white" />
+                  ) : (
+                    <Mic size={20} color={theme.colors.foreground} />
+                  )}
+                </Pressable>

-              {/* Realtime button */}
-              <Pressable
-                onPress={startRealtime}
-                disabled={!ws.isConnected}
-                style={[
-                  styles.realtimeButton,
-                  !ws.isConnected && styles.buttonDisabled,
-                ]}
-              >
-                <AudioLines size={20} color={theme.colors.background} />
-              </Pressable>
-            </>
-          )}
+                {/* Realtime button */}
+                <Pressable
+                  onPress={startRealtime}
+                  disabled={!ws.isConnected}
+                  style={[
+                    styles.realtimeButton,
+                    !ws.isConnected && styles.buttonDisabled,
+                  ]}
+                >
+                  <AudioLines size={20} color={theme.colors.background} />
+                </Pressable>
+              </>
+            )}
+        </View>
      </View>
    </View>
  );
@@ -178,13 +249,47 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {

 const styles = StyleSheet.create((theme) => ({
  container: {
-    flexDirection: "row",
-    alignItems: "center",
+    flexDirection: "column",
    paddingHorizontal: theme.spacing[4],
    paddingVertical: BASE_VERTICAL_PADDING,
-    gap: theme.spacing[3],
+    gap: theme.spacing[2],
    minHeight: FOOTER_HEIGHT,
  },
+  imagePreviewContainer: {
+    flexDirection: "row",
+    gap: theme.spacing[2],
+    flexWrap: "wrap",
+  },
+  imagePill: {
+    flexDirection: "row",
+    alignItems: "center",
+    backgroundColor: theme.colors.muted,
+    borderRadius: theme.borderRadius.lg,
+    padding: theme.spacing[1],
+    gap: theme.spacing[2],
+  },
+  imageThumbnail: {
+    width: 40,
+    height: 40,
+    borderRadius: theme.borderRadius.md,
+  },
+  removeImageButton: {
+    width: 24,
+    height: 24,
+    alignItems: "center",
+    justifyContent: "center",
+  },
+  inputRow: {
+    flexDirection: "row",
+    alignItems: "center",
+    gap: theme.spacing[2],
+  },
+  attachButton: {
+    width: 40,
+    height: 40,
+    alignItems: "center",
+    justifyContent: "center",
+  },
  textInput: {
    flex: 1,
    paddingHorizontal: theme.spacing[4],
@@ -158,7 +158,10 @@ export function GlobalFooter() {
        <View style={styles.threeButtonContainer}>
          <Pressable
            onPress={() => router.push("/")}
-            style={styles.footerButton}
+            style={({ pressed }) => [
+              styles.footerButton,
+              pressed && styles.buttonPressed,
+            ]}
          >
            <Users size={20} color={theme.colors.foreground} />
            <Text style={styles.footerButtonText}>Agents</Text>
@@ -166,7 +169,10 @@ export function GlobalFooter() {

          <Pressable
            onPress={() => setShowCreateModal(true)}
-            style={styles.footerButton}
+            style={({ pressed }) => [
+              styles.footerButton,
+              pressed && styles.buttonPressed,
+            ]}
          >
            <Plus size={20} color={theme.colors.foreground} />
            <Text style={styles.footerButtonText}>New Agent</Text>
@@ -175,9 +181,10 @@ export function GlobalFooter() {
          <Pressable
            onPress={startRealtime}
            disabled={!ws.isConnected}
-            style={[
+            style={({ pressed }) => [
              styles.footerButton,
              !ws.isConnected && styles.buttonDisabled,
+              pressed && !ws.isConnected && styles.buttonPressed,
            ]}
          >
            <AudioLines size={20} color={theme.colors.foreground} />
@@ -231,4 +238,7 @@ const styles = StyleSheet.create((theme) => ({
  buttonDisabled: {
    opacity: 0.5,
  },
+  buttonPressed: {
+    opacity: 0.5,
+  },
 }));
@@ -114,7 +114,7 @@ interface SessionContextValue {

  // Helpers
  initializeAgent: (params: { agentId: string; requestId?: string }) => void;
-  sendAgentMessage: (agentId: string, message: string) => void;
+  sendAgentMessage: (agentId: string, message: string, imageUris?: string[]) => Promise<void>;
  sendAgentAudio: (agentId: string, audioBlob: Blob) => Promise<void>;
  createAgent: (options: { cwd: string; initialMode?: string; requestId?: string }) => void;
  setAgentMode: (agentId: string, modeId: string) => void;
@@ -603,7 +603,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
    ws.send(msg);
  }, [ws]);

-  const sendAgentMessage = useCallback((agentId: string, message: string) => {
+  const sendAgentMessage = useCallback(async (agentId: string, message: string, imageUris?: string[]) => {
    // Generate unique message ID for deduplication
    const messageId = generateMessageId();

@@ -632,7 +632,32 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
      return updated;
    });

-    // Send to agent with messageId
+    // Convert images to base64 if provided
+    let imagesData: Array<{ data: string; mimeType: string }> | undefined;
+    if (imageUris && imageUris.length > 0) {
+      imagesData = [];
+      for (const imageUri of imageUris) {
+        try {
+          const response = await fetch(imageUri);
+          const blob = await response.blob();
+          const arrayBuffer = await blob.arrayBuffer();
+          const bytes = new Uint8Array(arrayBuffer);
+          let binary = '';
+          for (let i = 0; i < bytes.length; i++) {
+            binary += String.fromCharCode(bytes[i]);
+          }
+          const base64 = btoa(binary);
+          imagesData.push({
+            data: base64,
+            mimeType: blob.type || 'image/jpeg',
+          });
+        } catch (error) {
+          console.error('[Session] Failed to convert image:', error);
+        }
+      }
+    }
+
+    // Send to agent with messageId and optional images
    const msg: WSInboundMessage = {
      type: "session",
      message: {
@@ -640,6 +665,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
        agentId,
        text: message,
        messageId,
+        ...(imagesData && imagesData.length > 0 ? { images: imagesData } : {}),
      },
    };
    ws.send(msg);