Show image previews in optimistic user messages

This commit is contained in:
Mohamed Boudra
2026-02-16 12:19:38 +07:00
parent 75dc802a3a
commit f8f4c55976
10 changed files with 294 additions and 20 deletions
@@ -198,6 +198,7 @@ export function AgentInputArea({
id: messageId, id: messageId,
text, text,
timestamp: new Date(), timestamp: new Date(),
...(images && images.length > 0 ? { images } : {}),
}; };
// Append to head if streaming (keeps the user message with the current // Append to head if streaming (keeps the user message with the current
@@ -314,6 +314,7 @@ export function AgentStreamView({
return ( return (
<UserMessage <UserMessage
message={item.text} message={item.text}
images={item.images}
timestamp={item.timestamp.getTime()} timestamp={item.timestamp.getTime()}
isFirstInGroup={isFirstInGroup} isFirstInGroup={isFirstInGroup}
isLastInGroup={isLastInGroup} isLastInGroup={isLastInGroup}
+63 -16
View File
@@ -1,6 +1,7 @@
import { import {
View, View,
Text, Text,
Image,
Pressable, Pressable,
ActivityIndicator, ActivityIndicator,
type LayoutChangeEvent, type LayoutChangeEvent,
@@ -62,7 +63,7 @@ import {
} from "@/styles/markdown-styles"; } from "@/styles/markdown-styles";
import { Colors, Fonts } from "@/constants/theme"; import { Colors, Fonts } from "@/constants/theme";
import * as Clipboard from "expo-clipboard"; import * as Clipboard from "expo-clipboard";
import type { TodoEntry } from "@/types/stream"; import type { TodoEntry, UserMessageImageAttachment } from "@/types/stream";
import type { ToolCallDetail } from "@server/server/agent/agent-sdk-types"; import type { ToolCallDetail } from "@server/server/agent/agent-sdk-types";
import { import {
buildToolCallDisplayModel, buildToolCallDisplayModel,
@@ -77,6 +78,7 @@ import { ToolCallDetailsContent } from "./tool-call-details";
interface UserMessageProps { interface UserMessageProps {
message: string; message: string;
images?: UserMessageImageAttachment[];
timestamp: number; timestamp: number;
isFirstInGroup?: boolean; isFirstInGroup?: boolean;
isLastInGroup?: boolean; isLastInGroup?: boolean;
@@ -177,6 +179,24 @@ const userMessageStylesheet = StyleSheet.create((theme) => ({
lineHeight: 22, lineHeight: 22,
overflowWrap: "anywhere", overflowWrap: "anywhere",
}, },
imagePreviewContainer: {
flexDirection: "row",
gap: theme.spacing[2],
flexWrap: "wrap",
},
imagePreviewSpacing: {
marginBottom: theme.spacing[2],
},
imagePill: {
borderRadius: theme.borderRadius.md,
borderWidth: 1,
borderColor: theme.colors.borderAccent,
overflow: "hidden",
},
imageThumbnail: {
width: 48,
height: 48,
},
copyButton: { copyButton: {
alignSelf: "flex-end", alignSelf: "flex-end",
padding: theme.spacing[1], padding: theme.spacing[1],
@@ -192,6 +212,7 @@ const userMessageStylesheet = StyleSheet.create((theme) => ({
export const UserMessage = memo(function UserMessage({ export const UserMessage = memo(function UserMessage({
message, message,
images = [],
timestamp, timestamp,
isFirstInGroup = true, isFirstInGroup = true,
isLastInGroup = true, isLastInGroup = true,
@@ -201,8 +222,10 @@ export const UserMessage = memo(function UserMessage({
const [copyButtonHovered, setCopyButtonHovered] = useState(false); const [copyButtonHovered, setCopyButtonHovered] = useState(false);
const resolvedDisableOuterSpacing = const resolvedDisableOuterSpacing =
useDisableOuterSpacing(disableOuterSpacing); useDisableOuterSpacing(disableOuterSpacing);
const hasText = message.trim().length > 0;
const hasImages = images.length > 0;
const showCopyButton = const showCopyButton =
Platform.OS !== "web" || messageHovered || copyButtonHovered; hasText && (Platform.OS !== "web" || messageHovered || copyButtonHovered);
return ( return (
<View <View
@@ -227,21 +250,45 @@ export const UserMessage = memo(function UserMessage({
} }
> >
<View style={userMessageStylesheet.bubble}> <View style={userMessageStylesheet.bubble}>
<Text selectable style={userMessageStylesheet.text}> {hasImages ? (
{message} <View
</Text> style={[
userMessageStylesheet.imagePreviewContainer,
hasText ? userMessageStylesheet.imagePreviewSpacing : undefined,
]}
>
{images.map((image, index) => (
<View
key={`${image.uri}-${index}`}
style={userMessageStylesheet.imagePill}
>
<Image
source={{ uri: image.uri }}
style={userMessageStylesheet.imageThumbnail}
/>
</View>
))}
</View>
) : null}
{hasText ? (
<Text selectable style={userMessageStylesheet.text}>
{message}
</Text>
) : null}
</View> </View>
<TurnCopyButton {hasText ? (
getContent={() => message} <TurnCopyButton
containerStyle={[ getContent={() => message}
userMessageStylesheet.copyButton, containerStyle={[
showCopyButton userMessageStylesheet.copyButton,
? userMessageStylesheet.copyButtonVisible showCopyButton
: userMessageStylesheet.copyButtonHidden, ? userMessageStylesheet.copyButtonVisible
]} : userMessageStylesheet.copyButtonHidden,
accessibilityLabel="Copy message" ]}
onHoverChange={setCopyButtonHovered} accessibilityLabel="Copy message"
/> onHoverChange={setCopyButtonHovered}
/>
) : null}
</Pressable> </Pressable>
</View> </View>
); );
@@ -72,6 +72,7 @@ import {
derivePendingPermissionKey, derivePendingPermissionKey,
normalizeAgentSnapshot, normalizeAgentSnapshot,
} from "@/utils/agent-snapshots"; } from "@/utils/agent-snapshots";
import { mergePendingCreateImages } from "@/utils/pending-create-images";
import type { FetchAgentsEntry } from "@server/client/daemon-client"; import type { FetchAgentsEntry } from "@server/client/daemon-client";
import { import {
DropdownMenu, DropdownMenu,
@@ -492,6 +493,7 @@ function AgentScreenContent({
(state) => state.sessions[serverId]?.pendingPermissions (state) => state.sessions[serverId]?.pendingPermissions
); );
const setAgents = useSessionStore((state) => state.setAgents); const setAgents = useSessionStore((state) => state.setAgents);
const setAgentStreamTail = useSessionStore((state) => state.setAgentStreamTail);
const setPendingPermissions = useSessionStore( const setPendingPermissions = useSessionStore(
(state) => state.setPendingPermissions (state) => state.setPendingPermissions
); );
@@ -637,6 +639,9 @@ function AgentScreenContent({
id: pendingCreate.messageId, id: pendingCreate.messageId,
text: pendingCreate.text, text: pendingCreate.text,
timestamp: new Date(pendingCreate.timestamp), timestamp: new Date(pendingCreate.timestamp),
...(pendingCreate.images && pendingCreate.images.length > 0
? { images: pendingCreate.images }
: {}),
}, },
]; ];
}, [isPendingCreateForRoute, pendingCreate]); }, [isPendingCreateForRoute, pendingCreate]);
@@ -718,6 +723,32 @@ function AgentScreenContent({
(item.id === pendingCreate.messageId || item.text === pendingCreate.text) (item.id === pendingCreate.messageId || item.text === pendingCreate.text)
); );
if (agent && hasUserMessage) { if (agent && hasUserMessage) {
if (
resolvedAgentId &&
pendingCreate.images &&
pendingCreate.images.length > 0
) {
setAgentStreamTail(serverId, (prev) => {
const current = prev.get(resolvedAgentId);
if (!current) {
return prev;
}
const merged = mergePendingCreateImages({
streamItems: current,
messageId: pendingCreate.messageId,
text: pendingCreate.text,
images: pendingCreate.images,
});
if (merged === current) {
return prev;
}
const next = new Map(prev);
next.set(resolvedAgentId, merged);
return next;
});
}
clearPendingCreate(); clearPendingCreate();
} }
}, [ }, [
@@ -725,6 +756,9 @@ function AgentScreenContent({
clearPendingCreate, clearPendingCreate,
isPendingCreateForRoute, isPendingCreateForRoute,
pendingCreate, pendingCreate,
resolvedAgentId,
serverId,
setAgentStreamTail,
streamItems, streamItems,
]); ]);
@@ -37,8 +37,11 @@ import { useCreateFlowStore } from "@/stores/create-flow-store";
import { MAX_CONTENT_WIDTH } from "@/constants/layout"; import { MAX_CONTENT_WIDTH } from "@/constants/layout";
import { WelcomeScreen } from "@/components/welcome-screen"; import { WelcomeScreen } from "@/components/welcome-screen";
import type { Agent } from "@/contexts/session-context"; import type { Agent } from "@/contexts/session-context";
import type { StreamItem } from "@/types/stream"; import {
import { generateMessageId } from "@/types/stream"; generateMessageId,
type StreamItem,
type UserMessageImageAttachment,
} from "@/types/stream";
import { encodeImages } from "@/utils/encode-images"; import { encodeImages } from "@/utils/encode-images";
import type { import type {
AgentProvider, AgentProvider,
@@ -259,6 +262,7 @@ export function DraftAgentScreen({
messageId: string; messageId: string;
text: string; text: string;
timestamp: Date; timestamp: Date;
images?: UserMessageImageAttachment[];
}; };
type DraftAgentMachineState = type DraftAgentMachineState =
@@ -727,6 +731,9 @@ export function DraftAgentScreen({
id: machine.attempt.messageId, id: machine.attempt.messageId,
text: machine.attempt.text, text: machine.attempt.text,
timestamp: machine.attempt.timestamp, timestamp: machine.attempt.timestamp,
...(machine.attempt.images && machine.attempt.images.length > 0
? { images: machine.attempt.images }
: {}),
}, },
]; ];
}, [machine]); }, [machine]);
@@ -789,7 +796,7 @@ export function DraftAgentScreen({
images, images,
}: { }: {
text: string; text: string;
images?: Array<{ uri: string; mimeType: string }>; images?: UserMessageImageAttachment[];
}) => { }) => {
if (isSubmitting) { if (isSubmitting) {
throw new Error("Already loading"); throw new Error("Already loading");
@@ -848,6 +855,7 @@ export function DraftAgentScreen({
messageId: generateMessageId(), messageId: generateMessageId(),
text: trimmedPrompt, text: trimmedPrompt,
timestamp: new Date(), timestamp: new Date(),
...(images && images.length > 0 ? { images } : {}),
}; };
setPendingCreateAttempt({ setPendingCreateAttempt({
serverId: selectedServerId, serverId: selectedServerId,
@@ -855,6 +863,9 @@ export function DraftAgentScreen({
messageId: attempt.messageId, messageId: attempt.messageId,
text: attempt.text, text: attempt.text,
timestamp: attempt.timestamp.getTime(), timestamp: attempt.timestamp.getTime(),
...(attempt.images && attempt.images.length > 0
? { images: attempt.images }
: {}),
}); });
const modeId = const modeId =
modeOptions.length > 0 && selectedMode !== "" ? selectedMode : undefined; modeOptions.length > 0 && selectedMode !== "" ? selectedMode : undefined;
+2 -1
View File
@@ -1,4 +1,5 @@
import { create } from "zustand"; import { create } from "zustand";
import type { UserMessageImageAttachment } from "@/types/stream";
type PendingCreateAttempt = { type PendingCreateAttempt = {
serverId: string; serverId: string;
@@ -6,6 +7,7 @@ type PendingCreateAttempt = {
messageId: string; messageId: string;
text: string; text: string;
timestamp: number; timestamp: number;
images?: UserMessageImageAttachment[];
}; };
type CreateFlowState = { type CreateFlowState = {
@@ -24,4 +26,3 @@ export const useCreateFlowStore = create<CreateFlowState>((set) => ({
), ),
clear: () => set({ pending: null }), clear: () => set({ pending: null }),
})); }));
+38
View File
@@ -3,6 +3,7 @@ import { describe, it } from "vitest";
import { import {
hydrateStreamState, hydrateStreamState,
reduceStreamUpdate,
type AgentToolCallItem, type AgentToolCallItem,
type StreamItem, type StreamItem,
isAgentToolCallItem, isAgentToolCallItem,
@@ -399,4 +400,41 @@ describe("stream reducer canonical tool calls", () => {
assert.ok(todos); assert.ok(todos);
assert.strictEqual(todos.items[0]?.text, "Task 1"); assert.strictEqual(todos.items[0]?.text, "Task 1");
}); });
it("preserves optimistic user message images when authoritative user message arrives", () => {
const messageId = "msg-user-images";
const optimisticImages = [
{ uri: "file:///tmp/optimistic.jpg", mimeType: "image/jpeg" },
];
const initialState: StreamItem[] = [
{
kind: "user_message",
id: messageId,
text: "Analyze this image",
timestamp: new Date("2025-01-01T11:10:00Z"),
images: optimisticImages,
},
];
const event: AgentStreamEventPayload = {
type: "timeline",
provider: "claude",
item: {
type: "user_message",
text: "Analyze this image",
messageId,
},
};
const authoritativeTimestamp = new Date("2025-01-01T11:10:01Z");
const state = reduceStreamUpdate(initialState, event, authoritativeTimestamp);
const message = state.find((item) => item.kind === "user_message");
assert.ok(message);
assert.strictEqual(message.id, messageId);
assert.deepStrictEqual(message.images, optimisticImages);
assert.strictEqual(
message.timestamp.getTime(),
authoritativeTimestamp.getTime()
);
});
}); });
+14
View File
@@ -59,11 +59,17 @@ export type StreamItem =
| ActivityLogItem | ActivityLogItem
| CompactionItem; | CompactionItem;
export interface UserMessageImageAttachment {
uri: string;
mimeType: string;
}
export interface UserMessageItem { export interface UserMessageItem {
kind: "user_message"; kind: "user_message";
id: string; id: string;
text: string; text: string;
timestamp: Date; timestamp: Date;
images?: UserMessageImageAttachment[];
} }
export interface AssistantMessageItem { export interface AssistantMessageItem {
@@ -198,12 +204,20 @@ function appendUserMessage(
const existingIndex = state.findIndex( const existingIndex = state.findIndex(
(entry) => entry.kind === "user_message" && entry.id === entryId (entry) => entry.kind === "user_message" && entry.id === entryId
); );
const existing =
existingIndex >= 0 && state[existingIndex]?.kind === "user_message"
? state[existingIndex]
: null;
const preservedImages = existing?.images;
const nextItem: UserMessageItem = { const nextItem: UserMessageItem = {
kind: "user_message", kind: "user_message",
id: entryId, id: entryId,
text: chunk, text: chunk,
timestamp, timestamp,
...(preservedImages && preservedImages.length > 0
? { images: preservedImages }
: {}),
}; };
if (existingIndex >= 0) { if (existingIndex >= 0) {
@@ -0,0 +1,88 @@
import { describe, expect, it } from "vitest";
import type { StreamItem } from "@/types/stream";
import { mergePendingCreateImages } from "./pending-create-images";
function userMessage(params: {
id: string;
text: string;
images?: Array<{ uri: string; mimeType: string }>;
}): StreamItem {
return {
kind: "user_message",
id: params.id,
text: params.text,
timestamp: new Date("2026-01-01T00:00:00Z"),
...(params.images ? { images: params.images } : {}),
};
}
describe("mergePendingCreateImages", () => {
it("returns same reference when pending images are absent", () => {
const streamItems = [userMessage({ id: "msg-1", text: "hello" })];
const result = mergePendingCreateImages({
streamItems,
messageId: "msg-1",
text: "hello",
images: [],
});
expect(result).toBe(streamItems);
});
it("merges images by messageId when the matched message has none", () => {
const streamItems = [userMessage({ id: "msg-1", text: "hello" })];
const images = [{ uri: "file:///tmp/image-1.jpg", mimeType: "image/jpeg" }];
const result = mergePendingCreateImages({
streamItems,
messageId: "msg-1",
text: "hello",
images,
});
expect(result).not.toBe(streamItems);
const updated = result[0];
expect(updated?.kind).toBe("user_message");
if (updated?.kind !== "user_message") {
throw new Error("Expected user_message item");
}
expect(updated.images).toEqual(images);
});
it("falls back to text matching when messageId does not match", () => {
const streamItems = [userMessage({ id: "msg-1", text: "same text" })];
const images = [{ uri: "file:///tmp/image-2.jpg", mimeType: "image/jpeg" }];
const result = mergePendingCreateImages({
streamItems,
messageId: "missing-id",
text: "same text",
images,
});
const updated = result[0];
expect(updated?.kind).toBe("user_message");
if (updated?.kind !== "user_message") {
throw new Error("Expected user_message item");
}
expect(updated.images).toEqual(images);
});
it("does not overwrite existing user message images", () => {
const existingImages = [{ uri: "file:///tmp/existing.jpg", mimeType: "image/jpeg" }];
const streamItems = [
userMessage({ id: "msg-1", text: "hello", images: existingImages }),
];
const result = mergePendingCreateImages({
streamItems,
messageId: "msg-1",
text: "hello",
images: [{ uri: "file:///tmp/new.jpg", mimeType: "image/jpeg" }],
});
expect(result).toBe(streamItems);
const unchanged = result[0];
expect(unchanged?.kind).toBe("user_message");
if (unchanged?.kind !== "user_message") {
throw new Error("Expected user_message item");
}
expect(unchanged.images).toEqual(existingImages);
});
});
@@ -0,0 +1,39 @@
import type { StreamItem, UserMessageImageAttachment } from "@/types/stream";
type MergePendingCreateImagesParams = {
streamItems: StreamItem[];
messageId: string;
text: string;
images?: UserMessageImageAttachment[];
};
export function mergePendingCreateImages({
streamItems,
messageId,
text,
images,
}: MergePendingCreateImagesParams): StreamItem[] {
if (!images || images.length === 0) {
return streamItems;
}
const targetIndex = streamItems.findIndex(
(item) =>
item.kind === "user_message" && (item.id === messageId || item.text === text)
);
if (targetIndex < 0) {
return streamItems;
}
const target = streamItems[targetIndex];
if (target.kind !== "user_message") {
return streamItems;
}
if (target.images && target.images.length > 0) {
return streamItems;
}
const next = [...streamItems];
next[targetIndex] = { ...target, images };
return next;
}