From b8b66816ca6c700ec52e2d28045302696f80674a Mon Sep 17 00:00:00 2001 From: huiliaoning <75320000+huiliaoning@users.noreply.github.com> Date: Wed, 1 Jul 2026 04:58:16 +0800 Subject: [PATCH] feat(browser): inspect, annotate, and grab page elements for the agent (#1708) * feat(browser): inspect, annotate, and grab page elements for the agent Build a design-review flow on top of the in-app Electron browser so users can send page elements to their coding agent with context. - Annotate: pick an element, write a comment, choose an intent (fix/change/question/approve); the element context + intent + comment go to the agent as text, and a cropped screenshot rides along as an image. - Grab: pick an element to copy its info + screenshot straight to the system clipboard (no comment), with a toast on success/failure. - Hover inspector: in select mode each element shows a floating label with tag, id/class, React component name, and pixel size. - Page markers: annotated elements on the current page get numbered badges that track scroll/resize. - Device sizes: a viewport-size menu (responsive + 13 common device presets) renders the page centered in a fixed-size frame. - Toolbar buttons now expose hover tooltips and no longer get squeezed out on narrow panes; the element selector is available to all desktop users (previously dev-only). Screenshots and clipboard writes go through new Electron main-process IPCs (capturePage + clipboard) so they work regardless of webview focus. Element screenshots are referenced by id in the workspace attachment store and are protected from the draft-store attachment GC. All new strings are translated across the six supported locales. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(browser): write element clipboard exactly once Resolve the image before writing so a combined text+image grab no longer does a redundant text-only writeText() first (which flashed an intermediate clipboard state). Addresses greptile review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) * i18n: add browser annotate/grab/devices strings for ja and pt-BR main added Japanese and Brazilian Portuguese locales after this branch was created; add the browser annotation, grab, and device-size keys to keep all locales in sync with en. resources.test.ts parity passes. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../attachments/attachment-pill-content.tsx | 12 +- packages/app/src/attachments/types.ts | 16 + .../src/components/browser-pane.electron.tsx | 1090 +++++++++++++++-- .../app/src/composer/attachments/submit.ts | 3 + .../composer/attachments/workspace-cleanup.ts | 2 + packages/app/src/desktop/host.ts | 7 + packages/app/src/i18n/resources/ar.ts | 19 + packages/app/src/i18n/resources/en.ts | 19 + packages/app/src/i18n/resources/es.ts | 19 + packages/app/src/i18n/resources/fr.ts | 19 + packages/app/src/i18n/resources/ja.ts | 19 + packages/app/src/i18n/resources/pt-BR.ts | 19 + packages/app/src/i18n/resources/ru.ts | 19 + packages/app/src/i18n/resources/zh-CN.ts | 19 + packages/app/src/stores/draft-store/index.ts | 13 + packages/desktop/src/main.ts | 106 ++ packages/desktop/src/preload.ts | 6 + 17 files changed, 1313 insertions(+), 94 deletions(-) diff --git a/packages/app/src/attachments/attachment-pill-content.tsx b/packages/app/src/attachments/attachment-pill-content.tsx index 540e0230a..61b2456a4 100644 --- a/packages/app/src/attachments/attachment-pill-content.tsx +++ b/packages/app/src/attachments/attachment-pill-content.tsx @@ -9,7 +9,7 @@ import { } from "lucide-react-native"; import { withUnistyles } from "react-native-unistyles"; import type { AgentAttachment } from "@getpaseo/protocol/messages"; -import type { WorkspaceComposerAttachment } from "@/attachments/types"; +import type { BrowserAnnotationIntent, WorkspaceComposerAttachment } from "@/attachments/types"; import { getFileTypeLabel } from "@/attachments/file-types"; import { isPullRequestContextAttachment } from "@/attachments/workspace-attachment-utils"; import { ICON_SIZE, type Theme } from "@/styles/theme"; @@ -36,6 +36,13 @@ function getPullRequestContextSubtitle(attachment: WorkspaceComposerAttachment): return "Review"; } +const BROWSER_INTENT_LABEL_KEYS: Record = { + fix: "workspace.browser.annotate.intents.fix", + change: "workspace.browser.annotate.intents.change", + question: "workspace.browser.annotate.intents.question", + approve: "workspace.browser.annotate.intents.approve", +}; + function getTextAttachmentSubtitle( attachment: Extract, t: TFunction, @@ -89,10 +96,11 @@ export function getWorkspaceAttachmentPillContent( t: TFunction, ): AttachmentPillContent { if (attachment.kind === "browser_element") { + const intent = attachment.attachment.intent; return { icon: attachmentBrowserIcon, title: attachment.attachment.tag, - subtitle: t("composer.attachments.element"), + subtitle: intent ? t(BROWSER_INTENT_LABEL_KEYS[intent]) : t("composer.attachments.element"), }; } if (isPullRequestContextAttachment(attachment)) { diff --git a/packages/app/src/attachments/types.ts b/packages/app/src/attachments/types.ts index e4dcde5ce..85a5b40b6 100644 --- a/packages/app/src/attachments/types.ts +++ b/packages/app/src/attachments/types.ts @@ -21,6 +21,12 @@ export interface AttachmentMetadata { createdAt: number; } +/** + * The kind of review feedback the user is attaching to a selected browser + * element, sent to the agent alongside the element context. + */ +export type BrowserAnnotationIntent = "fix" | "change" | "question" | "approve"; + export interface BrowserElementAttachment { url: string; selector: string; @@ -42,6 +48,16 @@ export interface BrowserElementAttachment { } | null; parentChain: string[]; children: string[]; + /** Free-text review note the user wrote about this element, if any. */ + comment?: string; + /** What the user wants the agent to do with this element, if annotated. */ + intent?: BrowserAnnotationIntent; + /** + * Cropped screenshot of the selected element, sent to the agent as an image + * alongside the textual element context. Persisted via the attachment store; + * referenced by id so the draft-store GC keeps it alive. + */ + screenshot?: AttachmentMetadata; formatted: string; } diff --git a/packages/app/src/components/browser-pane.electron.tsx b/packages/app/src/components/browser-pane.electron.tsx index 7cfa1c2d2..566fdb7fb 100644 --- a/packages/app/src/components/browser-pane.electron.tsx +++ b/packages/app/src/components/browser-pane.electron.tsx @@ -5,18 +5,47 @@ import { useRef, useState, type CSSProperties, + type ReactNode, createElement, } from "react"; -import { Pressable, Text, TextInput, View } from "react-native"; -import { ArrowLeft, ArrowRight, MousePointer2, PencilRuler, RotateCw } from "lucide-react-native"; -import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from "react-native"; +import { + ArrowLeft, + ArrowRight, + ChevronDown, + Copy, + Maximize, + Monitor, + MousePointer2, + PencilRuler, + RotateCw, + Smartphone, + Tablet, + X, + type LucideIcon, +} from "lucide-react-native"; +import { StyleSheet, useUnistyles, withUnistyles } from "react-native-unistyles"; import { useTranslation } from "react-i18next"; +import * as Clipboard from "expo-clipboard"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { useToast } from "@/contexts/toast-context"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { buildWorkspaceAttachmentScopeKey, useWorkspaceAttachments, useWorkspaceAttachmentsStore, } from "@/attachments/workspace-attachments-store"; -import type { BrowserElementAttachment } from "@/attachments/types"; +import type { + AttachmentMetadata, + BrowserAnnotationIntent, + BrowserElementAttachment, +} from "@/attachments/types"; +import { persistAttachmentFromDataUrl } from "@/attachments/service"; import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout"; import { getDesktopHost, @@ -50,10 +79,81 @@ type WebTextInput = TextInput & { getNativeRef?: () => unknown; }; -type BrowserElementSelection = Omit & { +type BrowserElementSelection = Omit< + BrowserElementAttachment, + "formatted" | "comment" | "intent" +> & { attributes?: Record; }; +interface BrowserElementAnnotation { + comment: string; + intent: BrowserAnnotationIntent; +} + +const BROWSER_ANNOTATION_INTENTS: readonly BrowserAnnotationIntent[] = [ + "fix", + "change", + "question", + "approve", +]; + +type DeviceSizeId = + | "responsive" + | "iphone-se" + | "iphone-14" + | "iphone-14-pro-max" + | "pixel-7" + | "galaxy-s20" + | "ipad-mini" + | "ipad-air" + | "ipad-pro-11" + | "ipad-pro-12" + | "surface-pro" + | "laptop" + | "desktop-1080" + | "desktop-1440"; + +interface DeviceSizePreset { + id: DeviceSizeId; + /** Display name (not translated — device names are proper nouns). */ + name: string; + /** Fixed CSS width, or null for "fill the available area". */ + width: number | null; + height: number | null; + icon: LucideIcon; +} + +// Viewport presets for the in-app browser. "responsive" fills the pane; the +// others render a fixed-size, centered frame so the user can preview how a page +// behaves at common device sizes. Content is centered (not left-aligned). +const DEVICE_SIZE_PRESETS: readonly DeviceSizePreset[] = [ + { id: "responsive", name: "Responsive", width: null, height: null, icon: Maximize }, + { id: "iphone-se", name: "iPhone SE", width: 375, height: 667, icon: Smartphone }, + { id: "iphone-14", name: "iPhone 14", width: 390, height: 844, icon: Smartphone }, + { id: "iphone-14-pro-max", name: "iPhone 14 Pro Max", width: 430, height: 932, icon: Smartphone }, + { id: "pixel-7", name: "Pixel 7", width: 412, height: 915, icon: Smartphone }, + { id: "galaxy-s20", name: "Galaxy S20", width: 360, height: 800, icon: Smartphone }, + { id: "ipad-mini", name: "iPad Mini", width: 768, height: 1024, icon: Tablet }, + { id: "ipad-air", name: "iPad Air", width: 820, height: 1180, icon: Tablet }, + { id: "ipad-pro-11", name: 'iPad Pro 11"', width: 834, height: 1194, icon: Tablet }, + { id: "ipad-pro-12", name: 'iPad Pro 12.9"', width: 1024, height: 1366, icon: Tablet }, + { id: "surface-pro", name: "Surface Pro", width: 912, height: 1368, icon: Tablet }, + { id: "laptop", name: "Laptop", width: 1366, height: 768, icon: Monitor }, + { id: "desktop-1080", name: "Desktop 1080p", width: 1920, height: 1080, icon: Monitor }, + { id: "desktop-1440", name: "Desktop 1440p", width: 2560, height: 1440, icon: Monitor }, +]; + +const RESPONSIVE_DEVICE_LABEL_KEY = "workspace.browser.devices.responsive"; + +function formatDevicePresetLabel(preset: DeviceSizePreset, responsiveLabel: string): string { + const name = preset.id === "responsive" ? responsiveLabel : preset.name; + if (preset.width && preset.height) { + return `${name} · ${preset.width}×${preset.height}`; + } + return name; +} + const ERR_ABORTED = -3; const ALLOWED_BROWSER_PROTOCOLS = new Set(["http:", "https:"]); @@ -115,11 +215,18 @@ function getUnsafeNavigationMessage( } } -function formatElementAttachment(selection: BrowserElementSelection): string { +function formatElementAttachment( + selection: BrowserElementSelection, + annotation?: BrowserElementAnnotation, +): string { const textPreview = truncateText(selection.text.trim(), 200); const html = truncateText(selection.outerHTML.trim(), 800); const parts: string[] = []; + if (annotation) { + parts.push(`intent: ${annotation.intent}`); + } + if (selection.reactSource?.fileName) { const loc = [ selection.reactSource.fileName, @@ -151,6 +258,11 @@ function formatElementAttachment(selection: BrowserElementSelection): string { parts.push(`parents: ${selection.parentChain.slice(0, 3).join(" > ")}`); } + const comment = annotation?.comment.trim(); + if (comment) { + parts.push(`feedback: ${comment}`); + } + return [ ``, parts.map((part) => ` ${part}`).join("\n"), @@ -161,7 +273,10 @@ function formatElementAttachment(selection: BrowserElementSelection): string { function buildBrowserElementAttachment( selection: BrowserElementSelection, + annotation?: BrowserElementAnnotation, + screenshot?: AttachmentMetadata, ): BrowserElementAttachment { + const comment = annotation?.comment.trim(); return { url: selection.url, selector: selection.selector, @@ -173,7 +288,10 @@ function buildBrowserElementAttachment( reactSource: selection.reactSource, parentChain: selection.parentChain, children: selection.children, - formatted: formatElementAttachment(selection), + ...(comment ? { comment } : {}), + ...(annotation ? { intent: annotation.intent } : {}), + ...(screenshot ? { screenshot } : {}), + formatted: formatElementAttachment(selection, annotation), }; } @@ -219,6 +337,90 @@ function clearWebviewSelector(webview: ElectronWebview): void { ).catch(ignoreWebviewJavaScriptError); } +interface BrowserAnnotationMarker { + index: number; + selector: string; +} + +// Draws numbered badges over annotated elements inside the guest page. The +// overlay is a fixed, pointer-events:none layer that re-measures element rects +// on scroll/resize via rAF. Markers are matched by the CSS selector captured at +// annotation time; unmatched selectors are simply skipped. +function buildAnnotationMarkerScript(markers: readonly BrowserAnnotationMarker[]): string { + const payload = JSON.stringify( + markers.map((marker) => ({ index: marker.index, selector: marker.selector })), + ); + return ` + (function() { + var markers = ${payload}; + if (window.__paseoAnnotationMarkers) { window.__paseoAnnotationMarkers.update(markers); return true; } + var host = document.createElement('div'); + host.style.cssText = 'position:fixed;top:0;left:0;width:0;height:0;z-index:2147483646;pointer-events:none;'; + (document.body || document.documentElement).appendChild(host); + var badges = []; + var current = markers; + function clearBadges() { + for (var i = 0; i < badges.length; i++) { if (badges[i].parentNode) badges[i].parentNode.removeChild(badges[i]); } + badges = []; + } + function reposition() { + clearBadges(); + for (var i = 0; i < current.length; i++) { + var m = current[i]; + var el = null; + try { el = document.querySelector(m.selector); } catch (e) { el = null; } + if (!el) continue; + var rect = el.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) continue; + var badge = document.createElement('div'); + badge.textContent = String(m.index); + badge.style.cssText = 'position:fixed;min-width:18px;height:18px;padding:0 4px;border-radius:9px;background:#2563eb;color:#fff;font:600 11px/18px -apple-system,system-ui,sans-serif;text-align:center;box-shadow:0 1px 3px rgba(0,0,0,0.4);pointer-events:none;box-sizing:border-box;'; + badge.style.left = Math.max(0, rect.left) + 'px'; + badge.style.top = Math.max(0, rect.top) + 'px'; + host.appendChild(badge); + badges.push(badge); + } + } + var scheduled = false; + function schedule() { + if (scheduled) return; + scheduled = true; + requestAnimationFrame(function() { scheduled = false; reposition(); }); + } + window.addEventListener('scroll', schedule, true); + window.addEventListener('resize', schedule, true); + window.__paseoAnnotationMarkers = { + update: function(next) { current = next; schedule(); }, + destroy: function() { + window.removeEventListener('scroll', schedule, true); + window.removeEventListener('resize', schedule, true); + clearBadges(); + if (host.parentNode) host.parentNode.removeChild(host); + window.__paseoAnnotationMarkers = null; + } + }; + reposition(); + return true; + })() + `; +} + +function applyAnnotationMarkers( + webview: ElectronWebview, + markers: readonly BrowserAnnotationMarker[], +): void { + void executeWebviewJavaScript(webview, buildAnnotationMarkerScript(markers)).catch( + ignoreWebviewJavaScriptError, + ); +} + +function clearAnnotationMarkers(webview: ElectronWebview): void { + void executeWebviewJavaScript( + webview, + "if(window.__paseoAnnotationMarkers) window.__paseoAnnotationMarkers.destroy();", + ).catch(ignoreWebviewJavaScriptError); +} + function getTextInputNativeElement(current: WebTextInput | null): HTMLInputElement | null { const native = current?.getNativeRef?.() ?? current; return native instanceof HTMLInputElement ? native : null; @@ -275,6 +477,139 @@ function startSelectorResultPolling(input: { return poll; } +function ToolbarButton({ + label, + children, + active, + disabled, + onPress, + style, +}: { + label: string; + children: ReactNode; + active?: boolean; + disabled?: boolean; + onPress: () => void; + style: (state: { hovered?: boolean; pressed?: boolean }) => StyleProp; +}) { + const accessibilityState = useMemo( + () => ({ disabled: Boolean(disabled), selected: Boolean(active) }), + [active, disabled], + ); + return ( + + + + {children} + + + + {label} + + + ); +} + +// Lucide icons themed via withUnistyles so their color stays theme-reactive +// without a banned useUnistyles() call. +const ThemedMaximize = withUnistyles(Maximize); +const ThemedSmartphone = withUnistyles(Smartphone); +const ThemedTablet = withUnistyles(Tablet); +const ThemedMonitor = withUnistyles(Monitor); +const ThemedChevronDown = withUnistyles(ChevronDown); +const deviceMutedIconMapping = (theme: { colors: { foregroundMuted: string } }) => ({ + color: theme.colors.foregroundMuted, +}); + +function resolveThemedDeviceIcon(icon: LucideIcon): typeof ThemedMaximize { + if (icon === Smartphone) return ThemedSmartphone; + if (icon === Tablet) return ThemedTablet; + if (icon === Monitor) return ThemedMonitor; + return ThemedMaximize; +} + +function DeviceSizeMenuItem({ + preset, + selected, + label, + onSelect, +}: { + preset: DeviceSizePreset; + selected: boolean; + label: string; + onSelect: (id: DeviceSizeId) => void; +}) { + const ThemedIcon = resolveThemedDeviceIcon(preset.icon); + const handleSelect = useCallback(() => { + onSelect(preset.id); + }, [onSelect, preset.id]); + const leading = useMemo( + () => , + [ThemedIcon], + ); + return ( + + {label} + + ); +} + +function DeviceSizeMenu({ + selectedId, + onSelect, + triggerStyle, +}: { + selectedId: DeviceSizeId; + onSelect: (id: DeviceSizeId) => void; + triggerStyle: (state: { hovered?: boolean; pressed?: boolean }) => StyleProp; +}) { + const { t } = useTranslation(); + const selectedPreset = + DEVICE_SIZE_PRESETS.find((preset) => preset.id === selectedId) ?? DEVICE_SIZE_PRESETS[0]; + const SelectedIcon = resolveThemedDeviceIcon(selectedPreset.icon); + const label = t("workspace.browser.devices.label"); + return ( + + + + + + + + + + + + {label} + + + + {DEVICE_SIZE_PRESETS.map((preset) => ( + + ))} + + + ); +} + // eslint-disable-next-line complexity export function BrowserPane({ browserId, @@ -305,7 +640,19 @@ export function BrowserPane({ browserRef.current = browser; const pendingNavigationUrlRef = useRef(null); const domReadyRef = useRef(false); + const annotationMarkersRef = useRef([]); const [selectorActive, setSelectorActive] = useState(false); + // Which action the active selector performs on click: open the annotation card + // ("annotate") or copy the element to the clipboard ("grab"). + const selectorModeRef = useRef<"annotate" | "grab">("annotate"); + const toast = useToast(); + const toastRef = useRef(toast); + toastRef.current = toast; + const [deviceSizeId, setDeviceSizeId] = useState("responsive"); + const [pendingSelection, setPendingSelection] = useState(null); + // Screenshot is captured at selection time (overlay already torn down, no + // scroll drift) and reused when the annotation card is submitted. + const pendingScreenshotRef = useRef(undefined); const [draftUrl, setDraftUrl] = useState(browser?.url ?? "https://example.com"); const workspaceAttachmentScopeKey = useMemo( () => buildBrowserAttachmentScopeKey({ cwd, serverId, workspaceId }), @@ -495,6 +842,12 @@ export function BrowserPane({ const handleDomReady = () => { domReadyRef.current = true; syncNavigationState(); + // The previous page's overlay is gone after a load; re-apply markers for + // the freshly loaded document. + const markers = annotationMarkersRef.current; + if (markers.length > 0) { + applyAnnotationMarkers(webview, markers); + } }; const handleWebviewFocus = () => { onFocusPane?.(); @@ -671,7 +1024,11 @@ export function BrowserPane({ }, [draftUrl, navigate]); const addElementAttachment = useCallback( - (selection: BrowserElementSelection) => { + ( + selection: BrowserElementSelection, + annotation: BrowserElementAnnotation, + screenshot?: AttachmentMetadata, + ) => { if (!workspaceAttachmentScopeKey) { return; } @@ -681,7 +1038,7 @@ export function BrowserPane({ ...workspaceAttachments, { kind: "browser_element", - attachment: buildBrowserElementAttachment(selection), + attachment: buildBrowserElementAttachment(selection, annotation, screenshot), }, ], }); @@ -689,12 +1046,126 @@ export function BrowserPane({ [setWorkspaceAttachments, workspaceAttachmentScopeKey, workspaceAttachments], ); - const startElementSelector = useCallback(() => { - const webview = webviewRef.current; - if (!webview || !domReadyRef.current || !workspaceAttachmentScopeKey) return; - setSelectorActive(true); + const captureElementScreenshot = useCallback( + async (selection: BrowserElementSelection): Promise => { + const captureElement = getDesktopHost()?.browser?.captureElement; + if (typeof captureElement !== "function") { + return undefined; + } + const { x, y, width, height } = selection.boundingRect; + if (width <= 0 || height <= 0) { + return undefined; + } + try { + const dataUrl = await captureElement(browserIdRef.current, { x, y, width, height }); + if (!dataUrl) { + return undefined; + } + return await persistAttachmentFromDataUrl({ + dataUrl, + mimeType: "image/png", + fileName: `element-${selection.tag}.png`, + }); + } catch (error) { + console.warn("[browser-pane] captureElement failed", error); + return undefined; + } + }, + [], + ); - const js = ` + const grabElementToClipboard = useCallback( + async (selection: BrowserElementSelection) => { + const text = formatElementAttachment(selection); + const copyElement = getDesktopHost()?.browser?.copyElement; + const captureElement = getDesktopHost()?.browser?.captureElement; + const { x, y, width, height } = selection.boundingRect; + + let imageDataUrl: string | undefined; + if (typeof captureElement === "function" && width > 0 && height > 0) { + try { + const dataUrl = await captureElement(browserIdRef.current, { x, y, width, height }); + imageDataUrl = dataUrl ?? undefined; + } catch (error) { + console.warn("[browser-pane] capture element for grab failed", error); + } + } + + // Copy via the main process; the renderer's navigator.clipboard rejects + // with NotAllowedError because focus is inside the guest . + if (typeof copyElement === "function") { + try { + const ok = await copyElement({ text, imageDataUrl }); + if (ok) { + toastRef.current?.copied(t("workspace.browser.controls.grabElementLabel")); + } else { + toastRef.current?.error(t("workspace.browser.controls.grabFailed")); + } + return; + } catch (error) { + console.warn("[browser-pane] copyElement bridge failed", error); + } + } + + // Fallback to expo-clipboard (text only) when the bridge is unavailable. + try { + await Clipboard.setStringAsync(text); + toastRef.current?.copied(t("workspace.browser.controls.grabElementLabel")); + } catch (error) { + console.warn("[browser-pane] clipboard fallback failed", error); + toastRef.current?.error(t("workspace.browser.controls.grabFailed")); + } + }, + [t], + ); + + const handleSelectorResult = useCallback( + (selection: BrowserElementSelection) => { + if (selectorModeRef.current === "grab") { + void grabElementToClipboard(selection); + return; + } + pendingScreenshotRef.current = undefined; + setPendingSelection(selection); + void captureElementScreenshot(selection).then((screenshot) => { + pendingScreenshotRef.current = screenshot; + return undefined; + }); + }, + [captureElementScreenshot, grabElementToClipboard], + ); + + const submitAnnotation = useCallback( + (annotation: BrowserElementAnnotation) => { + const selection = pendingSelection; + const screenshot = pendingScreenshotRef.current; + pendingScreenshotRef.current = undefined; + setPendingSelection(null); + if (!selection) { + return; + } + addElementAttachment(selection, annotation, screenshot); + }, + [addElementAttachment, pendingSelection], + ); + + const cancelAnnotation = useCallback(() => { + pendingScreenshotRef.current = undefined; + setPendingSelection(null); + }, []); + + const startElementSelector = useCallback( + (mode: "annotate" | "grab") => { + const webview = webviewRef.current; + if (!webview || !domReadyRef.current) return; + // Annotate needs a workspace scope to attach to; grab only copies. + if (mode === "annotate" && !workspaceAttachmentScopeKey) return; + selectorModeRef.current = mode; + pendingScreenshotRef.current = undefined; + setPendingSelection(null); + setSelectorActive(true); + + const js = ` (function() { if (window.__paseoSelector) { window.__paseoSelector.destroy(); } var overlay = null; @@ -705,16 +1176,74 @@ export function BrowserPane({ '.__paseo-select-mode *, .__paseo-select-mode *::before, .__paseo-select-mode *::after { animation: none !important; transition: none !important; }', '.__paseo-select-mode a, .__paseo-select-mode button, .__paseo-select-mode input, .__paseo-select-mode select, .__paseo-select-mode textarea, .__paseo-select-mode [role="button"], .__paseo-select-mode [onclick] { pointer-events: none !important; }', '.__paseo-select-mode iframe, .__paseo-select-mode video, .__paseo-select-mode audio { pointer-events: none !important; }', + '.__paseo-hover-label { position: fixed; z-index: 2147483647; pointer-events: none; max-width: 360px; padding: 4px 8px; border-radius: 6px; background: rgba(24,24,27,0.96); color: #fff; font: 500 11px/1.45 ui-monospace,SFMono-Regular,Menlo,monospace; box-shadow: 0 2px 10px rgba(0,0,0,0.35); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }', + '.__paseo-hover-label .__paseo-tag { color: #93c5fd; }', + '.__paseo-hover-label .__paseo-id { color: #fca5a5; }', + '.__paseo-hover-label .__paseo-cls { color: #fcd34d; }', + '.__paseo-hover-label .__paseo-dim { color: #a1a1aa; margin-left: 6px; }', + '.__paseo-hover-label .__paseo-comp { color: #86efac; margin-left: 6px; }', ].join('\\n'); document.head.appendChild(style); document.documentElement.classList.add('__paseo-select-mode'); + var hoverLabel = document.createElement('div'); + hoverLabel.className = '__paseo-hover-label'; + hoverLabel.style.display = 'none'; + document.documentElement.appendChild(hoverLabel); var last = null; + function escapeHtml(value) { + return String(value).replace(/[&<>"]/g, function(ch) { + return ch === '&' ? '&' : ch === '<' ? '<' : ch === '>' ? '>' : '"'; + }); + } + function describeElement(el) { + var tag = el.tagName ? el.tagName.toLowerCase() : 'node'; + var parts = ['' + escapeHtml(tag) + '']; + if (el.id) { + parts.push('#' + escapeHtml(el.id) + ''); + } + if (el.classList && el.classList.length) { + var cls = Array.prototype.slice.call(el.classList, 0, 2) + .filter(function(c) { return c.indexOf('__paseo') !== 0; }) + .map(function(c) { return '.' + escapeHtml(c); }) + .join(''); + if (cls) parts.push('' + cls + ''); + } + var comp = getReactSource(el); + if (comp && comp.componentName) { + parts.push('<' + escapeHtml(comp.componentName) + '>'); + } + var rect = el.getBoundingClientRect(); + parts.push('' + Math.round(rect.width) + '×' + Math.round(rect.height) + ''); + return { html: parts.join(''), rect: rect }; + } + function positionLabel(rect, e) { + var pad = 12; + var lw = hoverLabel.offsetWidth || 0; + var lh = hoverLabel.offsetHeight || 0; + var top = rect.top - lh - 6; + if (top < 4) top = rect.bottom + 6; + if (top + lh > window.innerHeight - 4) top = Math.max(4, e.clientY - lh - 6); + var left = rect.left; + if (left + lw > window.innerWidth - 4) left = Math.max(4, window.innerWidth - lw - 4); + if (left < 4) left = 4; + hoverLabel.style.top = Math.round(top) + 'px'; + hoverLabel.style.left = Math.round(left) + 'px'; + } function onMove(e) { e.preventDefault(); e.stopPropagation(); if (last) last.classList.remove('__paseo-hover'); - e.target.classList.add('__paseo-hover'); - last = e.target; + var el = e.target; + el.classList.add('__paseo-hover'); + last = el; + try { + var info = describeElement(el); + hoverLabel.innerHTML = info.html; + hoverLabel.style.display = 'block'; + positionLabel(info.rect, e); + } catch (err) { + hoverLabel.style.display = 'none'; + } } function buildSelector(el) { if (el.id) return '#' + el.id; @@ -791,6 +1320,7 @@ export function BrowserPane({ e.stopImmediatePropagation(); var el = e.target; if (last) last.classList.remove('__paseo-hover'); + hoverLabel.style.display = 'none'; var attrs = {}; for (var i = 0; i < el.attributes.length; i++) { attrs[el.attributes[i].name] = el.attributes[i].value; @@ -834,6 +1364,7 @@ export function BrowserPane({ document.removeEventListener('submit', blockEvent, true); document.documentElement.classList.remove('__paseo-select-mode'); if (last) last.classList.remove('__paseo-hover'); + if (hoverLabel.parentNode) hoverLabel.parentNode.removeChild(hoverLabel); style.remove(); window.__paseoSelector = null; } @@ -852,31 +1383,33 @@ export function BrowserPane({ })() `; - try { - void executeWebviewJavaScript(webview, js) - .then(() => { - const poll = startSelectorResultPolling({ - webview, - onSelection: addElementAttachment, - onDone: () => setSelectorActive(false), - }); - window.setTimeout(() => { - window.clearInterval(poll); + try { + void executeWebviewJavaScript(webview, js) + .then(() => { + const poll = startSelectorResultPolling({ + webview, + onSelection: handleSelectorResult, + onDone: () => setSelectorActive(false), + }); + window.setTimeout(() => { + window.clearInterval(poll); + setSelectorActive(false); + if (webviewRef.current !== webview || !domReadyRef.current) { + return; + } + destroyWebviewSelector(webview); + }, 30000); + return undefined; + }) + .catch(() => { setSelectorActive(false); - if (webviewRef.current !== webview || !domReadyRef.current) { - return; - } - destroyWebviewSelector(webview); - }, 30000); - return undefined; - }) - .catch(() => { - setSelectorActive(false); - }); - } catch { - setSelectorActive(false); - } - }, [addElementAttachment, workspaceAttachmentScopeKey]); + }); + } catch { + setSelectorActive(false); + } + }, + [handleSelectorResult, workspaceAttachmentScopeKey], + ); const cancelElementSelector = useCallback(() => { const webview = webviewRef.current; @@ -888,12 +1421,61 @@ export function BrowserPane({ } }, []); + const currentPageUrl = browser?.url ?? null; + const annotationMarkers = useMemo(() => { + if (!currentPageUrl) { + return []; + } + const normalizedCurrent = normalizeWorkspaceBrowserUrl(currentPageUrl); + const markers: BrowserAnnotationMarker[] = []; + let index = 0; + for (const attachment of workspaceAttachments) { + if (attachment.kind !== "browser_element") { + continue; + } + index += 1; + if (normalizeWorkspaceBrowserUrl(attachment.attachment.url) !== normalizedCurrent) { + continue; + } + markers.push({ index, selector: attachment.attachment.selector }); + } + return markers; + }, [currentPageUrl, workspaceAttachments]); + + const markersKey = useMemo(() => JSON.stringify(annotationMarkers), [annotationMarkers]); + annotationMarkersRef.current = annotationMarkers; + + useEffect(() => { + if (!isElectronRuntime()) { + return; + } + const webview = webviewRef.current; + if (!webview || !domReadyRef.current) { + return; + } + if (annotationMarkers.length === 0) { + clearAnnotationMarkers(webview); + return; + } + applyAnnotationMarkers(webview, annotationMarkers); + // markersKey captures the marker contents; re-run when they change. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [markersKey, currentPageUrl]); + const handleToggleElementSelector = useCallback(() => { if (selectorActive) { cancelElementSelector(); return; } - startElementSelector(); + startElementSelector("annotate"); + }, [cancelElementSelector, selectorActive, startElementSelector]); + + const handleToggleGrab = useCallback(() => { + if (selectorActive) { + cancelElementSelector(); + return; + } + startElementSelector("grab"); }, [cancelElementSelector, selectorActive, startElementSelector]); const handleOpenDevTools = useCallback(() => { @@ -948,17 +1530,46 @@ export function BrowserPane({ [selectorActive], ); - const webviewHostStyle = useMemo( - () => ({ - display: "flex", - flex: 1, - width: "100%", - height: "100%", - minHeight: 0, - background: theme.colors.surface0, - }), - [theme.colors.surface0], + const devicePreset = useMemo( + () => + DEVICE_SIZE_PRESETS.find((preset) => preset.id === deviceSizeId) ?? DEVICE_SIZE_PRESETS[0], + [deviceSizeId], ); + const isResponsiveDevice = devicePreset.width === null; + + const webviewHostStyle = useMemo( + () => + isResponsiveDevice + ? { + display: "flex", + flex: 1, + width: "100%", + height: "100%", + minHeight: 0, + background: theme.colors.surface0, + } + : { + // Fixed-size device frame, centered within webviewWrap (see styles). + display: "flex", + width: devicePreset.width ?? undefined, + maxWidth: "100%", + height: devicePreset.height ?? undefined, + maxHeight: "100%", + minHeight: 0, + background: theme.colors.surface0, + boxShadow: "0 2px 16px rgba(0,0,0,0.25)", + }, + [devicePreset.height, devicePreset.width, isResponsiveDevice, theme.colors.surface0], + ); + + const webviewWrapStyle = useMemo( + () => [styles.webviewWrap, !isResponsiveDevice && styles.webviewWrapDeviceFrame], + [isResponsiveDevice], + ); + + const setWebviewHostNode = useCallback((node: HTMLDivElement | null) => { + webviewHostRef.current = node; + }, []); if (!isElectronRuntime()) { return ( @@ -973,27 +1584,24 @@ export function BrowserPane({ - - - + - - + - + + {isDev ? ( - <> - - - - - - - + + + ) : null} + + + + + + {browser?.lastError ? ( @@ -1056,18 +1673,180 @@ export function BrowserPane({ ) : null} - + {createElement("div", { - ref: (node: HTMLDivElement | null) => { - webviewHostRef.current = node; - }, + ref: setWebviewHostNode, style: webviewHostStyle, })} + {pendingSelection ? ( + + ) : null} ); } +const INTENT_LABEL_KEYS: Record = { + fix: "workspace.browser.annotate.intents.fix", + change: "workspace.browser.annotate.intents.change", + question: "workspace.browser.annotate.intents.question", + approve: "workspace.browser.annotate.intents.approve", +}; + +function IntentChip({ + active, + intent, + label, + onSelect, +}: { + active: boolean; + intent: BrowserAnnotationIntent; + label: string; + onSelect: (intent: BrowserAnnotationIntent) => void; +}) { + const handlePress = useCallback(() => { + onSelect(intent); + }, [intent, onSelect]); + const chipStyle = useMemo(() => [styles.intentChip, active && styles.intentChipActive], [active]); + const textStyle = useMemo( + () => [styles.intentChipText, active && styles.intentChipTextActive], + [active], + ); + const accessibilityState = useMemo(() => ({ selected: active }), [active]); + return ( + + {label} + + ); +} + +function BrowserElementAnnotationCard({ + selection, + onSubmit, + onCancel, +}: { + selection: BrowserElementSelection; + onSubmit: (annotation: BrowserElementAnnotation) => void; + onCancel: () => void; +}) { + const { t } = useTranslation(); + const [comment, setComment] = useState(""); + const [intent, setIntent] = useState("fix"); + const commentRef = useRef(comment); + commentRef.current = comment; + const intentRef = useRef(intent); + intentRef.current = intent; + + const handleSubmit = useCallback(() => { + onSubmit({ comment: commentRef.current, intent: intentRef.current }); + }, [onSubmit]); + + useEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + event.preventDefault(); + event.stopPropagation(); + onCancel(); + return; + } + if (event.key === "Enter" && (event.metaKey || event.ctrlKey)) { + event.preventDefault(); + event.stopPropagation(); + handleSubmit(); + } + }; + window.addEventListener("keydown", handleKeyDown, true); + return () => { + window.removeEventListener("keydown", handleKeyDown, true); + }; + }, [handleSubmit, onCancel]); + + const elementText = truncateText(selection.text.trim().replace(/\s+/g, " "), 60); + const elementLabel = elementText ? `${selection.tag} · ${elementText}` : selection.tag; + + return ( + + + + + {t("workspace.browser.annotate.title")} + + + + + + + {elementLabel} + + + {BROWSER_ANNOTATION_INTENTS.map((option) => ( + + ))} + + + + + + {t("workspace.browser.annotate.cancel")} + + + + + {t("workspace.browser.annotate.submit")} + + + + + + ); +} + +const ThemedCloseIcon = withUnistyles(X); +const ThemedAnnotationInput = withUnistyles(TextInput); +const iconForegroundMutedMapping = (theme: { colors: { foregroundMuted: string } }) => ({ + color: theme.colors.foregroundMuted, +}); +const annotationInputMapping = (theme: { colors: { foregroundMuted: string } }) => ({ + placeholderTextColor: theme.colors.foregroundMuted, +}); + const styles = StyleSheet.create((theme) => ({ container: { flex: 1, @@ -1088,11 +1867,13 @@ const styles = StyleSheet.create((theme) => ({ flexDirection: "row", alignItems: "center", gap: theme.spacing[1], + flexShrink: 0, }, chromeRight: { flexDirection: "row", alignItems: "center", gap: theme.spacing[1], + flexShrink: 0, }, iconButton: { width: 28, @@ -1144,6 +1925,131 @@ const styles = StyleSheet.create((theme) => ({ minHeight: 0, overflow: "hidden", }, + // When a fixed device size is active, center the framed webview both axes over + // a muted backdrop instead of left-aligning it. + webviewWrapDeviceFrame: { + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.surface1, + padding: theme.spacing[3], + }, + deviceTrigger: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[1], + }, + toolbarTooltipText: { + fontSize: theme.fontSize.xs, + color: theme.colors.popoverForeground, + }, + annotationOverlay: { + position: "absolute", + left: 0, + right: 0, + bottom: 0, + padding: theme.spacing[3], + alignItems: "center", + }, + annotationCard: { + width: "100%", + maxWidth: 420, + gap: theme.spacing[2], + padding: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + borderWidth: 1, + borderColor: theme.colors.border, + backgroundColor: theme.colors.surface0, + shadowColor: "#000", + shadowOpacity: 0.18, + shadowRadius: 16, + shadowOffset: { width: 0, height: 6 }, + }, + annotationHeader: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + gap: theme.spacing[2], + }, + annotationTitle: { + flex: 1, + fontSize: theme.fontSize.sm, + fontWeight: "600", + color: theme.colors.foreground, + }, + annotationCloseButton: { + width: 24, + height: 24, + alignItems: "center", + justifyContent: "center", + borderRadius: theme.borderRadius.md, + }, + annotationElement: { + fontSize: theme.fontSize.xs, + color: theme.colors.foregroundMuted, + }, + annotationIntents: { + flexDirection: "row", + flexWrap: "wrap", + gap: theme.spacing[1], + }, + intentChip: { + paddingHorizontal: theme.spacing[2], + paddingVertical: theme.spacing[1], + borderRadius: theme.borderRadius.md, + borderWidth: 1, + borderColor: theme.colors.border, + backgroundColor: theme.colors.surface1, + }, + intentChipActive: { + borderColor: theme.colors.accent, + backgroundColor: `${String(theme.colors.accent)}20`, + }, + intentChipText: { + fontSize: theme.fontSize.xs, + fontWeight: "500", + color: theme.colors.foregroundMuted, + }, + intentChipTextActive: { + color: theme.colors.accent, + }, + annotationInput: { + minHeight: 64, + fontSize: theme.fontSize.sm, + color: theme.colors.foreground, + borderRadius: theme.borderRadius.md, + borderWidth: 1, + borderColor: theme.colors.border, + backgroundColor: theme.colors.surface1, + paddingHorizontal: theme.spacing[2], + paddingVertical: theme.spacing[2], + textAlignVertical: "top", + }, + annotationActions: { + flexDirection: "row", + justifyContent: "flex-end", + gap: theme.spacing[2], + }, + annotationSecondaryButton: { + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.md, + }, + annotationSecondaryText: { + fontSize: theme.fontSize.sm, + fontWeight: "500", + color: theme.colors.foregroundMuted, + }, + annotationPrimaryButton: { + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.md, + backgroundColor: theme.colors.accent, + }, + annotationPrimaryText: { + fontSize: theme.fontSize.sm, + fontWeight: "600", + color: theme.colors.accentForeground ?? "#ffffff", + }, unavailableState: { flex: 1, alignItems: "center", diff --git a/packages/app/src/composer/attachments/submit.ts b/packages/app/src/composer/attachments/submit.ts index ccfae9e60..add962ec3 100644 --- a/packages/app/src/composer/attachments/submit.ts +++ b/packages/app/src/composer/attachments/submit.ts @@ -26,6 +26,9 @@ export function splitComposerAttachmentsForSubmit(attachments: ComposerAttachmen } if (isWorkspaceAttachment(attachment)) { + if (attachment.kind === "browser_element" && attachment.attachment.screenshot) { + images.push(attachment.attachment.screenshot); + } const workspaceAttachment = workspaceAttachmentToSubmitAttachment(attachment); if (workspaceAttachment) { agentAttachments.push(workspaceAttachment); diff --git a/packages/app/src/composer/attachments/workspace-cleanup.ts b/packages/app/src/composer/attachments/workspace-cleanup.ts index c39ce5be7..7d1b9fb11 100644 --- a/packages/app/src/composer/attachments/workspace-cleanup.ts +++ b/packages/app/src/composer/attachments/workspace-cleanup.ts @@ -11,6 +11,8 @@ export function getAttachmentKey(attachment: WorkspaceComposerAttachment): strin tag: attachment.attachment.tag, text: attachment.attachment.text, html: attachment.attachment.outerHTML, + intent: attachment.attachment.intent ?? null, + comment: attachment.attachment.comment ?? null, }); } if (isPullRequestContextAttachment(attachment)) { diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 2af6d692e..bcb5d4839 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -138,6 +138,13 @@ export interface DesktopBrowserBridge { executeAutomationCommand?: ( request: BrowserAutomationExecuteRequest, ) => Promise; + /** Capture a PNG screenshot of the guest viewport cropped to `rect`. */ + captureElement?: ( + browserId: string, + rect: { x: number; y: number; width: number; height: number }, + ) => Promise; + /** Copy element text and/or an image to the system clipboard from main. */ + copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise; } export interface DesktopInvokeBridge { diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index 039fb9a60..cb9fd0844 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -423,6 +423,25 @@ export const ar: TranslationResources = { openDevTools: "افتح أدوات تطوير المتصفح", cancelSelector: "إلغاء محدد العنصر", selectElement: "حدد العنصر", + grabElement: "نسخ العنصر إلى الحافظة", + grabElementLabel: "العنصر", + grabFailed: "تعذّر نسخ العنصر", + }, + annotate: { + title: "إرسال ملاحظات إلى الوكيل", + placeholder: "صف ما الذي ينبغي تغييره…", + submit: "إرفاق", + cancel: "إلغاء", + intents: { + fix: "إصلاح", + change: "تغيير", + question: "سؤال", + approve: "موافقة", + }, + }, + devices: { + label: "حجم الجهاز", + responsive: "متجاوب", }, errors: { failedToLoad: "فشل تحميل الصفحة", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 39a648c5e..ff2702b90 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -423,6 +423,25 @@ export const en = { openDevTools: "Open browser dev tools", cancelSelector: "Cancel element selector", selectElement: "Select element", + grabElement: "Copy element to clipboard", + grabElementLabel: "element", + grabFailed: "Couldn't copy element", + }, + annotate: { + title: "Send feedback to agent", + placeholder: "Describe what should change…", + submit: "Attach", + cancel: "Cancel", + intents: { + fix: "Fix", + change: "Change", + question: "Question", + approve: "Approve", + }, + }, + devices: { + label: "Device size", + responsive: "Responsive", }, errors: { failedToLoad: "Failed to load page", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index acf532a4c..1fbb47b2b 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -427,6 +427,25 @@ export const es: TranslationResources = { openDevTools: "Abrir herramientas de desarrollo del navegador", cancelSelector: "Cancelar selector de elementos", selectElement: "Seleccionar elemento", + grabElement: "Copiar elemento al portapapeles", + grabElementLabel: "elemento", + grabFailed: "No se pudo copiar el elemento", + }, + annotate: { + title: "Enviar comentarios al agente", + placeholder: "Describe qué debería cambiar…", + submit: "Adjuntar", + cancel: "Cancelar", + intents: { + fix: "Corregir", + change: "Cambiar", + question: "Pregunta", + approve: "Aprobar", + }, + }, + devices: { + label: "Tamaño del dispositivo", + responsive: "Adaptable", }, errors: { failedToLoad: "No se pudo cargar la página", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 4f3e314dc..54939303b 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -427,6 +427,25 @@ export const fr: TranslationResources = { openDevTools: "Outils de développement du navigateur ouvert", cancelSelector: "Annuler le sélecteur d'élément", selectElement: "Sélectionner un élément", + grabElement: "Copier l'élément dans le presse-papiers", + grabElementLabel: "élément", + grabFailed: "Impossible de copier l'élément", + }, + annotate: { + title: "Envoyer un retour à l'agent", + placeholder: "Décrivez ce qui doit changer…", + submit: "Joindre", + cancel: "Annuler", + intents: { + fix: "Corriger", + change: "Modifier", + question: "Question", + approve: "Approuver", + }, + }, + devices: { + label: "Taille de l'appareil", + responsive: "Adaptatif", }, errors: { failedToLoad: "Échec du chargement de la page", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index 017313fdc..f854c8b6f 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -427,6 +427,25 @@ export const ja: TranslationResources = { openDevTools: "ブラウザ開発ツールを開く", cancelSelector: "要素セレクターをキャンセル", selectElement: "要素を選択", + grabElement: "要素をクリップボードにコピー", + grabElementLabel: "要素", + grabFailed: "要素をコピーできませんでした", + }, + annotate: { + title: "エージェントにフィードバックを送信", + placeholder: "変更すべき内容を記述…", + submit: "添付", + cancel: "キャンセル", + intents: { + fix: "修正", + change: "変更", + question: "質問", + approve: "承認", + }, + }, + devices: { + label: "デバイスサイズ", + responsive: "レスポンシブ", }, errors: { failedToLoad: "ページの読み込みに失敗しました", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 4dd79464c..8b8ba6929 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -427,6 +427,25 @@ export const ptBR: TranslationResources = { openDevTools: "Abrir ferramentas de desenvolvedor do navegador", cancelSelector: "Cancelar seletor de elemento", selectElement: "Selecionar elemento", + grabElement: "Copiar elemento para a área de transferência", + grabElementLabel: "elemento", + grabFailed: "Não foi possível copiar o elemento", + }, + annotate: { + title: "Enviar feedback ao agente", + placeholder: "Descreva o que deve mudar…", + submit: "Anexar", + cancel: "Cancelar", + intents: { + fix: "Corrigir", + change: "Alterar", + question: "Pergunta", + approve: "Aprovar", + }, + }, + devices: { + label: "Tamanho do dispositivo", + responsive: "Responsivo", }, errors: { failedToLoad: "Falha ao carregar página", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index efea47b14..9418288a6 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -427,6 +427,25 @@ export const ru: TranslationResources = { openDevTools: "Открыть инструменты разработки браузера", cancelSelector: "Отменить выбор элемента", selectElement: "Выберите элемент", + grabElement: "Скопировать элемент в буфер обмена", + grabElementLabel: "элемент", + grabFailed: "Не удалось скопировать элемент", + }, + annotate: { + title: "Отправить отзыв агенту", + placeholder: "Опишите, что нужно изменить…", + submit: "Прикрепить", + cancel: "Отмена", + intents: { + fix: "Исправить", + change: "Изменить", + question: "Вопрос", + approve: "Одобрить", + }, + }, + devices: { + label: "Размер устройства", + responsive: "Адаптивный", }, errors: { failedToLoad: "Не удалось загрузить страницу", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 7bbab211d..2da0f70e1 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -423,6 +423,25 @@ export const zhCN: TranslationResources = { openDevTools: "打开浏览器开发者工具", cancelSelector: "取消元素选择器", selectElement: "选择元素", + grabElement: "复制元素到剪贴板", + grabElementLabel: "元素", + grabFailed: "复制元素失败", + }, + annotate: { + title: "发送反馈给智能体", + placeholder: "描述需要修改的内容…", + submit: "附加", + cancel: "取消", + intents: { + fix: "修复", + change: "修改", + question: "提问", + approve: "认可", + }, + }, + devices: { + label: "设备尺寸", + responsive: "自适应", }, errors: { failedToLoad: "页面加载失败", diff --git a/packages/app/src/stores/draft-store/index.ts b/packages/app/src/stores/draft-store/index.ts index 6199772ac..5c116a8c5 100644 --- a/packages/app/src/stores/draft-store/index.ts +++ b/packages/app/src/stores/draft-store/index.ts @@ -9,6 +9,7 @@ import { } from "@/attachments/service"; import { useCreateFlowStore } from "@/stores/create-flow-store"; import { useSessionStore, type SessionState } from "@/stores/session-store"; +import { useWorkspaceAttachmentsStore } from "@/attachments/workspace-attachments-store"; import { applyClearDraftRecord, collectReferencedAttachmentIdsFromState, @@ -141,6 +142,18 @@ async function runAttachmentGc(): Promise { collectStreamUserImageIds(session.agentStreamHead, referencedIds); } + // Browser-element screenshots live in the workspace attachment store, not in + // drafts, so collect their ids here to keep them from being garbage collected + // before the user sends the message. + const attachmentsByScope = useWorkspaceAttachmentsStore.getState().attachmentsByScope; + for (const attachments of Object.values(attachmentsByScope)) { + for (const attachment of attachments) { + if (attachment.kind === "browser_element" && attachment.attachment.screenshot) { + referencedIds.add(attachment.attachment.screenshot.id); + } + } + } + try { await garbageCollectAttachments({ referencedIds }); } catch (error) { diff --git a/packages/desktop/src/main.ts b/packages/desktop/src/main.ts index 3b83ea004..f6a779000 100644 --- a/packages/desktop/src/main.ts +++ b/packages/desktop/src/main.ts @@ -13,6 +13,7 @@ import { execFileSync } from "node:child_process"; import { app, BrowserWindow, + clipboard, Menu, ipcMain, nativeImage, @@ -301,6 +302,39 @@ ipcMain.handle("paseo:get-pending-open-project", (event) => { return result; }); +function normalizeBrowserCaptureRect( + rect: unknown, +): { x: number; y: number; width: number; height: number } | null { + if (!rect || typeof rect !== "object") { + return null; + } + const candidate = rect as Record; + const x = candidate.x; + const y = candidate.y; + const width = candidate.width; + const height = candidate.height; + if ( + typeof x !== "number" || + typeof y !== "number" || + typeof width !== "number" || + typeof height !== "number" || + !Number.isFinite(x) || + !Number.isFinite(y) || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return null; + } + return { + x: Math.max(0, Math.round(x)), + y: Math.max(0, Math.round(y)), + width: Math.round(width), + height: Math.round(height), + }; +} + ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: unknown) => { const input = readBrowserWorkspaceInput(rawInput); if (input) { @@ -371,6 +405,78 @@ ipcMain.handle("paseo:browser:clear-partition", async (_event, browserId: unknow await session.fromPartition(partition).clearStorageData(); }); +ipcMain.handle( + "paseo:browser:capture-element", + async (_event, browserId: unknown, rect: unknown) => { + if (typeof browserId !== "string" || browserId.trim().length === 0) { + return null; + } + const contents = getPaseoBrowserWebContents(browserId); + if (!contents || contents.isDestroyed()) { + return null; + } + const captureRect = normalizeBrowserCaptureRect(rect); + if (!captureRect) { + return null; + } + try { + // capturePage expects an integer rect in CSS pixels relative to the + // guest viewport, which matches getBoundingClientRect() on the page. + const image = await contents.capturePage(captureRect); + if (image.isEmpty()) { + return null; + } + return image.toDataURL(); + } catch (error) { + log.warn("[browser-capture] capture-element.failed", { + browserId, + error: error instanceof Error ? error.message : String(error), + }); + return null; + } + }, +); + +ipcMain.handle("paseo:browser:copy-element", (_event, payload: unknown): boolean => { + if (!payload || typeof payload !== "object") { + return false; + } + const { text, imageDataUrl } = payload as { text?: unknown; imageDataUrl?: unknown }; + const copyText = typeof text === "string" && text.length > 0 ? text : null; + + // Resolve the image first so we can write the clipboard exactly once and + // avoid flashing an intermediate text-only state. + let image: Electron.NativeImage | null = null; + if (typeof imageDataUrl === "string" && imageDataUrl.startsWith("data:image")) { + try { + const candidate = nativeImage.createFromDataURL(imageDataUrl); + if (!candidate.isEmpty()) { + image = candidate; + } + } catch (error) { + log.warn("[browser-capture] copy-element.image-failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + // Writing from the main process avoids the renderer's navigator.clipboard + // NotAllowedError, which fires when focus is inside the guest . + if (copyText && image) { + clipboard.write({ text: copyText, image }); + return true; + } + if (image) { + clipboard.writeImage(image); + return true; + } + if (copyText) { + clipboard.writeText(copyText); + return true; + } + return false; +}); + protocol.registerSchemesAsPrivileged([ { scheme: APP_SCHEME, diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index d0bbd5dfe..bf63cf5ae 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -86,5 +86,11 @@ contextBridge.exposeInMainWorld("paseoDesktop", { ipcRenderer.invoke("paseo:browser:clear-partition", browserId), executeAutomationCommand: (request: Record) => ipcRenderer.invoke("paseo:browser:execute-automation-command", request), + captureElement: ( + browserId: string, + rect: { x: number; y: number; width: number; height: number }, + ) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect), + copyElement: (payload: { text?: string; imageDataUrl?: string }) => + ipcRenderer.invoke("paseo:browser:copy-element", payload), }, });