feat(browser): inspect, annotate, and grab page elements for the agent (#1708)

* feat(browser): inspect, annotate, and grab page elements for the agent

Build a design-review flow on top of the in-app Electron browser so users
can send page elements to their coding agent with context.

- Annotate: pick an element, write a comment, choose an intent
  (fix/change/question/approve); the element context + intent + comment go
  to the agent as text, and a cropped screenshot rides along as an image.
- Grab: pick an element to copy its info + screenshot straight to the
  system clipboard (no comment), with a toast on success/failure.
- Hover inspector: in select mode each element shows a floating label with
  tag, id/class, React component name, and pixel size.
- Page markers: annotated elements on the current page get numbered badges
  that track scroll/resize.
- Device sizes: a viewport-size menu (responsive + 13 common device presets)
  renders the page centered in a fixed-size frame.
- Toolbar buttons now expose hover tooltips and no longer get squeezed out
  on narrow panes; the element selector is available to all desktop users
  (previously dev-only).

Screenshots and clipboard writes go through new Electron main-process IPCs
(capturePage + clipboard) so they work regardless of webview focus. Element
screenshots are referenced by id in the workspace attachment store and are
protected from the draft-store attachment GC. All new strings are
translated across the six supported locales.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix(browser): write element clipboard exactly once

Resolve the image before writing so a combined text+image grab no longer
does a redundant text-only writeText() first (which flashed an intermediate
clipboard state). Addresses greptile review feedback.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* i18n: add browser annotate/grab/devices strings for ja and pt-BR

main added Japanese and Brazilian Portuguese locales after this branch was
created; add the browser annotation, grab, and device-size keys to keep all
locales in sync with en. resources.test.ts parity passes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

packages/app/src/attachments/attachment-pill-content.tsx

@@ -9,7 +9,7 @@ import {
 } from "lucide-react-native";
 import { withUnistyles } from "react-native-unistyles";
 import type { AgentAttachment } from "@getpaseo/protocol/messages";
-import type { WorkspaceComposerAttachment } from "@/attachments/types";
+import type { BrowserAnnotationIntent, WorkspaceComposerAttachment } from "@/attachments/types";
 import { getFileTypeLabel } from "@/attachments/file-types";
 import { isPullRequestContextAttachment } from "@/attachments/workspace-attachment-utils";
 import { ICON_SIZE, type Theme } from "@/styles/theme";
@@ -36,6 +36,13 @@ function getPullRequestContextSubtitle(attachment: WorkspaceComposerAttachment):
   return "Review";
 }
 
+const BROWSER_INTENT_LABEL_KEYS: Record<BrowserAnnotationIntent, string> = {
+  fix: "workspace.browser.annotate.intents.fix",
+  change: "workspace.browser.annotate.intents.change",
+  question: "workspace.browser.annotate.intents.question",
+  approve: "workspace.browser.annotate.intents.approve",
+};
+
 function getTextAttachmentSubtitle(
   attachment: Extract<AgentAttachment, { type: "text" }>,
   t: TFunction,
@@ -89,10 +96,11 @@ export function getWorkspaceAttachmentPillContent(
   t: TFunction,
 ): AttachmentPillContent {
   if (attachment.kind === "browser_element") {
+    const intent = attachment.attachment.intent;
     return {
       icon: attachmentBrowserIcon,
       title: attachment.attachment.tag,
-      subtitle: t("composer.attachments.element"),
+      subtitle: intent ? t(BROWSER_INTENT_LABEL_KEYS[intent]) : t("composer.attachments.element"),
     };
   }
   if (isPullRequestContextAttachment(attachment)) {

packages/app/src/attachments/types.ts

@@ -21,6 +21,12 @@ export interface AttachmentMetadata {
   createdAt: number;
 }
 
+/**
+ * The kind of review feedback the user is attaching to a selected browser
+ * element, sent to the agent alongside the element context.
+ */
+export type BrowserAnnotationIntent = "fix" | "change" | "question" | "approve";
+
 export interface BrowserElementAttachment {
   url: string;
   selector: string;
@@ -42,6 +48,16 @@ export interface BrowserElementAttachment {
   } | null;
   parentChain: string[];
   children: string[];
+  /** Free-text review note the user wrote about this element, if any. */
+  comment?: string;
+  /** What the user wants the agent to do with this element, if annotated. */
+  intent?: BrowserAnnotationIntent;
+  /**
+   * Cropped screenshot of the selected element, sent to the agent as an image
+   * alongside the textual element context. Persisted via the attachment store;
+   * referenced by id so the draft-store GC keeps it alive.
+   */
+  screenshot?: AttachmentMetadata;
   formatted: string;
 }
 

packages/app/src/composer/attachments/submit.ts

@@ -26,6 +26,9 @@ export function splitComposerAttachmentsForSubmit(attachments: ComposerAttachmen
     }
 
     if (isWorkspaceAttachment(attachment)) {
+      if (attachment.kind === "browser_element" && attachment.attachment.screenshot) {
+        images.push(attachment.attachment.screenshot);
+      }
       const workspaceAttachment = workspaceAttachmentToSubmitAttachment(attachment);
       if (workspaceAttachment) {
         agentAttachments.push(workspaceAttachment);

packages/app/src/composer/attachments/workspace-cleanup.ts

@@ -11,6 +11,8 @@ export function getAttachmentKey(attachment: WorkspaceComposerAttachment): strin
       tag: attachment.attachment.tag,
       text: attachment.attachment.text,
       html: attachment.attachment.outerHTML,
+      intent: attachment.attachment.intent ?? null,
+      comment: attachment.attachment.comment ?? null,
     });
   }
   if (isPullRequestContextAttachment(attachment)) {

packages/app/src/desktop/host.ts

@@ -138,6 +138,13 @@ export interface DesktopBrowserBridge {
   executeAutomationCommand?: (
     request: BrowserAutomationExecuteRequest,
   ) => Promise<BrowserAutomationExecuteResponse["payload"]>;
+  /** Capture a PNG screenshot of the guest viewport cropped to `rect`. */
+  captureElement?: (
+    browserId: string,
+    rect: { x: number; y: number; width: number; height: number },
+  ) => Promise<string | null>;
+  /** Copy element text and/or an image to the system clipboard from main. */
+  copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
 }
 
 export interface DesktopInvokeBridge {

packages/app/src/i18n/resources/ar.ts

@@ -423,6 +423,25 @@ export const ar: TranslationResources = {
         openDevTools: "افتح أدوات تطوير المتصفح",
         cancelSelector: "إلغاء محدد العنصر",
         selectElement: "حدد العنصر",
+        grabElement: "نسخ العنصر إلى الحافظة",
+        grabElementLabel: "العنصر",
+        grabFailed: "تعذّر نسخ العنصر",
+      },
+      annotate: {
+        title: "إرسال ملاحظات إلى الوكيل",
+        placeholder: "صف ما الذي ينبغي تغييره…",
+        submit: "إرفاق",
+        cancel: "إلغاء",
+        intents: {
+          fix: "إصلاح",
+          change: "تغيير",
+          question: "سؤال",
+          approve: "موافقة",
+        },
+      },
+      devices: {
+        label: "حجم الجهاز",
+        responsive: "متجاوب",
       },
       errors: {
         failedToLoad: "فشل تحميل الصفحة",

packages/app/src/i18n/resources/en.ts

@@ -423,6 +423,25 @@ export const en = {
         openDevTools: "Open browser dev tools",
         cancelSelector: "Cancel element selector",
         selectElement: "Select element",
+        grabElement: "Copy element to clipboard",
+        grabElementLabel: "element",
+        grabFailed: "Couldn't copy element",
+      },
+      annotate: {
+        title: "Send feedback to agent",
+        placeholder: "Describe what should change…",
+        submit: "Attach",
+        cancel: "Cancel",
+        intents: {
+          fix: "Fix",
+          change: "Change",
+          question: "Question",
+          approve: "Approve",
+        },
+      },
+      devices: {
+        label: "Device size",
+        responsive: "Responsive",
       },
       errors: {
         failedToLoad: "Failed to load page",

packages/app/src/i18n/resources/es.ts

@@ -427,6 +427,25 @@ export const es: TranslationResources = {
         openDevTools: "Abrir herramientas de desarrollo del navegador",
         cancelSelector: "Cancelar selector de elementos",
         selectElement: "Seleccionar elemento",
+        grabElement: "Copiar elemento al portapapeles",
+        grabElementLabel: "elemento",
+        grabFailed: "No se pudo copiar el elemento",
+      },
+      annotate: {
+        title: "Enviar comentarios al agente",
+        placeholder: "Describe qué debería cambiar…",
+        submit: "Adjuntar",
+        cancel: "Cancelar",
+        intents: {
+          fix: "Corregir",
+          change: "Cambiar",
+          question: "Pregunta",
+          approve: "Aprobar",
+        },
+      },
+      devices: {
+        label: "Tamaño del dispositivo",
+        responsive: "Adaptable",
       },
       errors: {
         failedToLoad: "No se pudo cargar la página",

packages/app/src/i18n/resources/fr.ts

@@ -427,6 +427,25 @@ export const fr: TranslationResources = {
         openDevTools: "Outils de développement du navigateur ouvert",
         cancelSelector: "Annuler le sélecteur d'élément",
         selectElement: "Sélectionner un élément",
+        grabElement: "Copier l'élément dans le presse-papiers",
+        grabElementLabel: "élément",
+        grabFailed: "Impossible de copier l'élément",
+      },
+      annotate: {
+        title: "Envoyer un retour à l'agent",
+        placeholder: "Décrivez ce qui doit changer…",
+        submit: "Joindre",
+        cancel: "Annuler",
+        intents: {
+          fix: "Corriger",
+          change: "Modifier",
+          question: "Question",
+          approve: "Approuver",
+        },
+      },
+      devices: {
+        label: "Taille de l'appareil",
+        responsive: "Adaptatif",
       },
       errors: {
         failedToLoad: "Échec du chargement de la page",

packages/app/src/i18n/resources/ja.ts

@@ -427,6 +427,25 @@ export const ja: TranslationResources = {
         openDevTools: "ブラウザ開発ツールを開く",
         cancelSelector: "要素セレクターをキャンセル",
         selectElement: "要素を選択",
+        grabElement: "要素をクリップボードにコピー",
+        grabElementLabel: "要素",
+        grabFailed: "要素をコピーできませんでした",
+      },
+      annotate: {
+        title: "エージェントにフィードバックを送信",
+        placeholder: "変更すべき内容を記述…",
+        submit: "添付",
+        cancel: "キャンセル",
+        intents: {
+          fix: "修正",
+          change: "変更",
+          question: "質問",
+          approve: "承認",
+        },
+      },
+      devices: {
+        label: "デバイスサイズ",
+        responsive: "レスポンシブ",
       },
       errors: {
         failedToLoad: "ページの読み込みに失敗しました",

packages/app/src/i18n/resources/pt-BR.ts

@@ -427,6 +427,25 @@ export const ptBR: TranslationResources = {
         openDevTools: "Abrir ferramentas de desenvolvedor do navegador",
         cancelSelector: "Cancelar seletor de elemento",
         selectElement: "Selecionar elemento",
+        grabElement: "Copiar elemento para a área de transferência",
+        grabElementLabel: "elemento",
+        grabFailed: "Não foi possível copiar o elemento",
+      },
+      annotate: {
+        title: "Enviar feedback ao agente",
+        placeholder: "Descreva o que deve mudar…",
+        submit: "Anexar",
+        cancel: "Cancelar",
+        intents: {
+          fix: "Corrigir",
+          change: "Alterar",
+          question: "Pergunta",
+          approve: "Aprovar",
+        },
+      },
+      devices: {
+        label: "Tamanho do dispositivo",
+        responsive: "Responsivo",
       },
       errors: {
         failedToLoad: "Falha ao carregar página",

packages/app/src/i18n/resources/ru.ts

@@ -427,6 +427,25 @@ export const ru: TranslationResources = {
         openDevTools: "Открыть инструменты разработки браузера",
         cancelSelector: "Отменить выбор элемента",
         selectElement: "Выберите элемент",
+        grabElement: "Скопировать элемент в буфер обмена",
+        grabElementLabel: "элемент",
+        grabFailed: "Не удалось скопировать элемент",
+      },
+      annotate: {
+        title: "Отправить отзыв агенту",
+        placeholder: "Опишите, что нужно изменить…",
+        submit: "Прикрепить",
+        cancel: "Отмена",
+        intents: {
+          fix: "Исправить",
+          change: "Изменить",
+          question: "Вопрос",
+          approve: "Одобрить",
+        },
+      },
+      devices: {
+        label: "Размер устройства",
+        responsive: "Адаптивный",
       },
       errors: {
         failedToLoad: "Не удалось загрузить страницу",

packages/app/src/i18n/resources/zh-CN.ts

@@ -423,6 +423,25 @@ export const zhCN: TranslationResources = {
         openDevTools: "打开浏览器开发者工具",
         cancelSelector: "取消元素选择器",
         selectElement: "选择元素",
+        grabElement: "复制元素到剪贴板",
+        grabElementLabel: "元素",
+        grabFailed: "复制元素失败",
+      },
+      annotate: {
+        title: "发送反馈给智能体",
+        placeholder: "描述需要修改的内容…",
+        submit: "附加",
+        cancel: "取消",
+        intents: {
+          fix: "修复",
+          change: "修改",
+          question: "提问",
+          approve: "认可",
+        },
+      },
+      devices: {
+        label: "设备尺寸",
+        responsive: "自适应",
       },
       errors: {
         failedToLoad: "页面加载失败",

packages/app/src/stores/draft-store/index.ts

@@ -9,6 +9,7 @@ import {
 } from "@/attachments/service";
 import { useCreateFlowStore } from "@/stores/create-flow-store";
 import { useSessionStore, type SessionState } from "@/stores/session-store";
+import { useWorkspaceAttachmentsStore } from "@/attachments/workspace-attachments-store";
 import {
   applyClearDraftRecord,
   collectReferencedAttachmentIdsFromState,
@@ -141,6 +142,18 @@ async function runAttachmentGc(): Promise<void> {
     collectStreamUserImageIds(session.agentStreamHead, referencedIds);
   }
 
+  // Browser-element screenshots live in the workspace attachment store, not in
+  // drafts, so collect their ids here to keep them from being garbage collected
+  // before the user sends the message.
+  const attachmentsByScope = useWorkspaceAttachmentsStore.getState().attachmentsByScope;
+  for (const attachments of Object.values(attachmentsByScope)) {
+    for (const attachment of attachments) {
+      if (attachment.kind === "browser_element" && attachment.attachment.screenshot) {
+        referencedIds.add(attachment.attachment.screenshot.id);
+      }
+    }
+  }
+
   try {
     await garbageCollectAttachments({ referencedIds });
   } catch (error) {

packages/desktop/src/main.ts

@@ -13,6 +13,7 @@ import { execFileSync } from "node:child_process";
 import {
   app,
   BrowserWindow,
+  clipboard,
   Menu,
   ipcMain,
   nativeImage,
@@ -301,6 +302,39 @@ ipcMain.handle("paseo:get-pending-open-project", (event) => {
   return result;
 });
 
+function normalizeBrowserCaptureRect(
+  rect: unknown,
+): { x: number; y: number; width: number; height: number } | null {
+  if (!rect || typeof rect !== "object") {
+    return null;
+  }
+  const candidate = rect as Record<string, unknown>;
+  const x = candidate.x;
+  const y = candidate.y;
+  const width = candidate.width;
+  const height = candidate.height;
+  if (
+    typeof x !== "number" ||
+    typeof y !== "number" ||
+    typeof width !== "number" ||
+    typeof height !== "number" ||
+    !Number.isFinite(x) ||
+    !Number.isFinite(y) ||
+    !Number.isFinite(width) ||
+    !Number.isFinite(height) ||
+    width <= 0 ||
+    height <= 0
+  ) {
+    return null;
+  }
+  return {
+    x: Math.max(0, Math.round(x)),
+    y: Math.max(0, Math.round(y)),
+    width: Math.round(width),
+    height: Math.round(height),
+  };
+}
+
 ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: unknown) => {
   const input = readBrowserWorkspaceInput(rawInput);
   if (input) {
@@ -371,6 +405,78 @@ ipcMain.handle("paseo:browser:clear-partition", async (_event, browserId: unknow
   await session.fromPartition(partition).clearStorageData();
 });
 
+ipcMain.handle(
+  "paseo:browser:capture-element",
+  async (_event, browserId: unknown, rect: unknown) => {
+    if (typeof browserId !== "string" || browserId.trim().length === 0) {
+      return null;
+    }
+    const contents = getPaseoBrowserWebContents(browserId);
+    if (!contents || contents.isDestroyed()) {
+      return null;
+    }
+    const captureRect = normalizeBrowserCaptureRect(rect);
+    if (!captureRect) {
+      return null;
+    }
+    try {
+      // capturePage expects an integer rect in CSS pixels relative to the
+      // guest viewport, which matches getBoundingClientRect() on the page.
+      const image = await contents.capturePage(captureRect);
+      if (image.isEmpty()) {
+        return null;
+      }
+      return image.toDataURL();
+    } catch (error) {
+      log.warn("[browser-capture] capture-element.failed", {
+        browserId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return null;
+    }
+  },
+);
+
+ipcMain.handle("paseo:browser:copy-element", (_event, payload: unknown): boolean => {
+  if (!payload || typeof payload !== "object") {
+    return false;
+  }
+  const { text, imageDataUrl } = payload as { text?: unknown; imageDataUrl?: unknown };
+  const copyText = typeof text === "string" && text.length > 0 ? text : null;
+
+  // Resolve the image first so we can write the clipboard exactly once and
+  // avoid flashing an intermediate text-only state.
+  let image: Electron.NativeImage | null = null;
+  if (typeof imageDataUrl === "string" && imageDataUrl.startsWith("data:image")) {
+    try {
+      const candidate = nativeImage.createFromDataURL(imageDataUrl);
+      if (!candidate.isEmpty()) {
+        image = candidate;
+      }
+    } catch (error) {
+      log.warn("[browser-capture] copy-element.image-failed", {
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
+  // Writing from the main process avoids the renderer's navigator.clipboard
+  // NotAllowedError, which fires when focus is inside the guest <webview>.
+  if (copyText && image) {
+    clipboard.write({ text: copyText, image });
+    return true;
+  }
+  if (image) {
+    clipboard.writeImage(image);
+    return true;
+  }
+  if (copyText) {
+    clipboard.writeText(copyText);
+    return true;
+  }
+  return false;
+});
+
 protocol.registerSchemesAsPrivileged([
   {
     scheme: APP_SCHEME,

packages/desktop/src/preload.ts

@@ -86,5 +86,11 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
       ipcRenderer.invoke("paseo:browser:clear-partition", browserId),
     executeAutomationCommand: (request: Record<string, unknown>) =>
       ipcRenderer.invoke("paseo:browser:execute-automation-command", request),
+    captureElement: (
+      browserId: string,
+      rect: { x: number; y: number; width: number; height: number },
+    ) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
+    copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
+      ipcRenderer.invoke("paseo:browser:copy-element", payload),
   },
 });