feat(browser): inspect, annotate, and grab page elements for the agent (#1708)
* feat(browser): inspect, annotate, and grab page elements for the agent Build a design-review flow on top of the in-app Electron browser so users can send page elements to their coding agent with context. - Annotate: pick an element, write a comment, choose an intent (fix/change/question/approve); the element context + intent + comment go to the agent as text, and a cropped screenshot rides along as an image. - Grab: pick an element to copy its info + screenshot straight to the system clipboard (no comment), with a toast on success/failure. - Hover inspector: in select mode each element shows a floating label with tag, id/class, React component name, and pixel size. - Page markers: annotated elements on the current page get numbered badges that track scroll/resize. - Device sizes: a viewport-size menu (responsive + 13 common device presets) renders the page centered in a fixed-size frame. - Toolbar buttons now expose hover tooltips and no longer get squeezed out on narrow panes; the element selector is available to all desktop users (previously dev-only). Screenshots and clipboard writes go through new Electron main-process IPCs (capturePage + clipboard) so they work regardless of webview focus. Element screenshots are referenced by id in the workspace attachment store and are protected from the draft-store attachment GC. All new strings are translated across the six supported locales. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(browser): write element clipboard exactly once Resolve the image before writing so a combined text+image grab no longer does a redundant text-only writeText() first (which flashed an intermediate clipboard state). Addresses greptile review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * i18n: add browser annotate/grab/devices strings for ja and pt-BR main added Japanese and Brazilian Portuguese locales after this branch was created; add the browser annotation, grab, and device-size keys to keep all locales in sync with en. resources.test.ts parity passes. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
huiliaoning
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
2b46caa20e
commit
b8b66816ca
@@ -9,7 +9,7 @@ import {
|
||||
} from "lucide-react-native";
|
||||
import { withUnistyles } from "react-native-unistyles";
|
||||
import type { AgentAttachment } from "@getpaseo/protocol/messages";
|
||||
import type { WorkspaceComposerAttachment } from "@/attachments/types";
|
||||
import type { BrowserAnnotationIntent, WorkspaceComposerAttachment } from "@/attachments/types";
|
||||
import { getFileTypeLabel } from "@/attachments/file-types";
|
||||
import { isPullRequestContextAttachment } from "@/attachments/workspace-attachment-utils";
|
||||
import { ICON_SIZE, type Theme } from "@/styles/theme";
|
||||
@@ -36,6 +36,13 @@ function getPullRequestContextSubtitle(attachment: WorkspaceComposerAttachment):
|
||||
return "Review";
|
||||
}
|
||||
|
||||
const BROWSER_INTENT_LABEL_KEYS: Record<BrowserAnnotationIntent, string> = {
|
||||
fix: "workspace.browser.annotate.intents.fix",
|
||||
change: "workspace.browser.annotate.intents.change",
|
||||
question: "workspace.browser.annotate.intents.question",
|
||||
approve: "workspace.browser.annotate.intents.approve",
|
||||
};
|
||||
|
||||
function getTextAttachmentSubtitle(
|
||||
attachment: Extract<AgentAttachment, { type: "text" }>,
|
||||
t: TFunction,
|
||||
@@ -89,10 +96,11 @@ export function getWorkspaceAttachmentPillContent(
|
||||
t: TFunction,
|
||||
): AttachmentPillContent {
|
||||
if (attachment.kind === "browser_element") {
|
||||
const intent = attachment.attachment.intent;
|
||||
return {
|
||||
icon: attachmentBrowserIcon,
|
||||
title: attachment.attachment.tag,
|
||||
subtitle: t("composer.attachments.element"),
|
||||
subtitle: intent ? t(BROWSER_INTENT_LABEL_KEYS[intent]) : t("composer.attachments.element"),
|
||||
};
|
||||
}
|
||||
if (isPullRequestContextAttachment(attachment)) {
|
||||
|
||||
@@ -21,6 +21,12 @@ export interface AttachmentMetadata {
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The kind of review feedback the user is attaching to a selected browser
|
||||
* element, sent to the agent alongside the element context.
|
||||
*/
|
||||
export type BrowserAnnotationIntent = "fix" | "change" | "question" | "approve";
|
||||
|
||||
export interface BrowserElementAttachment {
|
||||
url: string;
|
||||
selector: string;
|
||||
@@ -42,6 +48,16 @@ export interface BrowserElementAttachment {
|
||||
} | null;
|
||||
parentChain: string[];
|
||||
children: string[];
|
||||
/** Free-text review note the user wrote about this element, if any. */
|
||||
comment?: string;
|
||||
/** What the user wants the agent to do with this element, if annotated. */
|
||||
intent?: BrowserAnnotationIntent;
|
||||
/**
|
||||
* Cropped screenshot of the selected element, sent to the agent as an image
|
||||
* alongside the textual element context. Persisted via the attachment store;
|
||||
* referenced by id so the draft-store GC keeps it alive.
|
||||
*/
|
||||
screenshot?: AttachmentMetadata;
|
||||
formatted: string;
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -26,6 +26,9 @@ export function splitComposerAttachmentsForSubmit(attachments: ComposerAttachmen
|
||||
}
|
||||
|
||||
if (isWorkspaceAttachment(attachment)) {
|
||||
if (attachment.kind === "browser_element" && attachment.attachment.screenshot) {
|
||||
images.push(attachment.attachment.screenshot);
|
||||
}
|
||||
const workspaceAttachment = workspaceAttachmentToSubmitAttachment(attachment);
|
||||
if (workspaceAttachment) {
|
||||
agentAttachments.push(workspaceAttachment);
|
||||
|
||||
@@ -11,6 +11,8 @@ export function getAttachmentKey(attachment: WorkspaceComposerAttachment): strin
|
||||
tag: attachment.attachment.tag,
|
||||
text: attachment.attachment.text,
|
||||
html: attachment.attachment.outerHTML,
|
||||
intent: attachment.attachment.intent ?? null,
|
||||
comment: attachment.attachment.comment ?? null,
|
||||
});
|
||||
}
|
||||
if (isPullRequestContextAttachment(attachment)) {
|
||||
|
||||
@@ -138,6 +138,13 @@ export interface DesktopBrowserBridge {
|
||||
executeAutomationCommand?: (
|
||||
request: BrowserAutomationExecuteRequest,
|
||||
) => Promise<BrowserAutomationExecuteResponse["payload"]>;
|
||||
/** Capture a PNG screenshot of the guest viewport cropped to `rect`. */
|
||||
captureElement?: (
|
||||
browserId: string,
|
||||
rect: { x: number; y: number; width: number; height: number },
|
||||
) => Promise<string | null>;
|
||||
/** Copy element text and/or an image to the system clipboard from main. */
|
||||
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface DesktopInvokeBridge {
|
||||
|
||||
@@ -423,6 +423,25 @@ export const ar: TranslationResources = {
|
||||
openDevTools: "افتح أدوات تطوير المتصفح",
|
||||
cancelSelector: "إلغاء محدد العنصر",
|
||||
selectElement: "حدد العنصر",
|
||||
grabElement: "نسخ العنصر إلى الحافظة",
|
||||
grabElementLabel: "العنصر",
|
||||
grabFailed: "تعذّر نسخ العنصر",
|
||||
},
|
||||
annotate: {
|
||||
title: "إرسال ملاحظات إلى الوكيل",
|
||||
placeholder: "صف ما الذي ينبغي تغييره…",
|
||||
submit: "إرفاق",
|
||||
cancel: "إلغاء",
|
||||
intents: {
|
||||
fix: "إصلاح",
|
||||
change: "تغيير",
|
||||
question: "سؤال",
|
||||
approve: "موافقة",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "حجم الجهاز",
|
||||
responsive: "متجاوب",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "فشل تحميل الصفحة",
|
||||
|
||||
@@ -423,6 +423,25 @@ export const en = {
|
||||
openDevTools: "Open browser dev tools",
|
||||
cancelSelector: "Cancel element selector",
|
||||
selectElement: "Select element",
|
||||
grabElement: "Copy element to clipboard",
|
||||
grabElementLabel: "element",
|
||||
grabFailed: "Couldn't copy element",
|
||||
},
|
||||
annotate: {
|
||||
title: "Send feedback to agent",
|
||||
placeholder: "Describe what should change…",
|
||||
submit: "Attach",
|
||||
cancel: "Cancel",
|
||||
intents: {
|
||||
fix: "Fix",
|
||||
change: "Change",
|
||||
question: "Question",
|
||||
approve: "Approve",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Device size",
|
||||
responsive: "Responsive",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "Failed to load page",
|
||||
|
||||
@@ -427,6 +427,25 @@ export const es: TranslationResources = {
|
||||
openDevTools: "Abrir herramientas de desarrollo del navegador",
|
||||
cancelSelector: "Cancelar selector de elementos",
|
||||
selectElement: "Seleccionar elemento",
|
||||
grabElement: "Copiar elemento al portapapeles",
|
||||
grabElementLabel: "elemento",
|
||||
grabFailed: "No se pudo copiar el elemento",
|
||||
},
|
||||
annotate: {
|
||||
title: "Enviar comentarios al agente",
|
||||
placeholder: "Describe qué debería cambiar…",
|
||||
submit: "Adjuntar",
|
||||
cancel: "Cancelar",
|
||||
intents: {
|
||||
fix: "Corregir",
|
||||
change: "Cambiar",
|
||||
question: "Pregunta",
|
||||
approve: "Aprobar",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Tamaño del dispositivo",
|
||||
responsive: "Adaptable",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "No se pudo cargar la página",
|
||||
|
||||
@@ -427,6 +427,25 @@ export const fr: TranslationResources = {
|
||||
openDevTools: "Outils de développement du navigateur ouvert",
|
||||
cancelSelector: "Annuler le sélecteur d'élément",
|
||||
selectElement: "Sélectionner un élément",
|
||||
grabElement: "Copier l'élément dans le presse-papiers",
|
||||
grabElementLabel: "élément",
|
||||
grabFailed: "Impossible de copier l'élément",
|
||||
},
|
||||
annotate: {
|
||||
title: "Envoyer un retour à l'agent",
|
||||
placeholder: "Décrivez ce qui doit changer…",
|
||||
submit: "Joindre",
|
||||
cancel: "Annuler",
|
||||
intents: {
|
||||
fix: "Corriger",
|
||||
change: "Modifier",
|
||||
question: "Question",
|
||||
approve: "Approuver",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Taille de l'appareil",
|
||||
responsive: "Adaptatif",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "Échec du chargement de la page",
|
||||
|
||||
@@ -427,6 +427,25 @@ export const ja: TranslationResources = {
|
||||
openDevTools: "ブラウザ開発ツールを開く",
|
||||
cancelSelector: "要素セレクターをキャンセル",
|
||||
selectElement: "要素を選択",
|
||||
grabElement: "要素をクリップボードにコピー",
|
||||
grabElementLabel: "要素",
|
||||
grabFailed: "要素をコピーできませんでした",
|
||||
},
|
||||
annotate: {
|
||||
title: "エージェントにフィードバックを送信",
|
||||
placeholder: "変更すべき内容を記述…",
|
||||
submit: "添付",
|
||||
cancel: "キャンセル",
|
||||
intents: {
|
||||
fix: "修正",
|
||||
change: "変更",
|
||||
question: "質問",
|
||||
approve: "承認",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "デバイスサイズ",
|
||||
responsive: "レスポンシブ",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "ページの読み込みに失敗しました",
|
||||
|
||||
@@ -427,6 +427,25 @@ export const ptBR: TranslationResources = {
|
||||
openDevTools: "Abrir ferramentas de desenvolvedor do navegador",
|
||||
cancelSelector: "Cancelar seletor de elemento",
|
||||
selectElement: "Selecionar elemento",
|
||||
grabElement: "Copiar elemento para a área de transferência",
|
||||
grabElementLabel: "elemento",
|
||||
grabFailed: "Não foi possível copiar o elemento",
|
||||
},
|
||||
annotate: {
|
||||
title: "Enviar feedback ao agente",
|
||||
placeholder: "Descreva o que deve mudar…",
|
||||
submit: "Anexar",
|
||||
cancel: "Cancelar",
|
||||
intents: {
|
||||
fix: "Corrigir",
|
||||
change: "Alterar",
|
||||
question: "Pergunta",
|
||||
approve: "Aprovar",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Tamanho do dispositivo",
|
||||
responsive: "Responsivo",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "Falha ao carregar página",
|
||||
|
||||
@@ -427,6 +427,25 @@ export const ru: TranslationResources = {
|
||||
openDevTools: "Открыть инструменты разработки браузера",
|
||||
cancelSelector: "Отменить выбор элемента",
|
||||
selectElement: "Выберите элемент",
|
||||
grabElement: "Скопировать элемент в буфер обмена",
|
||||
grabElementLabel: "элемент",
|
||||
grabFailed: "Не удалось скопировать элемент",
|
||||
},
|
||||
annotate: {
|
||||
title: "Отправить отзыв агенту",
|
||||
placeholder: "Опишите, что нужно изменить…",
|
||||
submit: "Прикрепить",
|
||||
cancel: "Отмена",
|
||||
intents: {
|
||||
fix: "Исправить",
|
||||
change: "Изменить",
|
||||
question: "Вопрос",
|
||||
approve: "Одобрить",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Размер устройства",
|
||||
responsive: "Адаптивный",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "Не удалось загрузить страницу",
|
||||
|
||||
@@ -423,6 +423,25 @@ export const zhCN: TranslationResources = {
|
||||
openDevTools: "打开浏览器开发者工具",
|
||||
cancelSelector: "取消元素选择器",
|
||||
selectElement: "选择元素",
|
||||
grabElement: "复制元素到剪贴板",
|
||||
grabElementLabel: "元素",
|
||||
grabFailed: "复制元素失败",
|
||||
},
|
||||
annotate: {
|
||||
title: "发送反馈给智能体",
|
||||
placeholder: "描述需要修改的内容…",
|
||||
submit: "附加",
|
||||
cancel: "取消",
|
||||
intents: {
|
||||
fix: "修复",
|
||||
change: "修改",
|
||||
question: "提问",
|
||||
approve: "认可",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "设备尺寸",
|
||||
responsive: "自适应",
|
||||
},
|
||||
errors: {
|
||||
failedToLoad: "页面加载失败",
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
} from "@/attachments/service";
|
||||
import { useCreateFlowStore } from "@/stores/create-flow-store";
|
||||
import { useSessionStore, type SessionState } from "@/stores/session-store";
|
||||
import { useWorkspaceAttachmentsStore } from "@/attachments/workspace-attachments-store";
|
||||
import {
|
||||
applyClearDraftRecord,
|
||||
collectReferencedAttachmentIdsFromState,
|
||||
@@ -141,6 +142,18 @@ async function runAttachmentGc(): Promise<void> {
|
||||
collectStreamUserImageIds(session.agentStreamHead, referencedIds);
|
||||
}
|
||||
|
||||
// Browser-element screenshots live in the workspace attachment store, not in
|
||||
// drafts, so collect their ids here to keep them from being garbage collected
|
||||
// before the user sends the message.
|
||||
const attachmentsByScope = useWorkspaceAttachmentsStore.getState().attachmentsByScope;
|
||||
for (const attachments of Object.values(attachmentsByScope)) {
|
||||
for (const attachment of attachments) {
|
||||
if (attachment.kind === "browser_element" && attachment.attachment.screenshot) {
|
||||
referencedIds.add(attachment.attachment.screenshot.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await garbageCollectAttachments({ referencedIds });
|
||||
} catch (error) {
|
||||
|
||||
Reference in New Issue
Block a user