feat(browser): inspect, annotate, and grab page elements for the agent (#1708)
* feat(browser): inspect, annotate, and grab page elements for the agent Build a design-review flow on top of the in-app Electron browser so users can send page elements to their coding agent with context. - Annotate: pick an element, write a comment, choose an intent (fix/change/question/approve); the element context + intent + comment go to the agent as text, and a cropped screenshot rides along as an image. - Grab: pick an element to copy its info + screenshot straight to the system clipboard (no comment), with a toast on success/failure. - Hover inspector: in select mode each element shows a floating label with tag, id/class, React component name, and pixel size. - Page markers: annotated elements on the current page get numbered badges that track scroll/resize. - Device sizes: a viewport-size menu (responsive + 13 common device presets) renders the page centered in a fixed-size frame. - Toolbar buttons now expose hover tooltips and no longer get squeezed out on narrow panes; the element selector is available to all desktop users (previously dev-only). Screenshots and clipboard writes go through new Electron main-process IPCs (capturePage + clipboard) so they work regardless of webview focus. Element screenshots are referenced by id in the workspace attachment store and are protected from the draft-store attachment GC. All new strings are translated across the six supported locales. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(browser): write element clipboard exactly once Resolve the image before writing so a combined text+image grab no longer does a redundant text-only writeText() first (which flashed an intermediate clipboard state). Addresses greptile review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * i18n: add browser annotate/grab/devices strings for ja and pt-BR main added Japanese and Brazilian Portuguese locales after this branch was created; add the browser annotation, grab, and device-size keys to keep all locales in sync with en. resources.test.ts parity passes. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
2b46caa20e
commit
b8b66816ca
@@ -9,7 +9,7 @@ import {
|
||||
} from "lucide-react-native";
|
||||
import { withUnistyles } from "react-native-unistyles";
|
||||
import type { AgentAttachment } from "@getpaseo/protocol/messages";
|
||||
import type { WorkspaceComposerAttachment } from "@/attachments/types";
|
||||
import type { BrowserAnnotationIntent, WorkspaceComposerAttachment } from "@/attachments/types";
|
||||
import { getFileTypeLabel } from "@/attachments/file-types";
|
||||
import { isPullRequestContextAttachment } from "@/attachments/workspace-attachment-utils";
|
||||
import { ICON_SIZE, type Theme } from "@/styles/theme";
|
||||
@@ -36,6 +36,13 @@ function getPullRequestContextSubtitle(attachment: WorkspaceComposerAttachment):
|
||||
return "Review";
|
||||
}
|
||||
|
||||
const BROWSER_INTENT_LABEL_KEYS: Record<BrowserAnnotationIntent, string> = {
|
||||
fix: "workspace.browser.annotate.intents.fix",
|
||||
change: "workspace.browser.annotate.intents.change",
|
||||
question: "workspace.browser.annotate.intents.question",
|
||||
approve: "workspace.browser.annotate.intents.approve",
|
||||
};
|
||||
|
||||
function getTextAttachmentSubtitle(
|
||||
attachment: Extract<AgentAttachment, { type: "text" }>,
|
||||
t: TFunction,
|
||||
@@ -89,10 +96,11 @@ export function getWorkspaceAttachmentPillContent(
|
||||
t: TFunction,
|
||||
): AttachmentPillContent {
|
||||
if (attachment.kind === "browser_element") {
|
||||
const intent = attachment.attachment.intent;
|
||||
return {
|
||||
icon: attachmentBrowserIcon,
|
||||
title: attachment.attachment.tag,
|
||||
subtitle: t("composer.attachments.element"),
|
||||
subtitle: intent ? t(BROWSER_INTENT_LABEL_KEYS[intent]) : t("composer.attachments.element"),
|
||||
};
|
||||
}
|
||||
if (isPullRequestContextAttachment(attachment)) {
|
||||
|
||||
@@ -21,6 +21,12 @@ export interface AttachmentMetadata {
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The kind of review feedback the user is attaching to a selected browser
|
||||
* element, sent to the agent alongside the element context.
|
||||
*/
|
||||
export type BrowserAnnotationIntent = "fix" | "change" | "question" | "approve";
|
||||
|
||||
export interface BrowserElementAttachment {
|
||||
url: string;
|
||||
selector: string;
|
||||
@@ -42,6 +48,16 @@ export interface BrowserElementAttachment {
|
||||
} | null;
|
||||
parentChain: string[];
|
||||
children: string[];
|
||||
/** Free-text review note the user wrote about this element, if any. */
|
||||
comment?: string;
|
||||
/** What the user wants the agent to do with this element, if annotated. */
|
||||
intent?: BrowserAnnotationIntent;
|
||||
/**
|
||||
* Cropped screenshot of the selected element, sent to the agent as an image
|
||||
* alongside the textual element context. Persisted via the attachment store;
|
||||
* referenced by id so the draft-store GC keeps it alive.
|
||||
*/
|
||||
screenshot?: AttachmentMetadata;
|
||||
formatted: string;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user