feat(browser): inspect, annotate, and grab page elements for the agent (#1708)

* feat(browser): inspect, annotate, and grab page elements for the agent

Build a design-review flow on top of the in-app Electron browser so users
can send page elements to their coding agent with context.

- Annotate: pick an element, write a comment, choose an intent
  (fix/change/question/approve); the element context + intent + comment go
  to the agent as text, and a cropped screenshot rides along as an image.
- Grab: pick an element to copy its info + screenshot straight to the
  system clipboard (no comment), with a toast on success/failure.
- Hover inspector: in select mode each element shows a floating label with
  tag, id/class, React component name, and pixel size.
- Page markers: annotated elements on the current page get numbered badges
  that track scroll/resize.
- Device sizes: a viewport-size menu (responsive + 13 common device presets)
  renders the page centered in a fixed-size frame.
- Toolbar buttons now expose hover tooltips and no longer get squeezed out
  on narrow panes; the element selector is available to all desktop users
  (previously dev-only).

Screenshots and clipboard writes go through new Electron main-process IPCs
(capturePage + clipboard) so they work regardless of webview focus. Element
screenshots are referenced by id in the workspace attachment store and are
protected from the draft-store attachment GC. All new strings are
translated across the six supported locales.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix(browser): write element clipboard exactly once

Resolve the image before writing so a combined text+image grab no longer
does a redundant text-only writeText() first (which flashed an intermediate
clipboard state). Addresses greptile review feedback.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* i18n: add browser annotate/grab/devices strings for ja and pt-BR

main added Japanese and Brazilian Portuguese locales after this branch was
created; add the browser annotation, grab, and device-size keys to keep all
locales in sync with en. resources.test.ts parity passes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
huiliaoning
2026-06-30 22:58:16 +02:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 2b46caa20e
commit b8b66816ca
17 changed files with 1313 additions and 94 deletions
+106
View File
@@ -13,6 +13,7 @@ import { execFileSync } from "node:child_process";
import {
app,
BrowserWindow,
clipboard,
Menu,
ipcMain,
nativeImage,
@@ -301,6 +302,39 @@ ipcMain.handle("paseo:get-pending-open-project", (event) => {
return result;
});
function normalizeBrowserCaptureRect(
rect: unknown,
): { x: number; y: number; width: number; height: number } | null {
if (!rect || typeof rect !== "object") {
return null;
}
const candidate = rect as Record<string, unknown>;
const x = candidate.x;
const y = candidate.y;
const width = candidate.width;
const height = candidate.height;
if (
typeof x !== "number" ||
typeof y !== "number" ||
typeof width !== "number" ||
typeof height !== "number" ||
!Number.isFinite(x) ||
!Number.isFinite(y) ||
!Number.isFinite(width) ||
!Number.isFinite(height) ||
width <= 0 ||
height <= 0
) {
return null;
}
return {
x: Math.max(0, Math.round(x)),
y: Math.max(0, Math.round(y)),
width: Math.round(width),
height: Math.round(height),
};
}
ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: unknown) => {
const input = readBrowserWorkspaceInput(rawInput);
if (input) {
@@ -371,6 +405,78 @@ ipcMain.handle("paseo:browser:clear-partition", async (_event, browserId: unknow
await session.fromPartition(partition).clearStorageData();
});
ipcMain.handle(
"paseo:browser:capture-element",
async (_event, browserId: unknown, rect: unknown) => {
if (typeof browserId !== "string" || browserId.trim().length === 0) {
return null;
}
const contents = getPaseoBrowserWebContents(browserId);
if (!contents || contents.isDestroyed()) {
return null;
}
const captureRect = normalizeBrowserCaptureRect(rect);
if (!captureRect) {
return null;
}
try {
// capturePage expects an integer rect in CSS pixels relative to the
// guest viewport, which matches getBoundingClientRect() on the page.
const image = await contents.capturePage(captureRect);
if (image.isEmpty()) {
return null;
}
return image.toDataURL();
} catch (error) {
log.warn("[browser-capture] capture-element.failed", {
browserId,
error: error instanceof Error ? error.message : String(error),
});
return null;
}
},
);
ipcMain.handle("paseo:browser:copy-element", (_event, payload: unknown): boolean => {
if (!payload || typeof payload !== "object") {
return false;
}
const { text, imageDataUrl } = payload as { text?: unknown; imageDataUrl?: unknown };
const copyText = typeof text === "string" && text.length > 0 ? text : null;
// Resolve the image first so we can write the clipboard exactly once and
// avoid flashing an intermediate text-only state.
let image: Electron.NativeImage | null = null;
if (typeof imageDataUrl === "string" && imageDataUrl.startsWith("data:image")) {
try {
const candidate = nativeImage.createFromDataURL(imageDataUrl);
if (!candidate.isEmpty()) {
image = candidate;
}
} catch (error) {
log.warn("[browser-capture] copy-element.image-failed", {
error: error instanceof Error ? error.message : String(error),
});
}
}
// Writing from the main process avoids the renderer's navigator.clipboard
// NotAllowedError, which fires when focus is inside the guest <webview>.
if (copyText && image) {
clipboard.write({ text: copyText, image });
return true;
}
if (image) {
clipboard.writeImage(image);
return true;
}
if (copyText) {
clipboard.writeText(copyText);
return true;
}
return false;
});
protocol.registerSchemesAsPrivileged([
{
scheme: APP_SCHEME,
+6
View File
@@ -86,5 +86,11 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
ipcRenderer.invoke("paseo:browser:clear-partition", browserId),
executeAutomationCommand: (request: Record<string, unknown>) =>
ipcRenderer.invoke("paseo:browser:execute-automation-command", request),
captureElement: (
browserId: string,
rect: { x: number; y: number; width: number; height: number },
) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
ipcRenderer.invoke("paseo:browser:copy-element", payload),
},
});