feat(browser): inspect, annotate, and grab page elements for the agent (#1708)
* feat(browser): inspect, annotate, and grab page elements for the agent Build a design-review flow on top of the in-app Electron browser so users can send page elements to their coding agent with context. - Annotate: pick an element, write a comment, choose an intent (fix/change/question/approve); the element context + intent + comment go to the agent as text, and a cropped screenshot rides along as an image. - Grab: pick an element to copy its info + screenshot straight to the system clipboard (no comment), with a toast on success/failure. - Hover inspector: in select mode each element shows a floating label with tag, id/class, React component name, and pixel size. - Page markers: annotated elements on the current page get numbered badges that track scroll/resize. - Device sizes: a viewport-size menu (responsive + 13 common device presets) renders the page centered in a fixed-size frame. - Toolbar buttons now expose hover tooltips and no longer get squeezed out on narrow panes; the element selector is available to all desktop users (previously dev-only). Screenshots and clipboard writes go through new Electron main-process IPCs (capturePage + clipboard) so they work regardless of webview focus. Element screenshots are referenced by id in the workspace attachment store and are protected from the draft-store attachment GC. All new strings are translated across the six supported locales. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(browser): write element clipboard exactly once Resolve the image before writing so a combined text+image grab no longer does a redundant text-only writeText() first (which flashed an intermediate clipboard state). Addresses greptile review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * i18n: add browser annotate/grab/devices strings for ja and pt-BR main added Japanese and Brazilian Portuguese locales after this branch was created; add the browser annotation, grab, and device-size keys to keep all locales in sync with en. resources.test.ts parity passes. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
2b46caa20e
commit
b8b66816ca
@@ -13,6 +13,7 @@ import { execFileSync } from "node:child_process";
|
||||
import {
|
||||
app,
|
||||
BrowserWindow,
|
||||
clipboard,
|
||||
Menu,
|
||||
ipcMain,
|
||||
nativeImage,
|
||||
@@ -301,6 +302,39 @@ ipcMain.handle("paseo:get-pending-open-project", (event) => {
|
||||
return result;
|
||||
});
|
||||
|
||||
function normalizeBrowserCaptureRect(
|
||||
rect: unknown,
|
||||
): { x: number; y: number; width: number; height: number } | null {
|
||||
if (!rect || typeof rect !== "object") {
|
||||
return null;
|
||||
}
|
||||
const candidate = rect as Record<string, unknown>;
|
||||
const x = candidate.x;
|
||||
const y = candidate.y;
|
||||
const width = candidate.width;
|
||||
const height = candidate.height;
|
||||
if (
|
||||
typeof x !== "number" ||
|
||||
typeof y !== "number" ||
|
||||
typeof width !== "number" ||
|
||||
typeof height !== "number" ||
|
||||
!Number.isFinite(x) ||
|
||||
!Number.isFinite(y) ||
|
||||
!Number.isFinite(width) ||
|
||||
!Number.isFinite(height) ||
|
||||
width <= 0 ||
|
||||
height <= 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
x: Math.max(0, Math.round(x)),
|
||||
y: Math.max(0, Math.round(y)),
|
||||
width: Math.round(width),
|
||||
height: Math.round(height),
|
||||
};
|
||||
}
|
||||
|
||||
ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: unknown) => {
|
||||
const input = readBrowserWorkspaceInput(rawInput);
|
||||
if (input) {
|
||||
@@ -371,6 +405,78 @@ ipcMain.handle("paseo:browser:clear-partition", async (_event, browserId: unknow
|
||||
await session.fromPartition(partition).clearStorageData();
|
||||
});
|
||||
|
||||
ipcMain.handle(
|
||||
"paseo:browser:capture-element",
|
||||
async (_event, browserId: unknown, rect: unknown) => {
|
||||
if (typeof browserId !== "string" || browserId.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
const contents = getPaseoBrowserWebContents(browserId);
|
||||
if (!contents || contents.isDestroyed()) {
|
||||
return null;
|
||||
}
|
||||
const captureRect = normalizeBrowserCaptureRect(rect);
|
||||
if (!captureRect) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
// capturePage expects an integer rect in CSS pixels relative to the
|
||||
// guest viewport, which matches getBoundingClientRect() on the page.
|
||||
const image = await contents.capturePage(captureRect);
|
||||
if (image.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
return image.toDataURL();
|
||||
} catch (error) {
|
||||
log.warn("[browser-capture] capture-element.failed", {
|
||||
browserId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return null;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
ipcMain.handle("paseo:browser:copy-element", (_event, payload: unknown): boolean => {
|
||||
if (!payload || typeof payload !== "object") {
|
||||
return false;
|
||||
}
|
||||
const { text, imageDataUrl } = payload as { text?: unknown; imageDataUrl?: unknown };
|
||||
const copyText = typeof text === "string" && text.length > 0 ? text : null;
|
||||
|
||||
// Resolve the image first so we can write the clipboard exactly once and
|
||||
// avoid flashing an intermediate text-only state.
|
||||
let image: Electron.NativeImage | null = null;
|
||||
if (typeof imageDataUrl === "string" && imageDataUrl.startsWith("data:image")) {
|
||||
try {
|
||||
const candidate = nativeImage.createFromDataURL(imageDataUrl);
|
||||
if (!candidate.isEmpty()) {
|
||||
image = candidate;
|
||||
}
|
||||
} catch (error) {
|
||||
log.warn("[browser-capture] copy-element.image-failed", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Writing from the main process avoids the renderer's navigator.clipboard
|
||||
// NotAllowedError, which fires when focus is inside the guest <webview>.
|
||||
if (copyText && image) {
|
||||
clipboard.write({ text: copyText, image });
|
||||
return true;
|
||||
}
|
||||
if (image) {
|
||||
clipboard.writeImage(image);
|
||||
return true;
|
||||
}
|
||||
if (copyText) {
|
||||
clipboard.writeText(copyText);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
|
||||
protocol.registerSchemesAsPrivileged([
|
||||
{
|
||||
scheme: APP_SCHEME,
|
||||
|
||||
@@ -86,5 +86,11 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
ipcRenderer.invoke("paseo:browser:clear-partition", browserId),
|
||||
executeAutomationCommand: (request: Record<string, unknown>) =>
|
||||
ipcRenderer.invoke("paseo:browser:execute-automation-command", request),
|
||||
captureElement: (
|
||||
browserId: string,
|
||||
rect: { x: number; y: number; width: number; height: number },
|
||||
) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
|
||||
copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
|
||||
ipcRenderer.invoke("paseo:browser:copy-element", payload),
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user