Keep browser input from submitting Paseo prompts (#1982)
* fix(desktop): keep browser input out of the composer Unhandled webview keys could be redispatched into the active host window, allowing agent Enter to submit a draft prompt. Give pages first refusal for ordinary shortcuts and contain automation at the guest boundary. * fix(desktop): tighten browser shortcut validation * fix(desktop): respect browser shortcut ownership * fix(desktop): retain browser keys across windows * fix(desktop): scope browser webviews by host * Reshape browser shortcut ownership Use the browser webview registry as the single guest identity authority, scope browser operations to their host window, publish chord continuations only while pending, and restore focus to the originating browser after command center dismissal. * Fix host-scoped browser shortcut follow-ups * Fix browser keyboard review follow-ups * Fix browser keyboard lifecycle regressions * Fix browser shortcut review regressions * Fix desktop browser review regressions * Fix browser shortcut frame regressions * Fix rebase integration regressions * Preserve browser-native shortcut ownership * Isolate browser shortcuts and automation by context
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import type { TabImage } from "./service.js";
|
||||
import { adaptWebContents } from "./ipc.js";
|
||||
import { adaptWebContents, HostSnapshotEngineRegistry } from "./ipc.js";
|
||||
import type { IsolatedKeyboardInputEvent } from "./trusted-input.js";
|
||||
|
||||
class FakeImage implements TabImage {
|
||||
public toPNG(): Uint8Array {
|
||||
@@ -90,6 +91,7 @@ type ConsoleMessageListener = (
|
||||
|
||||
class FakeWebContents {
|
||||
public readonly debugger = new FakeDebugger();
|
||||
public readonly inputEvents: IsolatedKeyboardInputEvent[] = [];
|
||||
public readonly captures: Array<{
|
||||
rect: Rectangle | undefined;
|
||||
options: { stayHidden?: boolean } | undefined;
|
||||
@@ -99,7 +101,14 @@ class FakeWebContents {
|
||||
private destroyedListener: (() => void) | null = null;
|
||||
public destroyed = false;
|
||||
|
||||
public constructor(public readonly id: number) {}
|
||||
public constructor(private readonly webContentsId: number) {}
|
||||
|
||||
public get id(): number {
|
||||
if (this.destroyed) {
|
||||
throw new TypeError("Object has been destroyed");
|
||||
}
|
||||
return this.webContentsId;
|
||||
}
|
||||
|
||||
public getURL(): string {
|
||||
return "https://example.com";
|
||||
@@ -149,6 +158,10 @@ class FakeWebContents {
|
||||
this.invalidations.push("invalidate");
|
||||
}
|
||||
|
||||
public sendInputEvent(event: IsolatedKeyboardInputEvent): void {
|
||||
this.inputEvents.push(event);
|
||||
}
|
||||
|
||||
public on(event: "console-message", listener: ConsoleMessageListener): void {
|
||||
expect(event).toBe("console-message");
|
||||
this.consoleMessageListener = listener;
|
||||
@@ -178,6 +191,30 @@ class FakeWebContents {
|
||||
}
|
||||
|
||||
describe("browser automation IPC adapter", () => {
|
||||
test("isolates snapshot refs by host window and releases them on destruction", () => {
|
||||
const registry = new HostSnapshotEngineRegistry();
|
||||
const firstHost = new FakeHostWebContents(1);
|
||||
const secondHost = new FakeHostWebContents(2);
|
||||
|
||||
const firstEngine = registry.get(firstHost);
|
||||
expect(registry.get(firstHost)).toBe(firstEngine);
|
||||
expect(registry.get(secondHost)).not.toBe(firstEngine);
|
||||
|
||||
firstHost.destroy();
|
||||
expect(registry.get(new FakeHostWebContents(1))).not.toBe(firstEngine);
|
||||
});
|
||||
|
||||
test("sends contained keyboard input directly to the guest", () => {
|
||||
const contents = new FakeWebContents(19);
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
tab.sendInputEvent({ type: "keyDown", keyCode: "Enter", skipIfUnhandled: true });
|
||||
|
||||
expect(contents.inputEvents).toEqual([
|
||||
{ type: "keyDown", keyCode: "Enter", skipIfUnhandled: true },
|
||||
]);
|
||||
});
|
||||
|
||||
test("delegates viewport capture to the guest without a renderer prep bridge", async () => {
|
||||
const contents = new FakeWebContents(20);
|
||||
const tab = adaptWebContents(contents);
|
||||
@@ -211,7 +248,7 @@ describe("browser automation IPC adapter", () => {
|
||||
},
|
||||
]);
|
||||
|
||||
contents.destroy();
|
||||
expect(() => contents.destroy()).not.toThrow();
|
||||
|
||||
expect(tab.getConsoleMessages?.()).toEqual([]);
|
||||
});
|
||||
@@ -536,6 +573,21 @@ describe("browser automation IPC adapter", () => {
|
||||
});
|
||||
});
|
||||
|
||||
class FakeHostWebContents {
|
||||
private destroyedListener: (() => void) | null = null;
|
||||
|
||||
public constructor(public readonly id: number) {}
|
||||
|
||||
public once(event: "destroyed", listener: () => void): void {
|
||||
expect(event).toBe("destroyed");
|
||||
this.destroyedListener = listener;
|
||||
}
|
||||
|
||||
public destroy(): void {
|
||||
this.destroyedListener?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function flushMicrotasks(): Promise<void> {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
BrowserAutomationDialogEvent,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
|
||||
import type { IsolatedKeyboardInputEvent } from "./trusted-input.js";
|
||||
import { CdpSessionQueue } from "./cdp-session-queue.js";
|
||||
import {
|
||||
dialogAcceptValue,
|
||||
@@ -16,11 +17,12 @@ import {
|
||||
promptShimRestoreScript,
|
||||
} from "./dialog-handling.js";
|
||||
import { executeAutomationCommand } from "./service.js";
|
||||
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
|
||||
import {
|
||||
listRegisteredPaseoBrowserIds,
|
||||
listRegisteredPaseoBrowserIdsForWorkspace,
|
||||
getPaseoBrowserWebContents,
|
||||
getWorkspaceActivePaseoBrowserId,
|
||||
getPaseoBrowserWebContentsForHostWindow,
|
||||
getWorkspaceActivePaseoBrowserIdForHostWindow,
|
||||
getPaseoBrowserWorkspaceId,
|
||||
} from "../browser-webviews/index.js";
|
||||
|
||||
@@ -34,6 +36,36 @@ interface IpcHandlerRegistry {
|
||||
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
|
||||
}
|
||||
|
||||
interface HostWebContents {
|
||||
readonly id: number;
|
||||
once(event: "destroyed", listener: () => void): void;
|
||||
}
|
||||
|
||||
export class HostSnapshotEngineRegistry {
|
||||
private readonly entries = new Map<
|
||||
number,
|
||||
{ hostContents: HostWebContents; snapshotEngine: BrowserSnapshotEngine }
|
||||
>();
|
||||
|
||||
public get(hostContents: HostWebContents): BrowserSnapshotEngine {
|
||||
const existing = this.entries.get(hostContents.id);
|
||||
if (existing) {
|
||||
return existing.snapshotEngine;
|
||||
}
|
||||
const snapshotEngine = new BrowserSnapshotEngine();
|
||||
const entry = { hostContents, snapshotEngine };
|
||||
this.entries.set(hostContents.id, entry);
|
||||
hostContents.once("destroyed", () => {
|
||||
if (this.entries.get(hostContents.id) === entry) {
|
||||
this.entries.delete(hostContents.id);
|
||||
}
|
||||
});
|
||||
return snapshotEngine;
|
||||
}
|
||||
}
|
||||
|
||||
const hostSnapshotEngines = new HostSnapshotEngineRegistry();
|
||||
|
||||
interface WebContentsDebugger {
|
||||
isAttached(): boolean;
|
||||
attach(protocolVersion?: string): void;
|
||||
@@ -74,14 +106,16 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
|
||||
reload(): void;
|
||||
capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise<TabImage>;
|
||||
invalidate(): void;
|
||||
sendInputEvent(event: IsolatedKeyboardInputEvent): void;
|
||||
}
|
||||
|
||||
export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents {
|
||||
observeConsoleMessages(contents);
|
||||
const cdpQueue = getCdpQueue(contents.id);
|
||||
const dialogMonitor = getDialogMonitor(contents, cdpQueue);
|
||||
const contentsId = contents.id;
|
||||
observeConsoleMessages(contents, contentsId);
|
||||
const cdpQueue = getCdpQueue(contentsId);
|
||||
const dialogMonitor = getDialogMonitor(contents, contentsId, cdpQueue);
|
||||
return {
|
||||
id: contents.id,
|
||||
id: contentsId,
|
||||
getURL: () => contents.getURL(),
|
||||
getTitle: () => contents.getTitle(),
|
||||
canGoBack: () => contents.canGoBack(),
|
||||
@@ -95,7 +129,8 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon
|
||||
reload: () => contents.reload(),
|
||||
capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
|
||||
invalidate: () => contents.invalidate(),
|
||||
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
|
||||
sendInputEvent: (event) => contents.sendInputEvent(event),
|
||||
getConsoleMessages: () => consoleMessagesByContentsId.get(contentsId) ?? [],
|
||||
captureDialogs: (task) => dialogMonitor.capture(task),
|
||||
sendDebugCommand: (command: string, params?: Record<string, unknown>) =>
|
||||
cdpQueue.run(async () => {
|
||||
@@ -117,35 +152,36 @@ function getCdpQueue(contentsId: number): CdpSessionQueue {
|
||||
return queue;
|
||||
}
|
||||
|
||||
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||
if (observedContentsIds.has(contents.id)) {
|
||||
function observeConsoleMessages(contents: BrowserAutomationWebContents, contentsId: number): void {
|
||||
if (observedContentsIds.has(contentsId)) {
|
||||
return;
|
||||
}
|
||||
observedContentsIds.add(contents.id);
|
||||
observedContentsIds.add(contentsId);
|
||||
contents.on("console-message", (_event, level, message, line, sourceId) => {
|
||||
const entry = normalizeConsoleMessage({ level, message, line, sourceId });
|
||||
const messages = consoleMessagesByContentsId.get(contents.id) ?? [];
|
||||
const messages = consoleMessagesByContentsId.get(contentsId) ?? [];
|
||||
messages.push(entry);
|
||||
consoleMessagesByContentsId.set(contents.id, messages.slice(-MAX_CONSOLE_MESSAGES_PER_TAB));
|
||||
consoleMessagesByContentsId.set(contentsId, messages.slice(-MAX_CONSOLE_MESSAGES_PER_TAB));
|
||||
});
|
||||
contents.once("destroyed", () => {
|
||||
observedContentsIds.delete(contents.id);
|
||||
consoleMessagesByContentsId.delete(contents.id);
|
||||
cdpQueuesByContentsId.delete(contents.id);
|
||||
dialogMonitorsByContentsId.delete(contents.id);
|
||||
observedContentsIds.delete(contentsId);
|
||||
consoleMessagesByContentsId.delete(contentsId);
|
||||
cdpQueuesByContentsId.delete(contentsId);
|
||||
dialogMonitorsByContentsId.delete(contentsId);
|
||||
});
|
||||
}
|
||||
|
||||
function getDialogMonitor(
|
||||
contents: BrowserAutomationWebContents,
|
||||
contentsId: number,
|
||||
cdpQueue: CdpSessionQueue,
|
||||
): DialogMonitor {
|
||||
const existing = dialogMonitorsByContentsId.get(contents.id);
|
||||
const existing = dialogMonitorsByContentsId.get(contentsId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const monitor = new DialogMonitor(contents, cdpQueue);
|
||||
dialogMonitorsByContentsId.set(contents.id, monitor);
|
||||
const monitor = new DialogMonitor(contents, contentsId, cdpQueue);
|
||||
dialogMonitorsByContentsId.set(contentsId, monitor);
|
||||
return monitor;
|
||||
}
|
||||
|
||||
@@ -156,6 +192,7 @@ class DialogMonitor {
|
||||
|
||||
public constructor(
|
||||
private readonly contents: BrowserAutomationWebContents,
|
||||
private readonly contentsId: number,
|
||||
private readonly cdpQueue: CdpSessionQueue,
|
||||
) {}
|
||||
|
||||
@@ -168,7 +205,7 @@ class DialogMonitor {
|
||||
await this.installPromptShim();
|
||||
} catch (error) {
|
||||
console.warn("[browser-automation] Dialog capture unavailable; running command without it", {
|
||||
contentsId: this.contents.id,
|
||||
contentsId: this.contentsId,
|
||||
error,
|
||||
});
|
||||
return { result: await task(), dialogs: [] };
|
||||
@@ -337,24 +374,38 @@ function normalizeConsoleMessage(input: {
|
||||
};
|
||||
}
|
||||
|
||||
function createRegistry(): BrowserRegistry {
|
||||
function createRegistry(hostWebContentsId: number): BrowserRegistry {
|
||||
return {
|
||||
listRegisteredBrowserIds: listRegisteredPaseoBrowserIds,
|
||||
listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace,
|
||||
getTabContents(browserId: string): TabContents | null {
|
||||
const contents = getPaseoBrowserWebContents(browserId);
|
||||
const contents = getPaseoBrowserWebContentsForHostWindow(browserId, hostWebContentsId);
|
||||
return contents ? adaptWebContents(contents) : null;
|
||||
},
|
||||
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
|
||||
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
|
||||
getWorkspaceActiveBrowserId(workspaceId: string): string | null {
|
||||
return getWorkspaceActivePaseoBrowserIdForHostWindow(workspaceId, hostWebContentsId);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function registerBrowserAutomationIpc(options?: { ipc?: IpcHandlerRegistry }): void {
|
||||
const ipc = options?.ipc ?? ipcMain;
|
||||
const registry = createRegistry();
|
||||
|
||||
ipc.handle("paseo:browser:execute-automation-command", async (_event, rawRequest: unknown) => {
|
||||
ipc.handle("paseo:browser:execute-automation-command", async (event, rawRequest: unknown) => {
|
||||
const hostContents = (event as { sender?: HostWebContents }).sender;
|
||||
const hostWebContentsId = hostContents?.id;
|
||||
if (!hostContents || typeof hostWebContentsId !== "number") {
|
||||
return {
|
||||
requestId: readRequestId(rawRequest),
|
||||
ok: false as const,
|
||||
error: {
|
||||
code: "browser_unsupported" as const,
|
||||
message: "Browser automation requires a host window.",
|
||||
},
|
||||
};
|
||||
}
|
||||
const registry = createRegistry(hostWebContentsId);
|
||||
const parsed = BrowserAutomationExecuteRequestSchema.safeParse(rawRequest);
|
||||
if (!parsed.success) {
|
||||
return {
|
||||
@@ -367,7 +418,9 @@ export function registerBrowserAutomationIpc(options?: { ipc?: IpcHandlerRegistr
|
||||
},
|
||||
};
|
||||
}
|
||||
return executeAutomationCommand(parsed.data, registry);
|
||||
return executeAutomationCommand(parsed.data, registry, {
|
||||
snapshotEngine: hostSnapshotEngines.get(hostContents),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
|
||||
import type { BrowserRegistry, TabContents, TabImage } from "./service.js";
|
||||
import { executeAutomationCommand } from "./service.js";
|
||||
import type { IsolatedKeyboardInputEvent } from "./trusted-input.js";
|
||||
|
||||
const BROWSER_A = "11111111-1111-4111-8111-111111111111";
|
||||
const BROWSER_B = "22222222-2222-4222-8222-222222222222";
|
||||
@@ -37,6 +38,7 @@ class FakeTab implements TabContents {
|
||||
public readonly actions: string[] = [];
|
||||
public readonly capturedViewports: Array<{ stayHidden?: boolean }> = [];
|
||||
public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = [];
|
||||
public readonly inputEvents: IsolatedKeyboardInputEvent[] = [];
|
||||
private readonly captureStartWaiters: Array<() => void> = [];
|
||||
private readonly deferredCaptures: Array<(image: TabImage) => void> = [];
|
||||
|
||||
@@ -223,6 +225,10 @@ class FakeTab implements TabContents {
|
||||
return {};
|
||||
}
|
||||
|
||||
public sendInputEvent(event: IsolatedKeyboardInputEvent): void {
|
||||
this.inputEvents.push(event);
|
||||
}
|
||||
|
||||
public waitForCaptureStart(count: number): Promise<void> {
|
||||
if (this.capturedViewports.length >= count) {
|
||||
return Promise.resolve();
|
||||
@@ -1120,30 +1126,19 @@ describe("executeAutomationCommand", () => {
|
||||
ok: true,
|
||||
result: { command: "keypress", browserId: BROWSER_A, key: "Enter" },
|
||||
});
|
||||
expect(browser.tab.debugCommands).toEqual([
|
||||
expect(browser.tab.inputEvents).toEqual([
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyDown",
|
||||
key: "Enter",
|
||||
code: "Enter",
|
||||
windowsVirtualKeyCode: 13,
|
||||
nativeVirtualKeyCode: 13,
|
||||
text: "\r",
|
||||
unmodifiedText: "\r",
|
||||
},
|
||||
type: "keyDown",
|
||||
keyCode: "Enter",
|
||||
skipIfUnhandled: true,
|
||||
},
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyUp",
|
||||
key: "Enter",
|
||||
code: "Enter",
|
||||
windowsVirtualKeyCode: 13,
|
||||
nativeVirtualKeyCode: 13,
|
||||
},
|
||||
type: "keyUp",
|
||||
keyCode: "Enter",
|
||||
skipIfUnhandled: true,
|
||||
},
|
||||
]);
|
||||
expect(browser.tab.debugCommands).toEqual([]);
|
||||
});
|
||||
|
||||
test("keypress focuses a non-editable ref without clicking before the trusted key", async () => {
|
||||
@@ -1163,22 +1158,44 @@ describe("executeAutomationCommand", () => {
|
||||
result: { command: "keypress", browserId: BROWSER_A, key: "Enter", ref: "@e4", x: 40, y: 30 },
|
||||
});
|
||||
expect(containsScript(browser.tab, "element.focus({ preventScroll: true })")).toBe(true);
|
||||
expect(browser.tab.debugCommands.map((entry) => entry.command)).toEqual([
|
||||
"Input.dispatchKeyEvent",
|
||||
"Input.dispatchKeyEvent",
|
||||
]);
|
||||
expect(browser.tab.debugCommands.at(-2)).toEqual({
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
expect(browser.tab.inputEvents).toEqual([
|
||||
{
|
||||
type: "keyDown",
|
||||
key: "Enter",
|
||||
code: "Enter",
|
||||
windowsVirtualKeyCode: 13,
|
||||
nativeVirtualKeyCode: 13,
|
||||
text: "\r",
|
||||
unmodifiedText: "\r",
|
||||
keyCode: "Enter",
|
||||
skipIfUnhandled: true,
|
||||
},
|
||||
{
|
||||
type: "keyUp",
|
||||
keyCode: "Enter",
|
||||
skipIfUnhandled: true,
|
||||
},
|
||||
]);
|
||||
expect(browser.tab.debugCommands).toEqual([]);
|
||||
});
|
||||
|
||||
test("keypress reports unsupported when focusing an editable ref requires unavailable CDP", async () => {
|
||||
const browser = new BrowserAutomationHarness();
|
||||
browser.tab.snapshotNodes = formElements();
|
||||
browser.tab.keypressTargetEditable = true;
|
||||
|
||||
requireSnapshotRefs(await browser.snapshot());
|
||||
Object.defineProperty(browser.tab, "sendDebugCommand", { value: undefined });
|
||||
|
||||
await expect(
|
||||
browser.execute({
|
||||
command: "keypress",
|
||||
args: { browserId: BROWSER_A, ref: "@e1", key: "Enter" },
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-keypress",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: "browser_keypress requires trusted browser input",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(browser.tab.inputEvents).toEqual([]);
|
||||
});
|
||||
|
||||
test("navigate loads the requested HTTP URL in the explicit tab", async () => {
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
dispatchTrustedScroll,
|
||||
dispatchTrustedText,
|
||||
type ClickInputOptions,
|
||||
type IsolatedKeyboardInputEvent,
|
||||
} from "./trusted-input.js";
|
||||
|
||||
export interface TabContents {
|
||||
@@ -36,6 +37,7 @@ export interface TabContents {
|
||||
reload(): void;
|
||||
capturePage(options?: TabCapturePageOptions): Promise<TabImage>;
|
||||
invalidate(): void;
|
||||
sendInputEvent(event: IsolatedKeyboardInputEvent): void;
|
||||
getConsoleMessages?(): BrowserAutomationConsoleLogEntry[];
|
||||
captureDialogs?<T>(
|
||||
task: () => Promise<T>,
|
||||
@@ -1066,13 +1068,6 @@ async function executeKeypress(
|
||||
return target;
|
||||
}
|
||||
return withDialogCapture(target.contents, async () => {
|
||||
if (!target.contents.sendDebugCommand) {
|
||||
return fail(
|
||||
requestId,
|
||||
"browser_unsupported",
|
||||
"browser_keypress requires trusted browser input",
|
||||
);
|
||||
}
|
||||
let actionable: ActionabilityResult | null = null;
|
||||
if (ref) {
|
||||
const elementExpression = snapshotEngine.runtimeElementExpression({
|
||||
@@ -1094,10 +1089,17 @@ async function executeKeypress(
|
||||
return staleRefFailure(requestId, ref);
|
||||
}
|
||||
if (focused === "editable") {
|
||||
if (!target.contents.sendDebugCommand) {
|
||||
return fail(
|
||||
requestId,
|
||||
"browser_unsupported",
|
||||
"browser_keypress requires trusted browser input",
|
||||
);
|
||||
}
|
||||
await dispatchTrustedClick(cdpSender(target.contents), actionable.target.point);
|
||||
}
|
||||
}
|
||||
await dispatchTrustedKey(cdpSender(target.contents), key);
|
||||
dispatchTrustedKey((event) => target.contents.sendInputEvent(event), key);
|
||||
return {
|
||||
requestId,
|
||||
ok: true,
|
||||
|
||||
@@ -1,38 +1,42 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import type { IsolatedKeyboardInputEvent } from "./trusted-input.js";
|
||||
import { dispatchTrustedKey } from "./trusted-input.js";
|
||||
|
||||
describe("trusted browser input", () => {
|
||||
test("Space dispatches a real space key event", async () => {
|
||||
const commands: Array<{ command: string; params?: Record<string, unknown> }> = [];
|
||||
test.each([
|
||||
["a", "a", ["keyDown", "char", "keyUp"]],
|
||||
["Z", "Z", ["keyDown", "char", "keyUp"]],
|
||||
["ArrowDown", "Down", ["keyDown", "keyUp"]],
|
||||
])(
|
||||
"sends %s as Electron key code %s with unhandled redispatch disabled",
|
||||
(key, keyCode, types) => {
|
||||
const events: IsolatedKeyboardInputEvent[] = [];
|
||||
|
||||
await dispatchTrustedKey(async (command, params) => {
|
||||
commands.push({ command, ...(params ? { params } : {}) });
|
||||
return {};
|
||||
dispatchTrustedKey((event) => {
|
||||
events.push(event);
|
||||
}, key);
|
||||
|
||||
expect(events).toEqual(
|
||||
types.map((type) => ({
|
||||
type,
|
||||
keyCode,
|
||||
skipIfUnhandled: true,
|
||||
})),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test("inserts a named Space keypress", () => {
|
||||
const events: IsolatedKeyboardInputEvent[] = [];
|
||||
|
||||
dispatchTrustedKey((event) => {
|
||||
events.push(event);
|
||||
}, "Space");
|
||||
|
||||
expect(commands).toEqual([
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyDown",
|
||||
key: " ",
|
||||
code: "Space",
|
||||
windowsVirtualKeyCode: 32,
|
||||
nativeVirtualKeyCode: 32,
|
||||
text: " ",
|
||||
unmodifiedText: " ",
|
||||
},
|
||||
},
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyUp",
|
||||
key: " ",
|
||||
code: "Space",
|
||||
windowsVirtualKeyCode: 32,
|
||||
nativeVirtualKeyCode: 32,
|
||||
},
|
||||
},
|
||||
expect(events).toEqual([
|
||||
{ type: "keyDown", keyCode: "Space", skipIfUnhandled: true },
|
||||
{ type: "char", keyCode: " ", skipIfUnhandled: true },
|
||||
{ type: "keyUp", keyCode: "Space", skipIfUnhandled: true },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { KeyboardInputEvent } from "electron";
|
||||
import type { ActionablePoint } from "./actionability.js";
|
||||
import type { CdpCommandSender } from "./cdp-session-queue.js";
|
||||
|
||||
@@ -17,24 +18,21 @@ const MODIFIER_MASKS: Record<InputModifier, number> = {
|
||||
Shift: 8,
|
||||
};
|
||||
|
||||
const SPECIAL_KEY_DEFINITIONS: Record<
|
||||
string,
|
||||
{ key: string; code: string; windowsVirtualKeyCode: number; text?: string }
|
||||
> = {
|
||||
Enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, text: "\r" },
|
||||
Space: { key: " ", code: "Space", windowsVirtualKeyCode: 32, text: " " },
|
||||
Tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9, text: "\t" },
|
||||
Escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
|
||||
Backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 },
|
||||
Delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 },
|
||||
ArrowUp: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
|
||||
ArrowDown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
|
||||
ArrowLeft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
|
||||
ArrowRight: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
|
||||
Home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 },
|
||||
End: { key: "End", code: "End", windowsVirtualKeyCode: 35 },
|
||||
PageUp: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 },
|
||||
PageDown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 },
|
||||
export interface IsolatedKeyboardInputEvent extends KeyboardInputEvent {
|
||||
type: "char" | "keyDown" | "keyUp";
|
||||
// Electron accepts this NativeWebKeyboardEvent flag even though its public
|
||||
// TypeScript declarations omit it. It stops an unhandled webview key from
|
||||
// being redispatched to the embedder's active DOM element or application menu.
|
||||
skipIfUnhandled: true;
|
||||
}
|
||||
|
||||
type KeyboardInputSender = (event: IsolatedKeyboardInputEvent) => void;
|
||||
|
||||
const ELECTRON_KEY_CODE_ALIASES: Record<string, string> = {
|
||||
ArrowDown: "Down",
|
||||
ArrowLeft: "Left",
|
||||
ArrowRight: "Right",
|
||||
ArrowUp: "Up",
|
||||
};
|
||||
|
||||
export async function dispatchTrustedClick(
|
||||
@@ -163,43 +161,31 @@ export async function dispatchTrustedText(send: CdpCommandSender, text: string):
|
||||
await send("Input.insertText", { text });
|
||||
}
|
||||
|
||||
export async function dispatchTrustedKey(send: CdpCommandSender, key: string): Promise<void> {
|
||||
const definition = keyDefinition(key);
|
||||
await send("Input.dispatchKeyEvent", {
|
||||
type: "keyDown",
|
||||
key: definition.key,
|
||||
code: definition.code,
|
||||
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
...(definition.text ? { text: definition.text, unmodifiedText: definition.text } : {}),
|
||||
});
|
||||
await send("Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: definition.key,
|
||||
code: definition.code,
|
||||
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
});
|
||||
}
|
||||
|
||||
function keyDefinition(key: string): {
|
||||
key: string;
|
||||
code: string;
|
||||
windowsVirtualKeyCode: number;
|
||||
text?: string;
|
||||
} {
|
||||
const special = SPECIAL_KEY_DEFINITIONS[key];
|
||||
if (special) {
|
||||
return special;
|
||||
export function dispatchTrustedKey(send: KeyboardInputSender, key: string): void {
|
||||
const keyCode = ELECTRON_KEY_CODE_ALIASES[key] ?? key;
|
||||
let character: string | null = null;
|
||||
if (key === "Space") {
|
||||
character = " ";
|
||||
} else if (key.length === 1) {
|
||||
character = key;
|
||||
}
|
||||
const text = key.length === 1 ? key : "";
|
||||
const upper = text.toUpperCase();
|
||||
return {
|
||||
key,
|
||||
code: upper ? `Key${upper}` : key,
|
||||
windowsVirtualKeyCode: upper ? upper.charCodeAt(0) : 0,
|
||||
...(text ? { text, unmodifiedText: text } : {}),
|
||||
};
|
||||
send({
|
||||
type: "keyDown",
|
||||
keyCode,
|
||||
skipIfUnhandled: true,
|
||||
});
|
||||
if (character !== null) {
|
||||
send({
|
||||
type: "char",
|
||||
keyCode: character,
|
||||
skipIfUnhandled: true,
|
||||
});
|
||||
}
|
||||
send({
|
||||
type: "keyUp",
|
||||
keyCode,
|
||||
skipIfUnhandled: true,
|
||||
});
|
||||
}
|
||||
|
||||
function modifierMask(modifiers: InputModifier[] | undefined): number {
|
||||
|
||||
Reference in New Issue
Block a user