407 lines
18 KiB
TypeScript
407 lines
18 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { spawnSync } from "node:child_process";
|
|
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
|
import { tmpdir } from "node:os";
|
|
import { join, dirname } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { createPrivateArtifactStore } from "./artifact-store.ts";
|
|
import { createBrowserSessionManager, type BrowserSession, type BrowserSessionManager } from "./browser-session.ts";
|
|
import type {
|
|
BrowserRuntime,
|
|
BrowserTab,
|
|
FailureEvidence,
|
|
PrivateArtifactMetadata,
|
|
RuntimeOperationContext,
|
|
} from "./contracts.ts";
|
|
import { createBrowserRuntimeFactory } from "./runtime.ts";
|
|
|
|
const ROOT = dirname(fileURLToPath(import.meta.url));
|
|
const FIXTURE = join(ROOT, "fixtures", "development-live-validation.html");
|
|
const CHROMIUM = "/usr/bin/chromium";
|
|
const PINNED_PYTHON = join(process.cwd(), ".pi", "fast-browser-demo", "venv", "bin", "python");
|
|
const READY_TIMEOUT_MS = 8_000;
|
|
|
|
type CaseStatus = "passed" | "failed" | "unavailable";
|
|
|
|
export interface RuntimeValidationCase {
|
|
name: string;
|
|
status: CaseStatus;
|
|
detail: string;
|
|
elapsedMs: number;
|
|
}
|
|
|
|
export interface RuntimeLiveValidationReport {
|
|
ok: boolean;
|
|
cases: RuntimeValidationCase[];
|
|
unavailableEvidence: boolean;
|
|
}
|
|
|
|
interface FixturePair {
|
|
originA: string;
|
|
originB: string;
|
|
close(): Promise<void>;
|
|
}
|
|
|
|
interface EvidenceState {
|
|
a?: FailureEvidence;
|
|
bBefore?: FailureEvidence;
|
|
b?: FailureEvidence;
|
|
narrowed?: FailureEvidence;
|
|
}
|
|
|
|
class EvidenceUnavailable extends Error {
|
|
override name = "EvidenceUnavailable";
|
|
}
|
|
|
|
function errorText(error: unknown): string {
|
|
return error instanceof Error ? error.message : String(error);
|
|
}
|
|
|
|
function boundedError(error: unknown): string {
|
|
return errorText(error).replace(/\s+/g, " ").slice(0, 400);
|
|
}
|
|
|
|
function requireRuntimePreflight(): void {
|
|
if (!existsSync(CHROMIUM) || !statSync(CHROMIUM).isFile()) throw new Error(`required executable is missing: ${CHROMIUM}`);
|
|
if (!existsSync(PINNED_PYTHON) || !statSync(PINNED_PYTHON).isFile()) throw new Error(`pinned Python is missing: ${PINNED_PYTHON}`);
|
|
}
|
|
|
|
function ownedProcessLines(root: string): string[] {
|
|
const result = spawnSync("ps", ["-eo", "pid=,args="], { encoding: "utf8", timeout: 5_000 });
|
|
if (result.status !== 0) return [];
|
|
return String(result.stdout).split(/\r?\n/).filter((line: string) => line.includes(root));
|
|
}
|
|
|
|
function fixtureResponse(
|
|
response: ServerResponse,
|
|
template: string,
|
|
tab: "A" | "B" | "NAV",
|
|
selfOrigin: string,
|
|
peerOrigin: string,
|
|
): void {
|
|
const content = template
|
|
.replaceAll("__TAB__", tab)
|
|
.replace("__SELF_ORIGIN__", JSON.stringify(selfOrigin))
|
|
.replace("__PEER_ORIGIN__", JSON.stringify(peerOrigin));
|
|
response.writeHead(200, {
|
|
"content-type": "text/html; charset=utf-8",
|
|
"cache-control": "no-store",
|
|
});
|
|
response.end(content);
|
|
}
|
|
|
|
async function listen(server: ReturnType<typeof createServer>): Promise<number> {
|
|
await new Promise<void>((resolve, reject) => {
|
|
server.once("error", reject);
|
|
server.listen(0, "127.0.0.1", () => resolve());
|
|
});
|
|
const address = server.address();
|
|
if (!address || typeof address === "string") throw new Error("fixture server did not expose a TCP port");
|
|
return address.port;
|
|
}
|
|
|
|
async function createFixturePair(): Promise<FixturePair> {
|
|
const template = readFileSync(FIXTURE, "utf8");
|
|
let originA = "";
|
|
let originB = "";
|
|
const makeServer = (which: "A" | "B") => createServer((request: IncomingMessage, response: ServerResponse) => {
|
|
const parsed = new URL(request.url ?? "/", "http://127.0.0.1");
|
|
if (parsed.pathname === "/runtime-live-validation.html") {
|
|
const requestedTab = parsed.searchParams.get("tab");
|
|
fixtureResponse(response, template, which === "A" && requestedTab === "NAV" ? "NAV" : which, which === "A" ? originA : originB, which === "A" ? originB : originA);
|
|
return;
|
|
}
|
|
if (parsed.pathname === "/runtime-live-third-party" || parsed.pathname === "/runtime-live-network-A" || parsed.pathname === "/runtime-live-network-B") {
|
|
response.writeHead(503, { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" });
|
|
response.end(`owned local failure ${which}`);
|
|
return;
|
|
}
|
|
response.writeHead(404, { "content-type": "text/plain; charset=utf-8" });
|
|
response.end("not found");
|
|
});
|
|
const serverA = makeServer("A");
|
|
const serverB = makeServer("B");
|
|
try {
|
|
const [portA, portB] = await Promise.all([listen(serverA), listen(serverB)]);
|
|
originA = `http://127.0.0.1:${portA}`;
|
|
originB = `http://127.0.0.1:${portB}`;
|
|
return {
|
|
originA,
|
|
originB,
|
|
async close() {
|
|
for (const server of [serverA, serverB]) {
|
|
server.closeAllConnections?.();
|
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
}
|
|
},
|
|
};
|
|
} catch (error) {
|
|
serverA.closeAllConnections?.();
|
|
serverB.closeAllConnections?.();
|
|
serverA.close();
|
|
serverB.close();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function launchConfig(origin: string, path: string): Parameters<BrowserSessionManager["open"]>[0] {
|
|
return {
|
|
allowedOrigins: "*",
|
|
browser: {
|
|
mode: "launch",
|
|
product: "chromium",
|
|
executablePath: CHROMIUM,
|
|
headless: true,
|
|
startUrl: `${origin}${path}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runCase(cases: RuntimeValidationCase[], name: string, operation: () => Promise<void>): Promise<void> {
|
|
const started = Date.now();
|
|
try {
|
|
await operation();
|
|
cases.push({ name, status: "passed", detail: "verified", elapsedMs: Date.now() - started });
|
|
} catch (error) {
|
|
const status: CaseStatus = error instanceof EvidenceUnavailable ? "unavailable" : "failed";
|
|
cases.push({ name, status, detail: boundedError(error), elapsedMs: Date.now() - started });
|
|
}
|
|
}
|
|
|
|
function tabByOrigin(tabs: readonly BrowserTab[], origin: string): BrowserTab {
|
|
const matches = tabs.filter((tab) => tab.origin === origin);
|
|
assert.equal(matches.length, 1, `expected one tab for owned origin ${origin}`);
|
|
return matches[0]!;
|
|
}
|
|
|
|
function candidateId(observation: Awaited<ReturnType<BrowserRuntime["observe"]>>, name: string, action: "click" | "fill"): string {
|
|
const matches = observation.candidates.filter((candidate) => candidate.name === name && candidate.actions.includes(action));
|
|
assert.equal(matches.length, 1, `expected one ${action} candidate named ${name}`);
|
|
return matches[0]!.id;
|
|
}
|
|
|
|
function consoleHas(evidence: FailureEvidence, marker: string): boolean {
|
|
return evidence.console.some((item) => item.message.includes(marker));
|
|
}
|
|
|
|
function networkHasOrigin(evidence: FailureEvidence, origin: string): boolean {
|
|
return evidence.failedRequests.some((item) => item.origin === origin);
|
|
}
|
|
|
|
function requireEvidenceMarkers(
|
|
evidence: FailureEvidence,
|
|
ownMarker: string,
|
|
otherMarker: string,
|
|
ownOrigin: string,
|
|
otherOrigin: string,
|
|
): void {
|
|
if (consoleHas(evidence, otherMarker) || networkHasOrigin(evidence, otherOrigin)) {
|
|
throw new Error("evidence mixed a prior or other-tab event into the selected tab");
|
|
}
|
|
if (!consoleHas(evidence, ownMarker) || !networkHasOrigin(evidence, ownOrigin)) {
|
|
throw new EvidenceUnavailable(
|
|
`real daemon session provenance is unavailable; expected ${ownMarker} and ${ownOrigin}, but zero-log evidence is not treated as success (status=${evidence.status})`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function requirePrivateMetadata(metadata: PrivateArtifactMetadata, root: string, contentType: PrivateArtifactMetadata["contentType"]): void {
|
|
assert.equal(metadata.contentType, contentType);
|
|
assert.equal(metadata.permissions, "0600");
|
|
assert.equal(metadata.localOnly, true);
|
|
assert.equal(metadata.modelForwarded, false);
|
|
assert.equal(metadata.bytes > 0, true);
|
|
assert.equal(metadata.path.startsWith(root), true);
|
|
const stats = statSync(metadata.path);
|
|
assert.equal((stats.mode & 0o777), 0o600);
|
|
}
|
|
|
|
async function requirePrivateEvidence(evidence: FailureEvidence, root: string): Promise<void> {
|
|
assert.ok(evidence.screenshot, "real screenshot evidence was unavailable");
|
|
assert.ok(evidence.diagnostics, "real diagnostics evidence was unavailable");
|
|
requirePrivateMetadata(evidence.screenshot, root, "image/png");
|
|
requirePrivateMetadata(evidence.diagnostics, root, "application/json");
|
|
const png = await readFile(evidence.screenshot.path);
|
|
assert.deepEqual([...png.subarray(0, 8)], [137, 80, 78, 71, 13, 10, 26, 10]);
|
|
const diagnostics = JSON.parse(await readFile(evidence.diagnostics.path, "utf8")) as Record<string, unknown>;
|
|
assert(Array.isArray(diagnostics.console));
|
|
assert(Array.isArray(diagnostics.failedRequests));
|
|
}
|
|
|
|
export async function runDevelopmentLiveValidation(): Promise<RuntimeLiveValidationReport> {
|
|
requireRuntimePreflight();
|
|
const root = await mkdtemp(join(tmpdir(), "fast-browser-runtime-live-"));
|
|
await mkdir(join(root, "leases"), { recursive: true, mode: 0o700 });
|
|
const fixture = await createFixturePair();
|
|
const manager = createBrowserSessionManager({
|
|
profileDirectory: join(root, "profiles"),
|
|
leaseDirectory: join(root, "leases"),
|
|
startupTimeoutMs: 20_000,
|
|
closeTimeoutMs: 4_000,
|
|
});
|
|
const controller = new AbortController();
|
|
const cases: RuntimeValidationCase[] = [];
|
|
const evidence: EvidenceState = {};
|
|
const artifactRoot = join(root, "artifacts");
|
|
const store = createPrivateArtifactStore(artifactRoot);
|
|
let session: BrowserSession | undefined;
|
|
let runtime: BrowserRuntime | undefined;
|
|
let selectedA: BrowserTab | undefined;
|
|
let selectedB: BrowserTab | undefined;
|
|
try {
|
|
session = await manager.open(launchConfig(fixture.originA, "/runtime-live-validation.html?tab=A"), controller.signal);
|
|
runtime = createBrowserRuntimeFactory().create({
|
|
sessionId: session.id,
|
|
cdpUrl: session.config.cdpUrl,
|
|
allowedOrigins: "*",
|
|
python: PINNED_PYTHON,
|
|
artifacts: store,
|
|
}, controller.signal);
|
|
|
|
await runCase(cases, "read excludes input sentinels; assert wait; navigation and reload", async () => {
|
|
const tabs = await runtime!.listTabs(controller.signal);
|
|
selectedA = tabByOrigin(tabs, fixture.originA);
|
|
const page = await runtime!.read({ tabId: selectedA.tabId }, controller.signal);
|
|
assert.match(page.visibleText, /READABLE_SENTINEL_A/);
|
|
assert.doesNotMatch(page.visibleText, /READ_INPUT_SENTINEL_A/);
|
|
const assertion = await runtime!.assert({
|
|
tabId: selectedA.tabId,
|
|
assertion: { kind: "visibleText", match: "contains", value: "ASSERT_READY_A" },
|
|
wait: "bounded",
|
|
}, controller.signal);
|
|
assert.equal(assertion.status, "satisfied");
|
|
const navigated = await runtime!.navigate({ tabId: selectedA.tabId, url: `${fixture.originA}/runtime-live-validation.html?tab=NAV` }, controller.signal);
|
|
assert.equal(navigated.status, "committed");
|
|
assert.equal(navigated.documentId.length > 0, true);
|
|
assert.notEqual(navigated.documentId, page.documentId);
|
|
const reloaded = await runtime!.reload({ tabId: selectedA.tabId }, controller.signal);
|
|
assert.equal(reloaded.status, "committed");
|
|
assert.equal(reloaded.documentId.length > 0, true);
|
|
const afterReload = await runtime!.read({ tabId: selectedA.tabId }, controller.signal);
|
|
assert.match(afterReload.visibleText, /NAVIGATION_READY/);
|
|
const restored = await runtime!.navigate({ tabId: selectedA.tabId, url: `${fixture.originA}/runtime-live-validation.html?tab=A` }, controller.signal);
|
|
assert.equal(restored.status, "committed");
|
|
});
|
|
|
|
await runCase(cases, "popup wait discovers the popup without automatic tab switching", async () => {
|
|
assert(selectedA);
|
|
const observation = await runtime!.observe(controller.signal);
|
|
const observedCandidate = candidateId(observation, "Open popup", "click");
|
|
const action = await runtime!.act({
|
|
observationId: observation.observationId,
|
|
candidateId: observedCandidate,
|
|
action: "click",
|
|
postcondition: { kind: "text", value: "POPUP_OPEN_REQUESTED" },
|
|
}, controller.signal);
|
|
assert.equal(action.verified, true);
|
|
const popup = await runtime!.waitPopup({ openerTabId: selectedA.tabId }, controller.signal);
|
|
assert.equal(popup.status, "found");
|
|
assert.equal(popup.autoSwitched, false);
|
|
assert.equal(popup.tab?.kind, "popup");
|
|
assert.equal(popup.tab?.openerTabId, selectedA.tabId);
|
|
selectedB = popup.tab;
|
|
assert(selectedB);
|
|
const tabs = await runtime!.listTabs(controller.signal);
|
|
const currentA = tabs.find((tab) => tab.tabId === selectedA!.tabId);
|
|
assert.equal(currentA?.selected, true);
|
|
const targetedPopupRead = await runtime!.read({ tabId: selectedB.tabId }, controller.signal);
|
|
assert.match(targetedPopupRead.visibleText, /READABLE_SENTINEL_B/);
|
|
const afterTargetedRead = await runtime!.listTabs(controller.signal);
|
|
assert.equal(afterTargetedRead.find((tab) => tab.tabId === selectedA!.tabId)?.selected, true);
|
|
});
|
|
|
|
await runCase(cases, "base-* evidence stays isolated per selected tab", async () => {
|
|
assert(selectedA && selectedB);
|
|
await runtime!.selectTab({ tabId: selectedA.tabId }, controller.signal);
|
|
await runtime!.reload({ tabId: selectedA.tabId }, controller.signal);
|
|
evidence.a = await runtime!.captureEvidence({ runId: "runtime-live-A", reason: "requested", operation: "captureEvidence" }, controller.signal);
|
|
await runtime!.selectTab({ tabId: selectedB.tabId }, controller.signal);
|
|
// Drain B before emitting a new B event. This specifically catches the
|
|
// old-buffer bug where A events reappear after switching targets.
|
|
evidence.bBefore = await runtime!.captureEvidence({ runId: "runtime-live-B-before", reason: "requested", operation: "captureEvidence" }, controller.signal);
|
|
assert.equal(consoleHas(evidence.bBefore, "RUNTIME_LIVE_CONSOLE_A"), false);
|
|
assert.equal(networkHasOrigin(evidence.bBefore, fixture.originA), false);
|
|
await runtime!.navigate({ tabId: selectedB.tabId, url: `${fixture.originB}/runtime-live-validation.html?tab=B&emit=1` }, controller.signal);
|
|
evidence.b = await runtime!.captureEvidence({ runId: "runtime-live-B", reason: "requested", operation: "captureEvidence" }, controller.signal);
|
|
requireEvidenceMarkers(evidence.a, "RUNTIME_LIVE_CONSOLE_A", "RUNTIME_LIVE_CONSOLE_B", fixture.originA, fixture.originB);
|
|
requireEvidenceMarkers(evidence.b, "RUNTIME_LIVE_CONSOLE_B", "RUNTIME_LIVE_CONSOLE_A", fixture.originB, fixture.originA);
|
|
});
|
|
|
|
await runCase(cases, "narrowed policy omits peer-origin request and omitted context restores base", async () => {
|
|
assert(selectedA && selectedB);
|
|
await runtime!.selectTab({ tabId: selectedA.tabId }, controller.signal);
|
|
const context: RuntimeOperationContext = await runtime!.narrowOriginPolicy({ allowedOrigins: [fixture.originA] }, controller.signal);
|
|
await runtime!.navigate({ tabId: selectedA.tabId, url: `${fixture.originA}/runtime-live-validation.html?tab=A&thirdparty=1` }, controller.signal, context);
|
|
evidence.narrowed = await runtime!.captureEvidence({ runId: "runtime-live-narrowed", reason: "requested", operation: "navigate" }, controller.signal, context);
|
|
if (!consoleHas(evidence.narrowed, "RUNTIME_LIVE_CONSOLE_A") || !networkHasOrigin(evidence.narrowed, fixture.originA)) {
|
|
throw new EvidenceUnavailable("real daemon session provenance is unavailable for narrowed evidence; zero-log evidence is not treated as success");
|
|
}
|
|
assert.equal(consoleHas(evidence.narrowed, "RUNTIME_LIVE_CONSOLE_B"), false);
|
|
assert.equal(networkHasOrigin(evidence.narrowed, fixture.originB), false);
|
|
const restoredTabs = await runtime!.listTabs(controller.signal);
|
|
assert.equal(restoredTabs.some((tab) => tab.tabId === selectedB!.tabId), true);
|
|
const restoredPage = await runtime!.read({ tabId: selectedB.tabId }, controller.signal);
|
|
assert.match(restoredPage.visibleText, /READABLE_SENTINEL_B/);
|
|
});
|
|
|
|
await runCase(cases, "captureEvidence stores private PNG and diagnostics artifacts", async () => {
|
|
assert(evidence.a);
|
|
await requirePrivateEvidence(evidence.a, artifactRoot);
|
|
});
|
|
|
|
await runCase(cases, "real daemon provenance is unavailable honestly, never silent zero-log success", async () => {
|
|
assert(evidence.a && evidence.b);
|
|
try {
|
|
requireEvidenceMarkers(evidence.a, "RUNTIME_LIVE_CONSOLE_A", "RUNTIME_LIVE_CONSOLE_B", fixture.originA, fixture.originB);
|
|
requireEvidenceMarkers(evidence.b, "RUNTIME_LIVE_CONSOLE_B", "RUNTIME_LIVE_CONSOLE_A", fixture.originB, fixture.originA);
|
|
} catch (error) {
|
|
if (error instanceof EvidenceUnavailable) throw error;
|
|
throw error;
|
|
}
|
|
});
|
|
} finally {
|
|
await runtime?.close().catch(() => {});
|
|
await store.close().catch(() => {});
|
|
if (session) await manager.close(session.id).catch(() => {});
|
|
await manager.closeAll().catch(() => {});
|
|
await fixture.close().catch(() => {});
|
|
const leftovers = ownedProcessLines(root);
|
|
if (leftovers.length === 0) {
|
|
await rm(root, { recursive: true, force: true });
|
|
} else {
|
|
cases.push({
|
|
name: "owned runtime cleanup",
|
|
status: "failed",
|
|
detail: `owned process descendants remained; temporary root preserved (${leftovers.length} process lines)`,
|
|
elapsedMs: 0,
|
|
});
|
|
}
|
|
}
|
|
const unavailableEvidence = cases.some((item) => item.status === "unavailable");
|
|
return { ok: cases.every((item) => item.status === "passed"), cases, unavailableEvidence };
|
|
}
|
|
|
|
function printReport(report: RuntimeLiveValidationReport): void {
|
|
console.log("Fast-browser development live validation");
|
|
for (const item of report.cases) console.log(`${item.status.toUpperCase()} ${item.name} (${item.elapsedMs}ms): ${item.detail}`);
|
|
console.log(`Evidence provenance unavailable: ${report.unavailableEvidence}`);
|
|
}
|
|
|
|
if (import.meta.main) {
|
|
if (process.env.FAST_BROWSER_DEVELOPMENT_LIVE !== "1") {
|
|
console.error("Development live validation is opt-in; set FAST_BROWSER_DEVELOPMENT_LIVE=1 after lifecycle review approval.");
|
|
process.exitCode = 2;
|
|
} else {
|
|
try {
|
|
const report = await runDevelopmentLiveValidation();
|
|
printReport(report);
|
|
if (!report.ok) process.exitCode = 1;
|
|
} catch (error) {
|
|
console.error(`BLOCKED runtime live validation: ${boundedError(error)}`);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
}
|