diff --git a/packages/app/e2e/browser/relay-deployment-reconnect.real.spec.ts b/packages/app/e2e/browser/relay-deployment-reconnect.real.spec.ts new file mode 100644 index 000000000..968342e16 --- /dev/null +++ b/packages/app/e2e/browser/relay-deployment-reconnect.real.spec.ts @@ -0,0 +1,71 @@ +import { test } from "@playwright/test"; +import { buildHostAgentDetailRoute } from "@/utils/host-routes"; +import { expectRunningAgentChrome } from "../support/helpers/agent-stream"; +import { startLocalElixirRelay } from "../support/helpers/local-elixir-relay"; +import { seedMockAgentWorkspace } from "../support/helpers/mock-agent"; +import { + startPackagedWebDaemon, + type PackagedWebDaemon, +} from "../support/helpers/packaged-web-daemon"; +import { + connectDaemonWebAppOnlyThroughRelay, + measureRelayRestartDuringStream, +} from "../support/helpers/relay-deployment"; + +test("a streaming chat recovers from a relay deployment without user action", async ({ + page, +}, testInfo) => { + test.setTimeout(180_000); + const relay = await startLocalElixirRelay(); + let daemon: PackagedWebDaemon | null = null; + let cleanupChat = async () => {}; + + try { + daemon = await startPackagedWebDaemon({ relayEndpoint: relay.endpoint }); + const runningDaemon = daemon; + await test.step("Connect the daemon-served app only through the relay", async () => { + await connectDaemonWebAppOnlyThroughRelay(page, runningDaemon); + }); + + await test.step("Open a running mock-provider chat", async () => { + const chat = await seedMockAgentWorkspace({ + repoPrefix: "relay-deployment-reconnect-", + title: "Relay deployment stream", + port: runningDaemon.port, + model: "five-minute-stream", + initialPrompt: "Stream while the relay is deployed.", + }); + cleanupChat = chat.cleanup; + const route = buildHostAgentDetailRoute( + runningDaemon.serverId, + chat.agentId, + chat.workspaceId, + ); + await page.goto(new URL(route, runningDaemon.origin).toString()); + await expectRunningAgentChrome(page, "Relay deployment stream"); + }); + + await test.step("Restart the relay and measure the visible interruption", async () => { + const measurements = await measureRelayRestartDuringStream({ + page, + relay, + agentTitle: "Relay deployment stream", + }); + await testInfo.attach("relay-deployment-measurements", { + body: JSON.stringify(measurements, null, 2), + contentType: "application/json", + }); + console.log(`Relay deployment measurements: ${JSON.stringify(measurements)}`); + }); + } finally { + try { + await cleanupChat(); + } finally { + try { + await daemon?.close(); + } finally { + await relay.close(); + } + } + } +}); diff --git a/packages/app/e2e/browser/viewed-agent-timelines.spec.ts b/packages/app/e2e/browser/viewed-agent-timelines.spec.ts index c597b445e..3ea9ef08c 100644 --- a/packages/app/e2e/browser/viewed-agent-timelines.spec.ts +++ b/packages/app/e2e/browser/viewed-agent-timelines.spec.ts @@ -12,6 +12,7 @@ import { expectTurnCopyButton, } from "../support/helpers/agent-stream"; import { + expectReconnectingToastDebounced, expectReconnectingToastGone, expectReconnectingToastVisible, } from "../support/helpers/workspace-ui"; @@ -179,6 +180,7 @@ test.describe("Viewed agent timelines", () => { "true", ); await gate.drop(); + await expectReconnectingToastDebounced(page); await expectReconnectingToastVisible(page); await commitMessage(scenario, scenario.firstAgentId, "Committed while the chat reconnects."); await expect( @@ -196,4 +198,23 @@ test.describe("Viewed agent timelines", () => { await scenario.cleanup(); } }); + + test("preserves reconnecting toast through retained tab switches", async ({ page }) => { + const gate = await installDaemonWebSocketGate(page); + const scenario = await seedViewedTimelineScenario(); + try { + await openAgent(page, scenario, scenario.firstAgentId); + await selectAgent(page, "Second viewed chat"); + await expect(page.getByRole("textbox", { name: "Message agent..." })).toBeVisible(); + await selectAgent(page, "First viewed chat"); + await gate.drop(); + await expectReconnectingToastVisible(page); + + await selectAgent(page, "Second viewed chat"); + await expectReconnectingToastVisible(page, { timeout: 500 }); + } finally { + gate.restore(); + await scenario.cleanup(); + } + }); }); diff --git a/packages/app/e2e/browser/workspace-navigation-regression.spec.ts b/packages/app/e2e/browser/workspace-navigation-regression.spec.ts index b778213fc..09c81ef3a 100644 --- a/packages/app/e2e/browser/workspace-navigation-regression.spec.ts +++ b/packages/app/e2e/browser/workspace-navigation-regression.spec.ts @@ -30,6 +30,7 @@ import { expectReconnectingToastGone, switchWorkspaceViaSidebar, waitForSidebarHydration, + waitForWorkspaceInSidebar, workspaceDeckEntryLocator, expectWorkspaceDeckEntryCount, } from "../support/helpers/workspace-ui"; @@ -37,7 +38,8 @@ import { clickSettingsBackToWorkspace } from "../support/helpers/settings"; import { getServerId } from "../support/helpers/server-id"; import { expectAppRoute } from "../support/helpers/route-assertions"; import { installDaemonWebSocketGate } from "../support/helpers/daemon-websocket-gate"; -import { addOfflineHostAndReload } from "../support/helpers/hosts"; +import { addConnectedHostAndReload, addOfflineHostAndReload } from "../support/helpers/hosts"; +import { startIsolatedHostDaemon } from "../support/helpers/isolated-host-daemon"; const LOADING_WORKSPACE_TEXT_PATTERN = /Loading workspace/i; async function expectNoLoadingWorkspacePane( @@ -199,6 +201,62 @@ test.describe("Workspace navigation regression", () => { } }); + test("does not show reconnecting for an inactive host workspace", async ({ page }) => { + const secondaryHost = await startIsolatedHostDaemon("inactive-reconnecting-host"); + const primaryWorkspace = await seedWorkspace({ repoPrefix: "active-reconnecting-host-" }); + const secondaryWorkspace = await seedWorkspace({ + repoPrefix: "inactive-reconnecting-host-", + port: secondaryHost.port, + }); + + try { + await Promise.all([ + createIdleAgent(primaryWorkspace.client, { + cwd: primaryWorkspace.repoPath, + workspaceId: primaryWorkspace.workspaceId, + title: "Active host agent", + }), + createIdleAgent(secondaryWorkspace.client, { + cwd: secondaryWorkspace.repoPath, + workspaceId: secondaryWorkspace.workspaceId, + title: "Inactive host agent", + }), + ]); + + await gotoAppShell(page); + await addConnectedHostAndReload(page, { + serverId: secondaryHost.serverId, + label: "Inactive host", + port: secondaryHost.port, + }); + await waitForWorkspaceInSidebar(page, { + serverId: secondaryHost.serverId, + workspaceId: secondaryWorkspace.workspaceId, + }); + await switchWorkspaceViaSidebar({ + page, + serverId: secondaryHost.serverId, + workspaceId: secondaryWorkspace.workspaceId, + }); + await waitForWorkspaceTabsVisible(page); + await switchWorkspaceViaSidebar({ + page, + serverId: getServerId(), + workspaceId: primaryWorkspace.workspaceId, + }); + await waitForWorkspaceTabsVisible(page); + + await secondaryHost.close(); + await page.waitForTimeout(1_500); + + await expectReconnectingToastGone(page, { timeout: 100 }); + } finally { + await secondaryHost.close(); + await secondaryWorkspace.cleanup(); + await primaryWorkspace.cleanup(); + } + }); + test("cold offline workspace route gates the screen interior but keeps settings reachable", async ({ page, }) => { diff --git a/packages/app/e2e/support/helpers/local-elixir-relay.ts b/packages/app/e2e/support/helpers/local-elixir-relay.ts new file mode 100644 index 000000000..14d9a8b22 --- /dev/null +++ b/packages/app/e2e/support/helpers/local-elixir-relay.ts @@ -0,0 +1,129 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { existsSync } from "node:fs"; +import net from "node:net"; +import path from "node:path"; + +export interface LocalElixirRelay { + endpoint: string; + port: number; + start(): Promise; + stop(): Promise; + close(): Promise; +} + +async function availablePort(): Promise { + return await new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(() => reject(new Error("Could not allocate a relay port"))); + return; + } + server.close(() => resolve(address.port)); + }); + }); +} + +async function stopProcess(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + const force = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, 5_000); + try { + await once(child, "exit"); + } finally { + clearTimeout(force); + } +} + +async function waitUntilReady( + port: number, + child: ChildProcess, + recentOutput: () => string, +): Promise { + const deadline = Date.now() + 60_000; + let lastError: unknown; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error( + `Elixir relay exited before becoming ready (exit ${child.exitCode}).\n${recentOutput()}`, + ); + } + try { + const response = await fetch(`http://127.0.0.1:${port}/ready`, { + signal: AbortSignal.timeout(1_000), + }); + if (response.ok) return; + lastError = new Error(`Relay readiness returned HTTP ${response.status}`); + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error( + `Elixir relay did not become ready on port ${port}: ${lastError instanceof Error ? lastError.message : String(lastError)}\n${recentOutput()}`, + ); +} + +export async function startLocalElixirRelay(): Promise { + const relayRoot = + process.env.PASEO_RELAY_CHECKOUT ?? path.resolve(__dirname, "../../../../../..", "paseo-relay"); + if (!existsSync(path.join(relayRoot, "mix.exs"))) { + throw new Error( + `Expected the Elixir relay checkout at ${relayRoot}. Set PASEO_RELAY_CHECKOUT to override it.`, + ); + } + + const port = await availablePort(); + let child: ChildProcess | null = null; + let output: string[] = []; + + const start = async () => { + if (child && child.exitCode === null && child.signalCode === null) { + throw new Error("Elixir relay is already running"); + } + output = []; + child = spawn("asdf", ["exec", "mix", "run", "--no-halt"], { + cwd: relayRoot, + env: { + ...process.env, + MIX_ENV: "prod", + PASEO_RELAY_HOST: "127.0.0.1", + PASEO_RELAY_PORT: String(port), + PASEO_RELAY_MIN_CLUSTER_SIZE: "1", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + const capture = (chunk: Buffer) => { + output.push(chunk.toString("utf8")); + output = output.join("").split("\n").slice(-80); + }; + child.stdout?.on("data", capture); + child.stderr?.on("data", capture); + try { + await waitUntilReady(port, child, () => output.join("\n")); + } catch (error) { + await stopProcess(child); + throw error; + } + }; + + const stop = async () => { + if (!child) return; + await stopProcess(child); + child = null; + }; + + await start(); + return { + endpoint: `127.0.0.1:${port}`, + port, + start, + stop, + close: stop, + }; +} diff --git a/packages/app/e2e/support/helpers/mock-agent.ts b/packages/app/e2e/support/helpers/mock-agent.ts index 6320ce3c8..bc964c429 100644 --- a/packages/app/e2e/support/helpers/mock-agent.ts +++ b/packages/app/e2e/support/helpers/mock-agent.ts @@ -14,6 +14,7 @@ export interface MockAgentWorkspace { export interface MockAgentOptions { repoPrefix: string; title: string; + port?: number; initialPrompt?: string; model?: string; modeId?: string; @@ -28,7 +29,7 @@ export interface MockAgentOptions { export async function seedMockAgentWorkspace( options: MockAgentOptions, ): Promise { - const workspace = await seedWorkspace({ repoPrefix: options.repoPrefix }); + const workspace = await seedWorkspace({ repoPrefix: options.repoPrefix, port: options.port }); try { const agent = await workspace.client.createAgent({ provider: "mock", diff --git a/packages/app/e2e/support/helpers/packaged-web-daemon.ts b/packages/app/e2e/support/helpers/packaged-web-daemon.ts new file mode 100644 index 000000000..33182df0d --- /dev/null +++ b/packages/app/e2e/support/helpers/packaged-web-daemon.ts @@ -0,0 +1,111 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export interface PackagedWebDaemon { + home: string; + origin: string; + port: number; + serverId: string; + pairingOfferUrl(): Promise; + close(): Promise; +} + +async function availablePort(): Promise { + return await new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(() => reject(new Error("Could not allocate a daemon port"))); + return; + } + server.close(() => resolve(address.port)); + }); + }); +} + +async function waitForWebUi(origin: string, home: string): Promise { + const deadline = Date.now() + 30_000; + let lastError: unknown; + while (Date.now() < deadline) { + try { + const response = await fetch(origin, { signal: AbortSignal.timeout(1_000) }); + const body = await response.text(); + if (response.ok && body.includes(" setTimeout(resolve, 100)); + } + const log = await readFile(path.join(home, "daemon.log"), "utf8").catch(() => ""); + throw new Error( + `Daemon web UI did not become ready: ${lastError instanceof Error ? lastError.message : String(lastError)}\n${log.split("\n").slice(-80).join("\n")}`, + ); +} + +export async function startPackagedWebDaemon(input: { + relayEndpoint: string; +}): Promise { + const port = await availablePort(); + const home = await mkdtemp(path.join(tmpdir(), "paseo-relay-deployment-e2e-")); + const serverId = `relay-deployment-${Date.now().toString(36)}`; + const paseo = path.resolve(__dirname, "../../../../../node_modules/.bin/paseo"); + const env: NodeJS.ProcessEnv = { + ...process.env, + CI: "true", + NODE_ENV: "development", + PASEO_NODE_ENV: "development", + PASEO_SERVER_ID: serverId, + PASEO_RELAY_ENDPOINT: input.relayEndpoint, + PASEO_RELAY_PUBLIC_ENDPOINT: input.relayEndpoint, + PASEO_RELAY_USE_TLS: "false", + PASEO_RELAY_PUBLIC_USE_TLS: "false", + }; + + try { + await execFileAsync( + paseo, + ["daemon", "start", "--home", home, "--port", String(port), "--relay", "--web-ui"], + { env }, + ); + const origin = `http://127.0.0.1:${port}`; + await waitForWebUi(origin, home); + + return { + home, + origin, + port, + serverId, + pairingOfferUrl: async () => { + const { stdout } = await execFileAsync( + paseo, + ["daemon", "pair", "--home", home, "--relay", "--json"], + { env }, + ); + const result = JSON.parse(stdout) as { url?: unknown }; + if (typeof result.url !== "string") { + throw new Error(`Pairing command returned no URL: ${stdout}`); + } + return result.url; + }, + close: async () => { + await execFileAsync(paseo, ["daemon", "stop", "--home", home], { env }).catch( + () => undefined, + ); + await rm(home, { recursive: true, force: true }); + }, + }; + } catch (error) { + await execFileAsync(paseo, ["daemon", "stop", "--home", home], { env }).catch(() => undefined); + await rm(home, { recursive: true, force: true }); + throw error; + } +} diff --git a/packages/app/e2e/support/helpers/relay-deployment.ts b/packages/app/e2e/support/helpers/relay-deployment.ts new file mode 100644 index 000000000..19e480d48 --- /dev/null +++ b/packages/app/e2e/support/helpers/relay-deployment.ts @@ -0,0 +1,131 @@ +import { expect, type Page } from "@playwright/test"; +import { parseConnectionOfferFromUrl } from "@getpaseo/protocol/connection-offer"; +import type { LocalElixirRelay } from "./local-elixir-relay"; +import type { PackagedWebDaemon } from "./packaged-web-daemon"; +import { expectRunningAgentChrome } from "./agent-stream"; +import { buildCreateAgentPreferences } from "./daemon-registry"; + +export interface RelayDeploymentMeasurements { + reconnectToastDelayMs: number; + reconnectToastVisibleMs: number; + totalReconnectMs: number; + relayStartupMs: number; + streamPausedWhileDisconnected: boolean; + runningStatePreserved: boolean; + streamResumedWithoutUserAction: boolean; +} + +export async function connectDaemonWebAppOnlyThroughRelay( + page: Page, + daemon: PackagedWebDaemon, +): Promise { + const offerUrl = await daemon.pairingOfferUrl(); + const offer = parseConnectionOfferFromUrl(offerUrl); + if (!offer) throw new Error("Daemon pairing command returned an invalid offer URL"); + const relayConnection = { + id: `relay:${offer.relay.endpoint}`, + type: "relay" as const, + relayEndpoint: offer.relay.endpoint, + useTls: offer.relay.useTls ?? false, + daemonPublicKeyB64: offer.daemonPublicKeyB64, + }; + const now = new Date().toISOString(); + const host = { + serverId: offer.serverId, + label: "Relay deployment host", + connections: [relayConnection], + preferredConnectionId: relayConnection.id, + createdAt: now, + updatedAt: now, + }; + + await page.route(/:(6767)\b/, (route) => route.abort()); + await page.routeWebSocket(/:(6767)\b/, async (socket) => { + await socket.close({ code: 1008, reason: "Blocked developer daemon during relay E2E" }); + }); + await page.route("**/*", async (route) => { + if (route.request().resourceType() !== "document") { + await route.continue(); + return; + } + const response = await route.fetch(); + const html = await response.text(); + await route.fulfill({ + response, + body: html.replace(/