relay: reset stuck control sockets; add prod prove script

2026-02-05 21:47:03 +07:00
parent 8dcf86a33f
commit 29bca7e7b9
2 changed files with 126 additions and 0 deletions
@@ -75,6 +75,45 @@ export class RelayDurableObject {

  }

+  private hasServerDataSocket(clientId: string): boolean {
+    try {
+      return this.state.getWebSockets(`server:${clientId}`).length > 0;
+    } catch {
+      return false;
+    }
+  }
+
+  private nudgeOrResetControlForClient(clientId: string): void {
+    // If the daemon's control WS becomes half-open, the DO can't reliably detect it via ws.send errors
+    // (Cloudflare may accept writes even if the other side is no longer reading).
+    //
+    // Instead, observe whether the daemon reacts by opening the per-client server-data socket.
+    // If it doesn't, nudge with a sync message; if still no reaction, force-close the control
+    // socket(s) so the daemon reconnects.
+    const initialDelayMs = 10_000;
+    const secondDelayMs = 5_000;
+
+    setTimeout(() => {
+      if (this.hasServerDataSocket(clientId)) return;
+
+      // First nudge: send a full sync list.
+      this.notifyControls({ type: "sync", clientIds: this.listConnectedClientIds() });
+
+      setTimeout(() => {
+        if (this.hasServerDataSocket(clientId)) return;
+
+        // Still nothing: assume control is stuck and force a reconnect.
+        for (const ws of this.state.getWebSockets("server-control")) {
+          try {
+            ws.close(1011, "Control unresponsive");
+          } catch {
+            // ignore
+          }
+        }
+      }, secondDelayMs);
+    }, initialDelayMs);
+  }
+
  private bufferClientFrame(clientId: string, message: string | ArrayBuffer): void {
    const existing = this.pendingClientFrames.get(clientId) ?? [];
    existing.push(message);
@@ -209,6 +248,7 @@ export class RelayDurableObject {

    if (role === "client") {
      this.notifyControls({ type: "client_connected", clientId });
+      this.nudgeOrResetControlForClient(clientId);
    }

    if (isServerControl) {
@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+import { chromium } from 'playwright';
+
+function parseArgs(argv) {
+  const args = {};
+  for (let i = 2; i < argv.length; i += 1) {
+    const part = argv[i];
+    if (!part.startsWith('--')) continue;
+    const key = part.slice(2);
+    const next = argv[i + 1];
+    if (!next || next.startsWith('--')) {
+      args[key] = true;
+      continue;
+    }
+    args[key] = next;
+    i += 1;
+  }
+  return args;
+}
+
+const args = parseArgs(process.argv);
+
+const serverId = args['server-id'] ?? process.env.PASEO_SERVER_ID;
+const daemonPublicKeyB64 = args['daemon-public-key-b64'] ?? process.env.PASEO_DAEMON_PUBLIC_KEY_B64;
+const relayEndpoint = args['relay-endpoint'] ?? process.env.PASEO_RELAY_ENDPOINT ?? 'relay.paseo.sh:443';
+const baseUrl = args['base-url'] ?? process.env.PASEO_APP_URL ?? 'https://app.paseo.sh';
+const timeoutMs = Number(args['timeout-ms'] ?? process.env.PASEO_PROVE_TIMEOUT_MS ?? 60_000);
+const stabilityMs = Number(args['stability-ms'] ?? process.env.PASEO_PROVE_STABILITY_MS ?? 30_000);
+
+if (!serverId || typeof serverId !== 'string') {
+  console.error('Missing server ID. Provide --server-id or PASEO_SERVER_ID.');
+  process.exit(2);
+}
+if (!daemonPublicKeyB64 || typeof daemonPublicKeyB64 !== 'string') {
+  console.error('Missing daemon public key. Provide --daemon-public-key-b64 or PASEO_DAEMON_PUBLIC_KEY_B64.');
+  process.exit(2);
+}
+
+const nowIso = new Date().toISOString();
+const daemon = {
+  serverId,
+  label: 'relay-daemon',
+  connections: [{ id: `relay:${relayEndpoint}`, type: 'relay', relayEndpoint, daemonPublicKeyB64 }],
+  preferredConnectionId: `relay:${relayEndpoint}`,
+  createdAt: nowIso,
+  updatedAt: nowIso,
+};
+
+const browser = await chromium.launch({ headless: true });
+const context = await browser.newContext();
+const page = await context.newPage();
+
+page.on('console', (m) => console.log(`[browser:${m.type()}] ${m.text()}`));
+page.on('pageerror', (e) => console.error(`[browser:pageerror] ${e.message}`));
+
+await page.addInitScript((seed) => {
+  localStorage.setItem('@paseo:daemon-registry', JSON.stringify([seed.daemon]));
+  localStorage.removeItem('@paseo:settings');
+}, { daemon });
+
+await page.goto(`${baseUrl}/settings`, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+
+const card = page.getByTestId(`daemon-card-${serverId}`);
+await card.waitFor({ timeout: timeoutMs });
+await card.getByText('Online', { exact: true }).waitFor({ timeout: timeoutMs });
+await card.getByText('Relay', { exact: true }).waitFor({ timeout: timeoutMs });
+
+// Stability window: ensure it doesn't flap.
+await page.waitForTimeout(stabilityMs);
+await card.getByText('Online', { exact: true }).waitFor({ timeout: 5_000 });
+
+console.log(
+  JSON.stringify(
+    {
+      ok: true,
+      serverId,
+      relayEndpoint,
+      baseUrl,
+      stabilityMs,
+    },
+    null,
+    2
+  )
+);
+
+await browser.close();