relay: reset stuck control sockets; add prod prove script
This commit is contained in:
@@ -75,6 +75,45 @@ export class RelayDurableObject {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private hasServerDataSocket(clientId: string): boolean {
|
||||||
|
try {
|
||||||
|
return this.state.getWebSockets(`server:${clientId}`).length > 0;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private nudgeOrResetControlForClient(clientId: string): void {
|
||||||
|
// If the daemon's control WS becomes half-open, the DO can't reliably detect it via ws.send errors
|
||||||
|
// (Cloudflare may accept writes even if the other side is no longer reading).
|
||||||
|
//
|
||||||
|
// Instead, observe whether the daemon reacts by opening the per-client server-data socket.
|
||||||
|
// If it doesn't, nudge with a sync message; if still no reaction, force-close the control
|
||||||
|
// socket(s) so the daemon reconnects.
|
||||||
|
const initialDelayMs = 10_000;
|
||||||
|
const secondDelayMs = 5_000;
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
if (this.hasServerDataSocket(clientId)) return;
|
||||||
|
|
||||||
|
// First nudge: send a full sync list.
|
||||||
|
this.notifyControls({ type: "sync", clientIds: this.listConnectedClientIds() });
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
if (this.hasServerDataSocket(clientId)) return;
|
||||||
|
|
||||||
|
// Still nothing: assume control is stuck and force a reconnect.
|
||||||
|
for (const ws of this.state.getWebSockets("server-control")) {
|
||||||
|
try {
|
||||||
|
ws.close(1011, "Control unresponsive");
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, secondDelayMs);
|
||||||
|
}, initialDelayMs);
|
||||||
|
}
|
||||||
|
|
||||||
private bufferClientFrame(clientId: string, message: string | ArrayBuffer): void {
|
private bufferClientFrame(clientId: string, message: string | ArrayBuffer): void {
|
||||||
const existing = this.pendingClientFrames.get(clientId) ?? [];
|
const existing = this.pendingClientFrames.get(clientId) ?? [];
|
||||||
existing.push(message);
|
existing.push(message);
|
||||||
@@ -209,6 +248,7 @@ export class RelayDurableObject {
|
|||||||
|
|
||||||
if (role === "client") {
|
if (role === "client") {
|
||||||
this.notifyControls({ type: "client_connected", clientId });
|
this.notifyControls({ type: "client_connected", clientId });
|
||||||
|
this.nudgeOrResetControlForClient(clientId);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isServerControl) {
|
if (isServerControl) {
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import { chromium } from 'playwright';
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
const args = {};
|
||||||
|
for (let i = 2; i < argv.length; i += 1) {
|
||||||
|
const part = argv[i];
|
||||||
|
if (!part.startsWith('--')) continue;
|
||||||
|
const key = part.slice(2);
|
||||||
|
const next = argv[i + 1];
|
||||||
|
if (!next || next.startsWith('--')) {
|
||||||
|
args[key] = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
args[key] = next;
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
|
||||||
|
const args = parseArgs(process.argv);
|
||||||
|
|
||||||
|
const serverId = args['server-id'] ?? process.env.PASEO_SERVER_ID;
|
||||||
|
const daemonPublicKeyB64 = args['daemon-public-key-b64'] ?? process.env.PASEO_DAEMON_PUBLIC_KEY_B64;
|
||||||
|
const relayEndpoint = args['relay-endpoint'] ?? process.env.PASEO_RELAY_ENDPOINT ?? 'relay.paseo.sh:443';
|
||||||
|
const baseUrl = args['base-url'] ?? process.env.PASEO_APP_URL ?? 'https://app.paseo.sh';
|
||||||
|
const timeoutMs = Number(args['timeout-ms'] ?? process.env.PASEO_PROVE_TIMEOUT_MS ?? 60_000);
|
||||||
|
const stabilityMs = Number(args['stability-ms'] ?? process.env.PASEO_PROVE_STABILITY_MS ?? 30_000);
|
||||||
|
|
||||||
|
if (!serverId || typeof serverId !== 'string') {
|
||||||
|
console.error('Missing server ID. Provide --server-id or PASEO_SERVER_ID.');
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
if (!daemonPublicKeyB64 || typeof daemonPublicKeyB64 !== 'string') {
|
||||||
|
console.error('Missing daemon public key. Provide --daemon-public-key-b64 or PASEO_DAEMON_PUBLIC_KEY_B64.');
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
const nowIso = new Date().toISOString();
|
||||||
|
const daemon = {
|
||||||
|
serverId,
|
||||||
|
label: 'relay-daemon',
|
||||||
|
connections: [{ id: `relay:${relayEndpoint}`, type: 'relay', relayEndpoint, daemonPublicKeyB64 }],
|
||||||
|
preferredConnectionId: `relay:${relayEndpoint}`,
|
||||||
|
createdAt: nowIso,
|
||||||
|
updatedAt: nowIso,
|
||||||
|
};
|
||||||
|
|
||||||
|
const browser = await chromium.launch({ headless: true });
|
||||||
|
const context = await browser.newContext();
|
||||||
|
const page = await context.newPage();
|
||||||
|
|
||||||
|
page.on('console', (m) => console.log(`[browser:${m.type()}] ${m.text()}`));
|
||||||
|
page.on('pageerror', (e) => console.error(`[browser:pageerror] ${e.message}`));
|
||||||
|
|
||||||
|
await page.addInitScript((seed) => {
|
||||||
|
localStorage.setItem('@paseo:daemon-registry', JSON.stringify([seed.daemon]));
|
||||||
|
localStorage.removeItem('@paseo:settings');
|
||||||
|
}, { daemon });
|
||||||
|
|
||||||
|
await page.goto(`${baseUrl}/settings`, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
||||||
|
|
||||||
|
const card = page.getByTestId(`daemon-card-${serverId}`);
|
||||||
|
await card.waitFor({ timeout: timeoutMs });
|
||||||
|
await card.getByText('Online', { exact: true }).waitFor({ timeout: timeoutMs });
|
||||||
|
await card.getByText('Relay', { exact: true }).waitFor({ timeout: timeoutMs });
|
||||||
|
|
||||||
|
// Stability window: ensure it doesn't flap.
|
||||||
|
await page.waitForTimeout(stabilityMs);
|
||||||
|
await card.getByText('Online', { exact: true }).waitFor({ timeout: 5_000 });
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
ok: true,
|
||||||
|
serverId,
|
||||||
|
relayEndpoint,
|
||||||
|
baseUrl,
|
||||||
|
stabilityMs,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
await browser.close();
|
||||||
Reference in New Issue
Block a user