perf(relay): reduce encrypted binary traffic overhead (#2480)

Preserve application frame identity through encryption so negotiated binary traffic avoids base64 expansion while mixed-version peers remain compatible.

packages/server/src/server/daemon-e2e/relay-transport.e2e.test.ts

@@ -83,10 +83,7 @@ function decodeCiphertext(text: string): ArrayBuffer {
 }
 
 function parseEncryptedJson(sharedKey: Uint8Array, text: string): unknown {
-  const plaintext = decrypt(sharedKey, decodeCiphertext(text));
-  if (typeof plaintext !== "string") {
-    throw new Error("Expected encrypted relay frame to contain UTF-8 JSON");
-  }
+  const plaintext = new TextDecoder().decode(decrypt(sharedKey, decodeCiphertext(text)));
   return JSON.parse(plaintext);
 }
 
@@ -291,8 +288,13 @@ async function waitForRelayWebSocketReady(port: number, timeout = 60000): Promis
           onerror: null,
         };
 
-        ws.on("message", (data) => {
-          transport.onmessage?.(typeof data === "string" ? data : data.toString());
+        ws.on("message", (data, isBinary) => {
+          transport.onmessage?.({
+            data: isBinary
+              ? data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)
+              : data.toString(),
+            isBinary,
+          });
         });
         ws.on("close", (code, reason) => {
           transport.onclose?.(code, reason.toString());
@@ -434,8 +436,13 @@ async function waitForRelayWebSocketReady(port: number, timeout = 60000): Promis
           onerror: null,
         };
 
-        ws.on("message", (data) => {
-          transport.onmessage?.(typeof data === "string" ? data : data.toString());
+        ws.on("message", (data, isBinary) => {
+          transport.onmessage?.({
+            data: isBinary
+              ? data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)
+              : data.toString(),
+            isBinary,
+          });
         });
         ws.on("close", (code, reason) => {
           transport.onclose?.(code, reason.toString());

packages/server/src/server/relay-transport.ts

@@ -484,7 +484,8 @@ function createRelayTransportAdapter(
   };
 
   socket.on("message", (data, isBinary) => {
-    relayTransport.onmessage?.(normalizeMessageData(data, isBinary === true));
+    const binary = isBinary === true;
+    relayTransport.onmessage?.({ data: normalizeMessageData(data, binary), isBinary: binary });
   });
   socket.on("close", (code, reason) => {
     const closeCode = typeof code === "number" ? code : 1006;

packages/server/src/server/websocket/encrypted-relay-socket.test.ts

@@ -26,12 +26,18 @@ class BlockingChannel implements EncryptedRelayChannel {
     this.closes.push({ code, reason });
   }
 
+  outboundWireByteLength(data: string | ArrayBuffer): number {
+    const plaintextBytes =
+      typeof data === "string" ? new TextEncoder().encode(data).byteLength : data.byteLength;
+    return plaintextBytes + 40;
+  }
+
   drain(): void {
     this.resolveSend?.();
   }
 }
 
-test("the encrypted send queue terminates its physical transport at the hard bound", async () => {
+test("negotiated binary ciphertext accepts the exact hard bound and rejects one byte over", async () => {
   const channel = new BlockingChannel();
   let terminations = 0;
   const socket = createEncryptedRelaySocket({
@@ -43,11 +49,11 @@ test("the encrypted send queue terminates its physical transport at the hard bou
     },
   });
 
-  socket.send(new Uint8Array(5 * 1024 * 1024));
+  socket.send(new Uint8Array(MAX_PHYSICAL_SOCKET_BUFFERED_BYTES - 40));
   expect(channel.sent).toHaveLength(1);
-  expect(socket.bufferedAmount).toBeGreaterThan(5 * 1024 * 1024);
+  expect(socket.bufferedAmount).toBe(MAX_PHYSICAL_SOCKET_BUFFERED_BYTES);
 
-  socket.send(new Uint8Array(2 * 1024 * 1024));
+  socket.send(new Uint8Array(1));
 
   expect(channel.sent).toHaveLength(1);
   expect(terminations).toBe(1);
@@ -93,7 +99,7 @@ test("pending encryption and underlying relay backpressure share one hard bound"
   socket.send(new Uint8Array(3 * 1024 * 1024));
   expect(channel.sent).toHaveLength(1);
 
-  transportBufferedAmount = 4 * 1024 * 1024;
+  transportBufferedAmount = 6 * 1024 * 1024;
   socket.send(new Uint8Array(1));
 
   expect(channel.sent).toHaveLength(1);

packages/server/src/server/websocket/encrypted-relay-socket.ts

@@ -1,12 +1,10 @@
 import { EventEmitter } from "node:events";
-import { MAX_PHYSICAL_SOCKET_BUFFERED_BYTES, outboundFrameByteLength } from "./physical-socket.js";
-
-// NaCl adds a 24-byte nonce and 16-byte authenticator before base64 encoding.
-const ENCRYPTED_FRAME_OVERHEAD_BYTES = 40;
+import { MAX_PHYSICAL_SOCKET_BUFFERED_BYTES } from "./physical-socket.js";
 
 export interface EncryptedRelayChannel {
   setState: (state: "open") => void;
   send: (data: string | ArrayBuffer) => Promise<void>;
+  outboundWireByteLength: (data: string | ArrayBuffer) => number;
   close: (code?: number, reason?: string) => void;
 }
 
@@ -58,7 +56,7 @@ export function createEncryptedRelaySocket(params: {
     send: (data) => {
       if (readyState !== 1) return;
       const outbound = normalizeRelaySendPayload(data);
-      const outboundBytes = encryptedRelayFrameByteLength(outbound);
+      const outboundBytes = channel.outboundWireByteLength(outbound);
       const queuedBytes = pendingEncryptedBytes + (getTransportBufferedAmount() ?? 0);
       if (queuedBytes + outboundBytes > MAX_PHYSICAL_SOCKET_BUFFERED_BYTES) {
         terminate();
@@ -93,8 +91,3 @@ function normalizeRelaySendPayload(data: string | Uint8Array | ArrayBuffer): str
   out.set(view);
   return out.buffer;
 }
-
-function encryptedRelayFrameByteLength(data: string | ArrayBuffer): number {
-  const encryptedBytes = outboundFrameByteLength(data) + ENCRYPTED_FRAME_OVERHEAD_BYTES;
-  return 4 * Math.ceil(encryptedBytes / 3);
-}