fix(relay): validate handshake keys (#3669)

packages/relay/src/crypto.test.ts

@@ -12,6 +12,10 @@ function decryptText(sharedKey: Uint8Array, ciphertext: ArrayBuffer): string {
   return new TextDecoder().decode(decrypt(sharedKey, ciphertext));
 }
 
+function bytesFromHex(hex: string): Uint8Array {
+  return Uint8Array.from(hex.match(/.{2}/g) ?? [], (byte) => Number.parseInt(byte, 16));
+}
+
 describe("crypto", () => {
   describe("generateKeyPair", () => {
     it("generates a valid keypair", () => {
@@ -36,6 +40,22 @@ describe("crypto", () => {
       const reExported = exportPublicKey(imported);
       expect(reExported).toBe(exported);
     });
+
+    it.each([
+      ["malformed", "!".repeat(43) + "="],
+      ["noncanonical", `${"A".repeat(42)}B=`],
+    ])("rejects %s base64", (_description, encoded) => {
+      expect(importPublicKey.bind(null, encoded)).toThrowError("Invalid public key encoding");
+    });
+
+    it.each([
+      ["short", new Uint8Array(31)],
+      ["oversized", new Uint8Array(33)],
+    ])("rejects a %s decoded key", (_description, key) => {
+      expect(importPublicKey.bind(null, Buffer.from(key).toString("base64"))).toThrowError(
+        "Invalid public key length (expected 32)",
+      );
+    });
   });
 
   describe("deriveSharedKey", () => {
@@ -63,6 +83,34 @@ describe("crypto", () => {
 
       expect(decrypted).toBe(testMessage);
     });
+
+    // Unsupported peer public keys.
+    it.each([
+      ["unsupported key 1", "00".repeat(32)],
+      ["unsupported key 2", `01${"00".repeat(31)}`],
+      ["unsupported key 3", "e0eb7a7c3b41b8ae1656e3faf19fc46ada098deb9c32b1fd866205165f49b800"],
+      ["unsupported key 4", "5f9c95bca3508c24b1d0b1559c83ef5b04445cc4581c8e86d8224eddd09f1157"],
+      ["unsupported key 5", `ec${"ff".repeat(30)}7f`],
+      ["unsupported key 6", `ed${"ff".repeat(30)}7f`],
+      ["unsupported key 7", `ee${"ff".repeat(30)}7f`],
+    ])("rejects the %s peer public key", (_description, publicKeyHex) => {
+      const { secretKey } = generateKeyPair();
+
+      expect(deriveSharedKey.bind(null, secretKey, bytesFromHex(publicKeyHex))).toThrowError(
+        "Invalid peer public key",
+      );
+    });
+
+    it.each([
+      ["short", new Uint8Array(31)],
+      ["oversized", new Uint8Array(33)],
+    ])("rejects a %s peer public key", (_description, publicKey) => {
+      const { secretKey } = generateKeyPair();
+
+      expect(deriveSharedKey.bind(null, secretKey, publicKey)).toThrowError(
+        "Invalid peer public key length (expected 32)",
+      );
+    });
   });
 
   describe("encrypt / decrypt", () => {

packages/relay/src/crypto.ts

@@ -23,6 +23,7 @@ export interface KeyPair {
 export type SharedKey = Uint8Array; // 32 bytes (box.before)
 
 const NONCE_LENGTH = nacl.box.nonceLength; // 24
+const ZERO_X25519_SHARED_RESULT = new Uint8Array(nacl.box.sharedKeyLength);
 
 let prngReady = false;
 
@@ -68,6 +69,22 @@ function decodeBase64(base64: string): Uint8Array {
   return toByteArray(base64);
 }
 
+function decodePublicKeyBase64(base64: string): Uint8Array {
+  if (
+    typeof base64 !== "string" ||
+    base64.length % 4 !== 0 ||
+    !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(base64)
+  ) {
+    throw new Error("Invalid public key encoding");
+  }
+
+  const bytes = decodeBase64(base64);
+  if (encodeBase64(bytes) !== base64) {
+    throw new Error("Invalid public key encoding");
+  }
+  return bytes;
+}
+
 function toUint8(data: string | ArrayBuffer): Uint8Array {
   return typeof data === "string" ? new TextEncoder().encode(data) : new Uint8Array(data);
 }
@@ -92,7 +109,7 @@ export function exportPublicKey(publicKey: Uint8Array): string {
 }
 
 export function importPublicKey(base64: string): Uint8Array {
-  const bytes = decodeBase64(base64);
+  const bytes = decodePublicKeyBase64(base64);
   if (bytes.byteLength !== nacl.box.publicKeyLength) {
     throw new Error(`Invalid public key length (expected ${nacl.box.publicKeyLength})`);
   }
@@ -121,6 +138,14 @@ export function deriveSharedKey(ourSecretKey: Uint8Array, peerPublicKey: Uint8Ar
   if (peerPublicKey.byteLength !== nacl.box.publicKeyLength) {
     throw new Error(`Invalid peer public key length (expected ${nacl.box.publicKeyLength})`);
   }
+
+  const rawSharedResult = nacl.scalarMult(ourSecretKey, peerPublicKey);
+  const isAllZero = nacl.verify(rawSharedResult, ZERO_X25519_SHARED_RESULT);
+  rawSharedResult.fill(0);
+  if (isAllZero) {
+    throw new Error("Invalid peer public key");
+  }
+
   return nacl.box.before(peerPublicKey, ourSecretKey);
 }