feat: add QUIC support

package.json

@@ -54,7 +54,7 @@
     "https-browserify": "^1.0.0",
     "jsdom": "^23.0.1",
     "lerna": "^7.3.0",
-    "libp2p": "2.8.2",
+    "libp2p": "2.9.0",
     "node-gyp": "^9.4.0",
     "npm-run-all": "^4.1.5",
     "path-browserify": "^1.0.1",

packages/beacon-node/package.json

@@ -98,6 +98,7 @@
     "@chainsafe/enr": "^5.0.0",
     "@chainsafe/libp2p-gossipsub": "^14.1.1",
     "@chainsafe/libp2p-noise": "^16.1.0",
+    "@chainsafe/libp2p-quic": "https://github.com/chainsafe/js-libp2p-quic/raw/lodestar-test/package.tgz",
     "@chainsafe/persistent-merkle-tree": "^1.2.0",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/pubkey-index-map": "^3.0.0",
@@ -111,7 +112,7 @@
     "@fastify/swagger-ui": "^5.0.1",
     "@libp2p/bootstrap": "^11.0.32",
     "@libp2p/crypto": "^5.0.15",
-    "@libp2p/identify": "3.0.27",
+    "@libp2p/identify": "^3.0.27",
     "@libp2p/interface": "^2.7.0",
     "@libp2p/mdns": "^11.0.32",
     "@libp2p/mplex": "^11.0.32",
@@ -139,7 +140,7 @@
     "it-all": "^3.0.4",
     "it-pipe": "^3.0.1",
     "jwt-simple": "0.5.6",
-    "libp2p": "2.8.2",
+    "libp2p": "2.9.0",
     "multiformats": "^11.0.1",
     "prom-client": "^15.1.0",
     "qs": "^6.11.1",

packages/beacon-node/src/network/discv5/worker.ts

@@ -80,6 +80,8 @@ const onDiscovered = (enr: ENR): void => {
     subject.next(enr.toObject());
   }
 };
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore wtf?
 discv5.addListener("discovered", onDiscovered);
 
 // Discv5 will now begin accepting request/responses
@@ -96,6 +98,8 @@ const module: Discv5WorkerApi = {
     return discv5.kadValues().map((enr) => enr.toObject());
   },
   async discoverKadValues(): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore wtf?
     discv5.kadValues().map(onDiscovered);
   },
   async findRandomNode(): Promise<ENRData[]> {

packages/beacon-node/src/network/libp2p/index.ts

@@ -1,13 +1,14 @@
 import {ENR} from "@chainsafe/enr";
 import {noise} from "@chainsafe/libp2p-noise";
+import {quic} from "@chainsafe/libp2p-quic";
 import {bootstrap} from "@libp2p/bootstrap";
 import {identify} from "@libp2p/identify";
 import {PrivateKey} from "@libp2p/interface";
 import {mdns} from "@libp2p/mdns";
 import {mplex} from "@libp2p/mplex";
 import {prometheusMetrics} from "@libp2p/prometheus-metrics";
 import {tcp} from "@libp2p/tcp";
-import {createLibp2p} from "libp2p";
+import {Libp2pInit, createLibp2p} from "libp2p";
 import {Registry} from "prom-client";
 import {Libp2p, LodestarComponents} from "../interface.js";
 import {NetworkOptions, defaultNetworkOptions} from "../options.js";
@@ -62,6 +63,33 @@ export async function createNodeJsLibp2p(
       peerDiscovery.push(mdns());
     }
   }
+  const transports: Libp2pInit["transports"] = [
+    // TCP is always enabled
+    // tcp({
+    //   // Reject connections when the server's connection count gets high
+    //   maxConnections: networkOpts.maxPeers,
+    //   // socket option: the maximum length of the queue of pending connections
+    //   // https://nodejs.org/dist/latest-v18.x/docs/api/net.html#serverlisten
+    //   // it's not safe if we increase this number
+    //   backlog: 5,
+    //   closeServerOnMaxConnections: {
+    //     closeAbove: networkOpts.maxPeers ?? Infinity,
+    //     listenBelow: networkOpts.maxPeers ?? Infinity,
+    //   },
+    // }),
+  ];
+  if (!networkOpts.disableQuic) {
+    transports.unshift(
+      quic({
+        handshakeTimeout: 5_000,
+        maxIdleTimeout: 10_000,
+        keepAliveInterval: 5_000,
+        maxConcurrentStreamLimit: 256,
+        maxStreamData: 10_000_000,
+        maxConnectionData: 15_000_000,
+      })
+    );
+  }
 
   return createLibp2p({
     privateKey,
@@ -70,20 +98,7 @@ export async function createNodeJsLibp2p(
       announce: [],
     },
     connectionEncrypters: [noise()],
-    // Reject connections when the server's connection count gets high
-    transports: [
-      tcp({
-        maxConnections: networkOpts.maxPeers,
-        // socket option: the maximum length of the queue of pending connections
-        // https://nodejs.org/dist/latest-v18.x/docs/api/net.html#serverlisten
-        // it's not safe if we increase this number
-        backlog: 5,
-        closeServerOnMaxConnections: {
-          closeAbove: networkOpts.maxPeers ?? Infinity,
-          listenBelow: networkOpts.maxPeers ?? Infinity,
-        },
-      }),
-    ],
+    transports,
     streamMuxers: [mplex({maxInboundStreams: 256, disconnectThreshold: networkOpts.disconnectThreshold})],
     peerDiscovery,
     metrics: nodeJsLibp2pOpts.metrics

packages/beacon-node/src/network/metadata.ts

@@ -8,6 +8,7 @@ import {getCurrentAndNextForkBoundary} from "./forks.js";
 
 export enum ENRKey {
   tcp = "tcp",
+  quic = "quic",
   eth2 = "eth2",
   attnets = "attnets",
   syncnets = "syncnets",

packages/beacon-node/src/network/options.ts

@@ -23,7 +23,6 @@ export interface NetworkOptions
   useWorker?: boolean;
   maxYoungGenerationSizeMb?: number;
   disableLightClientServer?: boolean;
-
   /**
    * During E2E tests observe a lot of following `missing stream`:
    *
@@ -42,12 +41,14 @@ export interface NetworkOptions
    * We need to increase this only for the testing purpose
    */
   disconnectThreshold?: number;
+  disableQuic?: boolean;
 }
 
 export const defaultNetworkOptions: NetworkOptions = {
   maxPeers: 110, // Allow some room above targetPeers for new inbound peers
   targetPeers: 100,
-  localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000"],
+  // this default is never used, in practice, it is always overridden by the cli
+  localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000", "/ip4/0.0.0.0/udp/9001/quic-v1"],
   bootMultiaddrs: [],
   /** disabled by default */
   discv5: null,
@@ -61,4 +62,6 @@ export const defaultNetworkOptions: NetworkOptions = {
   slotsToSubscribeBeforeAggregatorDuty: 2,
   // This will enable the light client server by default
   disableLightClientServer: false,
+  // enable quic by default
+  disableQuic: false,
 };

packages/beacon-node/src/network/peers/discover.ts

@@ -84,6 +84,7 @@ export type SubnetDiscvQueryMs = {
 type CachedENR = {
   peerId: PeerId;
   multiaddrTCP: Multiaddr;
+  multiaddrQUIC?: Multiaddr;
   subnets: Record<SubnetType, boolean[]>;
   addedUnixMs: number;
 };
@@ -105,6 +106,7 @@ export class PeerDiscovery {
     attnets: new Map(),
     syncnets: new Map(),
   };
+  private transports: string[];
 
   private discv5StartMs: number;
   private discv5FirstQueryDelayMs: number;
@@ -158,6 +160,8 @@ export class PeerDiscovery {
         }
       });
     }
+
+    this.transports = libp2p.services.components.transportManager.getTransports().map((t) => t[Symbol.toStringTag]);
   }
 
   static async init(modules: PeerDiscoveryModules, opts: PeerDiscoveryOpts): Promise<PeerDiscovery> {
@@ -314,9 +318,16 @@ export class PeerDiscovery {
       return;
     }
 
+    // There is at least one multiaddr
+    // We assume its the tcp multiaddr, we also assume that there may be an additional quic multiaddr
+    // In practice this will only be challenged for mdns discovered peers
+    // TODO: make this smarter
+    const multiaddrTCP = multiaddrs[0];
+    const multiaddrQUIC = multiaddrs[1];
+
     const attnets = zeroAttnets;
     const syncnets = zeroSyncnets;
-    const status = this.handleDiscoveredPeer(id, multiaddrs[0], attnets, syncnets);
+    const status = this.handleDiscoveredPeer(id, multiaddrTCP, multiaddrQUIC, attnets, syncnets);
     this.logger.debug("Discovered peer via libp2p", {peer: prettyPrintPeerId(id), status});
     this.metrics?.discovery.discoveredStatus.inc({status});
   };
@@ -347,7 +358,10 @@ export class PeerDiscovery {
     const attnets = attnetsBytes ? deserializeEnrSubnets(attnetsBytes, ATTESTATION_SUBNET_COUNT) : zeroAttnets;
     const syncnets = syncnetsBytes ? deserializeEnrSubnets(syncnetsBytes, SYNC_COMMITTEE_SUBNET_COUNT) : zeroSyncnets;
 
-    const status = this.handleDiscoveredPeer(peerId, multiaddrTCP, attnets, syncnets);
+    // quic multiaddr is optional
+    const multiaddrQUIC = enr.getLocationMultiaddr(ENRKey.quic);
+
+    const status = this.handleDiscoveredPeer(peerId, multiaddrTCP, multiaddrQUIC, attnets, syncnets);
     this.logger.debug("Discovered peer via discv5", {peer: prettyPrintPeerId(peerId), status});
     this.metrics?.discovery.discoveredStatus.inc({status});
   };
@@ -358,6 +372,7 @@ export class PeerDiscovery {
   private handleDiscoveredPeer(
     peerId: PeerId,
     multiaddrTCP: Multiaddr,
+    multiaddrQUIC: Multiaddr | undefined,
     attnets: boolean[],
     syncnets: boolean[]
   ): DiscoveredPeerStatus {
@@ -372,6 +387,11 @@ export class PeerDiscovery {
         return DiscoveredPeerStatus.already_connected;
       }
 
+      // ignore peers if they don't have the transport we are using
+      if (!this.transports.includes("tcp") && !multiaddrQUIC) {
+        return DiscoveredPeerStatus.no_multiaddrs;
+      }
+
       // Ignore dialing peers
       if (
         this.libp2p.services.components.connectionManager
@@ -385,6 +405,7 @@ export class PeerDiscovery {
       const cachedPeer: CachedENR = {
         peerId,
         multiaddrTCP,
+        multiaddrQUIC,
         subnets: {attnets, syncnets},
         addedUnixMs: Date.now(),
       };
@@ -448,19 +469,24 @@ export class PeerDiscovery {
     // are not successful.
     this.peersToConnect = Math.max(this.peersToConnect - 1, 0);
 
-    const {peerId, multiaddrTCP} = cachedPeer;
+    const {peerId, multiaddrTCP, multiaddrQUIC} = cachedPeer;
 
     // Must add the multiaddrs array to the address book before dialing
     // https://github.com/libp2p/js-libp2p/blob/aec8e3d3bb1b245051b60c2a890550d262d5b062/src/index.js#L638
-    const peer = await this.libp2p.peerStore.merge(peerId, {multiaddrs: [multiaddrTCP]});
+    const peer = await this.libp2p.peerStore.merge(peerId, {
+      multiaddrs: [multiaddrQUIC, multiaddrTCP].filter(Boolean) as Multiaddr[],
+    });
     if (peer.addresses.length === 0) {
       this.metrics?.discovery.notDialReason.inc({reason: NotDialReason.no_multiaddrs});
       return;
     }
 
-    // Note: PeerDiscovery adds the multiaddrTCP beforehand
+    // Note: PeerDiscovery adds the multiaddrs beforehand
     const peerIdShort = prettyPrintPeerId(peerId);
-    this.logger.debug("Dialing discovered peer", {peer: peerIdShort});
+    this.logger.debug("Dialing discovered peer", {
+      peer: peerIdShort,
+      addresses: peer.addresses.map((a) => a.multiaddr.toString()).join(", "),
+    });
 
     this.metrics?.discovery.dialAttempts.inc();
     const timer = this.metrics?.discovery.dialTime.startTimer();