feat: add lmdb and sqlite backend

packages/beacon-node/src/chain/archiveStore/archiveStore.ts

@@ -23,7 +23,11 @@ type ArchiveStoreModules = {
   metrics: Metrics | null;
 };
 
-type ArchiveStoreInitOpts = ArchiveStoreOpts & {dbName: string; anchorState: {finalizedCheckpoint: Checkpoint}};
+type ArchiveStoreInitOpts = ArchiveStoreOpts & {
+  dbName: string;
+  dbType: "level" | "lmdb";
+  anchorState: {finalizedCheckpoint: Checkpoint};
+};
 
 /**
  * Used for running tasks that depends on some events or are executed
@@ -110,6 +114,7 @@ export class ArchiveStore {
         opts: {
           genesisTime: this.chain.clock.genesisTime,
           dbLocation: this.opts.dbName,
+          dbType: this.opts.dbType,
         },
         config: this.chain.config,
         metrics: this.metrics,

packages/beacon-node/src/chain/archiveStore/historicalState/historicalStateRegen.ts

@@ -33,6 +33,7 @@ export class HistoricalStateRegen implements HistoricalStateWorkerApi {
       maxConcurrency: 1,
       maxLength: 50,
       dbLocation: modules.opts.dbLocation,
+      dbType: modules.opts.dbType,
       metricsEnabled: Boolean(modules.metrics),
       loggerOpts: modules.logger.toOpts(),
     };

packages/beacon-node/src/chain/archiveStore/historicalState/types.ts

@@ -7,6 +7,7 @@ export type HistoricalStateRegenInitModules = {
   opts: {
     genesisTime: number;
     dbLocation: string;
+    dbType: "level" | "lmdb";
   };
   config: BeaconConfig;
   logger: LoggerNode;
@@ -24,6 +25,7 @@ export type HistoricalStateWorkerData = {
   maxConcurrency: number;
   maxLength: number;
   dbLocation: string;
+  dbType: "level" | "lmdb";
   metricsEnabled: boolean;
   loggerOpts: LoggerNodeOpts;
 };

packages/beacon-node/src/chain/archiveStore/historicalState/worker.ts

@@ -3,6 +3,7 @@ import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {Transfer, expose} from "@chainsafe/threads/worker";
 import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config";
 import {LevelDbController} from "@lodestar/db/controller/level";
+import {LmdbController} from "@lodestar/db/controller/lmdb";
 import {getNodeLogger} from "@lodestar/logger/node";
 import {BeaconDb} from "../../../db/index.js";
 import {RegistryMetricCreator, collectNodeJSMetrics} from "../../../metrics/index.js";
@@ -26,7 +27,11 @@ logger.info("Historical state worker started");
 
 const config = createBeaconConfig(chainConfigFromJson(workerData.chainConfigJson), workerData.genesisValidatorsRoot);
 
-const db = new BeaconDb(config, await LevelDbController.create({name: workerData.dbLocation}, {logger}));
+const controller =
+  workerData.dbType === "level"
+    ? await LevelDbController.create({name: workerData.dbLocation}, {logger})
+    : await LmdbController.create({name: workerData.dbLocation}, {logger});
+const db = new BeaconDb(config, controller);
 
 const abortController = new AbortController();
 

packages/beacon-node/src/chain/chain.ts

@@ -203,6 +203,7 @@ export class BeaconChain implements IBeaconChain {
       config,
       db,
       dbName,
+      dbType,
       dataDir,
       logger,
       processShutdownCallback,
@@ -218,6 +219,7 @@ export class BeaconChain implements IBeaconChain {
       config: BeaconConfig;
       db: IBeaconDb;
       dbName: string;
+      dbType: "level" | "lmdb";
       dataDir: string;
       logger: Logger;
       processShutdownCallback: ProcessShutdownCallback;
@@ -403,7 +405,7 @@ export class BeaconChain implements IBeaconChain {
 
     this.archiveStore = new ArchiveStore(
       {db, chain: this, logger: logger as LoggerNode, metrics},
-      {...opts, dbName, anchorState: {finalizedCheckpoint: anchorState.finalizedCheckpoint}},
+      {...opts, dbName, dbType, anchorState: {finalizedCheckpoint: anchorState.finalizedCheckpoint}},
       signal
     );
 

packages/beacon-node/src/db/options.ts

@@ -1,7 +1,9 @@
 export type DatabaseOptions = {
   name: string;
+  type: "level" | "lmdb";
 };
 
 export const defaultDbOptions: DatabaseOptions = {
   name: "./.tmp/lodestar-db",
+  type: "level",
 };

packages/beacon-node/src/node/nodejs.ts

@@ -212,6 +212,7 @@ export class BeaconNode {
       dataDir,
       db,
       dbName: opts.db.name,
+      dbType: opts.db.type,
       logger: logger.child({module: LoggerModule.chain}),
       processShutdownCallback,
       metrics,

packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts

@@ -45,6 +45,7 @@ describe("produceBlockBody", () => {
         db,
         dataDir: ".",
         dbName: ".",
+        dbType: "level",
         logger,
         processShutdownCallback: () => {},
         metrics: null,

packages/beacon-node/test/perf/chain/verifyImportBlocks.test.ts

@@ -96,6 +96,7 @@ describe.skip("verify+import blocks - range sync perf test", () => {
           db,
           dataDir: ".",
           dbName: ".",
+          dbType: "level",
           logger,
           processShutdownCallback: () => {},
           metrics: null,

packages/beacon-node/test/spec/presets/fork_choice.test.ts

@@ -112,6 +112,7 @@ const forkChoiceTest =
             db: getMockedBeaconDb(),
             dataDir: ".",
             dbName: ",",
+            dbType: "level",
             logger,
             processShutdownCallback: () => {},
             clock,

packages/beacon-node/test/utils/networkWithMockDb.ts

@@ -65,6 +65,7 @@ export async function getNetworkForTest(
       db,
       dataDir: ".",
       dbName: ".",
+      dbType: "level",
       logger,
       processShutdownCallback: () => {},
       // set genesis time so that we are at ALTAIR_FORK_EPOCH

packages/cli/src/cmds/beacon/handler.ts

@@ -5,6 +5,7 @@ import {hasher} from "@chainsafe/persistent-merkle-tree";
 import {BeaconDb, BeaconNode} from "@lodestar/beacon-node";
 import {ChainForkConfig, createBeaconConfig} from "@lodestar/config";
 import {LevelDbController} from "@lodestar/db/controller/level";
+import {LmdbController} from "@lodestar/db/controller/lmdb";
 import {LoggerNode, getNodeLogger} from "@lodestar/logger/node";
 import {ACTIVE_PRESET, PresetName} from "@lodestar/params";
 import {ErrorAborted, bytesToInt} from "@lodestar/utils";
@@ -65,8 +66,17 @@ export async function beaconHandler(args: BeaconArgs & GlobalArgs): Promise<void
 
   if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset");
 
-  const db = new BeaconDb(config, await LevelDbController.create(options.db, {metrics: null, logger}));
-  logger.info("Connected to LevelDB database", {path: options.db.name});
+  let controller: LevelDbController | LmdbController;
+  if (options.db.type === "level") {
+    controller = await LevelDbController.create(options.db, {metrics: null, logger});
+    logger.info("Connected to LevelDB database", {path: options.db.name});
+  } else if (options.db.type === "lmdb") {
+    controller = await LmdbController.create(options.db, {metrics: null, logger});
+    logger.info("Connected to LMDB database", {path: options.db.name});
+  } else {
+    throw new Error(`Unsupported db type: ${options.db.type}`);
+  }
+  const db = new BeaconDb(config, controller);
 
   // BeaconNode setup
   try {
@@ -172,7 +182,7 @@ export async function beaconHandlerInit(args: BeaconArgs & GlobalArgs) {
   const {version, commit} = getVersionData();
   const beaconPaths = getBeaconPaths(args, network);
   // TODO: Rename db.name to db.path or db.location
-  beaconNodeOptions.set({db: {name: beaconPaths.dbDir}});
+  beaconNodeOptions.set({db: {name: beaconPaths.dbDir, type: args.dbType}});
   beaconNodeOptions.set({
     chain: {
       validatorMonitorLogs: args.validatorMonitorLogs,

packages/cli/src/cmds/beacon/options.ts

@@ -15,6 +15,7 @@ type BeaconExtraArgs = {
   ignoreWeakSubjectivityCheck?: boolean;
   beaconDir?: string;
   dbDir?: string;
+  dbType?: "level" | "lmdb";
   persistInvalidSszObjectsDir?: string;
   persistInvalidSszObjectsRetentionHours?: number;
   persistOrphanedBlocksDir?: string;
@@ -98,6 +99,14 @@ export const beaconExtraOptions: CliCommandOptions<BeaconExtraArgs> = {
     type: "string",
   },
 
+  dbType: {
+    description: "Database backend type",
+    choices: ["level", "lmdb"],
+    default: "level",
+    hidden: true,
+    type: "string",
+  },
+
   persistInvalidSszObjectsDir: {
     description: "Enable and specify a directory to persist invalid ssz objects",
     defaultDescription: defaultBeaconPaths.persistInvalidSszObjectsDir,

packages/db/package.json

@@ -20,6 +20,10 @@
     "./controller/level": {
       "bun": "./src/controller/level_bun.ts",
       "import": "./lib/controller/level.js"
+    },
+    "./controller/lmdb": {
+      "bun": "./src/controller/lmdb_bun.ts",
+      "import": "./lib/controller/lmdb.js"
     }
   },
   "imports": {

packages/db/src/controller/lmdb.ts

@@ -0,0 +1,68 @@
+import {Logger} from "@lodestar/utils";
+import {DatabaseController, DatabaseOptions, DbReqOpts, FilterOptions, KeyValue} from "./interface.ts";
+import {LevelDbControllerMetrics} from "./metrics.ts";
+
+export type LmdbControllerModules = {
+  logger: Logger;
+  metrics?: LevelDbControllerMetrics | null;
+};
+
+export class LmdbController implements DatabaseController<Uint8Array, Uint8Array> {
+  static async create(options: DatabaseOptions, {metrics}: LmdbControllerModules): Promise<LmdbController> {
+    return new LmdbController();
+  }
+  static async destroy(_location: string): Promise<void> {}
+  close(): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+  setMetrics(metrics: LevelDbControllerMetrics): void {
+    throw new Error("Method not implemented.");
+  }
+  get(key: Uint8Array<ArrayBufferLike>, opts?: DbReqOpts): Promise<Uint8Array<ArrayBufferLike> | null> {
+    throw new Error("Method not implemented.");
+  }
+  getMany(key: Uint8Array<ArrayBufferLike>[], opts?: DbReqOpts): Promise<(Uint8Array<ArrayBufferLike> | undefined)[]> {
+    throw new Error("Method not implemented.");
+  }
+  put(key: Uint8Array<ArrayBufferLike>, value: Uint8Array<ArrayBufferLike>, opts?: DbReqOpts): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+  delete(key: Uint8Array<ArrayBufferLike>, opts?: DbReqOpts): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+  batchPut(
+    items: KeyValue<Uint8Array<ArrayBufferLike>, Uint8Array<ArrayBufferLike>>[],
+    opts?: DbReqOpts
+  ): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+  batchDelete(keys: Uint8Array<ArrayBufferLike>[], opts?: DbReqOpts): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+  keysStream(
+    opts?: FilterOptions<Uint8Array<ArrayBufferLike>> | undefined
+  ): AsyncIterable<Uint8Array<ArrayBufferLike>> {
+    throw new Error("Method not implemented.");
+  }
+  keys(opts?: FilterOptions<Uint8Array<ArrayBufferLike>> | undefined): Promise<Uint8Array<ArrayBufferLike>[]> {
+    throw new Error("Method not implemented.");
+  }
+  valuesStream(
+    opts?: FilterOptions<Uint8Array<ArrayBufferLike>> | undefined
+  ): AsyncIterable<Uint8Array<ArrayBufferLike>> {
+    throw new Error("Method not implemented.");
+  }
+  values(opts?: FilterOptions<Uint8Array<ArrayBufferLike>> | undefined): Promise<Uint8Array<ArrayBufferLike>[]> {
+    throw new Error("Method not implemented.");
+  }
+  entriesStream(
+    opts?: FilterOptions<Uint8Array<ArrayBufferLike>> | undefined
+  ): AsyncIterable<KeyValue<Uint8Array<ArrayBufferLike>, Uint8Array<ArrayBufferLike>>> {
+    throw new Error("Method not implemented.");
+  }
+  entries(
+    opts?: FilterOptions<Uint8Array<ArrayBufferLike>> | undefined
+  ): Promise<KeyValue<Uint8Array<ArrayBufferLike>, Uint8Array<ArrayBufferLike>>[]> {
+    throw new Error("Method not implemented.");
+  }
+}