Allow Azks parallelism to be configurable (#457)

* Simplify append only proof parallelism

* Introduce configurable parallelism

* Prepare for 0.12.0-pre.11 release

* Adopt review suggestions

* Re-export parallelism structs at top-level for convenience

Author: afterdusk

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.12.0-pre.11 (Nov 21, 2024)
+* Added parallelization for node preloads during audit proof generation
+* Removed `parallel_azks` feature in favor of explicit parallelism configuration object allowing the
+setting of static or dynamic parallelism across insertion and preloads
+
 ## 0.12.0-pre.10 (Oct 17, 2024)
 * Added parallelization for node preloads during insertion
 * Added support for [tracing](https://docs.rs/tracing/latest/tracing/index.html)

README.md

@@ -22,7 +22,7 @@ Installation
 Add the following line to the dependencies of your `Cargo.toml`:
 
 ```
-akd = "0.12.0-pre.10"
+akd = "0.12.0-pre.11"
 ```
 
 ### Minimum Supported Rust Version

akd/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "akd"
-version = "0.12.0-pre.10"
+version = "0.12.0-pre.11"
 authors = ["akd contributors"]
 description = "An implementation of an auditable key directory"
 license = "MIT OR Apache-2.0"
@@ -18,7 +18,6 @@ experimental = ["akd_core/experimental"]
 default = [
     "public_auditing",
     "parallel_vrf",
-    "parallel_azks",
     "preload_history",
     "greedy_lookup_preload",
     "experimental",
@@ -28,8 +27,6 @@ bench = ["experimental", "public_tests", "tokio/rt-multi-thread"]
 # Greedy loading of lookup proof nodes
 greedy_lookup_preload = []
 public_auditing = ["dep:protobuf", "akd_core/protobuf"]
-# Parallelize node fetch and insertion during publish
-parallel_azks = []
 # Parallelize VRF calculations during publish
 parallel_vrf = ["akd_core/parallel_vrf"]
 # Enable pre-loading of the nodes when generating history proofs
@@ -56,7 +53,7 @@ tracing_instrument = ["tracing/attributes"]
 
 [dependencies]
 ## Required dependencies ##
-akd_core = { version = "0.12.0-pre.10", path = "../akd_core", default-features = false, features = [
+akd_core = { version = "0.12.0-pre.11", path = "../akd_core", default-features = false, features = [
     "vrf",
 ] }
 async-recursion = "1"

akd/benches/azks.rs

@@ -10,7 +10,7 @@ extern crate criterion;
 
 mod common;
 
-use akd::append_only_zks::InsertMode;
+use akd::append_only_zks::{AzksParallelismConfig, InsertMode};
 use akd::auditor;
 use akd::storage::manager::StorageManager;
 use akd::storage::memory::AsyncInMemoryDatabase;
@@ -57,6 +57,7 @@ fn batch_insertion<TC: NamedConfiguration>(c: &mut Criterion) {
                         &db,
                         initial_node_set.clone(),
                         InsertMode::Directory,
+                        AzksParallelismConfig::default(),
                     ))
                     .unwrap();
                 (azks, db, node_set.clone())
@@ -67,6 +68,7 @@ fn batch_insertion<TC: NamedConfiguration>(c: &mut Criterion) {
                         &db,
                         node_set,
                         InsertMode::Directory,
+                        AzksParallelismConfig::default(),
                     ))
                     .unwrap();
             },
@@ -107,6 +109,7 @@ fn audit_verify<TC: NamedConfiguration>(c: &mut Criterion) {
                         &db,
                         initial_node_set.clone(),
                         InsertMode::Directory,
+                        AzksParallelismConfig::default(),
                     ))
                     .unwrap();
 
@@ -118,12 +121,18 @@ fn audit_verify<TC: NamedConfiguration>(c: &mut Criterion) {
                         &db,
                         node_set.clone(),
                         InsertMode::Directory,
+                        AzksParallelismConfig::default(),
                     ))
                     .unwrap();
 
                 let end_hash = runtime.block_on(azks.get_root_hash::<TC, _>(&db)).unwrap();
                 let proof = runtime
-                    .block_on(azks.get_append_only_proof::<TC, _>(&db, 1, 2))
+                    .block_on(azks.get_append_only_proof::<TC, _>(
+                        &db,
+                        1,
+                        2,
+                        AzksParallelismConfig::default(),
+                    ))
                     .unwrap();
 
                 (start_hash, end_hash, proof)
@@ -157,7 +166,12 @@ fn audit_generate<TC: NamedConfiguration>(c: &mut Criterion) {
     for _epoch in 0..num_epochs {
         let node_set = gen_nodes(&mut rng, num_leaves);
         runtime
-            .block_on(azks.batch_insert_nodes::<TC, _>(&db, node_set, InsertMode::Directory))
+            .block_on(azks.batch_insert_nodes::<TC, _>(
+                &db,
+                node_set,
+                InsertMode::Directory,
+                AzksParallelismConfig::default(),
+            ))
             .unwrap();
     }
     let epoch = azks.get_latest_epoch();
@@ -172,7 +186,12 @@ fn audit_generate<TC: NamedConfiguration>(c: &mut Criterion) {
             || {},
             |_| {
                 let _proof = runtime
-                    .block_on(azks.get_append_only_proof::<TC, _>(&db, epoch - 1, epoch))
+                    .block_on(azks.get_append_only_proof::<TC, _>(
+                        &db,
+                        epoch - 1,
+                        epoch,
+                        AzksParallelismConfig::default(),
+                    ))
                     .unwrap();
             },
             BatchSize::PerIteration,

akd/benches/directory.rs

@@ -10,6 +10,7 @@ extern crate criterion;
 
 mod common;
 
+use akd::append_only_zks::AzksParallelismConfig;
 use akd::ecvrf::HardCodedAkdVRF;
 use akd::storage::manager::StorageManager;
 use akd::storage::memory::AsyncInMemoryDatabase;
@@ -56,7 +57,9 @@ fn history_generation<TC: NamedConfiguration>(c: &mut Criterion) {
                 );
                 let db_clone = db.clone();
                 let directory = runtime
-                    .block_on(async move { Directory::<TC, _, _>::new(db, vrf).await })
+                    .block_on(async move {
+                        Directory::<TC, _, _>::new(db, vrf, AzksParallelismConfig::default()).await
+                    })
                     .unwrap();
 
                 for _epoch in 1..num_updates {