Merge remote-tracking branch 'origin/main' into fan-out-dep

Author: wbbradley

.github/workflows/create-release-announce.yml

@@ -162,4 +162,5 @@ jobs:
           repository: MystenLabs/sui-operations
           token: ${{ secrets.SUI_OPS_DISPATCH_TOKEN }}
           event-type: walrus-release-announce
-          client-payload: '{"walrus_commit": "${{ needs.create-release.outputs.walrus_commit }}", "environment": "${{ inputs.network }}"}'
+          client-payload: >
+            {"walrus_commit": "${{ needs.create-release.outputs.walrus_commit }}", "environment": "${{ inputs.network }}"}

.github/workflows/lint.yml

@@ -53,7 +53,7 @@ jobs:
 
       - name: Install yamlfmt
         run: |
-          url="https://github.com/google/yamlfmt/releases/download/v0.14.0/yamlfmt_0.14.0_Linux_x86_64.tar.gz"
+          url="https://github.com/google/yamlfmt/releases/download/v0.17.0/yamlfmt_0.17.0_Linux_x86_64.tar.gz"
           curl -sSL "$url" | tar xvz yamlfmt
       - name: Check formatting of all YAML files
         run: ./yamlfmt -lint

.github/workflows/update-operators-cache.yml

@@ -14,7 +14,7 @@ on:
         type: string
         default: 'bfH-ZqYV5faXTiq2-FVBT2cLQD61VYRuDje0cIkMWMo'
   schedule:
-    - cron: '0 11 * * 1'  # Every Monday at 11:00 UTC
+    - cron: '0 11 * * 1' # Every Monday at 11:00 UTC
 
 env:
   MAINNET_BLOB_ID: ${{ github.event.inputs.mainnet-blob-id || 'bfH-ZqYV5faXTiq2-FVBT2cLQD61VYRuDje0cIkMWMo' }}

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
       - id: editorconfig-checker
         alias: ec
   - repo: https://github.com/google/yamlfmt
-    rev: v0.15.0
+    rev: v0.17.0
     hooks:
       - id: yamlfmt
   - repo: https://github.com/notken12/licensesnip

.yamlfmt.yml

@@ -10,3 +10,4 @@ formatter:
   scan_folded_as_literal: true
   max_line_length: 120
   drop_merge_tag: true
+  disable_alias_key_correction: true

Cargo.lock

@@ -88,7 +88,6 @@ proc-macro2 = "1.0.95"
 prometheus = "0.13.4"
 quote = "1.0"
 rand = "0.8.5"
-raptorq = "2.0.0"
 rayon = "1.10.0"
 rcgen = "0.13.2"
 reed-solomon-simd = "3.0.1"
@@ -154,9 +153,19 @@ walrus-utils = { path = "crates/walrus-utils" }
 x509-cert = "0.2.5"
 
 [workspace.lints.rust]
+future_incompatible = "warn"
 missing_debug_implementations = "warn"
 missing_docs = "warn"
-unexpected_cfgs = { level = "allow", check-cfg = ['cfg(msim)'] }
+nonstandard_style = "warn"
+unexpected_cfgs = { level = "allow", check-cfg = ['cfg(msim)'], priority = 1 }
+unused = "warn"
+
+[workspace.lints.clippy]
+cast_lossless = "warn"
+cast_possible_truncation = "warn"
+cast_possible_wrap = "warn"
+empty_structs_with_brackets = "warn"
+unwrap_used = "warn"
 
 [profile.release]
 panic = 'abort'

Cargo.toml

@@ -0,0 +1 @@
+allow-unwrap-in-tests = true

clippy.toml

@@ -230,7 +230,11 @@ impl ParallelCheckpointDownloaderInner {
                         .send(WorkerMessage::Shutdown)
                         .await?;
                     let new_count = worker_count.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
-                    config.metrics.num_workers.set(new_count as i64);
+                    config.metrics.num_workers.set(
+                        new_count
+                            .try_into()
+                            .expect("new_count should always fit into a i64"),
+                    );
                 }
             }
             std::cmp::Ordering::Less => {
@@ -249,7 +253,11 @@ impl ParallelCheckpointDownloaderInner {
                     );
                     *next_worker_id += 1;
                     let new_count = worker_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-                    config.metrics.num_workers.set(new_count as i64);
+                    config.metrics.num_workers.set(
+                        new_count
+                            .try_into()
+                            .expect("new_count should always fit into a i64"),
+                    );
                 }
             }
             std::cmp::Ordering::Equal => {}
@@ -318,7 +326,9 @@ impl ParallelCheckpointDownloaderInner {
                     };
 
                     consecutive_failures = 0;
-                    config.metrics.checkpoint_lag.set(lag as i64);
+                    config.metrics.checkpoint_lag.set(lag.try_into().expect(
+                        "lag should always fit into a i64"
+                    ));
 
                     let current = worker_count.load(std::sync::atomic::Ordering::Relaxed);
                     if lag > downloader_config.scale_up_lag_threshold &&
@@ -456,22 +466,20 @@ impl ParallelCheckpointDownloaderInner {
         };
 
         loop {
-            let result = client.get_full_checkpoint(sequence_number).await;
-            let Ok(checkpoint) = result else {
-                let err = result.unwrap_err();
-                handle_checkpoint_error(&err, sequence_number);
+            match client.get_full_checkpoint(sequence_number).await {
+                Ok(checkpoint) => return CheckpointEntry::new(sequence_number, Ok(checkpoint)),
+                Err(err) => {
+                    handle_checkpoint_error(&err, sequence_number);
+                    if let Some(delay) = backoff.next() {
+                        tokio::time::sleep(delay).await;
+                        continue;
+                    }
 
-                if let Some(delay) = backoff.next() {
-                    tokio::time::sleep(delay).await;
-                    continue;
+                    // Note that we only return error in test mode.
+                    assert!(cfg!(test));
+                    return CheckpointEntry::new(sequence_number, Err(err.into()));
                 }
-
-                // Note that we only return error in test mode.
-                assert!(cfg!(test));
-                return CheckpointEntry::new(sequence_number, Err(err.into()));
-            };
-
-            return CheckpointEntry::new(sequence_number, Ok(checkpoint));
+            }
         }
     }
 }

crates/checkpoint-downloader/src/downloader.rs

@@ -5,14 +5,13 @@
 //! This crate provides functionality for storing and retrieving typed data in RocksDB,
 //! with support for column families, batch operations, and metrics.
 
-#![warn(
-    future_incompatible,
-    nonstandard_style,
-    rust_2018_idioms,
-    rust_2021_compatibility,
-    unused,
-    missing_docs
+// TODO(WAL-869): Remove this attribute and fix corresponding warnings.
+#![allow(
+    clippy::cast_possible_truncation,
+    clippy::cast_possible_wrap,
+    clippy::unwrap_used
 )]
+#![warn(rust_2018_idioms)]
 
 /// Re-export rocksdb so that consumers can use the version of rocksdb via typed-store
 pub use rocksdb;

crates/typed-store/src/lib.rs

@@ -928,8 +928,10 @@ impl<K, V> DBMap<K, V> {
         let mut num_keys = 0;
         let mut key_bytes_total = 0;
         let mut value_bytes_total = 0;
-        let mut key_hist = hdrhistogram::Histogram::<u64>::new_with_max(100000, 2).unwrap();
-        let mut value_hist = hdrhistogram::Histogram::<u64>::new_with_max(100000, 2).unwrap();
+        let mut key_hist = hdrhistogram::Histogram::<u64>::new_with_max(100000, 2)
+            .expect("the function parameters are valid");
+        let mut value_hist = hdrhistogram::Histogram::<u64>::new_with_max(100000, 2)
+            .expect("the function parameters are valid");
         for item in self.safe_iter() {
             let (key, value) = item?;
             num_keys += 1;
@@ -987,7 +989,8 @@ impl<K, V> DBMap<K, V> {
         )
     }
 
-    // Creates a RocksDB read option with specified lower and upper bounds.
+    /// Creates a RocksDB read option with specified lower and upper bounds.
+    ///
     /// Lower bound is inclusive, and upper bound is exclusive.
     fn create_read_options_with_bounds(
         &self,

crates/typed-store/src/rocks.rs

@@ -7,9 +7,6 @@ edition.workspace = true
 license.workspace = true
 
 [features]
-default = ["rs2"]
-raptorq = []
-rs2 = []
 sui-types = ["dep:sui-types"]
 test-utils = ["walrus-test-utils"]
 
@@ -21,7 +18,6 @@ fastcrypto.workspace = true
 hex.workspace = true
 p256 = { workspace = true, features = ["pem", "pkcs8"] }
 rand.workspace = true
-raptorq.workspace = true
 reed-solomon-simd.workspace = true
 serde.workspace = true
 serde_with.workspace = true

crates/walrus-core/Cargo.toml

@@ -1,21 +1,21 @@
 // Copyright (c) Walrus Foundation
 // SPDX-License-Identifier: Apache-2.0
 
+// Allowing `unwrap`s in benchmarks.
+#![allow(clippy::unwrap_used)]
+
 //! Benchmarks for the basic encoding and decoding.
 
 use core::time::Duration;
 
 use criterion::{AxisScale, BatchSize, BenchmarkId, Criterion, PlotConfiguration};
 use fastcrypto::hash::Blake2b256;
-use raptorq::SourceBlockEncodingPlan;
 use walrus_core::{
-    encoding::{Decoder as _, DecodingSymbol, Primary, RaptorQDecoder, RaptorQEncoder},
+    encoding::{Decoder as _, DecodingSymbol, Primary, ReedSolomonDecoder, ReedSolomonEncoder},
     merkle::MerkleTree,
 };
 use walrus_test_utils::{random_data, random_subset};
 
-// TODO (WAL-610): Support both encoding types.
-
 const N_SHARDS: u16 = 1000;
 // Likely values for the number of source symbols for the primary and secondary encoding, which are
 // consistent with BFT.
@@ -31,8 +31,6 @@ fn basic_encoding(c: &mut Criterion) {
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
 
     for symbol_count in SYMBOL_COUNTS {
-        let encoding_plan = SourceBlockEncodingPlan::generate(symbol_count);
-
         for symbol_size in SYMBOL_SIZES {
             let data_length = usize::from(symbol_size) * usize::from(symbol_count);
             let data = random_data(data_length);
@@ -48,14 +46,13 @@ fn basic_encoding(c: &mut Criterion) {
                 &(symbol_count, data),
                 |b, (symbol_count, data)| {
                     b.iter(|| {
-                        let encoder = RaptorQEncoder::new(
+                        let mut encoder = ReedSolomonEncoder::new(
                             data,
                             (*symbol_count).try_into().unwrap(),
                             N_SHARDS.try_into().unwrap(),
-                            &encoding_plan,
                         )
                         .unwrap();
-                        let _encoded_symbols = encoder.encode_all().collect::<Vec<_>>();
+                        let _encoded_symbols = encoder.encode_all();
                     });
                 },
             );
@@ -70,25 +67,24 @@ fn basic_decoding(c: &mut Criterion) {
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
 
     for symbol_count in SYMBOL_COUNTS {
-        let encoding_plan = SourceBlockEncodingPlan::generate(symbol_count);
         for symbol_size in SYMBOL_SIZES {
             let data_length = usize::from(symbol_size) * usize::from(symbol_count);
             let data = random_data(data_length);
             group.throughput(criterion::Throughput::Bytes(
                 u64::try_from(data_length).unwrap(),
             ));
-            let encoder = RaptorQEncoder::new(
+            let mut encoder = ReedSolomonEncoder::new(
                 &data,
                 symbol_count.try_into().unwrap(),
                 N_SHARDS.try_into().unwrap(),
-                &encoding_plan,
             )
             .unwrap();
             let symbols: Vec<_> = random_subset(
                 encoder
                     .encode_all()
+                    .into_iter()
                     .enumerate()
-                    .map(|(i, s)| DecodingSymbol::<Primary>::new(i as u16, s)),
+                    .map(|(i, s)| DecodingSymbol::<Primary>::new(i.try_into().unwrap(), s)),
                 usize::from(symbol_count) + 1,
             )
             .collect();
@@ -102,7 +98,7 @@ fn basic_decoding(c: &mut Criterion) {
                     b.iter_batched(
                         || symbols.clone(),
                         |symbols| {
-                            let mut decoder = RaptorQDecoder::new(
+                            let mut decoder = ReedSolomonDecoder::new(
                                 (*symbol_count).try_into().unwrap(),
                                 N_SHARDS.try_into().unwrap(),
                                 (*symbol_size).try_into().unwrap(),
@@ -190,18 +186,16 @@ fn merkle_tree(c: &mut Criterion) {
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
 
     for symbol_count in SYMBOL_COUNTS {
-        let encoding_plan = SourceBlockEncodingPlan::generate(symbol_count);
-
         for symbol_size in SYMBOL_SIZES {
             let data_length = usize::from(symbol_size) * usize::from(symbol_count);
             let data = random_data(data_length);
-            let encoder = RaptorQEncoder::new(
+            let encoded_symbols = ReedSolomonEncoder::new(
                 &data,
                 symbol_count.try_into().unwrap(),
                 N_SHARDS.try_into().unwrap(),
-                &encoding_plan,
             )
-            .unwrap();
+            .unwrap()
+            .encode_all();
 
             group.throughput(criterion::Throughput::Bytes(
                 u64::try_from(data_length).unwrap(),
@@ -212,10 +206,9 @@ fn merkle_tree(c: &mut Criterion) {
                     "symbol_count={},symbol_size={}",
                     symbol_count, symbol_size
                 )),
-                &encoder,
-                |b, encoder| {
+                &encoded_symbols,
+                |b, encoded_symbols| {
                     b.iter(|| {
-                        let encoded_symbols = encoder.encode_all().collect::<Vec<_>>();
                         let _tree = MerkleTree::<Blake2b256>::build(encoded_symbols);
                     });
                 },

crates/walrus-core/benches/basic_encoding.rs

@@ -1,16 +1,17 @@
 // Copyright (c) Walrus Foundation
 // SPDX-License-Identifier: Apache-2.0
 
+// Allowing `unwrap`s in benchmarks.
+#![allow(clippy::unwrap_used)]
+
 //! Benchmarks for the blob encoding and decoding with and without authentication.
 
 use core::{num::NonZeroU16, time::Duration};
 
 use criterion::{AxisScale, BatchSize, BenchmarkId, Criterion, PlotConfiguration};
-use walrus_core::encoding::{EncodingConfigTrait as _, Primary, RaptorQEncodingConfig};
+use walrus_core::encoding::{EncodingConfigTrait as _, Primary, ReedSolomonEncodingConfig};
 use walrus_test_utils::{random_data, random_subset};
 
-// TODO (WAL-610): Support both encoding types.
-
 const N_SHARDS: u16 = 1000;
 
 // The maximum symbol size is `u16::MAX`, which means a maximum blob size of ~13 GiB.
@@ -24,8 +25,8 @@ const BLOB_SIZES: [(u64, &str); 6] = [
     (1 << 30, "1GiB"),
 ];
 
-fn encoding_config() -> RaptorQEncodingConfig {
-    RaptorQEncodingConfig::new(NonZeroU16::new(N_SHARDS).unwrap())
+fn encoding_config() -> ReedSolomonEncodingConfig {
+    ReedSolomonEncodingConfig::new(NonZeroU16::new(N_SHARDS).unwrap())
 }
 
 fn blob_encoding(c: &mut Criterion) {

crates/walrus-core/benches/blob_encoding.rs

@@ -8,6 +8,8 @@
 //!
 //! Additionally provides the [`by_axis::flat_map`][`flat_map`] macro that can be used to apply
 //! the same operation to the value stored in the primary or secondary variants of [`ByAxis`].
+
+#[cfg(feature = "utoipa")]
 use alloc::{string::String, vec::Vec};
 use core::fmt::{self, Display};