feat(checker): Firewood checker framework

firewood/src/v2/api.rs

@@ -8,7 +8,7 @@ use async_trait::async_trait;
 use futures::Stream;
 use std::fmt::Debug;
 use std::sync::Arc;
-use storage::{FileIoError, TrieHash};
+use storage::{CheckerError, FileIoError, TrieHash};
 
 /// A `KeyType` is something that can be xcast to a u8 reference,
 /// and can be sent and shared across threads. References with
@@ -150,6 +150,10 @@ pub enum Error {
     /// Revision not found
     #[error("revision not found")]
     RevisionNotFound,
+
+    /// Checker error
+    #[error("checker error")]
+    CheckerError(#[from] CheckerError),
 }
 
 impl From<RevisionManagerError> for Error {

fwdctl/Cargo.toml

@@ -6,19 +6,22 @@ rust-version = "1.85.0"
 
 [dependencies]
 firewood = { version = "0.0.5", path = "../firewood" }
+storage = { version = "0.0.5", path = "../storage" }
 clap = { version = "4.5.0", features = ["cargo", "derive"] }
 env_logger = "0.11.2"
 log = "0.4.20"
 tokio = { version = "1.36.0", features = ["full"] }
 futures-util = "0.3.30"
 hex = "0.4.3"
 csv = "1.3.1"
+nonzero_ext = "0.3.0"
 
 [dev-dependencies]
 anyhow = "1.0.79"
 assert_cmd = "2.0.13"
 predicates = "3.1.0"
 serial_test = "3.0.0"
+rand = "0.9.1"
 
 [lints.rust]
 unsafe_code = "deny"

fwdctl/src/check.rs

@@ -0,0 +1,41 @@
+// Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE.md for licensing terms.
+
+use std::path::Path;
+use std::sync::Arc;
+
+use clap::Args;
+use firewood::v2::api;
+use nonzero_ext::nonzero;
+use storage::{CacheReadStrategy, FileBacked, NodeStore};
+
+// TODO: (optionally) add a fix option
+#[derive(Args)]
+pub struct Options {
+    /// The database path (if no path is provided, return an error). Defaults to firewood.
+    #[arg(
+        long,
+        required = false,
+        value_name = "DB_NAME",
+        default_value_t = String::from("firewood"),
+        help = "Name of the database"
+    )]
+    pub db: String,
+}
+
+pub(super) async fn run(opts: &Options) -> Result<(), api::Error> {
+    let db_path = Path::new(&opts.db);
+    let node_cache_size = nonzero!(1usize);
+    let free_list_cache_size = nonzero!(1usize);
+
+    let storage = Arc::new(FileBacked::new(
+        db_path.to_path_buf(),
+        node_cache_size,
+        free_list_cache_size,
+        false,
+        CacheReadStrategy::WritesOnly, // cache none since this is a read-only workload - we don't want to cache any nodes since we won't read a node more than once
+    )?);
+
+    let node_store = NodeStore::open(storage)?;
+    node_store.check().await.map_err(Into::into)
+}

fwdctl/src/main.rs

@@ -6,6 +6,7 @@
 use clap::{Parser, Subcommand};
 use firewood::v2::api;
 
+pub mod check;
 pub mod create;
 pub mod delete;
 pub mod dump;
@@ -49,6 +50,8 @@ enum Commands {
     Dump(dump::Options),
     /// Produce a dot file of the database
     Graph(graph::Options),
+    /// Runs the checker on the database
+    Check(check::Options),
 }
 
 #[tokio::main]
@@ -68,5 +71,6 @@ async fn main() -> Result<(), api::Error> {
         Commands::Root(opts) => root::run(opts).await,
         Commands::Dump(opts) => dump::run(opts).await,
         Commands::Graph(opts) => graph::run(opts).await,
+        Commands::Check(opts) => check::run(opts).await,
     }
 }

fwdctl/tests/cli.rs

@@ -45,7 +45,7 @@ fn fwdctl_creates_database() -> Result<()> {
         .assert()
         .success();
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -71,7 +71,7 @@ fn fwdctl_insert_successful() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("year"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -106,7 +106,7 @@ fn fwdctl_get_successful() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("2023"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -140,7 +140,7 @@ fn fwdctl_delete_successful() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("key year deleted successfully"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -173,7 +173,7 @@ fn fwdctl_root_hash() -> Result<()> {
         .success()
         .stdout(predicate::str::is_empty().not());
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -205,8 +205,7 @@ fn fwdctl_dump() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("2023"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
+    fwdctl_delete_db()?;
     Ok(())
 }
 
@@ -313,7 +312,7 @@ fn fwdctl_dump_with_start_stop_and_max() -> Result<()> {
             "Next key is c, resume with \"--start-key=c\"",
         ));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -388,7 +387,7 @@ fn fwdctl_dump_with_csv_and_json() -> Result<()> {
     );
     fs::remove_file("dump.json").expect("Should remove dump.json file");
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -454,7 +453,7 @@ fn fwdctl_dump_with_file_name() -> Result<()> {
     assert_eq!(contents, "{\n  \"a\": \"1\"\n}\n");
     fs::remove_file("test.json").expect("Should remove test.json file");
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
 
     Ok(())
 }
@@ -534,7 +533,76 @@ fn fwdctl_dump_with_hex() -> Result<()> {
         .stdout(predicate::str::contains("--start-key=c"))
         .stdout(predicate::str::contains("--start-key-hex=63"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()?;
+
+    Ok(())
+}
+
+#[test]
+#[serial]
+fn fwdctl_check_empty_db() -> Result<()> {
+    Command::cargo_bin(PRG)?
+        .arg("create")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
+
+    Command::cargo_bin(PRG)?
+        .arg("check")
+        .arg("--db")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
+
+    fwdctl_delete_db()?;
+
+    Ok(())
+}
+
+#[test]
+#[serial]
+fn fwdctl_check_db_with_data() -> Result<()> {
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng, rng};
+
+    let seed = std::env::var("FIREWOOD_TEST_SEED")
+        .ok()
+        .map_or_else(
+            || None,
+            |s| Some(str::parse(&s).expect("couldn't parse FIREWOOD_TEST_SEED; must be a u64")),
+        )
+        .unwrap_or_else(|| rng().random());
+
+    eprintln!("Seed {seed}: to rerun with this data, export FIREWOOD_TEST_SEED={seed}");
+    let mut rng = StdRng::seed_from_u64(seed);
+
+    Command::cargo_bin(PRG)?
+        .arg("create")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
+
+    // TODO: bulk loading data instead of inserting one by one
+    for _ in 0..4 {
+        let key = format!("key_{}", rng.random::<u64>());
+        let value = format!("value_{}", rng.random::<u64>());
+        Command::cargo_bin(PRG)?
+            .arg("insert")
+            .args([key.clone()])
+            .args([value])
+            .args(["--db"])
+            .args([tmpdb::path()])
+            .assert()
+            .success()
+            .stdout(predicate::str::contains(&key));
+    }
+
+    Command::cargo_bin(PRG)?
+        .arg("check")
+        .arg("--db")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
 
     Ok(())
 }

storage/Cargo.toml

@@ -29,13 +29,17 @@ coarsetime = "0.1.35"
 rlp = { version = "0.6.1", optional = true }
 sha3 = { version = "0.10.8", optional = true }
 bytes = { version = "1.10.1", optional = true }
+thiserror = "2.0.3"
+btree-slab = "0.6.1"
 
 [dev-dependencies]
 rand = "0.9.0"
 test-case = "3.3.1"
 criterion = { version = "0.6.0", features = ["async_tokio", "html_reports"] }
 pprof = { version = "0.15.0", features = ["flamegraph"] }
 tempfile = "3.12.0"
+tokio = { version = "1.36.0", features = ["rt", "sync", "macros", "rt-multi-thread"] }
+nonzero_ext = "0.3.0"
 
 [features]
 logger = ["log"]

storage/src/lib.rs

@@ -11,11 +11,15 @@
 //!
 //! A [NodeStore] is backed by a [ReadableStorage] which is persisted storage.
 
+use std::ops::Range;
+use thiserror::Error;
+
 mod hashednode;
 mod hashers;
 mod linear;
 mod node;
 mod nodestore;
+mod range_set;
 mod trie_hash;
 
 /// Logger module for handling logging functionality
@@ -73,3 +77,45 @@ pub fn empty_trie_hash() -> TrieHash {
         .try_into()
         .expect("empty trie hash is 32 bytes")
 }
+
+/// Errors returned by the checker
+#[derive(Error, Debug)]
+#[non_exhaustive]
+pub enum CheckerError {
+    /// The file size is not valid
+    #[error("Invalid DB size ({db_size}): {description}")]
+    InvalidDBSize {
+        /// The size of the db
+        db_size: u64,
+        /// The description of the error
+        description: String,
+    },
+
+    /// The address is out of bounds
+    #[error("stored area at {start} with size {size} is out of bounds ({bounds:?})")]
+    AreaOutOfBounds {
+        /// Start of the StoredArea
+        start: LinearAddress,
+        /// Size of the StoredArea
+        size: u64,
+        /// Valid range of addresses
+        bounds: Range<LinearAddress>,
+    },
+
+    /// Stored areas intersect
+    #[error(
+        "stored area at {start} with size {size} intersects with other stored areas: {intersection:?}"
+    )]
+    AreaIntersects {
+        /// Start of the StoredArea
+        start: LinearAddress,
+        /// Size of the StoredArea
+        size: u64,
+        /// The intersection
+        intersection: Vec<Range<LinearAddress>>,
+    },
+
+    /// IO error
+    #[error("IO error")]
+    IO(#[from] FileIoError),
+}