feat(checker): Firewood checker framework

firewood/src/merkle.rs

@@ -201,7 +201,7 @@ impl<T: TrieReader> Merkle<T> {
             if let Some(branch) = root.as_branch() {
                 // TODO danlaine: can we avoid indexing?
                 #[expect(clippy::indexing_slicing)]
-                for (i, hash) in branch.children_iter() {
+                for (i, hash) in branch.children_hashes() {
                     child_hashes[i] = Some(hash.clone());
                 }
             }

firewood/src/proof.rs

@@ -122,7 +122,7 @@ impl From<PathIterItem> for ProofNode {
         if let Some(branch) = item.node.as_branch() {
             // TODO danlaine: can we avoid indexing?
             #[expect(clippy::indexing_slicing)]
-            for (i, hash) in branch.children_iter() {
+            for (i, hash) in branch.children_hashes() {
                 child_hashes[i] = Some(hash.clone());
             }
         }

firewood/src/v2/api.rs

@@ -14,7 +14,7 @@ use crate::manager::RevisionManagerError;
 use crate::proof::{Proof, ProofError, ProofNode};
 pub use crate::range_proof::RangeProof;
 use async_trait::async_trait;
-use firewood_storage::{FileIoError, TrieHash};
+use firewood_storage::{CheckerError, FileIoError, TrieHash};
 use futures::Stream;
 use std::fmt::Debug;
 use std::sync::Arc;
@@ -159,6 +159,10 @@ pub enum Error {
     /// Revision not found
     #[error("revision not found")]
     RevisionNotFound,
+
+    /// Checker error
+    #[error("checker error")]
+    CheckerError(#[from] CheckerError),
 }
 
 impl From<RevisionManagerError> for Error {

fwdctl/Cargo.toml

@@ -24,19 +24,22 @@ path = "src/main.rs"
 
 [dependencies]
 firewood = { version = "0.0.6", path = "../firewood" }
+firewood-storage = { version = "0.0.6", path = "../storage" }
 clap = { version = "4.5.0", features = ["cargo", "derive"] }
 env_logger = "0.11.2"
 log = "0.4.20"
 tokio = { version = "1.36.0", features = ["full"] }
 futures-util = "0.3.30"
 hex = "0.4.3"
 csv = "1.3.1"
+nonzero_ext = "0.3.0"
 
 [dev-dependencies]
 anyhow = "1.0.79"
 assert_cmd = "2.0.13"
 predicates = "3.1.0"
 serial_test = "3.0.0"
+rand = "0.9.1"
 
 [lints]
 workspace = true

fwdctl/src/check.rs

@@ -0,0 +1,41 @@
+// Copyright (C) 2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE.md for licensing terms.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use clap::Args;
+use firewood::v2::api;
+use firewood_storage::{CacheReadStrategy, FileBacked, NodeStore};
+use nonzero_ext::nonzero;
+
+// TODO: (optionally) add a fix option
+#[derive(Args)]
+pub struct Options {
+    /// The database path (if no path is provided, return an error). Defaults to firewood.
+    #[arg(
+        long,
+        required = false,
+        value_name = "DB_NAME",
+        default_value_t = String::from("firewood"),
+        help = "Name of the database"
+    )]
+    pub db: String,
+}
+
+#[allow(clippy::unused_async)]
+pub(super) async fn run(opts: &Options) -> Result<(), api::Error> {
+    let db_path = PathBuf::from(&opts.db);
+    let node_cache_size = nonzero!(1usize);
+    let free_list_cache_size = nonzero!(1usize);
+
+    let storage = Arc::new(FileBacked::new(
+        db_path,
+        node_cache_size,
+        free_list_cache_size,
+        false,
+        CacheReadStrategy::WritesOnly, // we scan the database once - no need to cache anything
+    )?);
+
+    NodeStore::open(storage)?.check().map_err(Into::into)
+}

fwdctl/src/main.rs

@@ -6,6 +6,7 @@
 use clap::{Parser, Subcommand};
 use firewood::v2::api;
 
+pub mod check;
 pub mod create;
 pub mod delete;
 pub mod dump;
@@ -49,6 +50,8 @@ enum Commands {
     Dump(dump::Options),
     /// Produce a dot file of the database
     Graph(graph::Options),
+    /// Runs the checker on the database
+    Check(check::Options),
 }
 
 #[tokio::main]
@@ -68,5 +71,6 @@ async fn main() -> Result<(), api::Error> {
         Commands::Root(opts) => root::run(opts).await,
         Commands::Dump(opts) => dump::run(opts).await,
         Commands::Graph(opts) => graph::run(opts).await,
+        Commands::Check(opts) => check::run(opts).await,
     }
 }

fwdctl/tests/cli.rs

@@ -45,9 +45,7 @@ fn fwdctl_creates_database() -> Result<()> {
         .assert()
         .success();
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -71,9 +69,7 @@ fn fwdctl_insert_successful() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("year"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -106,9 +102,7 @@ fn fwdctl_get_successful() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("2023"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -140,9 +134,7 @@ fn fwdctl_delete_successful() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("key year deleted successfully"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -173,9 +165,7 @@ fn fwdctl_root_hash() -> Result<()> {
         .success()
         .stdout(predicate::str::is_empty().not());
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -205,9 +195,7 @@ fn fwdctl_dump() -> Result<()> {
         .success()
         .stdout(predicate::str::contains("2023"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -313,9 +301,7 @@ fn fwdctl_dump_with_start_stop_and_max() -> Result<()> {
             "Next key is c, resume with \"--start-key=c\"",
         ));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -388,9 +374,7 @@ fn fwdctl_dump_with_csv_and_json() -> Result<()> {
     );
     fs::remove_file("dump.json").expect("Should remove dump.json file");
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -454,9 +438,7 @@ fn fwdctl_dump_with_file_name() -> Result<()> {
     assert_eq!(contents, "{\n  \"a\": \"1\"\n}\n");
     fs::remove_file("test.json").expect("Should remove test.json file");
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
-
-    Ok(())
+    fwdctl_delete_db()
 }
 
 #[test]
@@ -534,9 +516,73 @@ fn fwdctl_dump_with_hex() -> Result<()> {
         .stdout(predicate::str::contains("--start-key=c"))
         .stdout(predicate::str::contains("--start-key-hex=63"));
 
-    fwdctl_delete_db().map_err(|e| anyhow!(e))?;
+    fwdctl_delete_db()
+}
 
-    Ok(())
+#[test]
+#[serial]
+fn fwdctl_check_empty_db() -> Result<()> {
+    Command::cargo_bin(PRG)?
+        .arg("create")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
+
+    Command::cargo_bin(PRG)?
+        .arg("check")
+        .arg("--db")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
+
+    fwdctl_delete_db()
+}
+
+#[test]
+#[serial]
+fn fwdctl_check_db_with_data() -> Result<()> {
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng, rng};
+
+    let seed = std::env::var("FIREWOOD_TEST_SEED")
+        .ok()
+        .map_or_else(
+            || None,
+            |s| Some(str::parse(&s).expect("couldn't parse FIREWOOD_TEST_SEED; must be a u64")),
+        )
+        .unwrap_or_else(|| rng().random());
+
+    eprintln!("Seed {seed}: to rerun with this data, export FIREWOOD_TEST_SEED={seed}");
+    let mut rng = StdRng::seed_from_u64(seed);
+
+    Command::cargo_bin(PRG)?
+        .arg("create")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
+
+    // TODO: bulk loading data instead of inserting one by one
+    for _ in 0..4 {
+        let key = format!("key_{}", rng.random::<u64>());
+        let value = format!("value_{}", rng.random::<u64>());
+        Command::cargo_bin(PRG)?
+            .arg("insert")
+            .args([key])
+            .args([value])
+            .args(["--db"])
+            .args([tmpdb::path()])
+            .assert()
+            .success();
+    }
+
+    Command::cargo_bin(PRG)?
+        .arg("check")
+        .arg("--db")
+        .arg(tmpdb::path())
+        .assert()
+        .success();
+
+    fwdctl_delete_db()
 }
 
 // A module to create a temporary database name for use in

storage/Cargo.toml

@@ -38,6 +38,7 @@ coarsetime = "0.1.35"
 rlp = { version = "0.6.1", optional = true }
 sha3 = { version = "0.10.8", optional = true }
 bytes = { version = "1.10.1", optional = true }
+thiserror = "2.0.3"
 semver = "1.0.26"
 
 [dev-dependencies]
@@ -46,6 +47,7 @@ test-case = "3.3.1"
 criterion = { version = "0.6.0", features = ["async_tokio", "html_reports"] }
 pprof = { version = "0.15.0", features = ["flamegraph"] }
 tempfile = "3.12.0"
+nonzero_ext = "0.3.0"
 
 [features]
 logger = ["log"]