feat: support non consecutive blocks ryhope

inspect/src/index.rs

@@ -14,7 +14,7 @@ use crate::repl::PayloadFormatter;
 pub(crate) type IndexDb = MerkleTreeKvDb<
     BlockTree,
     IndexNode<BlockPrimaryIndex>,
-    PgsqlStorage<BlockTree, IndexNode<BlockPrimaryIndex>>,
+    PgsqlStorage<BlockTree, IndexNode<BlockPrimaryIndex>, false>,
 >;
 
 struct IndexPayloadFormatterDisplay {

inspect/src/main.rs

@@ -5,7 +5,7 @@ use repl::Repl;
 use rows::{RowDb, RowPayloadFormatter};
 use ryhope::{
     storage::pgsql::{SqlServerConnection, SqlStorageSettings, ToFromBytea},
-    Epoch, InitSettings,
+    InitSettings, UserEpoch,
 };
 use serde::Serialize;
 
@@ -26,7 +26,7 @@ struct Args {
 
     #[arg(short = 'E', long = "at")]
     /// If set, try to view the tree at this epoch
-    epoch: Option<Epoch>,
+    epoch: Option<UserEpoch>,
 
     #[command(subcommand)]
     /// The type of tree to load from the database
@@ -77,6 +77,8 @@ async fn main() -> Result<()> {
                 SqlStorageSettings {
                     source: SqlServerConnection::NewConnection(args.db_uri.clone()),
                     table: args.db_table,
+                    external_mapper: None, // not necessary even if there is an external epoch mapper,
+                                           // since we are initializing the tree with `InitSettings::MustExist`
                 },
             )
             .await?;
@@ -91,7 +93,7 @@ async fn main() -> Result<()> {
 
             let mut repl = Repl::new(tree_db, payload_fmt).await?;
             if let Some(epoch) = args.epoch {
-                repl.set_epoch(epoch)?;
+                repl.set_epoch(epoch).await?;
             }
             repl.run().await
         }
@@ -101,6 +103,7 @@ async fn main() -> Result<()> {
                 SqlStorageSettings {
                     source: SqlServerConnection::NewConnection(args.db_uri.clone()),
                     table: args.db_table,
+                    external_mapper: None,
                 },
             )
             .await?;
@@ -109,7 +112,7 @@ async fn main() -> Result<()> {
 
             let mut repl = Repl::new(tree_db, payload_fmt).await?;
             if let Some(epoch) = args.epoch {
-                repl.set_epoch(epoch)?;
+                repl.set_epoch(epoch).await?;
             }
             repl.run().await
         }

inspect/src/repl.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, bail};
+use anyhow::{anyhow, bail, Result};
 use colored::Colorize;
 use dialoguer::{console, theme::ColorfulTheme, FuzzySelect, Input};
 use itertools::Itertools;
@@ -8,7 +8,7 @@ use ryhope::{
         TreeStorage,
     },
     tree::{MutableTree, PrintableTree, TreeTopology},
-    Epoch, MerkleTreeKvDb, NodePayload,
+    MerkleTreeKvDb, NodePayload, UserEpoch,
 };
 use std::io::Write;
 use tabled::{builder::Builder, settings::Style};
@@ -57,7 +57,7 @@ pub(crate) struct Repl<
     F: PayloadFormatter<V>,
 > {
     current_key: T::Key,
-    current_epoch: Epoch,
+    current_epoch: UserEpoch,
     db: MerkleTreeKvDb<T, V, S>,
     tty: console::Term,
     payload_fmt: F,
@@ -77,7 +77,7 @@ impl<
 {
     pub async fn new(db: MerkleTreeKvDb<T, V, S>, payload_fmt: F) -> anyhow::Result<Self> {
         let current_key = db.root().await?.ok_or(anyhow!("tree is empty"))?;
-        let current_epoch = db.current_epoch();
+        let current_epoch = db.current_epoch().await?;
 
         Ok(Self {
             current_key,
@@ -105,19 +105,19 @@ impl<
         .unwrap();
     }
 
-    pub fn set_epoch(&mut self, epoch: Epoch) -> anyhow::Result<()> {
-        if epoch < self.db.initial_epoch() {
+    pub async fn set_epoch(&mut self, epoch: UserEpoch) -> Result<()> {
+        if epoch < self.db.initial_epoch().await {
             bail!(
                 "epoch `{}` is older than initial epoch `{}`",
                 epoch,
-                self.db.initial_epoch()
+                self.db.initial_epoch().await
             );
         }
-        if epoch > self.db.current_epoch() {
+        if epoch > self.db.current_epoch().await? {
             bail!(
                 "epoch `{}` is newer than latest epoch `{}`",
                 epoch,
-                self.db.current_epoch()
+                self.db.current_epoch().await?
             );
         }
 
@@ -147,9 +147,9 @@ impl<
 
     async fn travel(&mut self) -> anyhow::Result<()> {
         loop {
-            let epoch: Epoch = Input::new().with_prompt("target epoch:").interact_text()?;
+            let epoch: UserEpoch = Input::new().with_prompt("target epoch:").interact_text()?;
 
-            self.set_epoch(epoch)?;
+            self.set_epoch(epoch).await?;
         }
     }
 

inspect/src/rows.rs

@@ -17,7 +17,7 @@ use crate::repl::PayloadFormatter;
 pub(crate) type RowDb = MerkleTreeKvDb<
     RowTree,
     RowPayload<BlockPrimaryIndex>,
-    PgsqlStorage<RowTree, RowPayload<BlockPrimaryIndex>>,
+    PgsqlStorage<RowTree, RowPayload<BlockPrimaryIndex>, true>,
 >;
 
 struct RowPayloadFormatterDisplay {

mp2-test/src/cells_tree.rs

@@ -22,15 +22,15 @@ use plonky2::{
 use rand::{thread_rng, Rng};
 use ryhope::{
     storage::{memory::InMemory, updatetree::UpdateTree, EpochKvStorage, TreeTransactionalStorage},
-    tree::{sbbst, TreeTopology},
+    tree::{sbbst::IncrementalTree, TreeTopology},
     InitSettings, MerkleTreeKvDb, NodePayload,
 };
 use serde::{Deserialize, Serialize};
 use std::iter;
 
-pub type CellTree = sbbst::Tree;
+pub type CellTree = IncrementalTree;
 pub type CellTreeKey = <CellTree as TreeTopology>::Key;
-type CellStorage = InMemory<CellTree, TestCell>;
+type CellStorage = InMemory<CellTree, TestCell, false>;
 pub type MerkleCellTree = MerkleTreeKvDb<CellTree, TestCell, CellStorage>;
 
 /// Test node of the cells tree
@@ -116,7 +116,7 @@ impl NodePayload for TestCell {
 pub async fn build_cell_tree(
     row: Vec<TestCell>,
 ) -> Result<(MerkleCellTree, UpdateTree<<CellTree as TreeTopology>::Key>)> {
-    let mut cell_tree = MerkleCellTree::new(InitSettings::Reset(sbbst::Tree::empty()), ())
+    let mut cell_tree = MerkleCellTree::new(InitSettings::Reset(IncrementalTree::empty()), ())
         .await
         .unwrap();
     let update_tree = cell_tree

mp2-v1/src/indexing/block.rs

@@ -1,14 +1,60 @@
 //! Module to handle the block number as a primary index
-use ryhope::tree::{sbbst, TreeTopology};
-
-/// The index tree when the primary index is the block number of a blockchain is a sbbst since it
-/// is a highly optimized tree for monotonically increasing index. It produces very little
-/// tree-manipulating operations on update, and therefore, requires the least amount of reproving
-/// when adding a new index.
-/// NOTE: when dealing with another type of index, i.e. a general index such as what can happen on
-/// a result table, then this tree does not work anymore.
-pub type BlockTree = sbbst::Tree;
+use anyhow::anyhow;
+use ryhope::{
+    storage::{pgsql::PgsqlStorage, RoEpochKvStorage},
+    tree::{sbbst, TreeTopology},
+    MerkleTreeKvDb,
+};
+
+use crate::query::planner::TreeFetcher;
+
+use super::index::IndexNode;
+
+/// The index tree when the primary index is an epoch in a time-series DB, like the block number for a blockchain.
+/// It is a sbbst since it is a highly optimized tree for monotonically increasing index.
+/// It produces very little tree-manipulating operations on update, and therefore, requires the least amount
+/// of reproving when adding a new index.
+/// NOTE: it is still required that monotonically increasing indexes are inserted in the tree,
+/// i.e. a general index such as what can happen on a result table wouldn't work with this tree.
+pub type BlockTree = sbbst::EpochTree;
 /// The key used to refer to a table where the block number is the primary index.
 pub type BlockTreeKey = <BlockTree as TreeTopology>::Key;
 /// Just an alias that give more meaning depending on the context
 pub type BlockPrimaryIndex = BlockTreeKey;
+
+pub type IndexStorage = PgsqlStorage<BlockTree, IndexNode<BlockPrimaryIndex>, false>;
+pub type MerkleIndexTree = MerkleTreeKvDb<BlockTree, IndexNode<BlockPrimaryIndex>, IndexStorage>;
+
+/// Get the previous epoch of `epoch` in `tree`
+pub async fn get_previous_epoch(
+    tree: &MerkleIndexTree,
+    epoch: BlockPrimaryIndex,
+) -> anyhow::Result<Option<BlockPrimaryIndex>> {
+    let current_epoch = tree.current_epoch().await?;
+    let epoch_ctx = tree
+        .node_context(&epoch)
+        .await?
+        .ok_or(anyhow!("epoch {epoch} not found in the tree"))?;
+
+    Ok(tree
+        .get_predecessor(&epoch_ctx, current_epoch)
+        .await
+        .map(|(ctx, _)| ctx.node_id))
+}
+
+/// Get the next epoch of `epoch` in `tree`
+pub async fn get_next_epoch(
+    tree: &MerkleIndexTree,
+    epoch: BlockPrimaryIndex,
+) -> anyhow::Result<Option<BlockPrimaryIndex>> {
+    let current_epoch = tree.current_epoch().await?;
+    let epoch_ctx = tree
+        .node_context(&epoch)
+        .await?
+        .ok_or(anyhow!("epoch {epoch} not found in the tree"))?;
+
+    Ok(tree
+        .get_successor(&epoch_ctx, current_epoch)
+        .await
+        .map(|(ctx, _)| ctx.node_id))
+}

mp2-v1/src/indexing/cell.rs

@@ -25,13 +25,13 @@ use super::ColumnID;
 
 /// By default the cells tree is a sbbst tree since it is fixed for a given table and this is the
 /// simplest/fastest tree.
-pub type CellTree = sbbst::Tree;
+pub type CellTree = sbbst::IncrementalTree;
 /// The key used to refer to a cell in the tree
 pub type CellTreeKey = <CellTree as TreeTopology>::Key;
 /// The storage of cell tree is "in memory" since it is never really saved on disk. Rather, it is
 /// always reconstructed on the fly given it is very small. Moreover, storing it on disk would
 /// require as many sql tables as there would be rows, making this solution highly unpracticable.
-pub type CellStorage<PrimaryIndex> = InMemory<CellTree, MerkleCell<PrimaryIndex>>;
+pub type CellStorage<PrimaryIndex> = InMemory<CellTree, MerkleCell<PrimaryIndex>, false>;
 /// The cells tree is a Merkle tree with cryptographically secure hash function committing to its
 /// content.
 pub type MerkleCellTree<PrimaryIndex> =
@@ -50,7 +50,7 @@ pub async fn new_tree<
         + Serialize
         + for<'a> Deserialize<'a>,
 >() -> MerkleCellTree<PrimaryIndex> {
-    MerkleCellTree::new(InitSettings::Reset(sbbst::Tree::empty()), ())
+    MerkleCellTree::new(InitSettings::Reset(sbbst::IncrementalTree::empty()), ())
         .await
         .unwrap()
 }

mp2-v1/src/indexing/mod.rs

@@ -1,6 +1,15 @@
+use anyhow::Result;
+
 use crate::indexing::{index::IndexNode, row::RowPayload};
 use alloy::primitives::U256;
+use block::MerkleIndexTree;
 use mp2_common::types::HashOutput;
+use row::MerkleRowTree;
+use ryhope::{
+    storage::pgsql::{SqlServerConnection, SqlStorageSettings},
+    tree::scapegoat,
+    InitSettings, UserEpoch,
+};
 
 pub mod block;
 pub mod cell;
@@ -9,6 +18,77 @@ pub mod row;
 
 pub type ColumnID = u64;
 
+/// Build `MerkleIndexTree` and `MerkleRowTree` trees from tables
+/// `index_table_name` and `row_table_name` in the DB with URL `db_url`.
+pub async fn load_trees(
+    db_url: &str,
+    index_table_name: String,
+    row_table_name: String,
+) -> Result<(MerkleIndexTree, MerkleRowTree)> {
+    let index_tree = MerkleIndexTree::new(
+        InitSettings::MustExist,
+        SqlStorageSettings {
+            source: SqlServerConnection::NewConnection(db_url.to_string()),
+            table: index_table_name.clone(),
+            external_mapper: None,
+        },
+    )
+    .await?;
+    let row_tree = MerkleRowTree::new(
+        InitSettings::MustExist,
+        SqlStorageSettings {
+            table: row_table_name,
+            source: SqlServerConnection::NewConnection(db_url.to_string()),
+            external_mapper: Some(index_table_name),
+        },
+    )
+    .await?;
+
+    Ok((index_tree, row_tree))
+}
+
+/// Build `MerkleIndexTree` and `MerkleRowTree` trees starting from
+/// `genesis_block`. The tables employed in the DB with URL `db_url`
+/// to store the trees are `index_table_name` and `row_table_name`,
+/// respectively. The following additional parameters are required:
+/// - `alpha`: Parameter of the Scapegoat tree employed for the `MerkleRowTree`
+/// - `reset_if_exist`: if true, an existing tree would be deleted
+pub async fn build_trees(
+    db_url: &str,
+    index_table_name: String,
+    row_table_name: String,
+    genesis_block: UserEpoch,
+    alpha: scapegoat::Alpha,
+    reset_if_exist: bool,
+) -> Result<(MerkleIndexTree, MerkleRowTree)> {
+    let db_settings_index = SqlStorageSettings {
+        source: SqlServerConnection::NewConnection(db_url.to_string()),
+        table: index_table_name.clone(),
+        external_mapper: None,
+    };
+    let db_settings_row = SqlStorageSettings {
+        source: SqlServerConnection::NewConnection(db_url.to_string()),
+        table: row_table_name,
+        external_mapper: Some(index_table_name),
+    };
+
+    let index_tree = ryhope::new_index_tree(
+        genesis_block as UserEpoch,
+        db_settings_index,
+        reset_if_exist,
+    )
+    .await?;
+    let row_tree = ryhope::new_row_tree(
+        genesis_block as UserEpoch,
+        alpha,
+        db_settings_row,
+        reset_if_exist,
+    )
+    .await?;
+
+    Ok((index_tree, row_tree))
+}
+
 // NOTE this might be good to have on public API ?
 // cc/ @andrus
 pub trait LagrangeNode {