From 4255ab177fc6e7d5c1f4be55f62b1a380295b134 Mon Sep 17 00:00:00 2001 From: Nicolas Gailly Date: Thu, 7 Nov 2024 14:27:00 +0100 Subject: [PATCH] Export non existence query logic (#403) --- mp2-v1/Cargo.toml | 5 + mp2-v1/src/lib.rs | 5 +- mp2-v1/src/query/mod.rs | 1 + mp2-v1/src/query/planner.rs | 436 ++++++++++++++++++++++ mp2-v1/tests/common/block_extraction.rs | 9 +- mp2-v1/tests/common/cases/indexing.rs | 9 +- mp2-v1/tests/common/cases/planner.rs | 6 +- mp2-v1/tests/common/cases/query.rs | 350 +++-------------- mp2-v1/tests/common/cases/table_source.rs | 8 +- mp2-v1/tests/common/context.rs | 6 - mp2-v1/tests/common/length_extraction.rs | 1 + mp2-v1/tests/common/proof_storage.rs | 7 +- mp2-v1/tests/common/table.rs | 29 +- mp2-v1/tests/integrated_tests.rs | 5 +- 14 files changed, 511 insertions(+), 366 deletions(-) create mode 100644 mp2-v1/src/query/mod.rs create mode 100644 mp2-v1/src/query/planner.rs diff --git a/mp2-v1/Cargo.toml b/mp2-v1/Cargo.toml index 74ce2898b..eca166653 100644 --- a/mp2-v1/Cargo.toml +++ b/mp2-v1/Cargo.toml @@ -22,10 +22,15 @@ serde.workspace = true mp2_common = { path = "../mp2-common" } recursion_framework = { path = "../recursion-framework" } ryhope = { path = "../ryhope" } +parsil = { path = "../parsil" } verifiable-db = { path = "../verifiable-db" } derive_more = "0.99.18" hex.workspace = true serde_json.workspace = true +bb8 = "0.8.5" +bb8-postgres = "0.8.1" +tokio-postgres = "0.7.12" +futures = "0.3.30" [dev-dependencies] alloy.workspace = true diff --git a/mp2-v1/src/lib.rs b/mp2-v1/src/lib.rs index eaf85ee65..99dc7dc03 100644 --- a/mp2-v1/src/lib.rs +++ b/mp2-v1/src/lib.rs @@ -1,10 +1,12 @@ //! Circuits for v1 of Lagrange Proof Network (LPN) - +#![allow(incomplete_features)] // Add this to allow generic const expressions, e.g. `PAD_LEN(NODE_LEN)`. #![feature(generic_const_exprs)] // Add this so we don't need to always specify const generic in generic // parameters (i.e. use "_") #![feature(generic_arg_infer)] +// stylistic feature +#![feature(async_closure)] use mp2_common::mpt_sequential::PAD_LEN; pub const MAX_BRANCH_NODE_LEN: usize = 532; @@ -21,4 +23,5 @@ pub mod contract_extraction; pub mod final_extraction; pub mod indexing; pub mod length_extraction; +pub mod query; pub mod values_extraction; diff --git a/mp2-v1/src/query/mod.rs b/mp2-v1/src/query/mod.rs new file mode 100644 index 000000000..5e480858b --- /dev/null +++ b/mp2-v1/src/query/mod.rs @@ -0,0 +1 @@ +pub mod planner; diff --git a/mp2-v1/src/query/planner.rs b/mp2-v1/src/query/planner.rs new file mode 100644 index 000000000..c62c4889e --- /dev/null +++ b/mp2-v1/src/query/planner.rs @@ -0,0 +1,436 @@ +use alloy::primitives::U256; +use anyhow::Context; +use bb8::Pool; +use bb8_postgres::PostgresConnectionManager; +use futures::stream::TryStreamExt; +use itertools::Itertools; +use mp2_common::types::HashOutput; +use parsil::{bracketer::bracket_secondary_index, symbols::ContextProvider, ParsilSettings}; +use ryhope::{ + storage::{ + pgsql::{PgsqlStorage, ToFromBytea}, + updatetree::UpdateTree, + FromSettings, PayloadStorage, TransactionalStorage, TreeStorage, + }, + tree::{MutableTree, NodeContext, TreeTopology}, + Epoch, MerkleTreeKvDb, NodePayload, +}; +use std::fmt::Debug; +use tokio_postgres::{row::Row as PsqlRow, types::ToSql, NoTls}; +use verifiable_db::query::aggregation::{NodeInfo, QueryBounds}; + +use crate::indexing::{ + block::BlockPrimaryIndex, + row::{RowPayload, RowTree, RowTreeKey}, + LagrangeNode, +}; + +/// There is only the PSQL storage fully supported for the non existence case since one needs to +/// executor particular requests on the DB in this case. +pub type DBRowStorage = PgsqlStorage>; +/// The type of connection to psql backend +pub type DBPool = Pool>; + +pub struct NonExistenceInfo { + pub proving_plan: UpdateTree, +} + +/// Returns the proving plan to prove the non existence of node of the query in this row tree at +/// the epoch primary. It also returns the leaf node chosen. +/// +/// The row tree is given and specialized to psql storage since that is the only official storage +/// supported. +/// The `table_name` must be the one given to parsil settings, it is the human friendly table +/// name, i.e. the vTable name. +/// The pool is to issue specific query +/// Primary is indicating the primary index over which this row tree is looked at. +/// Settings are the parsil settings corresponding to the current SQL and current table looked at. +/// Pis contain the bounds and placeholders values. +/// TODO: we should extend ryhope to offer this API directly on the tree since it's very related. +pub async fn find_row_node_for_non_existence( + row_tree: &MerkleTreeKvDb, DBRowStorage>, + table_name: String, + pool: &DBPool, + primary: BlockPrimaryIndex, + settings: &ParsilSettings, + bounds: &QueryBounds, +) -> anyhow::Result<(RowTreeKey, UpdateTree)> +where + C: ContextProvider, +{ + let (query_for_min, query_for_max) = + bracket_secondary_index(&table_name, settings, primary as Epoch, &bounds); + + // try first with lower node than secondary min query bound + let to_be_proven_node = + match find_node_for_proof(pool, row_tree, query_for_min, primary, true).await? { + Some(node) => node, + None => find_node_for_proof(pool, row_tree, query_for_max, primary, false) + .await? + .expect("No valid node found to prove non-existence, something is wrong"), + }; + + let path = row_tree + // since the epoch starts at genesis we can directly give the block number ! + .lineage_at(&to_be_proven_node, primary as Epoch) + .await + .expect("node doesn't have a lineage?") + .into_full_path() + .collect_vec(); + let proving_tree = UpdateTree::from_paths([path], primary as Epoch); + Ok((to_be_proven_node.clone(), proving_tree)) +} + +// this method returns the `NodeContext` of the successor of the node provided as input, +// if the successor exists in the row tree and it stores the same value of the input node (i.e., `value`); +// returns `None` otherwise, as it means that the input node can be used to prove non-existence +async fn get_successor_node_with_same_value( + row_tree: &MerkleTreeKvDb, DBRowStorage>, + node_ctx: &NodeContext, + value: U256, + primary: BlockPrimaryIndex, +) -> Option> { + if node_ctx.right.is_some() { + let (right_child_ctx, payload) = row_tree + .fetch_with_context_at(node_ctx.right.as_ref().unwrap(), primary as Epoch) + .await; + // the value of the successor in this case is `payload.min`, since the successor is the + // minimum of the subtree rooted in the right child + if payload.min() != value { + // the value of successor is different from `value`, so we don't return the + // successor node + return None; + } + // find successor in the subtree rooted in the right child: it is + // the leftmost node in such a subtree + let mut successor_ctx = right_child_ctx; + while successor_ctx.left.is_some() { + successor_ctx = row_tree + .node_context_at(successor_ctx.left.as_ref().unwrap(), primary as Epoch) + .await + .expect( + format!( + "Node context not found for left child of node {:?}", + successor_ctx.node_id + ) + .as_str(), + ); + } + Some(successor_ctx) + } else { + // find successor among the ancestors of current node: we go up in the path + // until we either found a node whose left child is the previous node in the + // path, or we get to the root of the tree + let (mut candidate_successor_ctx, mut candidate_successor_val) = (node_ctx.clone(), value); + let mut successor_found = false; + while candidate_successor_ctx.parent.is_some() { + let (parent_ctx, parent_payload) = row_tree + .fetch_with_context_at( + candidate_successor_ctx.parent.as_ref().unwrap(), + primary as Epoch, + ) + .await; + candidate_successor_val = parent_payload.value(); + if parent_ctx + .iter_children() + .find_position(|child| { + child.is_some() && child.unwrap().clone() == candidate_successor_ctx.node_id + }) + .unwrap() + .0 + == 0 + { + // successor_ctx.node_id is left child of parent_ctx node, so parent_ctx is + // the successor + candidate_successor_ctx = parent_ctx; + successor_found = true; + break; + } else { + candidate_successor_ctx = parent_ctx; + } + } + if successor_found { + if candidate_successor_val != value { + // the value of successor is different from `value`, so we don't return the + // successor node + return None; + } + Some(candidate_successor_ctx) + } else { + // We got up to the root of the tree without finding the successor, + // which means that the input node has no successor; + // so we don't return any node + None + } + } +} + +// this method returns the `NodeContext` of the predecessor of the node provided as input, +// if the predecessor exists in the row tree and it stores the same value of the input node (i.e., `value`); +// returns `None` otherwise, as it means that the input node can be used to prove non-existence +async fn get_predecessor_node_with_same_value( + row_tree: &MerkleTreeKvDb, DBRowStorage>, + node_ctx: &NodeContext, + value: U256, + primary: BlockPrimaryIndex, +) -> Option> { + if node_ctx.left.is_some() { + let (left_child_ctx, payload) = row_tree + .fetch_with_context_at(node_ctx.right.as_ref().unwrap(), primary as Epoch) + .await; + // the value of the predecessor in this case is `payload.max`, since the predecessor is the + // maximum of the subtree rooted in the left child + if payload.max() != value { + // the value of predecessor is different from `value`, so we don't return the + // predecessor node + return None; + } + // find predecessor in the subtree rooted in the left child: it is + // the rightmost node in such a subtree + let mut predecessor_ctx = left_child_ctx; + while predecessor_ctx.right.is_some() { + predecessor_ctx = row_tree + .node_context_at(predecessor_ctx.right.as_ref().unwrap(), primary as Epoch) + .await + .expect( + format!( + "Node context not found for right child of node {:?}", + predecessor_ctx.node_id + ) + .as_str(), + ); + } + Some(predecessor_ctx) + } else { + // find successor among the ancestors of current node: we go up in the path + // until we either found a node whose right child is the previous node in the + // path, or we get to the root of the tree + let (mut candidate_predecessor_ctx, mut candidate_predecessor_val) = + (node_ctx.clone(), value); + let mut predecessor_found = false; + while candidate_predecessor_ctx.parent.is_some() { + let (parent_ctx, parent_payload) = row_tree + .fetch_with_context_at( + candidate_predecessor_ctx.parent.as_ref().unwrap(), + primary as Epoch, + ) + .await; + candidate_predecessor_val = parent_payload.value(); + if parent_ctx + .iter_children() + .find_position(|child| { + child.is_some() && child.unwrap().clone() == candidate_predecessor_ctx.node_id + }) + .unwrap() + .0 + == 1 + { + // predecessor_ctx.node_id is right child of parent_ctx node, so parent_ctx is + // the predecessor + candidate_predecessor_ctx = parent_ctx; + predecessor_found = true; + break; + } else { + candidate_predecessor_ctx = parent_ctx; + } + } + if predecessor_found { + if candidate_predecessor_val != value { + // the value of predecessor is different from `value`, so we don't return the + // predecessor node + return None; + } + Some(candidate_predecessor_ctx) + } else { + // We got up to the root of the tree without finding the predecessor, + // which means that the input node has no predecessor; + // so we don't return any node + None + } + } +} + +async fn find_node_for_proof( + db: &DBPool, + row_tree: &MerkleTreeKvDb, DBRowStorage>, + query: Option, + primary: BlockPrimaryIndex, + is_min_query: bool, +) -> anyhow::Result> { + if query.is_none() { + return Ok(None); + } + let rows = execute_row_query(db, &query.unwrap(), &[]).await?; + if rows.is_empty() { + // no node found, return None + return Ok(None); + } + let row_key = rows[0] + .get::<_, Option>>(0) + .map(RowTreeKey::from_bytea) + .context("unable to parse row key tree") + .expect(""); + // among the nodes with the same index value of the node with `row_key`, we need to find + // the one that satisfies the following property: all its successor nodes have values bigger + // than `max_query_secondary`, and all its predecessor nodes have values smaller than + // `min_query_secondary`. Such a node can be found differently, depending on the case: + // - if `is_min_query = true`, then we are looking among nodes with the highest value smaller + // than `min_query_secondary` bound (call this value `min_value`); + // therefore, we need to find the "last" node among the nodes with value `min_value`, that + // is the node whose successor (if exists) has a value bigger than `min_value`. Since there + // are no nodes in the tree in the range [`min_query_secondary, max_query_secondary`], then + // the value of the successor of the "last" node is necessarily bigger than `max_query_secondary`, + // and so it implies that we found the node satisfying the property mentioned above + // - if `is_min_query = false`, then we are looking among nodes with the smallest value higher + // than `max_query_secondary` bound (call this value `max_value`); + // therefore, we need to find the "first" node among the nodes with value `max_value`, that + // is the node whose predecessor (if exists) has a value smaller than `max_value`. Since there + // are no nodes in the tree in the range [`min_query_secondary, max_query_secondary`], then + // the value of the predecessor of the "first" node is necessarily smaller than `min_query_secondary`, + // and so it implies that we found the node satisfying the property mentioned above + let (mut node_ctx, node_value) = row_tree + .fetch_with_context_at(&row_key, primary as Epoch) + .await; + let value = node_value.value(); + + if is_min_query { + // starting from the node with key `row_key`, we iterate over its successor nodes in the tree, + // until we found a node that either has no successor or whose successor stores a value different + // from the value `value` stored in the node with key `row_key`; the node found is the one to be + // employed to generate the non-existence proof + let mut successor_ctx = + get_successor_node_with_same_value(&row_tree, &node_ctx, value, primary).await; + while successor_ctx.is_some() { + node_ctx = successor_ctx.unwrap(); + successor_ctx = + get_successor_node_with_same_value(&row_tree, &node_ctx, value, primary).await; + } + } else { + // starting from the node with key `row_key`, we iterate over its predecessor nodes in the tree, + // until we found a node that either has no predecessor or whose predecessor stores a value different + // from the value `value` stored in the node with key `row_key`; the node found is the one to be + // employed to generate the non-existence proof + let mut predecessor_ctx = + get_predecessor_node_with_same_value(&row_tree, &node_ctx, value, primary).await; + while predecessor_ctx.is_some() { + node_ctx = predecessor_ctx.unwrap(); + predecessor_ctx = + get_predecessor_node_with_same_value(&row_tree, &node_ctx, value, primary).await; + } + } + + Ok(Some(node_ctx.node_id)) +} +pub async fn execute_row_query2( + pool: &DBPool, + query: &str, + params: &[U256], +) -> anyhow::Result> { + // introduce this closure to coerce each param to have type `dyn ToSql + Sync` (required by pgSQL APIs) + let prepare_param = |param: U256| -> Box { Box::new(param) }; + let query_params = params + .iter() + .map(|param| prepare_param(*param)) + .collect_vec(); + let connection = pool.get().await.unwrap(); + let res = connection + .query( + query, + &query_params + .iter() + .map(|param| param.as_ref()) + .collect_vec(), + ) + .await + .context("while fetching current epoch")?; + Ok(res) +} +pub async fn execute_row_query( + pool: &DBPool, + query: &str, + params: &[U256], +) -> anyhow::Result> { + let connection = pool.get().await.unwrap(); + let res = connection + .query_raw(query, params) + .await + .context("while fetching current epoch")?; + let rows: Vec = res.try_collect().await?; + Ok(rows) +} + +pub async fn get_node_info( + lookup: &MerkleTreeKvDb, + k: &T::Key, + at: Epoch, +) -> (NodeInfo, Option, Option) +where + T: TreeTopology + MutableTree + Send, + V: NodePayload + Send + Sync + LagrangeNode, + S: TransactionalStorage + TreeStorage + PayloadStorage + FromSettings, + T::Key: Debug, +{ + // look at the left child first then right child, then build the node info + let (ctx, node_payload) = lookup + .try_fetch_with_context_at(k, at) + .await + .expect("cache not filled"); + // this looks at the value of a child node (left and right), and fetches the grandchildren + // information to be able to build their respective node info. + let fetch_ni = async |k: Option| -> (Option, Option) { + match k { + None => (None, None), + Some(child_k) => { + let (child_ctx, child_payload) = lookup + .try_fetch_with_context_at(&child_k, at) + .await + .expect("cache not filled"); + // we need the grand child hashes for constructing the node info of the + // children of the node in argument + let child_left_hash = match child_ctx.left { + Some(left_left_k) => { + let (_, payload) = lookup + .try_fetch_with_context_at(&left_left_k, at) + .await + .expect("cache not filled"); + Some(payload.hash()) + } + None => None, + }; + let child_right_hash = match child_ctx.right { + Some(left_right_k) => { + let (_, payload) = lookup + .try_fetch_with_context_at(&left_right_k, at) + .await + .expect("cache not full"); + Some(payload.hash()) + } + None => None, + }; + let left_ni = NodeInfo::new( + &child_payload.embedded_hash(), + child_left_hash.as_ref(), + child_right_hash.as_ref(), + child_payload.value(), + child_payload.min(), + child_payload.max(), + ); + (Some(left_ni), Some(child_payload.hash())) + } + } + }; + let (left_node, left_hash) = fetch_ni(ctx.left).await; + let (right_node, right_hash) = fetch_ni(ctx.right).await; + ( + NodeInfo::new( + &node_payload.embedded_hash(), + left_hash.as_ref(), + right_hash.as_ref(), + node_payload.value(), + node_payload.min(), + node_payload.max(), + ), + left_node, + right_node, + ) +} diff --git a/mp2-v1/tests/common/block_extraction.rs b/mp2-v1/tests/common/block_extraction.rs index 8f1a5cfdb..c13f19eb1 100644 --- a/mp2-v1/tests/common/block_extraction.rs +++ b/mp2-v1/tests/common/block_extraction.rs @@ -1,9 +1,8 @@ use alloy::primitives::U256; use anyhow::Result; use mp2_common::{ - eth::{left_pad_generic, BlockUtil}, + eth::BlockUtil, proof::deserialize_proof, - u256, utils::{Endianness, Packer, ToFields}, C, D, F, }; @@ -11,12 +10,6 @@ use mp2_v1::{api, block_extraction, indexing::block::BlockPrimaryIndex}; use super::TestContext; -pub(crate) fn block_number_to_u256_limbs(number: u64) -> Vec { - const NUM_LIMBS: usize = u256::NUM_LIMBS; - let block_number_buff = number.to_be_bytes(); - left_pad_generic::(&block_number_buff.pack(Endianness::Big)).to_fields() -} - impl TestContext { pub(crate) async fn prove_block_extraction(&self, bn: BlockPrimaryIndex) -> Result> { let block = self diff --git a/mp2-v1/tests/common/cases/indexing.rs b/mp2-v1/tests/common/cases/indexing.rs index d867d6c46..b25aa76d7 100644 --- a/mp2-v1/tests/common/cases/indexing.rs +++ b/mp2-v1/tests/common/cases/indexing.rs @@ -69,9 +69,6 @@ pub(crate) const MAPPING_VALUE_COLUMN: &str = "map_value"; pub(crate) const MAPPING_KEY_COLUMN: &str = "map_key"; impl TableIndexing { - pub fn table(&self) -> &Table { - &self.table - } pub(crate) async fn merge_table_test_case( ctx: &mut TestContext, ) -> Result<(Self, Vec>)> { @@ -429,7 +426,7 @@ impl TableIndexing { updates: Vec>, expected_metadata_hash: &HashOutput, ) -> Result<()> { - let current_block = ctx.block_number().await; + let current_block = ctx.block_number().await as BlockPrimaryIndex; // apply the new cells to the trees // NOTE ONLY the rest of the cells, not including the secondary one ! let mut rows_update = Vec::new(); @@ -462,7 +459,7 @@ impl TableIndexing { let row_payload = ctx .prove_cells_tree( &self.table, - current_block as usize, + current_block, previous_row, new_cell_collection, tree_update, @@ -495,7 +492,7 @@ impl TableIndexing { let row_payload = ctx .prove_cells_tree( &self.table, - current_block as usize, + current_block, Row { k: new_cells.previous_row_key.clone(), payload: old_row, diff --git a/mp2-v1/tests/common/cases/planner.rs b/mp2-v1/tests/common/cases/planner.rs index d7851bc84..34cc2e52e 100644 --- a/mp2-v1/tests/common/cases/planner.rs +++ b/mp2-v1/tests/common/cases/planner.rs @@ -121,7 +121,7 @@ impl TreeInfo> planner: &mut QueryPlanner<'a>, primary: BlockPrimaryIndex, k: &RowTreeKey, - v: &RowPayload, + _v: &RowPayload, ) -> Result>> { // TODO export that in single function Ok(if self.is_satisfying_query(k) { @@ -274,7 +274,7 @@ impl TreeInfo> &self, ctx: &mut TestContext, query_id: &QueryID, - primary: BlockPrimaryIndex, + _primary: BlockPrimaryIndex, key: &BlockPrimaryIndex, placeholder_values: PlaceholderValues, proof: Vec, @@ -345,7 +345,7 @@ impl<'b> TreeInfo> for IndexInfo &self, ctx: &mut TestContext, query_id: &QueryID, - primary: BlockPrimaryIndex, + _primary: BlockPrimaryIndex, key: &BlockPrimaryIndex, placeholder_values: PlaceholderValues, proof: Vec, diff --git a/mp2-v1/tests/common/cases/query.rs b/mp2-v1/tests/common/cases/query.rs index e875cffbf..f6833df27 100644 --- a/mp2-v1/tests/common/cases/query.rs +++ b/mp2-v1/tests/common/cases/query.rs @@ -45,11 +45,11 @@ use mp2_v1::{ row::{Row, RowPayload, RowTreeKey}, LagrangeNode, }, + query::planner::{execute_row_query, find_row_node_for_non_existence}, values_extraction::identifier_block_column, }; use parsil::{ assembler::{DynamicCircuitPis, StaticCircuitPis}, - bracketer::bracket_secondary_index, parse_and_validate, queries::{core_keys_for_index_tree, core_keys_for_row_tree}, ParsilSettings, PlaceholderSettings, DEFAULT_MAX_BLOCK_PLACEHOLDER, @@ -57,7 +57,6 @@ use parsil::{ }; use ryhope::{ storage::{ - pgsql::ToFromBytea, updatetree::{Next, UpdateTree, WorkplanItem}, EpochKvStorage, RoEpochKvStorage, TreeTransactionalStorage, WideLineage, }, @@ -168,14 +167,14 @@ async fn test_query_mapping( // the query to use to actually get the outputs expected let mut exec_query = parsil::executor::generate_query_execution(&mut parsed, &settings)?; let query_params = exec_query.convert_placeholders(&query_info.placeholders); - let res = table - .execute_row_query( - &exec_query - .normalize_placeholder_names() - .to_pgsql_string_with_placeholder(), - &query_params, - ) - .await?; + let res = execute_row_query( + &table.db_pool, + &exec_query + .normalize_placeholder_names() + .to_pgsql_string_with_placeholder(), + &query_params, + ) + .await?; let res = if is_empty_result(&res, SqlType::Numeric) { vec![] // empty results, but Postgres still return 1 row } else { @@ -235,7 +234,6 @@ async fn prove_query( // the query to use to fetch all the rows keys involved in the result tree. let pis = parsil::assembler::assemble_dynamic(&parsed, settings, &query.placeholders)?; let row_keys_per_epoch = row_cache.keys_by_epochs(); - let all_epochs = query.min_block as Epoch..=query.max_block as Epoch; let mut planner = QueryPlanner { ctx, query: query.clone(), @@ -349,15 +347,15 @@ async fn prove_query( // get number of matching rows let mut exec_query = parsil::executor::generate_query_keys(&mut parsed, &settings)?; let query_params = exec_query.convert_placeholders(&query.placeholders); - let num_touched_rows = table - .execute_row_query( - &exec_query - .normalize_placeholder_names() - .to_pgsql_string_with_placeholder(), - &query_params, - ) - .await? - .len(); + let num_touched_rows = execute_row_query( + &table.db_pool, + &exec_query + .normalize_placeholder_names() + .to_pgsql_string_with_placeholder(), + &query_params, + ) + .await? + .len(); check_final_outputs( proof, @@ -934,282 +932,29 @@ pub async fn prove_non_existence_row<'a>( planner: &mut QueryPlanner<'a>, primary: BlockPrimaryIndex, ) -> Result<()> { - let row_tree = &planner.table.row; - let (query_for_min, query_for_max) = bracket_secondary_index( - &planner.table.public_name, - planner.settings, - primary as Epoch, + let (chosen_node, plan) = find_row_node_for_non_existence( + &planner.table.row, + planner.table.public_name.clone(), + &planner.table.db_pool, + primary, + &planner.settings, &planner.pis.bounds, - ); - - // this method returns the `NodeContext` of the successor of the node provided as input, - // if the successor exists in the row tree and it stores the same value of the input node (i.e., `value`); - // returns `None` otherwise, as it means that the input node can be used to prove non-existence - async fn get_successor_node_with_same_value( - node_ctx: &NodeContext, - value: U256, - table: &Table, - primary: BlockPrimaryIndex, - ) -> Option> { - let row_tree = &table.row; - if node_ctx.right.is_some() { - let (right_child_ctx, payload) = row_tree - .fetch_with_context_at(node_ctx.right.as_ref().unwrap(), primary as Epoch) - .await; - // the value of the successor in this case is `payload.min`, since the successor is the - // minimum of the subtree rooted in the right child - if payload.min() != value { - // the value of successor is different from `value`, so we don't return the - // successor node - return None; - } - // find successor in the subtree rooted in the right child: it is - // the leftmost node in such a subtree - let mut successor_ctx = right_child_ctx; - while successor_ctx.left.is_some() { - successor_ctx = row_tree - .node_context_at(successor_ctx.left.as_ref().unwrap(), primary as Epoch) - .await - .expect( - format!( - "Node context not found for left child of node {:?}", - successor_ctx.node_id - ) - .as_str(), - ); - } - Some(successor_ctx) - } else { - // find successor among the ancestors of current node: we go up in the path - // until we either found a node whose left child is the previous node in the - // path, or we get to the root of the tree - let (mut candidate_successor_ctx, mut candidate_successor_val) = - (node_ctx.clone(), value); - let mut successor_found = false; - while candidate_successor_ctx.parent.is_some() { - let (parent_ctx, parent_payload) = row_tree - .fetch_with_context_at( - candidate_successor_ctx.parent.as_ref().unwrap(), - primary as Epoch, - ) - .await; - candidate_successor_val = parent_payload.value(); - if parent_ctx - .iter_children() - .find_position(|child| { - child.is_some() && child.unwrap().clone() == candidate_successor_ctx.node_id - }) - .unwrap() - .0 - == 0 - { - // successor_ctx.node_id is left child of parent_ctx node, so parent_ctx is - // the successor - candidate_successor_ctx = parent_ctx; - successor_found = true; - break; - } else { - candidate_successor_ctx = parent_ctx; - } - } - if successor_found { - if candidate_successor_val != value { - // the value of successor is different from `value`, so we don't return the - // successor node - return None; - } - Some(candidate_successor_ctx) - } else { - // We got up to the root of the tree without finding the successor, - // which means that the input node has no successor; - // so we don't return any node - None - } - } - } - - // this method returns the `NodeContext` of the predecessor of the node provided as input, - // if the predecessor exists in the row tree and it stores the same value of the input node (i.e., `value`); - // returns `None` otherwise, as it means that the input node can be used to prove non-existence - async fn get_predecessor_node_with_same_value( - node_ctx: &NodeContext, - value: U256, - table: &Table, - primary: BlockPrimaryIndex, - ) -> Option> { - let row_tree = &table.row; - if node_ctx.left.is_some() { - let (left_child_ctx, payload) = row_tree - .fetch_with_context_at(node_ctx.right.as_ref().unwrap(), primary as Epoch) - .await; - // the value of the predecessor in this case is `payload.max`, since the predecessor is the - // maximum of the subtree rooted in the left child - if payload.max() != value { - // the value of predecessor is different from `value`, so we don't return the - // predecessor node - return None; - } - // find predecessor in the subtree rooted in the left child: it is - // the rightmost node in such a subtree - let mut predecessor_ctx = left_child_ctx; - while predecessor_ctx.right.is_some() { - predecessor_ctx = row_tree - .node_context_at(predecessor_ctx.right.as_ref().unwrap(), primary as Epoch) - .await - .expect( - format!( - "Node context not found for right child of node {:?}", - predecessor_ctx.node_id - ) - .as_str(), - ); - } - Some(predecessor_ctx) - } else { - // find successor among the ancestors of current node: we go up in the path - // until we either found a node whose right child is the previous node in the - // path, or we get to the root of the tree - let (mut candidate_predecessor_ctx, mut candidate_predecessor_val) = - (node_ctx.clone(), value); - let mut predecessor_found = false; - while candidate_predecessor_ctx.parent.is_some() { - let (parent_ctx, parent_payload) = row_tree - .fetch_with_context_at( - candidate_predecessor_ctx.parent.as_ref().unwrap(), - primary as Epoch, - ) - .await; - candidate_predecessor_val = parent_payload.value(); - if parent_ctx - .iter_children() - .find_position(|child| { - child.is_some() - && child.unwrap().clone() == candidate_predecessor_ctx.node_id - }) - .unwrap() - .0 - == 1 - { - // predecessor_ctx.node_id is right child of parent_ctx node, so parent_ctx is - // the predecessor - candidate_predecessor_ctx = parent_ctx; - predecessor_found = true; - break; - } else { - candidate_predecessor_ctx = parent_ctx; - } - } - if predecessor_found { - if candidate_predecessor_val != value { - // the value of predecessor is different from `value`, so we don't return the - // predecessor node - return None; - } - Some(candidate_predecessor_ctx) - } else { - // We got up to the root of the tree without finding the predecessor, - // which means that the input node has no predecessor; - // so we don't return any node - None - } - } - } - - let find_node_for_proof = async |query: Option, - is_min_query: bool| - -> Result> { - if query.is_none() { - return Ok(None); - } - let rows = planner - .table - .execute_row_query(&query.unwrap(), &[]) - .await?; - if rows.is_empty() { - // no node found, return None - info!("Search node for non-existence circuit: no node found"); - return Ok(None); - } - let row_key = rows[0] - .get::<_, Option>>(0) - .map(RowTreeKey::from_bytea) - .context("unable to parse row key tree") - .expect(""); - // among the nodes with the same index value of the node with `row_key`, we need to find - // the one that satisfies the following property: all its successor nodes have values bigger - // than `max_query_secondary`, and all its predecessor nodes have values smaller than - // `min_query_secondary`. Such a node can be found differently, depending on the case: - // - if `is_min_query = true`, then we are looking among nodes with the highest value smaller - // than `min_query_secondary` bound (call this value `min_value`); - // therefore, we need to find the "last" node among the nodes with value `min_value`, that - // is the node whose successor (if exists) has a value bigger than `min_value`. Since there - // are no nodes in the tree in the range [`min_query_secondary, max_query_secondary`], then - // the value of the successor of the "last" node is necessarily bigger than `max_query_secondary`, - // and so it implies that we found the node satisfying the property mentioned above - // - if `is_min_query = false`, then we are looking among nodes with the smallest value higher - // than `max_query_secondary` bound (call this value `max_value`); - // therefore, we need to find the "first" node among the nodes with value `max_value`, that - // is the node whose predecessor (if exists) has a value smaller than `max_value`. Since there - // are no nodes in the tree in the range [`min_query_secondary, max_query_secondary`], then - // the value of the predecessor of the "first" node is necessarily smaller than `min_query_secondary`, - // and so it implies that we found the node satisfying the property mentioned above - let (mut node_ctx, node_value) = row_tree - .fetch_with_context_at(&row_key, primary as Epoch) - .await; - let value = node_value.value(); - - if is_min_query { - // starting from the node with key `row_key`, we iterate over its successor nodes in the tree, - // until we found a node that either has no successor or whose successor stores a value different - // from the value `value` stored in the node with key `row_key`; the node found is the one to be - // employed to generate the non-existence proof - let mut successor_ctx = - get_successor_node_with_same_value(&node_ctx, value, &planner.table, primary).await; - while successor_ctx.is_some() { - node_ctx = successor_ctx.unwrap(); - successor_ctx = - get_successor_node_with_same_value(&node_ctx, value, &planner.table, primary) - .await; - } - } else { - // starting from the node with key `row_key`, we iterate over its predecessor nodes in the tree, - // until we found a node that either has no predecessor or whose predecessor stores a value different - // from the value `value` stored in the node with key `row_key`; the node found is the one to be - // employed to generate the non-existence proof - let mut predecessor_ctx = - get_predecessor_node_with_same_value(&node_ctx, value, &planner.table, primary) - .await; - while predecessor_ctx.is_some() { - node_ctx = predecessor_ctx.unwrap(); - predecessor_ctx = - get_predecessor_node_with_same_value(&node_ctx, value, &planner.table, primary) - .await; - } - } - - Ok(Some(node_ctx.node_id)) - }; - // try first with lower node than secondary min query bound - let to_be_proven_node = match find_node_for_proof(query_for_min, true).await? { - Some(node) => node, - None => find_node_for_proof(query_for_max, false) - .await? - .expect("No valid node found to prove non-existence, something is wrong"), - }; - let (node_info, left_child_info, right_child_info) = get_node_info( - &RowInfo::no_satisfying_rows(row_tree), - &to_be_proven_node, - primary as Epoch, ) - .await; + .await?; + let (node_info, left_child_info, right_child_info) = + mp2_v1::query::planner::get_node_info(&planner.table.row, &chosen_node, primary as Epoch) + .await; let proof_key = ProofKey::QueryAggregateRow(( planner.query.query.clone(), planner.query.placeholders.placeholder_values(), primary, - to_be_proven_node.clone(), + chosen_node.clone(), )); - info!("Non-existence circuit proof RUNNING for {primary} -> {to_be_proven_node:?} "); + info!( + "Non-existence circuit proof RUNNING for {primary} -> {:?} ", + proof_key + ); let proof = generate_non_existence_proof( node_info, left_child_info, @@ -1219,24 +964,18 @@ pub async fn prove_non_existence_row<'a>( true, ) .unwrap_or_else(|_| { - panic!("unable to generate non-existence proof for {primary} -> {to_be_proven_node:?}") + panic!( + "unable to generate non-existence proof for {primary} -> {:?}", + chosen_node + ) }); - info!("Non-existence circuit proof DONE for {primary} -> {to_be_proven_node:?} "); + info!( + "Non-existence circuit proof DONE for {primary} -> {:?} ", + chosen_node + ); planner.ctx.storage.store_proof(proof_key, proof.clone())?; - // now generate the path up to the root of the row tree for the current epoch, as all nodes in such a path - // need to be proven - let path = planner - .table - .row - // since the epoch starts at genesis we can directly give the block number ! - .lineage_at(&to_be_proven_node, primary as Epoch) - .await - .expect("node doesn't have a lineage?") - .into_full_path() - .collect_vec(); - let proving_tree = UpdateTree::from_paths([path], primary as Epoch); - let info = RowInfo::no_satisfying_rows(&planner.table.row); + let tree_info = RowInfo::no_satisfying_rows(&planner.table.row); let mut planner = QueryPlanner { ctx: planner.ctx, table: planner.table, @@ -1245,7 +984,7 @@ pub async fn prove_non_existence_row<'a>( columns: planner.columns.clone(), settings: planner.settings, }; - prove_query_on_tree(&mut planner, info, proving_tree, primary).await?; + prove_query_on_tree(&mut planner, tree_info, plan, primary).await?; Ok(()) } @@ -1658,7 +1397,7 @@ async fn find_longest_lived_key( Some((k, l, start)) } }) - .max_by_key(|(k, l, start)| *l) + .max_by_key(|(_k, l, _start)| *l) .unwrap_or_else(|| { panic!( "unable to find longest row? -> length all _table {}, max {}", @@ -1721,6 +1460,7 @@ fn find_longest_consecutive_sequence(v: Vec) -> (usize, i64) { (longest, v[starting_idx]) } +#[allow(dead_code)] async fn check_correct_cells_tree( all_cells: &[ColumnCell], payload: &RowPayload, diff --git a/mp2-v1/tests/common/cases/table_source.rs b/mp2-v1/tests/common/cases/table_source.rs index fedbf3556..fdb742326 100644 --- a/mp2-v1/tests/common/cases/table_source.rs +++ b/mp2-v1/tests/common/cases/table_source.rs @@ -10,7 +10,7 @@ use alloy::{ providers::Provider, }; use anyhow::{bail, Result}; -use futures::{future::BoxFuture, FutureExt, StreamExt}; +use futures::{future::BoxFuture, FutureExt}; use log::{debug, info}; use mp2_common::{ digest::TableDimension, @@ -201,6 +201,7 @@ impl TableSource { } } + #[allow(elided_named_lifetimes)] pub fn init_contract_data<'a>( &'a mut self, ctx: &'a mut TestContext, @@ -241,6 +242,7 @@ impl TableSource { } } + #[allow(elided_named_lifetimes)] pub fn random_contract_update<'a>( &'a mut self, ctx: &'a mut TestContext, @@ -399,7 +401,7 @@ impl SingleValuesExtractionArgs { proof_key: ProofKey, ) -> Result<(ExtractionProofInput, HashOutput)> { let chain_id = ctx.rpc.get_chain_id().await?; - let ProofKey::ValueExtraction((id, bn)) = proof_key.clone() else { + let ProofKey::ValueExtraction((_id, bn)) = proof_key.clone() else { bail!("invalid proof key"); }; let single_value_proof = match ctx.storage.get_proof_exact(&proof_key) { @@ -625,7 +627,6 @@ impl MappingValuesExtractionArgs { .await .unwrap(); let new_block_number = ctx.block_number().await as BlockPrimaryIndex; - let chain_id = ctx.rpc.get_chain_id().await.unwrap(); // NOTE HERE is the interesting bit for dist system as this is the logic to execute // on receiving updates from scapper. This only needs to have the relevant // information from update and it will translate that to changes in the tree. @@ -942,6 +943,7 @@ impl MergeSource { } } + #[allow(elided_named_lifetimes)] pub fn generate_extraction_proof_inputs<'a>( &'a self, ctx: &'a mut TestContext, diff --git a/mp2-v1/tests/common/context.rs b/mp2-v1/tests/common/context.rs index 78305e421..2c8409957 100644 --- a/mp2-v1/tests/common/context.rs +++ b/mp2-v1/tests/common/context.rs @@ -262,12 +262,6 @@ impl TestContext { .unwrap() } - /// Reset the RPC provider. It could be used to query data from the - /// different RPCs during testing. - pub(crate) fn set_rpc(&mut self, rpc_url: &str) { - self.rpc = ProviderBuilder::new().on_http(rpc_url.parse().unwrap()); - } - pub fn run_query_proof( &self, name: &str, diff --git a/mp2-v1/tests/common/length_extraction.rs b/mp2-v1/tests/common/length_extraction.rs index 02cd57bcd..f036afa9d 100644 --- a/mp2-v1/tests/common/length_extraction.rs +++ b/mp2-v1/tests/common/length_extraction.rs @@ -12,6 +12,7 @@ use super::TestContext; impl TestContext { /// Generate the Values Extraction (C.2) proof for single variables. + #[allow(dead_code)] pub(crate) async fn prove_length_extraction( &self, contract_address: &Address, diff --git a/mp2-v1/tests/common/proof_storage.rs b/mp2-v1/tests/common/proof_storage.rs index 9bb7e7a46..f0cc91058 100644 --- a/mp2-v1/tests/common/proof_storage.rs +++ b/mp2-v1/tests/common/proof_storage.rs @@ -8,16 +8,11 @@ use alloy::primitives::{Address, U256}; use anyhow::{bail, Context, Result}; use envconfig::Envconfig; use mp2_test::cells_tree::CellTree; -use mp2_v1::indexing::{ - block::{BlockPrimaryIndex, BlockTree}, - row::RowTreeKey, -}; +use mp2_v1::indexing::{block::BlockPrimaryIndex, row::RowTreeKey}; use ryhope::tree::TreeTopology; use serde::{Deserialize, Serialize}; type CellTreeKey = ::Key; -type IndexTreeKey = ::Key; - type ContractKey = (Address, BlockPrimaryIndex); /// This is the identifier we store cells tree proof under in storage. This identifier diff --git a/mp2-v1/tests/common/table.rs b/mp2-v1/tests/common/table.rs index 2671494df..ea7e692c4 100644 --- a/mp2-v1/tests/common/table.rs +++ b/mp2-v1/tests/common/table.rs @@ -1,4 +1,3 @@ -use alloy::primitives::U256; use anyhow::{ensure, Context, Result}; use bb8::Pool; use bb8_postgres::{tokio_postgres::NoTls, PostgresConnectionManager}; @@ -9,7 +8,7 @@ use futures::{ use itertools::Itertools; use log::debug; use mp2_v1::indexing::{ - block::BlockPrimaryIndex, + block::{BlockPrimaryIndex, BlockTreeKey}, cell::{self, Cell, CellTreeKey, MerkleCell, MerkleCellTree}, index::IndexNode, row::{CellCollection, Row, RowTreeKey}, @@ -27,7 +26,6 @@ use ryhope::{ }; use serde::{Deserialize, Serialize}; use std::{hash::Hash, iter::once}; -use tokio_postgres::{row::Row as PsqlRow, types::ToSql}; use super::{index_tree::MerkleIndexTree, rowtree::MerkleRowTree, ColumnIdentifier}; @@ -433,12 +431,12 @@ impl Table { pub async fn apply_index_update( &mut self, updates: IndexUpdate, - ) -> Result> { + ) -> Result> { let plan = self .index .in_transaction(|t| { async move { - t.store(updates.added_index.0, updates.added_index.1) + t.store(updates.added_index.0 as usize, updates.added_index.1) .await?; Ok(()) } @@ -447,27 +445,6 @@ impl Table { .await?; Ok(IndexUpdateResult { plan }) } - - pub async fn execute_row_query(&self, query: &str, params: &[U256]) -> Result> { - // introduce this closure to coerce each param to have type `dyn ToSql + Sync` (required by pgSQL APIs) - let prepare_param = |param: U256| -> Box { Box::new(param) }; - let query_params = params - .iter() - .map(|param| prepare_param(*param)) - .collect_vec(); - let connection = self.db_pool.get().await.unwrap(); - let res = connection - .query( - query, - &query_params - .iter() - .map(|param| param.as_ref()) - .collect_vec(), - ) - .await - .context("while fetching current epoch")?; - Ok(res) - } } #[derive(Debug, Clone)] diff --git a/mp2-v1/tests/integrated_tests.rs b/mp2-v1/tests/integrated_tests.rs index c619795d2..ead38b76f 100644 --- a/mp2-v1/tests/integrated_tests.rs +++ b/mp2-v1/tests/integrated_tests.rs @@ -1,5 +1,6 @@ //! Database creation integration test // Used to fix the error: failed to evaluate generic const expression `PAD_LEN(NODE_LEN)`. +#![allow(incomplete_features)] #![feature(generic_const_exprs)] #![feature(let_chains)] #![feature(async_closure)] @@ -177,7 +178,7 @@ fn read_table_info(f: &str) -> Result { struct T(ZkTable); impl ContextProvider for T { - fn fetch_table(&self, table_name: &str) -> Result { + fn fetch_table(&self, _table_name: &str) -> Result { Ok(self.0.clone()) } } @@ -232,7 +233,7 @@ async fn test_andrus_query() -> Result<()> { pis_hash, )?; info!("Generating the revelation proof"); - let proof = ctx.run_query_proof("revelation", GlobalCircuitInput::Revelation(input))?; + let _proof = ctx.run_query_proof("revelation", GlobalCircuitInput::Revelation(input))?; info!("all good"); Ok(()) }