From 20dc0c7da0a12e31ab0c896ccf4208585658ad07 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:25:33 +0200 Subject: [PATCH 001/239] some fmt (#11933) --- crates/storage/provider/src/providers/blockchain_provider.rs | 2 +- crates/storage/provider/src/providers/mod.rs | 2 +- crates/transaction-pool/src/traits.rs | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 1866610e3f28..13215e11a8e2 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -71,7 +71,7 @@ impl BlockchainProvider2 { /// the database to initialize the provider. pub fn new(database: ProviderFactory) -> ProviderResult { let provider = database.provider()?; - let best: ChainInfo = provider.chain_info()?; + let best = provider.chain_info()?; match provider.header_by_number(best.best_number)? { Some(header) => { drop(provider); diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index b98bbf5be472..a67ebf89ba62 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -118,7 +118,7 @@ impl BlockchainProvider { /// the database to initialize the provider. pub fn new(database: ProviderFactory, tree: Arc) -> ProviderResult { let provider = database.provider()?; - let best: ChainInfo = provider.chain_info()?; + let best = provider.chain_info()?; let latest_header = provider .header_by_number(best.best_number)? .ok_or_else(|| ProviderError::HeaderNotFound(best.best_number.into()))?; diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index fcfdae4ed1b3..56da11fe696a 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -72,7 +72,6 @@ pub trait TransactionPool: Send + Sync + Clone { /// Imports all _external_ transactions /// - /// /// Consumer: Utility fn add_external_transactions( &self, @@ -83,7 +82,7 @@ pub trait TransactionPool: Send + Sync + Clone { /// Adds an _unvalidated_ transaction into the pool and subscribe to state changes. /// - /// This is the same as [TransactionPool::add_transaction] but returns an event stream for the + /// This is the same as [`TransactionPool::add_transaction`] but returns an event stream for the /// given transaction. /// /// Consumer: Custom From 3f2a41bd3f1d44c9a794a20876ca46a09499bb00 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 21 Oct 2024 20:05:06 +0200 Subject: [PATCH 002/239] chore: rm redundant trait bound (#11940) --- crates/rpc/rpc/src/debug.rs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 66465ef474aa..2d9d6f7822e1 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -21,8 +21,8 @@ use reth_evm::{ }; use reth_primitives::{Block, BlockId, BlockNumberOrTag, TransactionSignedEcRecovered}; use reth_provider::{ - BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, HeaderProvider, StateProofProvider, - StateProviderFactory, TransactionVariant, + BlockReaderIdExt, ChainSpecProvider, HeaderProvider, StateProofProvider, StateProviderFactory, + TransactionVariant, }; use reth_revm::database::StateProviderDatabase; use reth_rpc_api::DebugApiServer; @@ -81,7 +81,6 @@ where + HeaderProvider + ChainSpecProvider + StateProviderFactory - + EvmEnvProvider + 'static, Eth: EthApiTypes + TraceExt + 'static, BlockExecutor: BlockExecutorProvider, @@ -842,7 +841,6 @@ where + HeaderProvider + ChainSpecProvider + StateProviderFactory - + EvmEnvProvider + 'static, Eth: EthApiSpec + EthTransactions + TraceExt + 'static, BlockExecutor: BlockExecutorProvider, @@ -979,15 +977,6 @@ where .map_err(Into::into) } - /// Handler for `debug_executionWitness` - async fn debug_execution_witness( - &self, - block: BlockNumberOrTag, - ) -> RpcResult { - let _permit = self.acquire_trace_permit().await; - Self::debug_execution_witness(self, block).await.map_err(Into::into) - } - /// Handler for `debug_traceCall` async fn debug_trace_call( &self, @@ -1011,6 +1000,15 @@ where Self::debug_trace_call_many(self, bundles, state_context, opts).await.map_err(Into::into) } + /// Handler for `debug_executionWitness` + async fn debug_execution_witness( + &self, + block: BlockNumberOrTag, + ) -> RpcResult { + let _permit = self.acquire_trace_permit().await; + Self::debug_execution_witness(self, block).await.map_err(Into::into) + } + async fn debug_backtrace_at(&self, _location: &str) -> RpcResult<()> { Ok(()) } From d9d184d4984aec0099e3a832cf9dc9c6072af0df Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Mon, 21 Oct 2024 20:47:50 +0200 Subject: [PATCH 003/239] feat: default impl for some BlockExecutionStrategy methods (#11941) --- crates/ethereum/evm/src/execute.rs | 7 +------ crates/evm/src/execute.rs | 32 ++++++++++++++++++++++-------- crates/evm/src/test_utils.rs | 2 ++ crates/optimism/evm/src/execute.rs | 10 +--------- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index e8b238c6a7cb..c62949902aea 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -21,7 +21,7 @@ use reth_evm::{ ConfigureEvm, }; use reth_primitives::{BlockWithSenders, Receipt}; -use reth_revm::db::{states::bundle_state::BundleRetention, BundleState, State}; +use reth_revm::db::State; use revm_primitives::{ db::{Database, DatabaseCommit}, BlockEnv, CfgEnvWithHandlerCfg, EnvWithHandlerCfg, ResultAndState, U256, @@ -264,11 +264,6 @@ where self.system_caller.with_state_hook(hook); } - fn finish(&mut self) -> BundleState { - self.state.merge_transitions(BundleRetention::Reverts); - self.state.take_bundle() - } - fn validate_block_post_execution( &self, block: &BlockWithSenders, diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 2b3ce85e9b94..677a15dfa1b4 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -6,6 +6,7 @@ pub use reth_execution_errors::{ }; pub use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput, ExecutionOutcome}; pub use reth_storage_errors::provider::ProviderError; +use revm::db::states::bundle_state::BundleRetention; use crate::system_calls::OnStateHook; use alloc::{boxed::Box, vec::Vec}; @@ -176,7 +177,10 @@ pub struct ExecuteOutput { } /// Defines the strategy for executing a single block. -pub trait BlockExecutionStrategy { +pub trait BlockExecutionStrategy +where + DB: Database, +{ /// The error type returned by this strategy's methods. type Error: From + core::error::Error; @@ -209,18 +213,23 @@ pub trait BlockExecutionStrategy { fn state_mut(&mut self) -> &mut State; /// Sets a hook to be called after each state change during execution. - fn with_state_hook(&mut self, hook: Option>); + fn with_state_hook(&mut self, _hook: Option>) {} /// Returns the final bundle state. - fn finish(&mut self) -> BundleState; + fn finish(&mut self) -> BundleState { + self.state_mut().merge_transitions(BundleRetention::Reverts); + self.state_mut().take_bundle() + } /// Validate a block with regard to execution results. fn validate_block_post_execution( &self, - block: &BlockWithSenders, - receipts: &[Receipt], - requests: &Requests, - ) -> Result<(), ConsensusError>; + _block: &BlockWithSenders, + _receipts: &[Receipt], + _requests: &Requests, + ) -> Result<(), ConsensusError> { + Ok(()) + } } /// A strategy factory that can create block execution strategies. @@ -293,6 +302,7 @@ where pub struct BasicBlockExecutor where S: BlockExecutionStrategy, + DB: Database, { /// Block execution strategy. pub(crate) strategy: S, @@ -302,6 +312,7 @@ where impl BasicBlockExecutor where S: BlockExecutionStrategy, + DB: Database, { /// Creates a new `BasicBlockExecutor` with the given strategy. pub const fn new(strategy: S) -> Self { @@ -384,6 +395,7 @@ where pub struct BasicBatchExecutor where S: BlockExecutionStrategy, + DB: Database, { /// Batch execution strategy. pub(crate) strategy: S, @@ -395,6 +407,7 @@ where impl BasicBatchExecutor where S: BlockExecutionStrategy, + DB: Database, { /// Creates a new `BasicBatchExecutor` with the given strategy. pub const fn new(strategy: S, batch_record: BlockBatchRecord) -> Self { @@ -597,7 +610,10 @@ mod tests { } } - impl BlockExecutionStrategy for TestExecutorStrategy { + impl BlockExecutionStrategy for TestExecutorStrategy + where + DB: Database, + { type Error = BlockExecutionError; fn apply_pre_execution_changes( diff --git a/crates/evm/src/test_utils.rs b/crates/evm/src/test_utils.rs index c20f43dca9d4..a4dc906494ce 100644 --- a/crates/evm/src/test_utils.rs +++ b/crates/evm/src/test_utils.rs @@ -119,6 +119,7 @@ impl BatchExecutor for MockExecutorProvider { impl BasicBlockExecutor where S: BlockExecutionStrategy, + DB: Database, { /// Provides safe read access to the state pub fn with_state(&self, f: F) -> R @@ -140,6 +141,7 @@ where impl BasicBatchExecutor where S: BlockExecutionStrategy, + DB: Database, { /// Provides safe read access to the state pub fn with_state(&self, f: F) -> R diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 20ef64457bb7..748e57e6b33b 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -20,10 +20,7 @@ use reth_optimism_chainspec::OpChainSpec; use reth_optimism_consensus::validate_block_post_execution; use reth_optimism_forks::OptimismHardfork; use reth_primitives::{BlockWithSenders, Header, Receipt, TxType}; -use reth_revm::{ - db::{states::bundle_state::BundleRetention, BundleState}, - Database, State, -}; +use reth_revm::{Database, State}; use revm_primitives::{ db::DatabaseCommit, BlockEnv, CfgEnvWithHandlerCfg, EnvWithHandlerCfg, ResultAndState, U256, }; @@ -271,11 +268,6 @@ where self.system_caller.with_state_hook(hook); } - fn finish(&mut self) -> BundleState { - self.state.merge_transitions(BundleRetention::Reverts); - self.state.take_bundle() - } - fn validate_block_post_execution( &self, block: &BlockWithSenders, From bddd3202e4603251d76ecceb6bfacf40b26ec46e Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 21 Oct 2024 20:04:09 +0100 Subject: [PATCH 004/239] test(trie): narrow the range of keys for sparse trie fuzz (#11937) --- crates/trie/sparse/src/trie.rs | 73 ++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 0b9ffb5c0ed4..6680d7e9b650 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -265,15 +265,15 @@ impl RevealedSparseTrie { } /// Remove leaf node from the trie. - pub fn remove_leaf(&mut self, path: Nibbles) -> SparseTrieResult<()> { + pub fn remove_leaf(&mut self, path: &Nibbles) -> SparseTrieResult<()> { self.prefix_set.insert(path.clone()); - let existing = self.values.remove(&path); + let existing = self.values.remove(path); if existing.is_none() { // trie structure unchanged, return immediately return Ok(()) } - let mut removed_nodes = self.take_nodes_for_path(&path)?; + let mut removed_nodes = self.take_nodes_for_path(path)?; debug!(target: "trie::sparse", ?path, ?removed_nodes, "Removed nodes for path"); // Pop the first node from the stack which is the leaf node we want to remove. let mut child = removed_nodes.pop().expect("leaf exists"); @@ -282,7 +282,7 @@ impl RevealedSparseTrie { let mut child_path = child.path.clone(); let SparseNode::Leaf { key, .. } = &child.node else { panic!("expected leaf node") }; child_path.extend_from_slice_unchecked(key); - assert_eq!(child_path, path); + assert_eq!(&child_path, path); } // If we don't have any other removed nodes, insert an empty node at the root. @@ -727,6 +727,7 @@ mod tests { use super::*; use alloy_primitives::U256; use itertools::Itertools; + use prop::sample::SizeRange; use proptest::prelude::*; use rand::seq::IteratorRandom; use reth_testing_utils::generators; @@ -998,7 +999,7 @@ mod tests { ]) ); - sparse.remove_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3])).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -1049,7 +1050,7 @@ mod tests { ]) ); - sparse.remove_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1])).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -1085,7 +1086,7 @@ mod tests { ]) ); - sparse.remove_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2])).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -1118,7 +1119,7 @@ mod tests { ]) ); - sparse.remove_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0])).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -1140,7 +1141,7 @@ mod tests { ]) ); - sparse.remove_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3])).unwrap(); // Leaf (Key = 53302) pretty_assertions::assert_eq!( @@ -1151,7 +1152,7 @@ mod tests { ),]) ); - sparse.remove_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2])).unwrap(); // Empty pretty_assertions::assert_eq!( @@ -1162,34 +1163,31 @@ mod tests { #[test] fn sparse_trie_fuzz() { - proptest!(ProptestConfig::with_cases(10), |(updates: Vec>)| { + // Having only the first 3 nibbles set, we narrow down the range of keys + // to 4096 different hashes. It allows us to generate collisions more likely + // to test the sparse trie updates. + const KEY_NIBBLES_LEN: usize = 3; + + fn test(updates: Vec>>) { let mut rng = generators::rng(); let mut state = BTreeMap::default(); - let mut unpacked_state = BTreeMap::default(); let mut sparse = RevealedSparseTrie::default(); for update in updates { let keys_to_delete_len = update.len() / 2; - let unpacked_update = update.iter().map(|(key, value)| ( - Nibbles::unpack(key), - alloy_rlp::encode_fixed_size(value).to_vec() - )); - // Insert state updates into the sparse trie and calculate the root - for (key, value) in unpacked_update.clone() { + for (key, value) in update.clone() { sparse.update_leaf(key, value).unwrap(); } let sparse_root = sparse.root(); // Insert state updates into the hash builder and calculate the root - unpacked_state.extend(unpacked_update); state.extend(update); - let keys = state.keys().map(Nibbles::unpack).collect::>(); let (hash_builder_root, hash_builder_proof_nodes) = hash_builder_root_with_proofs( - unpacked_state.clone(), - keys, + state.clone(), + state.keys().cloned().collect::>(), ); // Assert that the sparse trie root matches the hash builder root @@ -1204,20 +1202,18 @@ mod tests { .keys() .choose_multiple(&mut rng, keys_to_delete_len) .into_iter() - .copied() + .cloned() .collect::>(); for key in keys_to_delete { state.remove(&key).unwrap(); - unpacked_state.remove(&Nibbles::unpack(key)).unwrap(); - sparse.remove_leaf(Nibbles::unpack(key)).unwrap(); + sparse.remove_leaf(&key).unwrap(); } let sparse_root = sparse.root(); - let keys = state.keys().map(Nibbles::unpack).collect::>(); let (hash_builder_root, hash_builder_proof_nodes) = hash_builder_root_with_proofs( - unpacked_state.clone(), - keys, + state.clone(), + state.keys().cloned().collect::>(), ); // Assert that the sparse trie root matches the hash builder root @@ -1225,6 +1221,25 @@ mod tests { // Assert that the sparse trie nodes match the hash builder proof nodes assert_eq_sparse_trie_proof_nodes(&sparse, hash_builder_proof_nodes); } - }); + } + + /// Pad nibbles of length [`KEY_NIBBLES_LEN`] with zeros to the length of a B256 hash. + fn pad_nibbles(nibbles: Nibbles) -> Nibbles { + let mut base = + Nibbles::from_nibbles_unchecked([0; { B256::len_bytes() / 2 - KEY_NIBBLES_LEN }]); + base.extend_from_slice_unchecked(&nibbles); + base + } + + proptest!(ProptestConfig::with_cases(10), |( + updates in proptest::collection::vec( + proptest::collection::hash_map( + any_with::(SizeRange::new(KEY_NIBBLES_LEN..=KEY_NIBBLES_LEN)).prop_map(pad_nibbles), + any::>(), + 1..100, + ), + 1..100, + ) + )| { test(updates) }); } } From 7f47ef0fd3e540232f24f07d31f3b12e8be4e8cd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 21 Oct 2024 21:18:24 +0200 Subject: [PATCH 005/239] chore: spawn eth_calls as blocking tasks (#11944) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index b43b34305bd9..64017f9f8fc5 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -544,6 +544,16 @@ pub trait Call: LoadState + SpawnBlocking { /// /// This returns the configured [`EnvWithHandlerCfg`] for the given [`TransactionRequest`] at /// the given [`BlockId`] and with configured call settings: `prepare_call_env`. + /// + /// This is primarily used by `eth_call`. + /// + /// # Blocking behaviour + /// + /// This assumes executing the call is relatively more expensive on IO than CPU because it + /// transacts a single transaction on an empty in memory database. Because `eth_call`s are + /// usually allowed to consume a lot of gas, this also allows a lot of memory operations so + /// we assume this is not primarily CPU bound and instead spawn the call on a regular tokio task + /// instead, where blocking IO is less problematic. fn spawn_with_call_at( &self, request: TransactionRequest, @@ -561,7 +571,7 @@ pub trait Call: LoadState + SpawnBlocking { async move { let (cfg, block_env, at) = self.evm_env_at(at).await?; let this = self.clone(); - self.spawn_tracing(move |_| { + self.spawn_blocking_io(move |_| { let state = this.state_at_block_id(at)?; let mut db = CacheDB::new(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))); From 1f4ca32e35a7cc4bfccf7cc5a974429e3df4a846 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 21 Oct 2024 21:18:30 +0200 Subject: [PATCH 006/239] refactor(primitives-traits): rm useless trait bounds for `Receipt` (#11942) --- crates/primitives-traits/src/receipt.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/primitives-traits/src/receipt.rs b/crates/primitives-traits/src/receipt.rs index e2d19e4d4ff5..5c317dc49a23 100644 --- a/crates/primitives-traits/src/receipt.rs +++ b/crates/primitives-traits/src/receipt.rs @@ -1,7 +1,5 @@ //! Receipt abstraction -use alloc::fmt; - use alloy_consensus::TxReceipt; use reth_codecs::Compact; use serde::{Deserialize, Serialize}; @@ -14,10 +12,6 @@ impl FullReceipt for T where T: Receipt + Compact {} /// Abstraction of a receipt. pub trait Receipt: TxReceipt - + Clone - + fmt::Debug - + PartialEq - + Eq + Default + alloy_rlp::Encodable + alloy_rlp::Decodable From 4d3b35dbd24c3a5c6b1a4f7bd86b1451e8efafcc Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 21 Oct 2024 21:31:40 +0200 Subject: [PATCH 007/239] test(tx-pool): add unit test for `remove_sender_count` (#11939) --- crates/transaction-pool/src/pool/parked.rs | 64 +++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/parked.rs b/crates/transaction-pool/src/pool/parked.rs index b591fdb539a9..407f04fd5be3 100644 --- a/crates/transaction-pool/src/pool/parked.rs +++ b/crates/transaction-pool/src/pool/parked.rs @@ -35,8 +35,8 @@ pub struct ParkedPool { best: BTreeSet>, /// Keeps track of last submission id for each sender. /// - /// This are sorted in Reverse order, so the last (highest) submission id is first, and the - /// lowest(oldest) is the last. + /// This are sorted in reverse order, so the last (highest) submission id is first, and the + /// lowest (oldest) is the last. last_sender_submission: BTreeSet, /// Keeps track of the number of transactions in the pool by the sender and the last submission /// id. @@ -856,4 +856,64 @@ mod tests { assert_eq!(submission_info2.sender_id, sender2); assert_eq!(submission_info2.submission_id, 2); } + + #[test] + fn test_remove_sender_count() { + // Initialize a mock transaction factory + let mut f = MockTransactionFactory::default(); + // Create an empty transaction pool + let mut pool = ParkedPool::>::default(); + // Generate two validated transactions and add them to the pool + let tx1 = f.validated_arc(MockTransaction::eip1559().inc_price()); + let tx2 = f.validated_arc(MockTransaction::eip1559().inc_price()); + pool.add_transaction(tx1); + pool.add_transaction(tx2); + + // Define two different sender IDs and their corresponding submission IDs + let sender1: SenderId = 11.into(); + let sender2: SenderId = 22.into(); + + // Add the sender counts to the pool + pool.add_sender_count(sender1, 1); + + // We add sender 2 multiple times to test the removal of sender counts + pool.add_sender_count(sender2, 2); + pool.add_sender_count(sender2, 3); + + // Before removing the sender count we should have 4 sender transaction counts + assert_eq!(pool.sender_transaction_count.len(), 4); + assert!(pool.sender_transaction_count.contains_key(&sender1)); + + // We should have 1 sender transaction count for sender 1 before removing the sender count + assert_eq!(pool.sender_transaction_count.get(&sender1).unwrap().count, 1); + + // Remove the sender count for sender 1 + pool.remove_sender_count(sender1); + + // After removing the sender count we should have 3 sender transaction counts remaining + assert_eq!(pool.sender_transaction_count.len(), 3); + assert!(!pool.sender_transaction_count.contains_key(&sender1)); + + // Check the sender transaction count for sender 2 before removing the sender count + assert_eq!( + *pool.sender_transaction_count.get(&sender2).unwrap(), + SenderTransactionCount { count: 2, last_submission_id: 3 } + ); + + // Remove the sender count for sender 2 + pool.remove_sender_count(sender2); + + // After removing the sender count for sender 2, we still have 3 sender transaction counts + // remaining. + // + // This is because we added sender 2 multiple times and we only removed the last submission. + assert_eq!(pool.sender_transaction_count.len(), 3); + assert!(pool.sender_transaction_count.contains_key(&sender2)); + + // Sender transaction count for sender 2 should be updated correctly + assert_eq!( + *pool.sender_transaction_count.get(&sender2).unwrap(), + SenderTransactionCount { count: 1, last_submission_id: 3 } + ); + } } From 387b0f8b361f0c42cee92cc598232fdf344487b9 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 21 Oct 2024 22:14:18 +0200 Subject: [PATCH 008/239] refactor(tx-pool): small refactor for `contains_conflicting_transaction` (#11935) --- crates/transaction-pool/src/pool/txpool.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index a85a9a1856b9..e5857d564571 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -1405,12 +1405,9 @@ impl AllTransactions { /// Caution: This assumes that mutually exclusive invariant is always true for the same sender. #[inline] fn contains_conflicting_transaction(&self, tx: &ValidPoolTransaction) -> bool { - let mut iter = self.txs_iter(tx.transaction_id.sender); - if let Some((_, existing)) = iter.next() { - return tx.tx_type_conflicts_with(&existing.transaction) - } - // no existing transaction for this sender - false + self.txs_iter(tx.transaction_id.sender) + .next() + .map_or(false, |(_, existing)| tx.tx_type_conflicts_with(&existing.transaction)) } /// Additional checks for a new transaction. From cc895e705249197778c2b467af603b6f8e0f7641 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Tue, 22 Oct 2024 00:14:48 +0400 Subject: [PATCH 009/239] fix: always accept transactions with current nonce (#11931) --- crates/transaction-pool/src/pool/txpool.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index e5857d564571..4fbec1105aa8 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -1421,11 +1421,15 @@ impl AllTransactions { fn ensure_valid( &self, transaction: ValidPoolTransaction, + on_chain_nonce: u64, ) -> Result, InsertErr> { if !self.local_transactions_config.is_local(transaction.origin, transaction.sender()) { let current_txs = self.tx_counter.get(&transaction.sender_id()).copied().unwrap_or_default(); - if current_txs >= self.max_account_slots { + + // Reject transactions if sender's capacity is exceeded. + // If transaction's nonce matches on-chain nonce always let it through + if current_txs >= self.max_account_slots && transaction.nonce() > on_chain_nonce { return Err(InsertErr::ExceededSenderTransactionsCapacity { transaction: Arc::new(transaction), }) @@ -1592,7 +1596,7 @@ impl AllTransactions { ) -> InsertResult { assert!(on_chain_nonce <= transaction.nonce(), "Invalid transaction"); - let mut transaction = self.ensure_valid(transaction)?; + let mut transaction = self.ensure_valid(transaction, on_chain_nonce)?; let inserted_tx_id = *transaction.id(); let mut state = TxState::default(); @@ -2631,6 +2635,7 @@ mod tests { let mut pool = AllTransactions::default(); let mut tx = MockTransaction::eip1559(); + let unblocked_tx = tx.clone(); for _ in 0..pool.max_account_slots { tx = tx.next(); pool.insert_tx(f.validated(tx.clone()), on_chain_balance, on_chain_nonce).unwrap(); @@ -2644,6 +2649,10 @@ mod tests { let err = pool.insert_tx(f.validated(tx.next()), on_chain_balance, on_chain_nonce).unwrap_err(); assert!(matches!(err, InsertErr::ExceededSenderTransactionsCapacity { .. })); + + assert!(pool + .insert_tx(f.validated(unblocked_tx), on_chain_balance, on_chain_nonce) + .is_ok()); } #[test] From 88a38de4d74cb8f01de405b67a65b098580434f0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 21 Oct 2024 22:46:16 +0200 Subject: [PATCH 010/239] chore(rpc): relax some types (#11946) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 11 ++--- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 2 +- crates/rpc/rpc-eth-types/src/cache/db.rs | 50 ++++++++++----------- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 64017f9f8fc5..0acf6646294f 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -655,14 +655,14 @@ pub trait Call: LoadState + SpawnBlocking { /// Returns the index of the target transaction in the given iterator. fn replay_transactions_until<'a, DB, I>( &self, - db: &mut CacheDB, + db: &mut DB, cfg: CfgEnvWithHandlerCfg, block_env: BlockEnv, transactions: I, target_tx_hash: B256, ) -> Result where - DB: DatabaseRef, + DB: Database + DatabaseCommit, EthApiError: From, I: IntoIterator, { @@ -929,14 +929,15 @@ pub trait Call: LoadState + SpawnBlocking { /// Executes the requests again after an out of gas error to check if the error is gas related /// or not #[inline] - fn map_out_of_gas_err( + fn map_out_of_gas_err( &self, env_gas_limit: U256, mut env: EnvWithHandlerCfg, - db: &mut CacheDB>, + db: &mut DB, ) -> Self::Error where - S: StateProvider, + DB: Database, + EthApiError: From, { let req_gas_limit = env.tx.gas_limit; env.tx.gas_limit = env_gas_limit.try_into().unwrap_or(u64::MAX); diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 981de8fa6c45..64056148cd38 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -117,7 +117,7 @@ pub trait Trace: LoadState { self.spawn_with_state_at_block(at, move |state| { let mut db = CacheDB::new(StateProviderDatabase::new(state)); let mut inspector = TracingInspector::new(config); - let (res, _) = this.inspect(StateCacheDbRefMutWrapper(&mut db), env, &mut inspector)?; + let (res, _) = this.inspect(&mut db, env, &mut inspector)?; f(inspector, res, db) }) } diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 7422dcfb8a7b..627fd2b2df72 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -114,6 +114,13 @@ impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> { self.0.block_hash(block_number) } + fn convert_block_hash( + &self, + hash_or_number: alloy_rpc_types::BlockHashOrNumber, + ) -> reth_errors::ProviderResult> { + self.0.convert_block_hash(hash_or_number) + } + fn canonical_hashes_range( &self, start: alloy_primitives::BlockNumber, @@ -121,21 +128,22 @@ impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> { ) -> reth_errors::ProviderResult> { self.0.canonical_hashes_range(start, end) } +} - fn convert_block_hash( +impl StateProvider for StateProviderTraitObjWrapper<'_> { + fn storage( &self, - hash_or_number: alloy_rpc_types::BlockHashOrNumber, - ) -> reth_errors::ProviderResult> { - self.0.convert_block_hash(hash_or_number) + account: revm_primitives::Address, + storage_key: alloy_primitives::StorageKey, + ) -> reth_errors::ProviderResult> { + self.0.storage(account, storage_key) } -} -impl StateProvider for StateProviderTraitObjWrapper<'_> { - fn account_balance( + fn bytecode_by_hash( &self, - addr: revm_primitives::Address, - ) -> reth_errors::ProviderResult> { - self.0.account_balance(addr) + code_hash: B256, + ) -> reth_errors::ProviderResult> { + self.0.bytecode_by_hash(code_hash) } fn account_code( @@ -145,26 +153,18 @@ impl StateProvider for StateProviderTraitObjWrapper<'_> { self.0.account_code(addr) } - fn account_nonce( + fn account_balance( &self, addr: revm_primitives::Address, - ) -> reth_errors::ProviderResult> { - self.0.account_nonce(addr) - } - - fn bytecode_by_hash( - &self, - code_hash: B256, - ) -> reth_errors::ProviderResult> { - self.0.bytecode_by_hash(code_hash) + ) -> reth_errors::ProviderResult> { + self.0.account_balance(addr) } - fn storage( + fn account_nonce( &self, - account: revm_primitives::Address, - storage_key: alloy_primitives::StorageKey, - ) -> reth_errors::ProviderResult> { - self.0.storage(account, storage_key) + addr: revm_primitives::Address, + ) -> reth_errors::ProviderResult> { + self.0.account_nonce(addr) } } From f2ac547666b3b79b20597571530768e7a4a2dc09 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 22 Oct 2024 00:02:44 +0300 Subject: [PATCH 011/239] Add custom beacon withdrawals example (#9497) Co-authored-by: Federico Gimenez --- Cargo.lock | 33 +- Cargo.toml | 1 + examples/custom-beacon-withdrawals/Cargo.toml | 26 ++ .../custom-beacon-withdrawals/src/main.rs | 286 ++++++++++++++++++ 4 files changed, 338 insertions(+), 8 deletions(-) create mode 100644 examples/custom-beacon-withdrawals/Cargo.toml create mode 100644 examples/custom-beacon-withdrawals/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 19287e6669a5..9082aaeb2749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,9 +609,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2395336745358cc47207442127c47c63801a7065ecc0aa928da844f8bb5576" +checksum = "b0900b83f4ee1f45c640ceee596afbc118051921b9438fdb5a3175c1a7e05f8b" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -623,9 +623,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed5047c9a241df94327879c2b0729155b58b941eae7805a7ada2e19436e6b39" +checksum = "a41b1e78dde06b5e12e6702fa8c1d30621bf07728ba75b801fb801c9c6a0ba10" dependencies = [ "alloy-sol-macro-input", "const-hex", @@ -641,9 +641,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dee02a81f529c415082235129f0df8b8e60aa1601b9c9298ffe54d75f57210b" +checksum = "91dc311a561a306664393407b88d3e53ae58581624128afd8a15faa5de3627dc" dependencies = [ "const-hex", "dunce", @@ -2783,6 +2783,23 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "example-custom-beacon-withdrawals" +version = "0.0.0" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-sol-macro", + "alloy-sol-types", + "eyre", + "reth", + "reth-chainspec", + "reth-evm", + "reth-evm-ethereum", + "reth-node-ethereum", + "reth-primitives", +] + [[package]] name = "example-custom-dev-node" version = "0.0.0" @@ -10355,9 +10372,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfc1bfd06acc78f16d8fd3ef846bc222ee7002468d10a7dce8d703d6eab89a3" +checksum = "9d5e0c2ea8db64b2898b62ea2fbd60204ca95e0b2c6bdf53ff768bbe916fbe4d" dependencies = [ "paste", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 5b6912c33f27..b32e2d0dfbe0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,6 +145,7 @@ members = [ "examples/rpc-db/", "examples/stateful-precompile/", "examples/txpool-tracing/", + "examples/custom-beacon-withdrawals", "testing/ef-tests/", "testing/testing-utils", ] diff --git a/examples/custom-beacon-withdrawals/Cargo.toml b/examples/custom-beacon-withdrawals/Cargo.toml new file mode 100644 index 000000000000..c396ca11df8b --- /dev/null +++ b/examples/custom-beacon-withdrawals/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "example-custom-beacon-withdrawals" +version = "0.0.0" +publish = false +edition.workspace = true +license.workspace = true + +[dependencies] +reth.workspace = true +reth-node-ethereum.workspace = true +reth-evm-ethereum.workspace = true +reth-chainspec.workspace = true +reth-evm.workspace = true +reth-primitives.workspace = true + +alloy-sol-macro = "0.8.9" +alloy-sol-types.workspace = true +alloy-eips.workspace = true +alloy-consensus.workspace = true + +eyre.workspace = true + +[features] +optimism = [ + "reth-primitives/optimism" +] \ No newline at end of file diff --git a/examples/custom-beacon-withdrawals/src/main.rs b/examples/custom-beacon-withdrawals/src/main.rs new file mode 100644 index 000000000000..09dad2f7007d --- /dev/null +++ b/examples/custom-beacon-withdrawals/src/main.rs @@ -0,0 +1,286 @@ +//! Example for how to modify a block post-execution step. It credits beacon withdrawals with a +//! custom mechanism instead of minting native tokens + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +use alloy_eips::eip7685::Requests; +use alloy_sol_macro::sol; +use alloy_sol_types::SolCall; +#[cfg(feature = "optimism")] +use reth::revm::primitives::OptimismFields; +use reth::{ + api::{ConfigureEvm, ConfigureEvmEnv, NodeTypesWithEngine}, + builder::{components::ExecutorBuilder, BuilderContext, FullNodeTypes}, + cli::Cli, + providers::ProviderError, + revm::{ + interpreter::Host, + primitives::{Env, TransactTo, TxEnv}, + Database, DatabaseCommit, Evm, State, + }, +}; +use reth_chainspec::{ChainSpec, EthereumHardforks}; +use reth_evm::execute::{ + BlockExecutionError, BlockExecutionStrategy, BlockExecutionStrategyFactory, ExecuteOutput, + InternalBlockExecutionError, +}; +use reth_evm_ethereum::EthEvmConfig; +use reth_node_ethereum::{node::EthereumAddOns, BasicBlockExecutorProvider, EthereumNode}; +use reth_primitives::{ + revm_primitives::{ + address, Address, BlockEnv, Bytes, CfgEnvWithHandlerCfg, EnvWithHandlerCfg, U256, + }, + BlockWithSenders, Receipt, Withdrawal, +}; +use std::{fmt::Display, sync::Arc}; + +pub const SYSTEM_ADDRESS: Address = address!("fffffffffffffffffffffffffffffffffffffffe"); +pub const WITHDRAWALS_ADDRESS: Address = address!("4200000000000000000000000000000000000000"); + +fn main() { + Cli::parse_args() + .run(|builder, _| async move { + let handle = builder + // use the default ethereum node types + .with_types::() + // Configure the components of the node + // use default ethereum components but use our custom pool + .with_components( + EthereumNode::components().executor(CustomExecutorBuilder::default()), + ) + .with_add_ons(EthereumAddOns::default()) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) + .unwrap(); +} + +/// A custom executor builder +#[derive(Debug, Default, Clone, Copy)] +#[non_exhaustive] +pub struct CustomExecutorBuilder; + +impl ExecutorBuilder for CustomExecutorBuilder +where + Types: NodeTypesWithEngine, + Node: FullNodeTypes, +{ + type EVM = EthEvmConfig; + type Executor = BasicBlockExecutorProvider; + + async fn build_evm( + self, + ctx: &BuilderContext, + ) -> eyre::Result<(Self::EVM, Self::Executor)> { + let chain_spec = ctx.chain_spec(); + let evm_config = EthEvmConfig::new(ctx.chain_spec()); + let strategy_factory = + CustomExecutorStrategyFactory { chain_spec, evm_config: evm_config.clone() }; + let executor = BasicBlockExecutorProvider::new(strategy_factory); + + Ok((evm_config, executor)) + } +} + +#[derive(Clone)] +pub struct CustomExecutorStrategyFactory { + /// The chainspec + chain_spec: Arc, + /// How to create an EVM. + evm_config: EthEvmConfig, +} + +impl BlockExecutionStrategyFactory for CustomExecutorStrategyFactory { + type Strategy + Display>> = CustomExecutorStrategy; + + fn create_strategy(&self, db: DB) -> Self::Strategy + where + DB: Database + Display>, + { + let state = + State::builder().with_database(db).with_bundle_update().without_state_clear().build(); + CustomExecutorStrategy { + state, + chain_spec: self.chain_spec.clone(), + evm_config: self.evm_config.clone(), + } + } +} + +pub struct CustomExecutorStrategy +where + DB: Database + Display>, +{ + /// The chainspec + chain_spec: Arc, + /// How to create an EVM. + evm_config: EthEvmConfig, + /// Current state for block execution. + state: State, +} + +impl CustomExecutorStrategy +where + DB: Database + Display>, +{ + /// Configures a new evm configuration and block environment for the given block. + /// + /// # Caution + /// + /// This does not initialize the tx environment. + fn evm_env_for_block( + &self, + header: &alloy_consensus::Header, + total_difficulty: U256, + ) -> EnvWithHandlerCfg { + let mut cfg = CfgEnvWithHandlerCfg::new(Default::default(), Default::default()); + let mut block_env = BlockEnv::default(); + self.evm_config.fill_cfg_and_block_env(&mut cfg, &mut block_env, header, total_difficulty); + + EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()) + } +} + +impl BlockExecutionStrategy for CustomExecutorStrategy +where + DB: Database + Display>, +{ + type Error = BlockExecutionError; + + fn apply_pre_execution_changes( + &mut self, + block: &BlockWithSenders, + _total_difficulty: U256, + ) -> Result<(), Self::Error> { + // Set state clear flag if the block is after the Spurious Dragon hardfork. + let state_clear_flag = + (*self.chain_spec).is_spurious_dragon_active_at_block(block.header.number); + self.state.set_state_clear_flag(state_clear_flag); + + Ok(()) + } + + fn execute_transactions( + &mut self, + _block: &BlockWithSenders, + _total_difficulty: U256, + ) -> Result { + Ok(ExecuteOutput { receipts: vec![], gas_used: 0 }) + } + + fn apply_post_execution_changes( + &mut self, + block: &BlockWithSenders, + total_difficulty: U256, + _receipts: &[Receipt], + ) -> Result { + let env = self.evm_env_for_block(&block.header, total_difficulty); + let mut evm = self.evm_config.evm_with_env(&mut self.state, env); + + if let Some(withdrawals) = block.body.withdrawals.as_ref() { + apply_withdrawals_contract_call(withdrawals, &mut evm)?; + } + + Ok(Requests::default()) + } + + fn state_ref(&self) -> &State { + &self.state + } + + fn state_mut(&mut self) -> &mut State { + &mut self.state + } +} + +sol!( + function withdrawals( + uint64[] calldata amounts, + address[] calldata addresses + ); +); + +/// Applies the post-block call to the withdrawal / deposit contract, using the given block, +/// [`ChainSpec`], EVM. +pub fn apply_withdrawals_contract_call( + withdrawals: &[Withdrawal], + evm: &mut Evm<'_, EXT, DB>, +) -> Result<(), BlockExecutionError> +where + DB::Error: std::fmt::Display, +{ + // get previous env + let previous_env = Box::new(evm.context.env().clone()); + + // modify env for pre block call + fill_tx_env_with_system_contract_call( + &mut evm.context.evm.env, + SYSTEM_ADDRESS, + WITHDRAWALS_ADDRESS, + withdrawalsCall { + amounts: withdrawals.iter().map(|w| w.amount).collect::>(), + addresses: withdrawals.iter().map(|w| w.address).collect::>(), + } + .abi_encode() + .into(), + ); + + let mut state = match evm.transact() { + Ok(res) => res.state, + Err(e) => { + evm.context.evm.env = previous_env; + return Err(BlockExecutionError::Internal(InternalBlockExecutionError::Other( + format!("withdrawal contract system call revert: {}", e).into(), + ))) + } + }; + + // Clean-up post system tx context + state.remove(&SYSTEM_ADDRESS); + state.remove(&evm.block().coinbase); + evm.context.evm.db.commit(state); + // re-set the previous env + evm.context.evm.env = previous_env; + + Ok(()) +} + +fn fill_tx_env_with_system_contract_call( + env: &mut Env, + caller: Address, + contract: Address, + data: Bytes, +) { + env.tx = TxEnv { + caller, + transact_to: TransactTo::Call(contract), + // Explicitly set nonce to None so revm does not do any nonce checks + nonce: None, + gas_limit: 30_000_000, + value: U256::ZERO, + data, + // Setting the gas price to zero enforces that no value is transferred as part of the call, + // and that the call will not count against the block's gas limit + gas_price: U256::ZERO, + // The chain ID check is not relevant here and is disabled if set to None + chain_id: None, + // Setting the gas priority fee to None ensures the effective gas price is derived from the + // `gas_price` field, which we need to be zero + gas_priority_fee: None, + access_list: Vec::new(), + // blob fields can be None for this tx + blob_hashes: Vec::new(), + max_fee_per_blob_gas: None, + authorization_list: None, + #[cfg(feature = "optimism")] + optimism: OptimismFields::default(), + }; + + // ensure the block gas limit is >= the tx + env.block.gas_limit = U256::from(env.tx.gas_limit); + + // disable the base fee check for this call by setting the base fee to zero + env.block.basefee = U256::ZERO; +} From 51594c9a68e437075f0aa0bfc13491dd82176d7c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 21 Oct 2024 23:36:44 +0200 Subject: [PATCH 012/239] chore: relax payload traits (#11947) --- crates/e2e-test-utils/src/payload.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/e2e-test-utils/src/payload.rs b/crates/e2e-test-utils/src/payload.rs index 946d9af57536..a5e4f56ac429 100644 --- a/crates/e2e-test-utils/src/payload.rs +++ b/crates/e2e-test-utils/src/payload.rs @@ -1,20 +1,20 @@ use futures_util::StreamExt; -use reth::api::{BuiltPayload, EngineTypes, PayloadBuilderAttributes}; +use reth::api::{BuiltPayload, PayloadBuilderAttributes}; use reth_payload_builder::{PayloadBuilderHandle, PayloadId}; -use reth_payload_primitives::{Events, PayloadBuilder}; +use reth_payload_primitives::{Events, PayloadBuilder, PayloadTypes}; use tokio_stream::wrappers::BroadcastStream; /// Helper for payload operations #[derive(Debug)] -pub struct PayloadTestContext { - pub payload_event_stream: BroadcastStream>, - payload_builder: PayloadBuilderHandle, +pub struct PayloadTestContext { + pub payload_event_stream: BroadcastStream>, + payload_builder: PayloadBuilderHandle, pub timestamp: u64, } -impl PayloadTestContext { +impl PayloadTestContext { /// Creates a new payload helper - pub async fn new(payload_builder: PayloadBuilderHandle) -> eyre::Result { + pub async fn new(payload_builder: PayloadBuilderHandle) -> eyre::Result { let payload_events = payload_builder.subscribe().await?; let payload_event_stream = payload_events.into_stream(); // Cancun timestamp @@ -24,10 +24,10 @@ impl PayloadTestContext { /// Creates a new payload job from static attributes pub async fn new_payload( &mut self, - attributes_generator: impl Fn(u64) -> E::PayloadBuilderAttributes, - ) -> eyre::Result { + attributes_generator: impl Fn(u64) -> T::PayloadBuilderAttributes, + ) -> eyre::Result { self.timestamp += 1; - let attributes: E::PayloadBuilderAttributes = attributes_generator(self.timestamp); + let attributes = attributes_generator(self.timestamp); self.payload_builder.send_new_payload(attributes.clone()).await.unwrap()?; Ok(attributes) } @@ -35,10 +35,10 @@ impl PayloadTestContext { /// Asserts that the next event is a payload attributes event pub async fn expect_attr_event( &mut self, - attrs: E::PayloadBuilderAttributes, + attrs: T::PayloadBuilderAttributes, ) -> eyre::Result<()> { let first_event = self.payload_event_stream.next().await.unwrap()?; - if let reth::payload::Events::Attributes(attr) = first_event { + if let Events::Attributes(attr) = first_event { assert_eq!(attrs.timestamp(), attr.timestamp()); } else { panic!("Expect first event as payload attributes.") @@ -59,9 +59,9 @@ impl PayloadTestContext { } /// Expects the next event to be a built payload event or panics - pub async fn expect_built_payload(&mut self) -> eyre::Result { + pub async fn expect_built_payload(&mut self) -> eyre::Result { let second_event = self.payload_event_stream.next().await.unwrap()?; - if let reth::payload::Events::BuiltPayload(payload) = second_event { + if let Events::BuiltPayload(payload) = second_event { Ok(payload) } else { panic!("Expect a built payload event."); From 2973f0c3e80705d0303a62874ec4fffadf3472aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:33:47 +0200 Subject: [PATCH 013/239] chore(deps): bump dawidd6/action-homebrew-bump-formula from 3 to 4 (#11951) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-dist.yml b/.github/workflows/release-dist.yml index 2142360e0396..f7df80e81f9f 100644 --- a/.github/workflows/release-dist.yml +++ b/.github/workflows/release-dist.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Update Homebrew formula - uses: dawidd6/action-homebrew-bump-formula@v3 + uses: dawidd6/action-homebrew-bump-formula@v4 with: token: ${{ secrets.HOMEBREW }} no_fork: true From e52f647644cb10fe6d310428d75ac3045140f311 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:34:53 +0200 Subject: [PATCH 014/239] chore(primitives): use alloy eth conversion constants (#11892) --- Cargo.lock | 3 ++ crates/consensus/common/src/calc.rs | 4 +-- crates/ethereum/evm/src/execute.rs | 6 ++-- crates/evm/Cargo.toml | 36 ++++++++++--------- crates/evm/src/state_change.rs | 2 +- crates/exex/exex/src/backfill/test_utils.rs | 5 ++- crates/node/events/Cargo.toml | 1 + crates/node/events/src/node.rs | 3 +- crates/primitives-traits/src/constants/mod.rs | 9 ----- crates/revm/Cargo.toml | 27 +++++++------- crates/rpc/rpc-eth-types/src/gas_oracle.rs | 3 +- crates/rpc/rpc-eth-types/src/revm_utils.rs | 2 +- 12 files changed, 50 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9082aaeb2749..37135dfd5459 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7441,6 +7441,7 @@ dependencies = [ name = "reth-evm" version = "1.1.0" dependencies = [ + "alloy-consensus", "alloy-eips", "alloy-primitives", "auto_impl", @@ -8053,6 +8054,7 @@ dependencies = [ name = "reth-node-events" version = "1.1.0" dependencies = [ + "alloy-consensus", "alloy-primitives", "alloy-rpc-types-engine", "futures", @@ -8590,6 +8592,7 @@ dependencies = [ name = "reth-revm" version = "1.1.0" dependencies = [ + "alloy-consensus", "alloy-eips", "alloy-primitives", "reth-ethereum-forks", diff --git a/crates/consensus/common/src/calc.rs b/crates/consensus/common/src/calc.rs index 3f519332fec0..e30c5b715f5a 100644 --- a/crates/consensus/common/src/calc.rs +++ b/crates/consensus/common/src/calc.rs @@ -1,6 +1,6 @@ +use alloy_consensus::constants::ETH_TO_WEI; use alloy_primitives::{BlockNumber, U256}; use reth_chainspec::{EthereumHardfork, Hardforks}; -use reth_primitives::constants::ETH_TO_WEI; /// Calculates the base block reward. /// @@ -57,7 +57,7 @@ pub fn base_block_reward_pre_merge(chain_spec: impl Hardforks, block_number: Blo /// ``` /// # use reth_chainspec::MAINNET; /// # use reth_consensus_common::calc::{base_block_reward, block_reward}; -/// # use reth_primitives::constants::ETH_TO_WEI; +/// # use alloy_consensus::constants::ETH_TO_WEI; /// # use alloy_primitives::U256; /// # /// // This is block 126 on mainnet. diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index c62949902aea..f082a3a707ea 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -296,7 +296,7 @@ impl EthExecutorProvider { #[cfg(test)] mod tests { use super::*; - use alloy_consensus::{Header, TxLegacy}; + use alloy_consensus::{constants::ETH_TO_WEI, Header, TxLegacy}; use alloy_eips::{ eip2935::{HISTORY_STORAGE_ADDRESS, HISTORY_STORAGE_CODE}, eip4788::{BEACON_ROOTS_ADDRESS, BEACON_ROOTS_CODE, SYSTEM_ADDRESS}, @@ -309,9 +309,7 @@ mod tests { BasicBlockExecutorProvider, BatchExecutor, BlockExecutorProvider, Executor, }; use reth_execution_types::BlockExecutionOutput; - use reth_primitives::{ - constants::ETH_TO_WEI, public_key_to_address, Account, Block, BlockBody, Transaction, - }; + use reth_primitives::{public_key_to_address, Account, Block, BlockBody, Transaction}; use reth_revm::{ database::StateProviderDatabase, test_utils::StateProviderTest, TransitionState, }; diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index 6a1e1fe0d727..d97a57864193 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -39,27 +39,29 @@ parking_lot = { workspace = true, optional = true } [dev-dependencies] parking_lot.workspace = true reth-ethereum-forks.workspace = true +alloy-consensus.workspace = true [features] default = ["std"] std = [ - "dep:metrics", - "dep:reth-metrics", - "reth-consensus/std", - "reth-primitives/std", - "reth-primitives-traits/std", - "reth-revm/std", - "alloy-eips/std", - "alloy-primitives/std", - "revm-primitives/std", - "revm/std" + "dep:metrics", + "dep:reth-metrics", + "reth-consensus/std", + "reth-primitives/std", + "reth-primitives-traits/std", + "reth-revm/std", + "alloy-eips/std", + "alloy-primitives/std", + "alloy-consensus/std", + "revm-primitives/std", + "revm/std", ] test-utils = [ - "dep:parking_lot", - "reth-chainspec/test-utils", - "reth-consensus/test-utils", - "reth-primitives/test-utils", - "reth-primitives-traits/test-utils", - "reth-revm/test-utils", - "revm/test-utils" + "dep:parking_lot", + "reth-chainspec/test-utils", + "reth-consensus/test-utils", + "reth-primitives/test-utils", + "reth-primitives-traits/test-utils", + "reth-revm/test-utils", + "revm/test-utils", ] diff --git a/crates/evm/src/state_change.rs b/crates/evm/src/state_change.rs index 2d5209015271..2a3d93f94d9c 100644 --- a/crates/evm/src/state_change.rs +++ b/crates/evm/src/state_change.rs @@ -91,9 +91,9 @@ pub fn insert_post_block_withdrawals_balance_increments Date: Tue, 22 Oct 2024 12:35:09 +0200 Subject: [PATCH 015/239] fix: spawn network manager on test exex ctx (#11907) --- crates/exex/test-utils/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index 9b86da7c77a5..1f6ea75ce6d9 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -268,15 +268,16 @@ pub async fn test_exex_context_with_chain_spec( let network_manager = NetworkManager::new( NetworkConfigBuilder::new(SecretKey::new(&mut rand::thread_rng())) .with_unused_discovery_port() + .with_unused_listener_port() .build(provider_factory.clone()), ) .await?; let network = network_manager.handle().clone(); - - let (_, payload_builder) = NoopPayloadBuilderService::::new(); - let tasks = TaskManager::current(); let task_executor = tasks.executor(); + tasks.executor().spawn(network_manager); + + let (_, payload_builder) = NoopPayloadBuilderService::::new(); let components = NodeAdapter::, _>, _> { components: Components { From 4a68c5e2d49a4789fcccbce2ed645a40f86e4844 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:35:27 +0200 Subject: [PATCH 016/239] refactor(rpc): small refactor in `block_with_senders` (#11950) --- crates/rpc/rpc-eth-api/src/helpers/block.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 9993b477a662..9bf35d850af6 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -219,17 +219,17 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching - let maybe_pending = LoadPendingBlock::provider(self) + if let Some(pending_block) = LoadPendingBlock::provider(self) .pending_block_with_senders() - .map_err(Self::Error::from_eth_err)?; - return if maybe_pending.is_some() { - Ok(maybe_pending.map(Arc::new)) - } else { - // If no pending block from provider, try to get local pending block - return match self.local_pending_block().await? { - Some((block, _)) => Ok(Some(Arc::new(block))), - None => Ok(None), - }; + .map_err(Self::Error::from_eth_err)? + { + return Ok(Some(Arc::new(pending_block))); + } + + // If no pending block from provider, try to get local pending block + return match self.local_pending_block().await? { + Some((block, _)) => Ok(Some(Arc::new(block))), + None => Ok(None), }; } From 0df7f65f3d0e9ab5af616c8d5913d9d660113775 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Oct 2024 13:09:44 +0200 Subject: [PATCH 017/239] chore: serde 1.0.210 (#11963) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b32e2d0dfbe0..91e3ac6e5010 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -503,7 +503,7 @@ rand = "0.8.5" rayon = "1.7" rustc-hash = { version = "2.0", default-features = false } schnellru = "0.2" -serde = { version = "1.0", default-features = false } +serde = { version = "=1.0.210", default-features = false } serde_json = "1.0.94" serde_with = "3.3.0" sha2 = { version = "0.10", default-features = false } From 75b39bc2647b28a62dd583a56090305338264a4c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Oct 2024 13:28:27 +0200 Subject: [PATCH 018/239] chore: run clippy locked (#11964) --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1921859c2729..4723d8a4d57f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,10 +18,10 @@ jobs: matrix: include: - type: ethereum - args: --bin reth --workspace + args: --bin reth --workspace --locked features: "ethereum asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" - type: optimism - args: --bin op-reth --workspace + args: --bin op-reth --workspace --locked features: "optimism asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" - type: book args: --manifest-path book/sources/Cargo.toml --workspace --bins From e8205936daea4419191d68f0a8aa82517a025033 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:38:45 +0200 Subject: [PATCH 019/239] primitive-traits: rm unused `SELECTOR_LEN` (#11959) --- crates/primitives-traits/src/constants/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index 12ef300f6677..2874c596a8a4 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -10,9 +10,6 @@ pub use gas_units::{GIGAGAS, KILOGAS, MEGAGAS}; /// The client version: `reth/v{major}.{minor}.{patch}` pub const RETH_CLIENT_VERSION: &str = concat!("reth/v", env!("CARGO_PKG_VERSION")); -/// The first four bytes of the call data for a function call specifies the function to be called. -pub const SELECTOR_LEN: usize = 4; - /// An EPOCH is a series of 32 slots. pub const EPOCH_SLOTS: u64 = 32; From cab76f2083be680afa4ddcff44420334dd0e71a5 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 22 Oct 2024 20:39:08 +0900 Subject: [PATCH 020/239] fix(engine): run pruner after saving blocks (#11927) --- crates/engine/tree/src/persistence.rs | 38 +++++++++------------------ 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 25c1f0ed7030..f4650a047b4f 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -77,20 +77,22 @@ impl PersistenceService { } PersistenceAction::SaveBlocks(blocks, sender) => { let result = self.on_save_blocks(blocks)?; - if let Some(ref num_hash) = result { + let result_number = result.map(|r| r.number); + + // we ignore the error because the caller may or may not care about the result + let _ = sender.send(result); + + if let Some(block_number) = result_number { // send new sync metrics based on saved blocks let _ = self .sync_metrics_tx - .send(MetricEvent::SyncHeight { height: num_hash.number }); - } - // we ignore the error because the caller may or may not care about the result - let _ = sender.send(result); - } - PersistenceAction::PruneBefore(block_num, sender) => { - let res = self.prune_before(block_num)?; + .send(MetricEvent::SyncHeight { height: block_number }); - // we ignore the error because the caller may or may not care about the result - let _ = sender.send(res); + if self.pruner.is_pruning_needed(block_number) { + // We log `PrunerOutput` inside the `Pruner` + let _ = self.prune_before(block_number)?; + } + } } PersistenceAction::SaveFinalizedBlock(finalized_block) => { let provider = self.provider.database_provider_rw()?; @@ -175,10 +177,6 @@ pub enum PersistenceAction { /// static files. RemoveBlocksAbove(u64, oneshot::Sender>), - /// Prune associated block data before the given block number, according to already-configured - /// prune modes. - PruneBefore(u64, oneshot::Sender), - /// Update the persisted finalized block on disk SaveFinalizedBlock(u64), @@ -279,18 +277,6 @@ impl PersistenceHandle { ) -> Result<(), SendError> { self.send_action(PersistenceAction::RemoveBlocksAbove(block_num, tx)) } - - /// Tells the persistence service to remove block data before the given hash, according to the - /// configured prune config. - /// - /// The resulting [`PrunerOutput`] is returned in the receiver end of the sender argument. - pub fn prune_before( - &self, - block_num: u64, - tx: oneshot::Sender, - ) -> Result<(), SendError> { - self.send_action(PersistenceAction::PruneBefore(block_num, tx)) - } } #[cfg(test)] From 90aaad8285ea549734e978f17fcb7622fb137800 Mon Sep 17 00:00:00 2001 From: James Prestwich Date: Tue, 22 Oct 2024 07:53:39 -0400 Subject: [PATCH 021/239] opt: reduce allocs for `parse_deposits_from_receipts` (#11949) --- crates/ethereum/evm/src/eip6110.rs | 114 ++++++++++++++++++----------- 1 file changed, 73 insertions(+), 41 deletions(-) diff --git a/crates/ethereum/evm/src/eip6110.rs b/crates/ethereum/evm/src/eip6110.rs index 4cf1c6ae9da5..d57002081955 100644 --- a/crates/ethereum/evm/src/eip6110.rs +++ b/crates/ethereum/evm/src/eip6110.rs @@ -1,12 +1,18 @@ //! EIP-6110 deposit requests parsing use alloc::{string::ToString, vec::Vec}; use alloy_eips::eip6110::MAINNET_DEPOSIT_CONTRACT_ADDRESS; -use alloy_primitives::{Bytes, Log}; +use alloy_primitives::{Address, Bytes, Log}; use alloy_sol_types::{sol, SolEvent}; -use reth_chainspec::ChainSpec; +use reth_chainspec::{ChainSpec, EthChainSpec}; use reth_evm::execute::BlockValidationError; use reth_primitives::Receipt; +/// The size of a deposit request in bytes. While the event fields emit +/// bytestrings, those bytestrings are fixed size. The fields are: 48-byte +/// pubkey, 32-byte withdrawal credentials, 8-byte amount, 96-byte signature, +/// and 8-byte index. +const DEPOSIT_BYTES_SIZE: usize = 48 + 32 + 8 + 96 + 8; + sol! { #[allow(missing_docs)] event DepositEvent( @@ -18,53 +24,79 @@ sol! { ); } -/// Parse [deposit contract](https://etherscan.io/address/0x00000000219ab540356cbb839cbe05303d7705fa) -/// (address is from the passed [`ChainSpec`]) deposits from receipts, and return them as a -/// [vector](Vec) of (requests)[`alloy_eips::eip7685::Requests`]. -pub fn parse_deposits_from_receipts<'a, I>( - chain_spec: &ChainSpec, - receipts: I, -) -> Result -where - I: IntoIterator, -{ - let mut requests = Vec::new(); - let deposit_contract_address = chain_spec - .deposit_contract - .as_ref() - .map_or(MAINNET_DEPOSIT_CONTRACT_ADDRESS, |contract| contract.address); - let logs: Vec<_> = receipts - .into_iter() - .flat_map(|receipt| &receipt.logs) - // No need to filter for topic because there's only one event and that's the Deposit - // event in the deposit contract. - .filter(|log| log.address == deposit_contract_address) - .collect(); +/// Accumulate a deposit request from a log. containing a [`DepositEvent`]. +pub fn accumulate_deposit_from_log(log: &Log, out: &mut Vec) { + out.reserve(DEPOSIT_BYTES_SIZE); + out.extend_from_slice(log.pubkey.as_ref()); + out.extend_from_slice(log.withdrawal_credentials.as_ref()); + out.extend_from_slice(log.amount.as_ref()); + out.extend_from_slice(log.signature.as_ref()); + out.extend_from_slice(log.index.as_ref()); +} - for log in &logs { +/// Accumulate deposits from an iterator of logs. +pub fn accumulate_deposits_from_logs<'a>( + address: Address, + logs: impl IntoIterator, + out: &mut Vec, +) -> Result<(), BlockValidationError> { + logs.into_iter().filter(|log| log.address == address).try_for_each(|log| { + // We assume that the log is valid because it was emitted by the + // deposit contract. let decoded_log = DepositEvent::decode_log(log, false).map_err(|err: alloy_sol_types::Error| { BlockValidationError::DepositRequestDecode(err.to_string()) })?; - requests.extend(parse_deposit_from_log(&decoded_log).as_ref()) - } + accumulate_deposit_from_log(&decoded_log, out); + Ok(()) + }) +} + +/// Accumulate deposits from a receipt. Iterates over the logs in the receipt +/// and accumulates the deposit request bytestrings. +pub fn accumulate_deposits_from_receipt( + address: Address, + receipt: &Receipt, + out: &mut Vec, +) -> Result<(), BlockValidationError> { + accumulate_deposits_from_logs(address, &receipt.logs, out) +} - Ok(requests.into()) +/// Accumulate deposits from a list of receipts. Iterates over the logs in the +/// receipts and accumulates the deposit request bytestrings. +pub fn accumulate_deposits_from_receipts<'a, I>( + address: Address, + receipts: I, + out: &mut Vec, +) -> Result<(), BlockValidationError> +where + I: IntoIterator, +{ + receipts + .into_iter() + .try_for_each(|receipt| accumulate_deposits_from_receipt(address, receipt, out)) } -fn parse_deposit_from_log(log: &Log) -> Bytes { - // SAFETY: These `expect` https://github.com/ethereum/consensus-specs/blob/5f48840f4d768bf0e0a8156a3ed06ec333589007/solidity_deposit_contract/deposit_contract.sol#L107-L110 - // are safe because the `DepositEvent` is the only event in the deposit contract and the length - // checks are done there. - [ - log.pubkey.as_ref(), - log.withdrawal_credentials.as_ref(), - log.amount.as_ref(), - log.signature.as_ref(), - log.index.as_ref(), - ] - .concat() - .into() +/// Find deposit logs in a list of receipts, and return the concatenated +/// deposit request bytestring. +/// +/// The address of the deposit contract is taken from the chain spec, and +/// defaults to [`MAINNET_DEPOSIT_CONTRACT_ADDRESS`] if not specified in +/// the chain spec. +pub fn parse_deposits_from_receipts<'a, I>( + chainspec: &ChainSpec, + receipts: I, +) -> Result +where + I: IntoIterator, +{ + let mut out = Vec::new(); + accumulate_deposits_from_receipts( + chainspec.deposit_contract().map(|c| c.address).unwrap_or(MAINNET_DEPOSIT_CONTRACT_ADDRESS), + receipts, + &mut out, + )?; + Ok(out.into()) } #[cfg(test)] From df57aedba64af2cdc1b006792fb147454529bbe3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Oct 2024 14:34:03 +0200 Subject: [PATCH 022/239] chore: bump discv5 (#11966) --- Cargo.lock | 153 +++++++++++++++++++++-------------------------------- Cargo.toml | 2 +- 2 files changed, 62 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37135dfd5459..59c582fec3f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,7 +343,7 @@ dependencies = [ "futures", "futures-utils-wasm", "lru", - "parking_lot 0.12.3", + "parking_lot", "pin-project", "reqwest", "schnellru", @@ -2085,7 +2085,7 @@ dependencies = [ "bitflags 2.6.0", "crossterm_winapi", "libc", - "parking_lot 0.12.3", + "parking_lot", "winapi", ] @@ -2098,7 +2098,7 @@ dependencies = [ "bitflags 2.6.0", "crossterm_winapi", "mio 1.0.2", - "parking_lot 0.12.3", + "parking_lot", "rustix", "signal-hook", "signal-hook-mio", @@ -2255,7 +2255,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.10", + "parking_lot_core", ] [[package]] @@ -2269,7 +2269,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.10", + "parking_lot_core", "serde", ] @@ -2316,11 +2316,12 @@ dependencies = [ [[package]] name = "delay_map" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4355c25cbf99edcb6b4a0e906f6bdc6956eda149e84455bea49696429b2f8e8" +checksum = "df941644b671f05f59433e481ba0d31ac10e3667de725236a4c0d587c496fba1" dependencies = [ "futures", + "tokio", "tokio-util", ] @@ -2470,9 +2471,9 @@ dependencies = [ [[package]] name = "discv5" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f569b8c367554666c8652305621e8bae3634a2ff5c6378081d5bd8c399c99f23" +checksum = "23e6b70634e26c909d1edbb3142b3eaf3b89da0e52f284f00ca7c80d9901ad9e" dependencies = [ "aes", "aes-gcm", @@ -2491,13 +2492,13 @@ dependencies = [ "lru", "more-asserts", "multiaddr", - "parking_lot 0.11.2", + "parking_lot", "rand 0.8.5", "smallvec", - "socket2 0.4.10", + "socket2", "tokio", "tracing", - "uint", + "uint 0.10.0", "zeroize", ] @@ -3029,7 +3030,7 @@ dependencies = [ "alloy-genesis", "alloy-primitives", "eyre", - "parking_lot 0.12.3", + "parking_lot", "reth", "reth-chainspec", "reth-node-api", @@ -3528,9 +3529,9 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.8.4" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ "hashbrown 0.14.5", ] @@ -3776,7 +3777,7 @@ dependencies = [ "http-body", "hyper", "pin-project-lite", - "socket2 0.5.7", + "socket2", "tokio", "tower-service", "tracing", @@ -4166,7 +4167,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.7", + "socket2", "widestring", "windows-sys 0.48.0", "winreg", @@ -4324,7 +4325,7 @@ dependencies = [ "http-body", "http-body-util", "jsonrpsee-types", - "parking_lot 0.12.3", + "parking_lot", "pin-project", "rand 0.8.5", "rustc-hash 2.0.0", @@ -4590,7 +4591,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall 0.5.7", + "redox_syscall", ] [[package]] @@ -5433,17 +5434,6 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.3" @@ -5451,21 +5441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core 0.9.10", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -5476,7 +5452,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.7", + "redox_syscall", "smallvec", "windows-targets 0.52.6", ] @@ -5706,7 +5682,7 @@ dependencies = [ "log", "nix", "once_cell", - "parking_lot 0.12.3", + "parking_lot", "smallvec", "symbolic-demangle", "tempfile", @@ -5785,7 +5761,7 @@ checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ "fixed-hash", "impl-codec", - "uint", + "uint 0.9.5", ] [[package]] @@ -5992,7 +5968,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.0.0", "rustls", - "socket2 0.5.7", + "socket2", "thiserror", "tokio", "tracing", @@ -6023,7 +5999,7 @@ checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", - "socket2 0.5.7", + "socket2", "tracing", "windows-sys 0.59.0", ] @@ -6180,15 +6156,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.7" @@ -6544,7 +6511,7 @@ dependencies = [ "assert_matches", "linked_hash_set", "metrics", - "parking_lot 0.12.3", + "parking_lot", "reth-blockchain-tree-api", "reth-chainspec", "reth-consensus", @@ -6594,7 +6561,7 @@ dependencies = [ "auto_impl", "derive_more 1.0.0", "metrics", - "parking_lot 0.12.3", + "parking_lot", "pin-project", "rand 0.8.5", "reth-chainspec", @@ -6843,7 +6810,7 @@ dependencies = [ "iai-callgrind", "metrics", "page_size", - "parking_lot 0.12.3", + "parking_lot", "paste", "pprof", "proptest", @@ -6951,7 +6918,7 @@ dependencies = [ "discv5", "enr", "generic-array", - "parking_lot 0.12.3", + "parking_lot", "rand 0.8.5", "reth-ethereum-forks", "reth-net-banlist", @@ -7001,7 +6968,7 @@ dependencies = [ "data-encoding", "enr", "linked_hash_set", - "parking_lot 0.12.3", + "parking_lot", "rand 0.8.5", "reth-chainspec", "reth-ethereum-forks", @@ -7447,7 +7414,7 @@ dependencies = [ "auto_impl", "futures-util", "metrics", - "parking_lot 0.12.3", + "parking_lot", "reth-chainspec", "reth-consensus", "reth-consensus-common", @@ -7531,7 +7498,7 @@ dependencies = [ "futures", "itertools 0.13.0", "metrics", - "parking_lot 0.12.3", + "parking_lot", "rand 0.8.5", "reth-blockchain-tree", "reth-chain-state", @@ -7675,7 +7642,7 @@ dependencies = [ "dashmap 6.1.0", "derive_more 1.0.0", "indexmap 2.6.0", - "parking_lot 0.12.3", + "parking_lot", "pprof", "rand 0.8.5", "rand_xorshift", @@ -7745,7 +7712,7 @@ dependencies = [ "futures", "itertools 0.13.0", "metrics", - "parking_lot 0.12.3", + "parking_lot", "pin-project", "pprof", "rand 0.8.5", @@ -7817,7 +7784,7 @@ dependencies = [ "auto_impl", "derive_more 1.0.0", "futures", - "parking_lot 0.12.3", + "parking_lot", "reth-consensus", "reth-eth-wire-types", "reth-network-peers", @@ -8090,7 +8057,7 @@ dependencies = [ "reth-metrics", "reth-provider", "reth-tasks", - "socket2 0.5.7", + "socket2", "tikv-jemalloc-ctl", "tokio", "tower 0.4.13", @@ -8238,7 +8205,7 @@ dependencies = [ "eyre", "op-alloy-consensus", "op-alloy-rpc-types-engine", - "parking_lot 0.12.3", + "parking_lot", "reth", "reth-auto-seal-consensus", "reth-basic-payload-builder", @@ -8327,7 +8294,7 @@ dependencies = [ "op-alloy-consensus", "op-alloy-network", "op-alloy-rpc-types", - "parking_lot 0.12.3", + "parking_lot", "reqwest", "reth-chainspec", "reth-evm", @@ -8505,7 +8472,7 @@ dependencies = [ "itertools 0.13.0", "metrics", "notify", - "parking_lot 0.12.3", + "parking_lot", "rand 0.8.5", "rayon", "reth-blockchain-tree-api", @@ -8635,7 +8602,7 @@ dependencies = [ "jsonrpsee", "jsonrpsee-types", "jsonwebtoken", - "parking_lot 0.12.3", + "parking_lot", "pin-project", "rand 0.8.5", "reth-chainspec", @@ -8822,7 +8789,7 @@ dependencies = [ "futures", "jsonrpsee", "jsonrpsee-types", - "parking_lot 0.12.3", + "parking_lot", "reth-chainspec", "reth-errors", "reth-evm", @@ -9035,7 +9002,7 @@ version = "1.1.0" dependencies = [ "alloy-primitives", "assert_matches", - "parking_lot 0.12.3", + "parking_lot", "rayon", "reth-db", "reth-db-api", @@ -9161,7 +9128,7 @@ dependencies = [ "criterion", "futures-util", "metrics", - "parking_lot 0.12.3", + "parking_lot", "paste", "pprof", "proptest", @@ -9997,7 +9964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b4b487fe2acf240a021cf57c6b2b4903b1e78ca0ecd862a71b71d2a51fed77d" dependencies = [ "once_cell", - "parking_lot 0.12.3", + "parking_lot", "scc", "serial_test_derive", ] @@ -10202,16 +10169,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.7" @@ -10685,10 +10642,10 @@ dependencies = [ "bytes", "libc", "mio 1.0.2", - "parking_lot 0.12.3", + "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.7", + "socket2", "tokio-macros", "windows-sys 0.52.0", ] @@ -11057,7 +11014,7 @@ dependencies = [ "ipconfig", "lru-cache", "once_cell", - "parking_lot 0.12.3", + "parking_lot", "rand 0.8.5", "resolv-conf", "serde", @@ -11118,6 +11075,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "uint" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909988d098b2f738727b161a106cfc7cab00c539c2687a8836f8e565976fb53e" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + [[package]] name = "unarray" version = "0.1.4" @@ -11413,7 +11382,7 @@ checksum = "c7ed9d8b15c7fb594d72bfb4b5a276f3d2029333cd93a932f376f5937f6f80ee" dependencies = [ "futures", "js-sys", - "parking_lot 0.12.3", + "parking_lot", "pin-utils", "slab", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 91e3ac6e5010..48a6525d5efd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -548,7 +548,7 @@ tower = "0.4" tower-http = "0.5" # p2p -discv5 = "0.7.0" +discv5 = "0.8.0" if-addrs = "0.13" # rpc From 60337d9614b25d20347599d590f83646b1de3e7f Mon Sep 17 00:00:00 2001 From: Francis Li Date: Tue, 22 Oct 2024 05:43:25 -0700 Subject: [PATCH 023/239] feat(rpc): Add flags to disable read tx timeout (#11856) --- book/cli/reth/db.md | 3 +++ book/cli/reth/db/diff.md | 3 +++ book/cli/reth/debug/build-block.md | 3 +++ book/cli/reth/debug/execution.md | 3 +++ book/cli/reth/debug/in-memory-merkle.md | 3 +++ book/cli/reth/debug/merkle.md | 3 +++ book/cli/reth/debug/replay-engine.md | 3 +++ book/cli/reth/import.md | 3 +++ book/cli/reth/init-state.md | 3 +++ book/cli/reth/init.md | 3 +++ book/cli/reth/node.md | 3 +++ book/cli/reth/p2p.md | 3 +++ book/cli/reth/prune.md | 3 +++ book/cli/reth/recover/storage-tries.md | 3 +++ book/cli/reth/stage/drop.md | 3 +++ book/cli/reth/stage/dump.md | 3 +++ book/cli/reth/stage/run.md | 3 +++ book/cli/reth/stage/unwind.md | 3 +++ crates/node/core/src/args/database.rs | 14 +++++++++++++- 19 files changed, 67 insertions(+), 1 deletion(-) diff --git a/book/cli/reth/db.md b/book/cli/reth/db.md index 9e3b32cc0b39..f9a8a158adc8 100644 --- a/book/cli/reth/db.md +++ b/book/cli/reth/db.md @@ -81,6 +81,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/book/cli/reth/db/diff.md b/book/cli/reth/db/diff.md index ea4c29612ff7..f57c6ac364fe 100644 --- a/book/cli/reth/db/diff.md +++ b/book/cli/reth/db/diff.md @@ -45,6 +45,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + --table The table name to diff. If not specified, all tables are diffed. diff --git a/book/cli/reth/debug/build-block.md b/book/cli/reth/debug/build-block.md index 76ddac306ce2..2e6d637d52c2 100644 --- a/book/cli/reth/debug/build-block.md +++ b/book/cli/reth/debug/build-block.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + --trusted-setup-file Overrides the KZG trusted setup by reading from the supplied file diff --git a/book/cli/reth/debug/execution.md b/book/cli/reth/debug/execution.md index 202e1452a8ae..9ca74897c5ea 100644 --- a/book/cli/reth/debug/execution.md +++ b/book/cli/reth/debug/execution.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Networking: -d, --disable-discovery Disable the discovery service diff --git a/book/cli/reth/debug/in-memory-merkle.md b/book/cli/reth/debug/in-memory-merkle.md index 534e6d46c69d..3e322a6913dd 100644 --- a/book/cli/reth/debug/in-memory-merkle.md +++ b/book/cli/reth/debug/in-memory-merkle.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Networking: -d, --disable-discovery Disable the discovery service diff --git a/book/cli/reth/debug/merkle.md b/book/cli/reth/debug/merkle.md index 19bc38acceb9..d701803b81ca 100644 --- a/book/cli/reth/debug/merkle.md +++ b/book/cli/reth/debug/merkle.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Networking: -d, --disable-discovery Disable the discovery service diff --git a/book/cli/reth/debug/replay-engine.md b/book/cli/reth/debug/replay-engine.md index 7a14b9cf09d4..dd587620a868 100644 --- a/book/cli/reth/debug/replay-engine.md +++ b/book/cli/reth/debug/replay-engine.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Networking: -d, --disable-discovery Disable the discovery service diff --git a/book/cli/reth/import.md b/book/cli/reth/import.md index 7bd8a0079ec8..28e085bda718 100644 --- a/book/cli/reth/import.md +++ b/book/cli/reth/import.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + --no-state Disables stages that require state. diff --git a/book/cli/reth/init-state.md b/book/cli/reth/init-state.md index cb221634c40e..ddcd3cece378 100644 --- a/book/cli/reth/init-state.md +++ b/book/cli/reth/init-state.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + JSONL file with state dump. diff --git a/book/cli/reth/init.md b/book/cli/reth/init.md index cc889e5e35aa..cd01accc0471 100644 --- a/book/cli/reth/init.md +++ b/book/cli/reth/init.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 34d32209ada9..4cd55db1fe08 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -590,6 +590,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Dev testnet: --dev Start the node in dev mode diff --git a/book/cli/reth/p2p.md b/book/cli/reth/p2p.md index 01253705b233..603b451d9405 100644 --- a/book/cli/reth/p2p.md +++ b/book/cli/reth/p2p.md @@ -247,6 +247,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/book/cli/reth/prune.md b/book/cli/reth/prune.md index e0641256f1c9..ed16197a76c3 100644 --- a/book/cli/reth/prune.md +++ b/book/cli/reth/prune.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/book/cli/reth/recover/storage-tries.md b/book/cli/reth/recover/storage-tries.md index 1f639cb095ae..ecdaabe77817 100644 --- a/book/cli/reth/recover/storage-tries.md +++ b/book/cli/reth/recover/storage-tries.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/book/cli/reth/stage/drop.md b/book/cli/reth/stage/drop.md index ae21a8918306..399b3818c28f 100644 --- a/book/cli/reth/stage/drop.md +++ b/book/cli/reth/stage/drop.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Possible values: - headers: The headers stage within the pipeline diff --git a/book/cli/reth/stage/dump.md b/book/cli/reth/stage/dump.md index 291d896902d3..4b3de3fb1cb5 100644 --- a/book/cli/reth/stage/dump.md +++ b/book/cli/reth/stage/dump.md @@ -76,6 +76,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/book/cli/reth/stage/run.md b/book/cli/reth/stage/run.md index bfe5ff9d6c63..9da3ce0deb6d 100644 --- a/book/cli/reth/stage/run.md +++ b/book/cli/reth/stage/run.md @@ -69,6 +69,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + --metrics Enable Prometheus metrics. diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index d181b3bcade6..700ab3d7e7ce 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -74,6 +74,9 @@ Database: [possible values: true, false] + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + Networking: -d, --disable-discovery Disable the discovery service diff --git a/crates/node/core/src/args/database.rs b/crates/node/core/src/args/database.rs index da96deb70c1b..0eec6639a117 100644 --- a/crates/node/core/src/args/database.rs +++ b/crates/node/core/src/args/database.rs @@ -1,12 +1,14 @@ //! clap [Args](clap::Args) for database configuration +use std::time::Duration; + use crate::version::default_client_version; use clap::{ builder::{PossibleValue, TypedValueParser}, error::ErrorKind, Arg, Args, Command, Error, }; -use reth_db::ClientVersion; +use reth_db::{mdbx::MaxReadTransactionDuration, ClientVersion}; use reth_storage_errors::db::LogLevel; /// Parameters for database configuration @@ -20,6 +22,9 @@ pub struct DatabaseArgs { /// NFS volume. #[arg(long = "db.exclusive")] pub exclusive: Option, + /// Read transaction timeout in seconds, 0 means no timeout. + #[arg(long = "db.read-transaction-timeout")] + pub read_transaction_timeout: Option, } impl DatabaseArgs { @@ -33,9 +38,16 @@ impl DatabaseArgs { &self, client_version: ClientVersion, ) -> reth_db::mdbx::DatabaseArguments { + let max_read_transaction_duration = match self.read_transaction_timeout { + None => None, // if not specified, use default value + Some(0) => Some(MaxReadTransactionDuration::Unbounded), // if 0, disable timeout + Some(secs) => Some(MaxReadTransactionDuration::Set(Duration::from_secs(secs))), + }; + reth_db::mdbx::DatabaseArguments::new(client_version) .with_log_level(self.log_level) .with_exclusive(self.exclusive) + .with_max_read_transaction_duration(max_read_transaction_duration) } } From b3015c75b1f56fdfbbcce8cdb97a3cde1c642985 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 22 Oct 2024 14:31:37 +0100 Subject: [PATCH 024/239] fix(trie): removing a blinded leaf should result in an error (#11869) Co-authored-by: Roman Krasiuk --- crates/trie/sparse/src/trie.rs | 68 ++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 6680d7e9b650..42f90d34c8a7 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -267,11 +267,11 @@ impl RevealedSparseTrie { /// Remove leaf node from the trie. pub fn remove_leaf(&mut self, path: &Nibbles) -> SparseTrieResult<()> { self.prefix_set.insert(path.clone()); - let existing = self.values.remove(path); - if existing.is_none() { - // trie structure unchanged, return immediately - return Ok(()) - } + self.values.remove(path); + + // If the path wasn't present in `values`, we still need to walk the trie and ensure that + // there is no node at the path. When a leaf node is a blinded `Hash`, it will have an entry + // in `nodes`, but not in the `values`. let mut removed_nodes = self.take_nodes_for_path(path)?; debug!(target: "trie::sparse", ?path, ?removed_nodes, "Removed nodes for path"); @@ -726,6 +726,7 @@ mod tests { use super::*; use alloy_primitives::U256; + use assert_matches::assert_matches; use itertools::Itertools; use prop::sample::SizeRange; use proptest::prelude::*; @@ -960,7 +961,7 @@ mod tests { pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), BTreeMap::from_iter([ - (Nibbles::new(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1101.into())), ( Nibbles::from_nibbles([0x5, 0x0]), @@ -972,11 +973,11 @@ mod tests { ), ( Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), - SparseNode::new_leaf(Nibbles::new()) + SparseNode::new_leaf(Nibbles::default()) ), ( Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), - SparseNode::new_leaf(Nibbles::new()) + SparseNode::new_leaf(Nibbles::default()) ), ( Nibbles::from_nibbles([0x5, 0x2]), @@ -1015,7 +1016,7 @@ mod tests { pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), BTreeMap::from_iter([ - (Nibbles::new(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), ( Nibbles::from_nibbles([0x5, 0x0]), @@ -1027,11 +1028,11 @@ mod tests { ), ( Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), - SparseNode::new_leaf(Nibbles::new()) + SparseNode::new_leaf(Nibbles::default()) ), ( Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), - SparseNode::new_leaf(Nibbles::new()) + SparseNode::new_leaf(Nibbles::default()) ), (Nibbles::from_nibbles([0x5, 0x3]), SparseNode::new_branch(0b1010.into())), ( @@ -1063,7 +1064,7 @@ mod tests { pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), BTreeMap::from_iter([ - (Nibbles::new(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), ( Nibbles::from_nibbles([0x5, 0x0]), @@ -1097,7 +1098,7 @@ mod tests { pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), BTreeMap::from_iter([ - (Nibbles::new(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), ( Nibbles::from_nibbles([0x5, 0x0]), @@ -1128,7 +1129,7 @@ mod tests { pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), BTreeMap::from_iter([ - (Nibbles::new(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), ( Nibbles::from_nibbles([0x5, 0x0]), @@ -1147,7 +1148,7 @@ mod tests { pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), BTreeMap::from_iter([( - Nibbles::new(), + Nibbles::default(), SparseNode::new_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2])) ),]) ); @@ -1157,7 +1158,42 @@ mod tests { // Empty pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), - BTreeMap::from_iter([(Nibbles::new(), SparseNode::Empty),]) + BTreeMap::from_iter([(Nibbles::default(), SparseNode::Empty),]) + ); + } + + #[test] + fn sparse_trie_remove_leaf_blinded() { + let mut sparse = RevealedSparseTrie::default(); + + let leaf = LeafNode::new( + Nibbles::default(), + alloy_rlp::encode_fixed_size(&U256::from(1)).to_vec(), + ); + + // Reveal a branch node and one of its children + // + // Branch (Mask = 11) + // ├── 0 -> Hash (Path = 0) + // └── 1 -> Leaf (Path = 1) + sparse + .reveal_node( + Nibbles::default(), + TrieNode::Branch(BranchNode::new( + vec![ + RlpNode::word_rlp(&B256::repeat_byte(1)), + RlpNode::from_raw_rlp(&alloy_rlp::encode(leaf.clone())).unwrap(), + ], + TrieMask::new(0b11), + )), + ) + .unwrap(); + sparse.reveal_node(Nibbles::from_nibbles([0x1]), TrieNode::Leaf(leaf)).unwrap(); + + // Removing a blinded leaf should result in an error + assert_matches!( + sparse.remove_leaf(&Nibbles::from_nibbles([0x0])), + Err(SparseTrieError::BlindedNode { path, hash }) if path == Nibbles::from_nibbles([0x0]) && hash == B256::repeat_byte(1) ); } From b20a2715514a320c8633356fb7430d7f37a847e9 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Tue, 22 Oct 2024 16:56:04 +0200 Subject: [PATCH 025/239] chore(tree): improved debug logging for block insertion (#11958) --- crates/engine/tree/src/tree/mod.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index a2abd3f531df..021b3149ad5d 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -259,6 +259,7 @@ impl TreeState { } } } + debug!(target: "engine::tree", ?upper_bound, ?last_persisted_hash, "Removed canonical blocks from the tree"); } /// Removes all blocks that are below the finalized block, as well as removing non-canonical @@ -1593,7 +1594,7 @@ where /// Returns an error if we failed to fetch the state from the database. fn state_provider(&self, hash: B256) -> ProviderResult> { if let Some((historical, blocks)) = self.state.tree_state.blocks_by_hash(hash) { - trace!(target: "engine::tree", %hash, "found canonical state for block in memory"); + debug!(target: "engine::tree", %hash, %historical, "found canonical state for block in memory"); // the block leads back to the canonical chain let historical = self.provider.state_by_block_hash(historical)?; return Ok(Some(Box::new(MemoryOverlayStateProvider::new(historical, blocks)))) @@ -1601,13 +1602,13 @@ where // the hash could belong to an unknown block or a persisted block if let Some(header) = self.provider.header(&hash)? { - trace!(target: "engine::tree", %hash, number = %header.number, "found canonical state for block in database"); + debug!(target: "engine::tree", %hash, number = %header.number, "found canonical state for block in database"); // the block is known and persisted let historical = self.provider.state_by_block_hash(hash)?; return Ok(Some(historical)) } - trace!(target: "engine::tree", %hash, "no canonical state found for block"); + debug!(target: "engine::tree", %hash, "no canonical state found for block"); Ok(None) } @@ -2137,7 +2138,8 @@ where &mut self, block: SealedBlockWithSenders, ) -> Result { - debug!(target: "engine::tree", block=?block.num_hash(), "Inserting new block into tree"); + debug!(target: "engine::tree", block=?block.num_hash(), parent = ?block.parent_hash, state_root = ?block.state_root, "Inserting new block into tree"); + if self.block_by_hash(block.hash())?.is_some() { return Ok(InsertPayloadOk2::AlreadySeen(BlockStatus2::Valid)) } @@ -2206,7 +2208,7 @@ where let hashed_state = HashedPostState::from_bundle_state(&output.state.state); - trace!(target: "engine::tree", block=?BlockNumHash::new(block_number, block_hash), "Calculating block state root"); + trace!(target: "engine::tree", block=?sealed_block.num_hash(), "Calculating block state root"); let root_time = Instant::now(); let mut state_root_result = None; @@ -2232,7 +2234,7 @@ where let (state_root, trie_output) = if let Some(result) = state_root_result { result } else { - debug!(target: "engine::tree", persistence_in_progress, "Failed to compute state root in parallel"); + debug!(target: "engine::tree", block=?sealed_block.num_hash(), persistence_in_progress, "Failed to compute state root in parallel"); state_provider.state_root_with_updates(hashed_state.clone())? }; @@ -2252,7 +2254,7 @@ where let root_elapsed = root_time.elapsed(); self.metrics.block_validation.record_state_root(&trie_output, root_elapsed.as_secs_f64()); - debug!(target: "engine::tree", ?root_elapsed, ?block_number, "Calculated state root"); + debug!(target: "engine::tree", ?root_elapsed, block=?sealed_block.num_hash(), "Calculated state root"); let executed = ExecutedBlock { block: sealed_block.clone(), @@ -2301,6 +2303,7 @@ where let mut input = TrieInput::default(); if let Some((historical, blocks)) = self.state.tree_state.blocks_by_hash(parent_hash) { + debug!(target: "engine::tree", %parent_hash, %historical, "Calculating state root in parallel, parent found in memory"); // Retrieve revert state for historical block. let revert_state = consistent_view.revert_state(historical)?; input.append(revert_state); @@ -2311,6 +2314,7 @@ where } } else { // The block attaches to canonical persisted parent. + debug!(target: "engine::tree", %parent_hash, "Calculating state root in parallel, parent found in disk"); let revert_state = consistent_view.revert_state(parent_hash)?; input.append(revert_state); } From 8b1dfcca3fd1edcbf574e969ce30493f997e6d19 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 22 Oct 2024 15:56:17 +0100 Subject: [PATCH 026/239] test(trie): proptest <-> alloy maps integration (#11962) --- book/sources/Cargo.toml | 1 - crates/trie/sparse/benches/root.rs | 2 ++ crates/trie/sparse/src/trie.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/book/sources/Cargo.toml b/book/sources/Cargo.toml index 1529af952b92..32fb13990b96 100644 --- a/book/sources/Cargo.toml +++ b/book/sources/Cargo.toml @@ -8,4 +8,3 @@ members = [ # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html resolver = "2" - diff --git a/crates/trie/sparse/benches/root.rs b/crates/trie/sparse/benches/root.rs index 248e3caeeeee..bc221a8f8313 100644 --- a/crates/trie/sparse/benches/root.rs +++ b/crates/trie/sparse/benches/root.rs @@ -198,6 +198,8 @@ fn generate_test_data(size: usize) -> HashMap { .new_tree(&mut runner) .unwrap() .current() + .into_iter() + .collect() } criterion_group!(root, calculate_root_from_leaves, calculate_root_from_leaves_repeated); diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 42f90d34c8a7..a0f7a0c3051e 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -1276,6 +1276,6 @@ mod tests { ), 1..100, ) - )| { test(updates) }); + )| { test(updates.into_iter().collect()) }); } } From 468ac0d43b5ff9538d89061bdb15604041948562 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Oct 2024 19:00:10 +0200 Subject: [PATCH 027/239] chore: log enode (#11974) --- crates/ethereum/node/src/node.rs | 4 ++-- crates/optimism/node/src/node.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 3df46b4856f6..dbd6ce0a134a 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -11,7 +11,7 @@ use reth_ethereum_engine_primitives::{ }; use reth_evm::execute::BasicBlockExecutorProvider; use reth_evm_ethereum::execute::EthExecutionStrategyFactory; -use reth_network::NetworkHandle; +use reth_network::{NetworkHandle, PeersInfo}; use reth_node_api::{ AddOnsContext, ConfigureEvm, EngineValidator, FullNodeComponents, NodePrimitives, NodeTypesWithDB, @@ -314,7 +314,7 @@ where ) -> eyre::Result { let network = ctx.network_builder().await?; let handle = ctx.start_network(network, pool); - + info!(target: "reth::cli", enode=%handle.local_node_record(), "P2P networking initialized"); Ok(handle) } } diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 22fc1a88ff73..9492bb8c429a 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use reth_basic_payload_builder::{BasicPayloadJobGenerator, BasicPayloadJobGeneratorConfig}; use reth_chainspec::{EthChainSpec, Hardforks}; use reth_evm::{execute::BasicBlockExecutorProvider, ConfigureEvm}; -use reth_network::{NetworkConfig, NetworkHandle, NetworkManager}; +use reth_network::{NetworkConfig, NetworkHandle, NetworkManager, PeersInfo}; use reth_node_api::{ AddOnsContext, EngineValidator, FullNodeComponents, NodeAddOns, NodePrimitives, }; @@ -432,6 +432,7 @@ where let network_config = self.network_config(ctx)?; let network = NetworkManager::builder(network_config).await?; let handle = ctx.start_network(network, pool); + info!(target: "reth::cli", enode=%handle.local_node_record(), "P2P networking initialized"); Ok(handle) } From e70b112420b1b06fc0d451ec79d2c01a561dd8af Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 22 Oct 2024 19:04:58 +0100 Subject: [PATCH 028/239] feat(trie): update sparse trie hashes below level (#11969) --- crates/trie/sparse/src/trie.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index a0f7a0c3051e..39deb50e7a60 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -518,34 +518,35 @@ impl RevealedSparseTrie { } } - /// Update node hashes only if their path exceeds the provided level. - pub fn update_rlp_node_level(&mut self, min_len: usize) { - let mut paths = Vec::from([Nibbles::default()]); + /// Update hashes of the nodes that are located at a level deeper than or equal to the provided + /// depth. Root node has a level of 0. + pub fn update_rlp_node_level(&mut self, depth: usize) { + let mut paths = Vec::from([(Nibbles::default(), 0)]); let mut targets = HashSet::::default(); - while let Some(mut path) = paths.pop() { + while let Some((mut path, level)) = paths.pop() { match self.nodes.get(&path).unwrap() { SparseNode::Empty | SparseNode::Hash(_) => {} SparseNode::Leaf { .. } => { targets.insert(path); } SparseNode::Extension { key, .. } => { - if path.len() >= min_len { + if level >= depth { targets.insert(path); } else { path.extend_from_slice_unchecked(key); - paths.push(path); + paths.push((path, level + 1)); } } SparseNode::Branch { state_mask, .. } => { - if path.len() >= min_len { + if level >= depth { targets.insert(path); } else { for bit in CHILD_INDEX_RANGE { if state_mask.is_bit_set(bit) { let mut child_path = path.clone(); child_path.push_unchecked(bit); - paths.push(child_path); + paths.push((child_path, level + 1)); } } } From 3174bd5c913a76803403d83ded0a9623ac1d9493 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Oct 2024 20:12:15 +0200 Subject: [PATCH 029/239] chore: bump aquamarine (#11965) --- Cargo.lock | 30 +++--------------------------- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59c582fec3f2..e1cead609112 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -847,13 +847,13 @@ checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" [[package]] name = "aquamarine" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21cc1548309245035eb18aa7f0967da6bc65587005170c56e6ef2788a4cf3f4e" +checksum = "0f50776554130342de4836ba542aa85a4ddb361690d7e8df13774d7284c3d5c2" dependencies = [ "include_dir", "itertools 0.10.5", - "proc-macro-error", + "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.80", @@ -5773,30 +5773,6 @@ dependencies = [ "toml_edit", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro-error-attr2" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 48a6525d5efd..274af4cd79f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -466,7 +466,7 @@ op-alloy-network = "0.5" op-alloy-consensus = "0.5" # misc -aquamarine = "0.5" +aquamarine = "0.6" auto_impl = "1" backon = { version = "1.2", default-features = false, features = [ "std-blocking-sleep", From 28c61c15b42305edca436e980e29c2cf5087a7c5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Oct 2024 21:46:00 +0200 Subject: [PATCH 030/239] fix: invoke prometheus recorder on op-reth Cli::run (#11982) --- bin/reth/src/cli/mod.rs | 3 +-- crates/optimism/cli/src/lib.rs | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 01f8f73e7b1b..01662eb4dcb6 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -146,8 +146,7 @@ impl, Ext: clap::Args + fmt::Debug> Cl let _guard = self.init_tracing()?; info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory); - // Install the prometheus recorder to be sure to record task - // executor's metrics + // Install the prometheus recorder to be sure to record all metrics let _ = install_prometheus_recorder(); let runner = CliRunner::default(); diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index e6eed86bf7fc..235b44559696 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -54,6 +54,7 @@ use tracing::info; // This allows us to manually enable node metrics features, required for proper jemalloc metric // reporting use reth_node_metrics as _; +use reth_node_metrics::recorder::install_prometheus_recorder; /// The main op-reth cli interface. /// @@ -135,6 +136,9 @@ where let _guard = self.init_tracing()?; info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory); + // Install the prometheus recorder to be sure to record all metrics + let _ = install_prometheus_recorder(); + let runner = CliRunner::default(); match self.command { Commands::Node(command) => { From 22171d27bf575a45308063072ea7984846dbd8e1 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 22 Oct 2024 22:24:25 +0200 Subject: [PATCH 031/239] chore: unpin serde (#11977) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 274af4cd79f4..f1506366dfd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -503,7 +503,7 @@ rand = "0.8.5" rayon = "1.7" rustc-hash = { version = "2.0", default-features = false } schnellru = "0.2" -serde = { version = "=1.0.210", default-features = false } +serde = { version = "1.0", default-features = false } serde_json = "1.0.94" serde_with = "3.3.0" sha2 = { version = "0.10", default-features = false } From 8bfbd977952cbcd42a66f46df235ee9694476a52 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Oct 2024 23:19:34 +0200 Subject: [PATCH 032/239] feat(discv4): add soft_remove_node (#11970) --- crates/net/discv4/Cargo.toml | 2 +- crates/net/discv4/src/lib.rs | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/crates/net/discv4/Cargo.toml b/crates/net/discv4/Cargo.toml index f008d03b56fa..6fb669388851 100644 --- a/crates/net/discv4/Cargo.toml +++ b/crates/net/discv4/Cargo.toml @@ -46,7 +46,7 @@ serde = { workspace = true, optional = true } [dev-dependencies] assert_matches.workspace = true rand.workspace = true -tokio = { workspace = true, features = ["macros"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } reth-tracing.workspace = true [features] diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index 7963c6e6fd65..c9b14d38b9d4 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -829,6 +829,24 @@ impl Discv4Service { /// table. Returns `true` if the node was in the table and `false` otherwise. pub fn remove_node(&mut self, node_id: PeerId) -> bool { let key = kad_key(node_id); + self.remove_key(node_id, key) + } + + /// Removes a `node_id` from the routing table but only if there are enough other nodes in the + /// bucket (bucket must be at least half full) + /// + /// Returns `true` if the node was removed + pub fn soft_remove_node(&mut self, node_id: PeerId) -> bool { + let key = kad_key(node_id); + let Some(bucket) = self.kbuckets.get_bucket(&key) else { return false }; + if bucket.num_entries() < MAX_NODES_PER_BUCKET / 2 { + // skip half empty bucket + return false; + } + self.remove_key(node_id, key) + } + + fn remove_key(&mut self, node_id: PeerId, key: discv5::Key) -> bool { let removed = self.kbuckets.remove(&key); if removed { trace!(target: "discv4", ?node_id, "removed node"); @@ -1491,13 +1509,7 @@ impl Discv4Service { // the table, but only if there are enough other nodes in the bucket (bucket must be at // least half full) if failures > (self.config.max_find_node_failures as usize) { - if let Some(bucket) = self.kbuckets.get_bucket(&key) { - if bucket.num_entries() < MAX_NODES_PER_BUCKET / 2 { - // skip half empty bucket - continue - } - } - self.remove_node(node_id); + self.soft_remove_node(node_id); } } } From 527d344ddab7397ede9f06874b9a00746fa2fa41 Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Tue, 22 Oct 2024 23:52:49 +0200 Subject: [PATCH 033/239] feat: Add reproducible build profile (#10459) Co-authored-by: Roman Krasiuk Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- Cargo.toml | 7 +++++++ Dockerfile.reproducible | 37 +++++++++++++++++++++++++++++++++++++ Makefile | 10 ++++++++++ 3 files changed, 54 insertions(+) create mode 100644 Dockerfile.reproducible diff --git a/Cargo.toml b/Cargo.toml index f1506366dfd5..ea90cd7fb64f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,6 +288,13 @@ codegen-units = 1 inherits = "release" lto = "fat" +[profile.reproducible] +inherits = "release" +debug = false +panic = "abort" +codegen-units = 1 +overflow-checks = true + [workspace.dependencies] # reth op-reth = { path = "crates/optimism/bin" } diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible new file mode 100644 index 000000000000..12c12dd7c7d4 --- /dev/null +++ b/Dockerfile.reproducible @@ -0,0 +1,37 @@ +# Use the Rust 1.82 image based on Debian Bullseye +FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af2672370559d7e854653c66d AS builder + +# Install specific version of libclang-dev +RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 + +# Clone the repository at the specific branch +RUN git clone https://github.com/paradigmxyz/reth /app +WORKDIR /app + +# Checkout the reproducible-build branch +RUN git checkout reproducible-build + +# Get the latest commit timestamp and set SOURCE_DATE_EPOCH +RUN SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> /etc/environment + +# Set environment variables for reproducibility +ARG RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $(pwd)=." +ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ + CARGO_INCREMENTAL=0 \ + LC_ALL=C \ + TZ=UTC \ + RUSTFLAGS="${RUSTFLAGS}" + +# Set the default features if not provided +ARG FEATURES="jemalloc asm-keccak" + +# Build the project with the reproducible settings +RUN . /etc/environment && \ + cargo build --bin reth --features "${FEATURES}" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu + +# Create a minimal final image with just the binary +FROM scratch AS binaries + +# Copy the compiled binary from the builder stage +COPY --from=builder /app/target/x86_64-unknown-linux-gnu/reproducible/reth /reth diff --git a/Makefile b/Makefile index 908f1ef24daa..5ad7abac6755 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,16 @@ install-op: ## Build and install the op-reth binary under `~/.cargo/bin`. build: ## Build the reth binary into `target` directory. cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)" +SOURCE_DATE_EPOCH := $(shell git log -1 --pretty=%ct) +.PHONY: reproducible +reproducible: ## Build the reth binary into `target` directory with reproducible builds. Only works for x86_64-unknown-linux-gnu currently + SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \ + CARGO_INCREMENTAL=0 \ + LC_ALL=C \ + TZ=UTC \ + RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $$(pwd)=." \ + cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu + .PHONY: build-debug build-debug: ## Build the reth binary into `target/debug` directory. cargo build --bin reth --features "$(FEATURES)" From d68dca1a26b69e4784b7b4671b05641025e657fc Mon Sep 17 00:00:00 2001 From: alpharush <0xalpharush@protonmail.com> Date: Tue, 22 Oct 2024 17:34:09 -0500 Subject: [PATCH 034/239] chore: fix cargo feature warning (#11900) Co-authored-by: Matthias Seitz Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- bin/reth/Cargo.toml | 2 +- crates/cli/commands/Cargo.toml | 7 ++++++- crates/cli/commands/src/lib.rs | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 8380915d463e..ffd1998b24ea 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -101,7 +101,7 @@ tempfile.workspace = true [features] default = ["jemalloc"] -dev = ["reth-cli-commands/dev"] +dev = ["reth-cli-commands/arbitrary"] asm-keccak = [ "reth-node-core/asm-keccak", diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index e307859dfd86..6f4b1008f29f 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -88,10 +88,15 @@ reth-discv4.workspace = true [features] default = [] -dev = [ +arbitrary = [ "dep:proptest", "dep:arbitrary", "dep:proptest-arbitrary-interop", "reth-primitives/arbitrary", "reth-db-api/arbitrary", + "reth-eth-wire/arbitrary", + "reth-db/arbitrary", + "reth-chainspec/arbitrary", + "alloy-eips/arbitrary", + "alloy-primitives/arbitrary", ] diff --git a/crates/cli/commands/src/lib.rs b/crates/cli/commands/src/lib.rs index 33a38ddbc010..166ea438fb97 100644 --- a/crates/cli/commands/src/lib.rs +++ b/crates/cli/commands/src/lib.rs @@ -20,7 +20,7 @@ pub mod p2p; pub mod prune; pub mod recover; pub mod stage; -#[cfg(feature = "dev")] +#[cfg(feature = "arbitrary")] pub mod test_vectors; pub use node::NodeCommand; From 74eb37523a8ad74b7748701703ea3f3542e5fec7 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 23 Oct 2024 02:38:44 +0400 Subject: [PATCH 035/239] fix: correctly poll `BasicBlockDownloader` (#11981) --- crates/engine/tree/src/engine.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/engine/tree/src/engine.rs b/crates/engine/tree/src/engine.rs index c1571ed82178..914121adce51 100644 --- a/crates/engine/tree/src/engine.rs +++ b/crates/engine/tree/src/engine.rs @@ -113,9 +113,11 @@ where } // advance the downloader - if let Poll::Ready(DownloadOutcome::Blocks(blocks)) = self.downloader.poll(cx) { - // delegate the downloaded blocks to the handler - self.handler.on_event(FromEngine::DownloadedBlocks(blocks)); + if let Poll::Ready(outcome) = self.downloader.poll(cx) { + if let DownloadOutcome::Blocks(blocks) = outcome { + // delegate the downloaded blocks to the handler + self.handler.on_event(FromEngine::DownloadedBlocks(blocks)); + } continue } From 6e5176221ff8b5cacea12c86dc52feba0de1593a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Oct 2024 01:20:19 +0200 Subject: [PATCH 036/239] fix: ping oldest more often (#11988) --- crates/net/discv4/src/config.rs | 8 +++----- crates/net/discv4/src/lib.rs | 6 ++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/net/discv4/src/config.rs b/crates/net/discv4/src/config.rs index 4fae31f585ae..c934f3361a16 100644 --- a/crates/net/discv4/src/config.rs +++ b/crates/net/discv4/src/config.rs @@ -8,8 +8,6 @@ use alloy_rlp::Encodable; use reth_net_banlist::BanList; use reth_net_nat::{NatResolver, ResolveNatInterval}; use reth_network_peers::NodeRecord; -#[cfg(feature = "serde")] -use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, time::Duration, @@ -17,7 +15,7 @@ use std::{ /// Configuration parameters that define the performance of the discovery network. #[derive(Clone, Debug)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct Discv4Config { /// Whether to enable the incoming packet filter. Default: false. pub enable_packet_filter: bool, @@ -118,7 +116,7 @@ impl Default for Discv4Config { // Every outgoing request will eventually lead to an incoming response udp_ingress_message_buffer: 1024, max_find_node_failures: 5, - ping_interval: Duration::from_secs(60 * 10), + ping_interval: Duration::from_secs(10), // Unified expiration and timeout durations, mirrors geth's `expiration` duration ping_expiration: Duration::from_secs(20), bond_expiration: Duration::from_secs(60 * 60), @@ -144,7 +142,7 @@ impl Default for Discv4Config { /// Builder type for [`Discv4Config`] #[derive(Clone, Debug, Default)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct Discv4ConfigBuilder { config: Discv4Config, } diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index c9b14d38b9d4..77f64ff82979 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -870,7 +870,9 @@ impl Discv4Service { false } - /// Update the entry on RE-ping + /// Update the entry on RE-ping. + /// + /// Invoked when we received the Pong to our [`PingReason::RePing`] ping. /// /// On re-ping we check for a changed `enr_seq` if eip868 is enabled and when it changed we sent /// a followup request to retrieve the updated ENR @@ -2259,7 +2261,7 @@ impl NodeEntry { impl NodeEntry { /// Returns true if the node should be re-pinged. fn is_expired(&self) -> bool { - self.last_seen.elapsed() > ENDPOINT_PROOF_EXPIRATION + self.last_seen.elapsed() > (ENDPOINT_PROOF_EXPIRATION / 2) } } From ee1260a1de29325c52a756cc84ef4fd49f7deb19 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:38:45 +0200 Subject: [PATCH 037/239] refactor(tx-pool): move `is_underpriced` to `ValidPoolTransaction` impl (#11938) --- crates/transaction-pool/src/pool/txpool.rs | 49 +------------------ crates/transaction-pool/src/validate/mod.rs | 53 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 4fbec1105aa8..03e69c39067c 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -1511,52 +1511,6 @@ impl AllTransactions { Ok(new_blob_tx) } - /// Returns true if the replacement candidate is underpriced and can't replace the existing - /// transaction. - #[inline] - fn is_underpriced( - existing_transaction: &ValidPoolTransaction, - maybe_replacement: &ValidPoolTransaction, - price_bumps: &PriceBumpConfig, - ) -> bool { - let price_bump = price_bumps.price_bump(existing_transaction.tx_type()); - - if maybe_replacement.max_fee_per_gas() <= - existing_transaction.max_fee_per_gas() * (100 + price_bump) / 100 - { - return true - } - - let existing_max_priority_fee_per_gas = - existing_transaction.transaction.max_priority_fee_per_gas().unwrap_or(0); - let replacement_max_priority_fee_per_gas = - maybe_replacement.transaction.max_priority_fee_per_gas().unwrap_or(0); - - if replacement_max_priority_fee_per_gas <= - existing_max_priority_fee_per_gas * (100 + price_bump) / 100 && - existing_max_priority_fee_per_gas != 0 && - replacement_max_priority_fee_per_gas != 0 - { - return true - } - - // check max blob fee per gas - if let Some(existing_max_blob_fee_per_gas) = - existing_transaction.transaction.max_fee_per_blob_gas() - { - // this enforces that blob txs can only be replaced by blob txs - let replacement_max_blob_fee_per_gas = - maybe_replacement.transaction.max_fee_per_blob_gas().unwrap_or(0); - if replacement_max_blob_fee_per_gas <= - existing_max_blob_fee_per_gas * (100 + price_bump) / 100 - { - return true - } - } - - false - } - /// Inserts a new _valid_ transaction into the pool. /// /// If the transaction already exists, it will be replaced if not underpriced. @@ -1671,8 +1625,7 @@ impl AllTransactions { let maybe_replacement = transaction.as_ref(); // Ensure the new transaction is not underpriced - if Self::is_underpriced(existing_transaction, maybe_replacement, &self.price_bumps) - { + if existing_transaction.is_underpriced(maybe_replacement, &self.price_bumps) { return Err(InsertErr::Underpriced { transaction: pool_tx.transaction, existing: *entry.get().transaction.hash(), diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index 4395cc97908b..4a82a1a148ff 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -4,6 +4,7 @@ use crate::{ error::InvalidPoolTransactionError, identifier::{SenderId, TransactionId}, traits::{PoolTransaction, TransactionOrigin}, + PriceBumpConfig, }; use alloy_primitives::{Address, TxHash, B256, U256}; use futures_util::future::Either; @@ -372,6 +373,58 @@ impl ValidPoolTransaction { pub(crate) fn tx_type_conflicts_with(&self, other: &Self) -> bool { self.is_eip4844() != other.is_eip4844() } + + /// Determines whether a candidate transaction (`maybe_replacement`) is underpriced compared to + /// an existing transaction in the pool. + /// + /// A transaction is considered underpriced if it doesn't meet the required fee bump threshold. + /// This applies to both standard gas fees and, for blob-carrying transactions (EIP-4844), + /// the blob-specific fees. + #[inline] + pub(crate) fn is_underpriced( + &self, + maybe_replacement: &Self, + price_bumps: &PriceBumpConfig, + ) -> bool { + // Retrieve the required price bump percentage for this type of transaction. + // + // The bump is different for EIP-4844 and other transactions. See `PriceBumpConfig`. + let price_bump = price_bumps.price_bump(self.tx_type()); + + // Check if the max fee per gas is underpriced. + if maybe_replacement.max_fee_per_gas() <= self.max_fee_per_gas() * (100 + price_bump) / 100 + { + return true + } + + let existing_max_priority_fee_per_gas = + self.transaction.max_priority_fee_per_gas().unwrap_or_default(); + let replacement_max_priority_fee_per_gas = + maybe_replacement.transaction.max_priority_fee_per_gas().unwrap_or_default(); + + // Check max priority fee per gas (relevant for EIP-1559 transactions only) + if existing_max_priority_fee_per_gas != 0 && + replacement_max_priority_fee_per_gas != 0 && + replacement_max_priority_fee_per_gas <= + existing_max_priority_fee_per_gas * (100 + price_bump) / 100 + { + return true + } + + // Check max blob fee per gas + if let Some(existing_max_blob_fee_per_gas) = self.transaction.max_fee_per_blob_gas() { + // This enforces that blob txs can only be replaced by blob txs + let replacement_max_blob_fee_per_gas = + maybe_replacement.transaction.max_fee_per_blob_gas().unwrap_or_default(); + if replacement_max_blob_fee_per_gas <= + existing_max_blob_fee_per_gas * (100 + price_bump) / 100 + { + return true + } + } + + false + } } impl>> ValidPoolTransaction { From 7fd28df2b4a01cc88a85ccb46cebbbff547123c4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Oct 2024 11:39:12 +0200 Subject: [PATCH 038/239] fix: re-establish bond on ping (#11989) --- crates/net/discv4/src/lib.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index 77f64ff82979..e955e45df613 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -1048,11 +1048,23 @@ impl Discv4Service { let old_enr = match self.kbuckets.entry(&key) { kbucket::Entry::Present(mut entry, _) => { - is_proven = entry.value().has_endpoint_proof; + if entry.value().is_expired() { + // If no communication with the sender has occurred within the last 12h, a ping + // should be sent in addition to pong in order to receive an endpoint proof. + needs_bond = true; + } else { + is_proven = entry.value().has_endpoint_proof; + } entry.value_mut().update_with_enr(ping.enr_sq) } kbucket::Entry::Pending(mut entry, _) => { - is_proven = entry.value().has_endpoint_proof; + if entry.value().is_expired() { + // If no communication with the sender has occurred within the last 12h, a ping + // should be sent in addition to pong in order to receive an endpoint proof. + needs_bond = true; + } else { + is_proven = entry.value().has_endpoint_proof; + } entry.value().update_with_enr(ping.enr_sq) } kbucket::Entry::Absent(entry) => { @@ -1225,7 +1237,8 @@ impl Discv4Service { self.update_on_pong(node, pong.enr_sq); } PingReason::EstablishBond => { - // nothing to do here + // same as `InitialInsert` which renews the bond if the peer is in the table + self.update_on_pong(node, pong.enr_sq); } PingReason::RePing => { self.update_on_reping(node, pong.enr_sq); @@ -2270,8 +2283,7 @@ impl NodeEntry { enum PingReason { /// Initial ping to a previously unknown peer that was inserted into the table. InitialInsert, - /// Initial ping to a previously unknown peer that didn't fit into the table. But we still want - /// to establish a bond. + /// A ping to a peer to establish a bond (endpoint proof). EstablishBond, /// Re-ping a peer. RePing, From ff04de380dc9f107d0aec4a4f0b1bc5027c891c4 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 23 Oct 2024 16:53:29 +0400 Subject: [PATCH 039/239] chore: bump alloy-eip7702 (#11986) Co-authored-by: Oliver Nordbjerg --- Cargo.lock | 133 +++++++++--------- Cargo.toml | 60 ++++---- book/sources/Cargo.toml | 13 +- book/sources/exex/hello-world/Cargo.toml | 8 +- book/sources/exex/tracking-state/Cargo.toml | 10 +- crates/primitives/src/transaction/sidecar.rs | 90 ++---------- .../codecs/src/alloy/authorization_list.rs | 14 +- crates/transaction-pool/src/blobstore/disk.rs | 13 +- 8 files changed, 141 insertions(+), 200 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1cead609112..7e10bdb0a675 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42642aed67f938363d9c7543e5ca4163cfb4205d9ec15fe933dc4e865d2932dd" +checksum = "cdf02dfacfc815214f9b54ff50d54900ba527a68fd73e2c5637ced3460005045" dependencies = [ "alloy-eips", "alloy-primitives", @@ -161,13 +161,14 @@ dependencies = [ [[package]] name = "alloy-eip7702" -version = "0.2.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeffd2590ce780ddfaa9d0ae340eb2b4e08627650c4676eef537cef0b4bf535d" +checksum = "64ffc577390ce50234e02d841214b3dc0bea6aaaae8e04bbf3cb82e9a45da9eb" dependencies = [ "alloy-primitives", "alloy-rlp", "arbitrary", + "derive_more 1.0.0", "k256", "rand 0.8.5", "serde", @@ -176,9 +177,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbc52a30df46f9831ed74557dfad0d94b12420393662a8b9ef90e2d6c8cb4b0" +checksum = "769da342b6bcd945013925ef4c40763cc82f11e002c60702dba8b444bb60e5a7" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -197,9 +198,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0787d1688b9806290313cc335d416cc7ee39b11e3245f3d218544c62572d92ba" +checksum = "c698ce0ada980b17f0323e1a28c7da8a2e9abc6dff5be9ee33d1525b28ac46b6" dependencies = [ "alloy-primitives", "alloy-serde", @@ -220,9 +221,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d55a16a5f9ca498a217c060414bcd1c43e934235dc8058b31b87dcd69ff4f105" +checksum = "c1050e1d65524c030b17442b6546b564da51fdab7f71bd534b001ba65f2ebb16" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -234,9 +235,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d236a8c3e1d5adc09b1b63c81815fc9b757d9a4ba9482cc899f9679b55dd437" +checksum = "da34a18446a27734473af3d77eb21c5ebbdf97ea8eb65c39c0b50916bc659023" dependencies = [ "alloy-consensus", "alloy-eips", @@ -255,9 +256,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15a0990fa8a56d85a42d6a689719aa4eebf5e2f1a5c5354658c0bfc52cac9a" +checksum = "9a968c063fcfcb937736665c865a71fc2242b68916156f5ffa41fee7b44bb695" dependencies = [ "alloy-consensus", "alloy-eips", @@ -268,9 +269,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2249f3c3ce446cf4063fe3d1aa7530823643c2706a1cc63045e0683ebc497a0a" +checksum = "439fc6a933b9f8e8b272a8cac35dbeabaf2b2eaf9590482bebedb5782153118e" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -317,9 +318,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "316f522bb6f9ac3805132112197957013b570e20cfdad058e8339dae6030c849" +checksum = "c45dbc0e3630becef9e988b69d43339f68d67e32a854e3c855bc28bd5031895b" dependencies = [ "alloy-chains", "alloy-consensus", @@ -358,9 +359,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222cd9b17b1c5ad48de51a88ffbdb17f17145170288f22662f80ac88739125e6" +checksum = "1e3961a56e10f44bfd69dd3f4b0854b90b84c612b0c43708e738933e8b47f93a" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -399,9 +400,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b2ab59712c594c9624aaa69e38e4d38f180cb569f1fa46cdaf8c21fd50793e5" +checksum = "917e5504e4f8f7e39bdc322ff81589ed54c1e462240adaeb58162c2d986a5a2b" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -424,9 +425,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba21284319e12d053baa204d438db6c1577aedd94c1298e4becefdac1f9cec87" +checksum = "07c7eb2dc6db1dd41e5e7bd2b98a38813854efc30e034afd90d1e420e7f3de2b" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -437,9 +438,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416cc9f391d0b876c4c8da85f7131e771a88a55b917cc9a35e1724d9409e3b1c" +checksum = "bd468a4e3eddcd9d612cad657852de4b7475ac2080e7af9224fbf1df20ddffe0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -449,9 +450,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba40bea86c3102b9ed9b3be579e32e0b3e54e766248d873de5fc0437238c8df2" +checksum = "2640928d9b1d43bb1cec7a0d615e10c2b407c5bd8ff1fcbe49e6318a2b62d731" dependencies = [ "alloy-primitives", "alloy-serde", @@ -460,9 +461,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b535781fe224c101c3d957b514cb9f438d165ff0280e5c0b2f87a0d9a2950593" +checksum = "64f731ad2ef8d7dd75a4d28214f4922a5b683feee1e6df35bd7b427315f94366" dependencies = [ "alloy-eips", "alloy-primitives", @@ -474,9 +475,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4303deacf4cbf12ed4431a5a1bbc3284f0defb4b8b72d9aa2b888656cc5ae657" +checksum = "06bd0757bfb3eccde06ee3f4e378f5839fe923d40956cff586018d4427a15bb5" dependencies = [ "alloy-primitives", "serde", @@ -484,9 +485,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44848fced3b42260b9cb61f22102246636dfe5a2d0132f8d10a617df3cb1a74b" +checksum = "7d3d95c3bf03efbb7bdc1d097e2931f520aac47438b709ccd8f065a7793dd371" dependencies = [ "alloy-consensus", "alloy-eips", @@ -505,9 +506,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35894711990019fafff0012b82b9176cbb744516eb2a9bbe6b8e5cae522163ee" +checksum = "e855b0daccf2320ba415753c3fed422abe9d3ad5d77b2d6cafcc9bcf32fe387f" dependencies = [ "alloy-consensus", "alloy-eips", @@ -526,9 +527,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac6250cad380a005ecb5ffc6d2facf03df0e72628d819a63dd8c3ade7a766ff" +checksum = "eca3753b9894235f915437f908644e737d8714c686ce4e8d03afbf585b23f074" dependencies = [ "alloy-eips", "alloy-primitives", @@ -539,9 +540,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f568c5624881896d8a25e19acbdcbabadd8df339427ea2f10b2ee447d57c4509" +checksum = "ae58a997afde032cd021547c960a53eef6245f47969dd71886e9f63fb45a6048" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -553,9 +554,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a37d2e1ed9b7daf20ad0b3e0092613cbae46737e0e988b23caa556c7067ce6" +checksum = "667e45c882fda207d4cc94c4bb35e24a23347955113dcb236a5e4e0eaddef826" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -565,9 +566,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2843c195675f06b29c09a4315cccdc233ab5bdc7c0a3775909f9f0cab5e9ae0f" +checksum = "35c2661ca6785add8fc37aff8005439c806ffad58254c19939c6f59ac0d6596e" dependencies = [ "alloy-primitives", "arbitrary", @@ -577,9 +578,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b2a00d9803dfef99963303ffe41a7bf2221f3342f0a503d6741a9f4a18e5e5" +checksum = "67eca011160d18a7dc6d8cdc1e8dc13e2e86c908f8e41b02aa76e429d6fe7085" dependencies = [ "alloy-primitives", "async-trait", @@ -591,9 +592,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a2505d4f8c98dcae86152d58d549cb4bcf953f8352fca903410e0a0ef535571" +checksum = "1c54b195a6ee5a83f32e7c697b4e6b565966737ed5a2ef9176bbbb39f720d023" dependencies = [ "alloy-consensus", "alloy-network", @@ -679,9 +680,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dc2c8f6b8c227ef0398f702d954c4ab572c2ead3c1ed4a5157aa1cbaf959747" +checksum = "3e4a136e733f55fef0870b81e1f8f1db28e78973d1b1ae5a5df642ba39538a07" dependencies = [ "alloy-json-rpc", "base64 0.22.1", @@ -699,9 +700,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd328e990d57f4c4e63899fb2c26877597d6503f8e0022a3d71b2d753ecbfc0c" +checksum = "1a6b358a89b6d107b92d09b61a61fbc04243942182709752c796f4b29402cead" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -714,9 +715,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89aea26aaf1d67904a7ff95ec4a24ddd5e7d419a6945f641b885962d7c2803e2" +checksum = "4a899c43b7f5e3bc83762dfe5128fccd9cfa99f1f03c5f26bbfb2495ae8dcd35" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -733,9 +734,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e222e950ecc4ea12fbfb524b9a2275cac2cd5f57c8ce25bcaf1bd3ff80dd8fc8" +checksum = "d27aac1246e13c9e6fa0c784fbb0c56872c6224f78dbde388bb2213ccdf8af02" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -9264,9 +9265,9 @@ dependencies = [ [[package]] name = "revm" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34e44692d5736cc44c697a372e507890f8797f06d1541c5f4b9bec594d90fd8a" +checksum = "2eab16cb0a8cd5ac88b11230b20df588b7e8aae7dfab4b3f830e98aebeb4b365" dependencies = [ "auto_impl", "cfg-if", @@ -9279,9 +9280,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64e2246ad480167548724eb9c9c66945241b867c7d50894de3ca860c9823a45" +checksum = "1e29c662f7887f3b659d4b0fd234673419a8fcbeaa1ecc29bf7034c0a75cc8ea" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -9298,9 +9299,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "12.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f89940d17d5d077570de1977f52f69049595322e237cb6c754c3d47f668f023" +checksum = "fac2034454f8bc69dc7d3c94cdb1b57559e27f5ef0518771f1787de543d7d6a1" dependencies = [ "revm-primitives", "serde", @@ -9308,9 +9309,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "13.0.0" +version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f816aaea3245cbdbe7fdd84955df33597f9322c7912c3e3ba7bc855e03211f" +checksum = "7a88c8c7c5f9b988a9e65fc0990c6ce859cdb74114db705bd118a96d22d08027" dependencies = [ "aurora-engine-modexp", "blst", @@ -9328,9 +9329,9 @@ dependencies = [ [[package]] name = "revm-primitives" -version = "12.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532411bbde45a46707c1d434dcdc29866cf261c1b748fb01b303ce3b4310b361" +checksum = "0d11fa1e195b0bebaf3fb18596f314a13ba3a4cb1fdd16d3465934d812fd921e" dependencies = [ "alloy-eip2930", "alloy-eip7702", diff --git a/Cargo.toml b/Cargo.toml index ea90cd7fb64f..e8f10229ed59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -418,9 +418,9 @@ reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } # revm -revm = { version = "16.0.0", features = ["std"], default-features = false } -revm-inspectors = "0.9.0" -revm-primitives = { version = "12.0.0", features = [ +revm = { version = "17.0.0", features = ["std"], default-features = false } +revm-inspectors = "0.10.0" +revm-primitives = { version = "13.0.0", features = [ "std", ], default-features = false } @@ -432,39 +432,39 @@ alloy-rlp = "0.3.4" alloy-sol-types = "0.8.0" alloy-trie = { version = "0.7", default-features = false } -alloy-consensus = { version = "0.5.2", default-features = false } -alloy-eips = { version = "0.5.2", default-features = false } -alloy-genesis = { version = "0.5.2", default-features = false } -alloy-json-rpc = { version = "0.5.2", default-features = false } -alloy-network = { version = "0.5.2", default-features = false } -alloy-network-primitives = { version = "0.5.2", default-features = false } -alloy-node-bindings = { version = "0.5.2", default-features = false } -alloy-provider = { version = "0.5.2", features = [ +alloy-consensus = { version = "0.5.3", default-features = false } +alloy-eips = { version = "0.5.3", default-features = false } +alloy-genesis = { version = "0.5.3", default-features = false } +alloy-json-rpc = { version = "0.5.3", default-features = false } +alloy-network = { version = "0.5.3", default-features = false } +alloy-network-primitives = { version = "0.5.3", default-features = false } +alloy-node-bindings = { version = "0.5.3", default-features = false } +alloy-provider = { version = "0.5.3", features = [ "reqwest", ], default-features = false } -alloy-pubsub = { version = "0.5.2", default-features = false } -alloy-rpc-client = { version = "0.5.2", default-features = false } -alloy-rpc-types = { version = "0.5.2", features = [ +alloy-pubsub = { version = "0.5.3", default-features = false } +alloy-rpc-client = { version = "0.5.3", default-features = false } +alloy-rpc-types = { version = "0.5.3", features = [ "eth", ], default-features = false } -alloy-rpc-types-admin = { version = "0.5.2", default-features = false } -alloy-rpc-types-anvil = { version = "0.5.2", default-features = false } -alloy-rpc-types-beacon = { version = "0.5.2", default-features = false } -alloy-rpc-types-debug = { version = "0.5.2", default-features = false } -alloy-rpc-types-engine = { version = "0.5.2", default-features = false } -alloy-rpc-types-eth = { version = "0.5.2", default-features = false } -alloy-rpc-types-mev = { version = "0.5.2", default-features = false } -alloy-rpc-types-trace = { version = "0.5.2", default-features = false } -alloy-rpc-types-txpool = { version = "0.5.2", default-features = false } -alloy-serde = { version = "0.5.2", default-features = false } -alloy-signer = { version = "0.5.2", default-features = false } -alloy-signer-local = { version = "0.5.2", default-features = false } -alloy-transport = { version = "0.5.2" } -alloy-transport-http = { version = "0.5.2", features = [ +alloy-rpc-types-admin = { version = "0.5.3", default-features = false } +alloy-rpc-types-anvil = { version = "0.5.3", default-features = false } +alloy-rpc-types-beacon = { version = "0.5.3", default-features = false } +alloy-rpc-types-debug = { version = "0.5.3", default-features = false } +alloy-rpc-types-engine = { version = "0.5.3", default-features = false } +alloy-rpc-types-eth = { version = "0.5.3", default-features = false } +alloy-rpc-types-mev = { version = "0.5.3", default-features = false } +alloy-rpc-types-trace = { version = "0.5.3", default-features = false } +alloy-rpc-types-txpool = { version = "0.5.3", default-features = false } +alloy-serde = { version = "0.5.3", default-features = false } +alloy-signer = { version = "0.5.3", default-features = false } +alloy-signer-local = { version = "0.5.3", default-features = false } +alloy-transport = { version = "0.5.3" } +alloy-transport-http = { version = "0.5.3", features = [ "reqwest-rustls-tls", ], default-features = false } -alloy-transport-ipc = { version = "0.5.2", default-features = false } -alloy-transport-ws = { version = "0.5.2", default-features = false } +alloy-transport-ipc = { version = "0.5.3", default-features = false } +alloy-transport-ws = { version = "0.5.3", default-features = false } # op op-alloy-rpc-types = "0.5" diff --git a/book/sources/Cargo.toml b/book/sources/Cargo.toml index 32fb13990b96..b374ad798b5d 100644 --- a/book/sources/Cargo.toml +++ b/book/sources/Cargo.toml @@ -1,10 +1,13 @@ [workspace] -members = [ - "exex/hello-world", - "exex/remote", - "exex/tracking-state", -] +members = ["exex/hello-world", "exex/remote", "exex/tracking-state"] # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html resolver = "2" + +[patch.'https://github.com/paradigmxyz/reth'] +reth = { path = "../../bin/reth" } +reth-exex = { path = "../../crates/exex/exex" } +reth-node-ethereum = { path = "../../crates/ethereum/node" } +reth-tracing = { path = "../../crates/tracing" } +reth-node-api = { path = "../../crates/node/api" } diff --git a/book/sources/exex/hello-world/Cargo.toml b/book/sources/exex/hello-world/Cargo.toml index e5d32a140549..c466018c667a 100644 --- a/book/sources/exex/hello-world/Cargo.toml +++ b/book/sources/exex/hello-world/Cargo.toml @@ -4,10 +4,10 @@ version = "0.1.0" edition = "2021" [dependencies] -reth = { git = "https://github.com/paradigmxyz/reth.git" } # Reth -reth-exex = { git = "https://github.com/paradigmxyz/reth.git" } # Execution Extensions +reth = { git = "https://github.com/paradigmxyz/reth.git" } # Reth +reth-exex = { git = "https://github.com/paradigmxyz/reth.git" } # Execution Extensions reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git" } # Ethereum Node implementation -reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } # Logging +reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } # Logging -eyre = "0.6" # Easy error handling +eyre = "0.6" # Easy error handling futures-util = "0.3" # Stream utilities for consuming notifications diff --git a/book/sources/exex/tracking-state/Cargo.toml b/book/sources/exex/tracking-state/Cargo.toml index 3ce21b0c3403..a8e862d0a735 100644 --- a/book/sources/exex/tracking-state/Cargo.toml +++ b/book/sources/exex/tracking-state/Cargo.toml @@ -5,10 +5,12 @@ edition = "2021" [dependencies] reth = { git = "https://github.com/paradigmxyz/reth.git" } -reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = ["serde"] } -reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git"} +reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = [ + "serde", +] } +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git" } reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } -eyre = "0.6" # Easy error handling -futures-util = "0.3" # Stream utilities for consuming notifications +eyre = "0.6" # Easy error handling +futures-util = "0.3" # Stream utilities for consuming notifications alloy-primitives = "0.8.7" diff --git a/crates/primitives/src/transaction/sidecar.rs b/crates/primitives/src/transaction/sidecar.rs index edc1427d1fe8..e901cbfc08dc 100644 --- a/crates/primitives/src/transaction/sidecar.rs +++ b/crates/primitives/src/transaction/sidecar.rs @@ -1,11 +1,9 @@ #![cfg_attr(docsrs, doc(cfg(feature = "c-kzg")))] use crate::{Signature, Transaction, TransactionSigned}; -use alloy_consensus::{ - constants::EIP4844_TX_TYPE_ID, transaction::TxEip4844, TxEip4844WithSidecar, -}; -use alloy_primitives::{keccak256, TxHash}; -use alloy_rlp::{Decodable, Error as RlpError, Header}; +use alloy_consensus::{constants::EIP4844_TX_TYPE_ID, TxEip4844WithSidecar}; +use alloy_primitives::TxHash; +use alloy_rlp::Header; use serde::{Deserialize, Serialize}; #[doc(inline)] @@ -14,8 +12,6 @@ pub use alloy_eips::eip4844::BlobTransactionSidecar; #[cfg(feature = "c-kzg")] pub use alloy_eips::eip4844::BlobTransactionValidationError; -use alloc::vec::Vec; - /// A response to `GetPooledTransactions` that includes blob data, their commitments, and their /// corresponding proofs. /// @@ -36,7 +32,7 @@ impl BlobTransaction { /// Constructs a new [`BlobTransaction`] from a [`TransactionSigned`] and a /// [`BlobTransactionSidecar`]. /// - /// Returns an error if the signed transaction is not [`TxEip4844`] + /// Returns an error if the signed transaction is not [`Transaction::Eip4844`] pub fn try_from_signed( tx: TransactionSigned, sidecar: BlobTransactionSidecar, @@ -57,7 +53,7 @@ impl BlobTransaction { /// Verifies that the transaction's blob data, commitments, and proofs are all valid. /// - /// See also [`TxEip4844::validate_blob`] + /// See also [`alloy_consensus::TxEip4844::validate_blob`] #[cfg(feature = "c-kzg")] pub fn validate( &self, @@ -163,7 +159,7 @@ impl BlobTransaction { // The payload length is the length of the `tranascation_payload_body` list, plus the // length of the blobs, commitments, and proofs. - let payload_length = tx_length + self.transaction.sidecar.fields_len(); + let payload_length = tx_length + self.transaction.sidecar.rlp_encoded_fields_length(); // We use the calculated payload len to construct the first list header, which encompasses // everything in the tx - the length of the second, inner list header is part of @@ -188,74 +184,17 @@ impl BlobTransaction { /// Note: this should be used only when implementing other RLP decoding methods, and does not /// represent the full RLP decoding of the `PooledTransactionsElement` type. pub(crate) fn decode_inner(data: &mut &[u8]) -> alloy_rlp::Result { - // decode the _first_ list header for the rest of the transaction - let outer_header = Header::decode(data)?; - if !outer_header.list { - return Err(RlpError::Custom("PooledTransactions blob tx must be encoded as a list")) - } - - let outer_remaining_len = data.len(); - - // Now we need to decode the inner 4844 transaction and its signature: - // - // `[chain_id, nonce, max_priority_fee_per_gas, ..., y_parity, r, s]` - let inner_header = Header::decode(data)?; - if !inner_header.list { - return Err(RlpError::Custom( - "PooledTransactions inner blob tx must be encoded as a list", - )) - } - - let inner_remaining_len = data.len(); - - // inner transaction - let transaction = TxEip4844::decode_fields(data)?; - - // signature - let signature = Signature::decode_rlp_vrs(data)?; - - // the inner header only decodes the transaction and signature, so we check the length here - let inner_consumed = inner_remaining_len - data.len(); - if inner_consumed != inner_header.payload_length { - return Err(RlpError::UnexpectedLength) - } - - // All that's left are the blobs, commitments, and proofs - let sidecar = BlobTransactionSidecar::decode(data)?; - - // # Calculating the hash - // - // The full encoding of the `PooledTransaction` response is: - // `tx_type (0x03) || rlp([tx_payload_body, blobs, commitments, proofs])` - // - // The transaction hash however, is: - // `keccak256(tx_type (0x03) || rlp(tx_payload_body))` - // - // Note that this is `tx_payload_body`, not `[tx_payload_body]`, which would be - // `[[chain_id, nonce, max_priority_fee_per_gas, ...]]`, i.e. a list within a list. - // - // Because the pooled transaction encoding is different than the hash encoding for - // EIP-4844 transactions, we do not use the original buffer to calculate the hash. - // - // Instead, we use `encode_with_signature`, which RLP encodes the transaction with a - // signature for hashing without a header. We then hash the result. - let mut buf = Vec::new(); - transaction.encode_with_signature(&signature, &mut buf, false); - let hash = keccak256(&buf); - - // the outer header is for the entire transaction, so we check the length here - let outer_consumed = outer_remaining_len - data.len(); - if outer_consumed != outer_header.payload_length { - return Err(RlpError::UnexpectedLength) - } + let (transaction, signature, hash) = + TxEip4844WithSidecar::decode_signed_fields(data)?.into_parts(); - Ok(Self { transaction: TxEip4844WithSidecar { tx: transaction, sidecar }, hash, signature }) + Ok(Self { transaction, hash, signature }) } } /// Generates a [`BlobTransactionSidecar`] structure containing blobs, commitments, and proofs. #[cfg(all(feature = "c-kzg", any(test, feature = "arbitrary")))] pub fn generate_blob_sidecar(blobs: Vec) -> BlobTransactionSidecar { + use alloc::vec::Vec; use alloy_eips::eip4844::env_settings::EnvKzgSettings; use c_kzg::{KzgCommitment, KzgProof}; @@ -285,12 +224,12 @@ pub fn generate_blob_sidecar(blobs: Vec) -> BlobTransactionSidecar mod tests { use super::*; use crate::{kzg::Blob, PooledTransactionsElement}; + use alloc::vec::Vec; use alloy_eips::{ eip2718::{Decodable2718, Encodable2718}, eip4844::Bytes48, }; use alloy_primitives::hex; - use alloy_rlp::Encodable; use std::{fs, path::PathBuf, str::FromStr}; #[test] @@ -392,7 +331,7 @@ mod tests { let mut encoded_rlp = Vec::new(); // Encode the inner data of the BlobTransactionSidecar into RLP - sidecar.encode(&mut encoded_rlp); + sidecar.rlp_encode_fields(&mut encoded_rlp); // Assert the equality between the expected RLP from the JSON and the encoded RLP assert_eq!(json_value.get("rlp").unwrap().as_str().unwrap(), hex::encode(&encoded_rlp)); @@ -423,10 +362,11 @@ mod tests { let mut encoded_rlp = Vec::new(); // Encode the inner data of the BlobTransactionSidecar into RLP - sidecar.encode(&mut encoded_rlp); + sidecar.rlp_encode_fields(&mut encoded_rlp); // Decode the RLP-encoded data back into a BlobTransactionSidecar - let decoded_sidecar = BlobTransactionSidecar::decode(&mut encoded_rlp.as_slice()).unwrap(); + let decoded_sidecar = + BlobTransactionSidecar::rlp_decode_fields(&mut encoded_rlp.as_slice()).unwrap(); // Assert the equality between the original BlobTransactionSidecar and the decoded one assert_eq!(sidecar, decoded_sidecar); diff --git a/crates/storage/codecs/src/alloy/authorization_list.rs b/crates/storage/codecs/src/alloy/authorization_list.rs index 2b013c0d3c52..6dc36956d249 100644 --- a/crates/storage/codecs/src/alloy/authorization_list.rs +++ b/crates/storage/codecs/src/alloy/authorization_list.rs @@ -44,11 +44,9 @@ impl Compact for SignedAuthorization { where B: bytes::BufMut + AsMut<[u8]>, { - let signature = self.signature(); - let (v, r, s) = (signature.v(), signature.r(), signature.s()); - buf.put_u8(v.y_parity_byte()); - buf.put_slice(r.as_le_slice()); - buf.put_slice(s.as_le_slice()); + buf.put_u8(self.y_parity()); + buf.put_slice(self.r().as_le_slice()); + buf.put_slice(self.s().as_le_slice()); // to_compact doesn't write the len to buffer. // By placing it as last, we don't need to store it either. @@ -56,17 +54,15 @@ impl Compact for SignedAuthorization { } fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { - let y = alloy_primitives::Parity::Parity(buf.get_u8() == 1); + let y_parity = buf.get_u8(); let r = U256::from_le_slice(&buf[0..32]); buf.advance(32); let s = U256::from_le_slice(&buf[0..32]); buf.advance(32); - let signature = alloy_primitives::Signature::from_rs_and_parity(r, s, y) - .expect("invalid authorization signature"); let (auth, buf) = AlloyAuthorization::from_compact(buf, len); - (auth.into_signed(signature), buf) + (Self::new_unchecked(auth, y_parity, r, s), buf) } } diff --git a/crates/transaction-pool/src/blobstore/disk.rs b/crates/transaction-pool/src/blobstore/disk.rs index e168a1c11116..787d4985ff1b 100644 --- a/crates/transaction-pool/src/blobstore/disk.rs +++ b/crates/transaction-pool/src/blobstore/disk.rs @@ -3,7 +3,6 @@ use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobStoreSize}; use alloy_eips::eip4844::BlobAndProofV1; use alloy_primitives::{TxHash, B256}; -use alloy_rlp::{Decodable, Encodable}; use parking_lot::{Mutex, RwLock}; use reth_primitives::BlobTransactionSidecar; use schnellru::{ByLength, LruMap}; @@ -204,8 +203,8 @@ impl DiskFileBlobStoreInner { /// Ensures blob is in the blob cache and written to the disk. fn insert_one(&self, tx: B256, data: BlobTransactionSidecar) -> Result<(), BlobStoreError> { - let mut buf = Vec::with_capacity(data.fields_len()); - data.encode(&mut buf); + let mut buf = Vec::with_capacity(data.rlp_encoded_fields_length()); + data.rlp_encode_fields(&mut buf); self.blob_cache.lock().insert(tx, data); let size = self.write_one_encoded(tx, &buf)?; @@ -219,8 +218,8 @@ impl DiskFileBlobStoreInner { let raw = txs .iter() .map(|(tx, data)| { - let mut buf = Vec::with_capacity(data.fields_len()); - data.encode(&mut buf); + let mut buf = Vec::with_capacity(data.rlp_encoded_fields_length()); + data.rlp_encode_fields(&mut buf); (self.blob_disk_file(*tx), buf) }) .collect::>(); @@ -312,7 +311,7 @@ impl DiskFileBlobStoreInner { } } }; - BlobTransactionSidecar::decode(&mut data.as_slice()) + BlobTransactionSidecar::rlp_decode_fields(&mut data.as_slice()) .map(Some) .map_err(BlobStoreError::DecodeError) } @@ -322,7 +321,7 @@ impl DiskFileBlobStoreInner { self.read_many_raw(txs) .into_iter() .filter_map(|(tx, data)| { - BlobTransactionSidecar::decode(&mut data.as_slice()) + BlobTransactionSidecar::rlp_decode_fields(&mut data.as_slice()) .map(|sidecar| (tx, sidecar)) .ok() }) From cf4e7745427c8e57e576acb2458f10e6a89d89c0 Mon Sep 17 00:00:00 2001 From: Hai | RISE <150876604+hai-rise@users.noreply.github.com> Date: Wed, 23 Oct 2024 20:12:44 +0700 Subject: [PATCH 040/239] chore: `Ethereum` -> `Optimism` in comment of `optimism_payload` (#11998) --- crates/optimism/payload/src/builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index b6ab9b87956b..3ed00c49aec5 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -146,12 +146,12 @@ where } } -/// Constructs an Ethereum transaction payload from the transactions sent through the +/// Constructs an Optimism transaction payload from the transactions sent through the /// Payload attributes by the sequencer. If the `no_tx_pool` argument is passed in /// the payload attributes, the transaction pool will be ignored and the only transactions /// included in the payload will be those sent through the attributes. /// -/// Given build arguments including an Ethereum client, transaction pool, +/// Given build arguments including an Optimism client, transaction pool, /// and configuration, this function creates a transaction payload. Returns /// a result indicating success with the payload or an error in case of failure. #[inline] From 386379efd55e491013d010b0a377771317a21e14 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:58:56 +0200 Subject: [PATCH 041/239] test(tokio-util): add unit tests for `EventSender` (#11980) --- crates/tokio-util/src/event_sender.rs | 93 +++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/crates/tokio-util/src/event_sender.rs b/crates/tokio-util/src/event_sender.rs index a4e9815388cd..16208ee19c0e 100644 --- a/crates/tokio-util/src/event_sender.rs +++ b/crates/tokio-util/src/event_sender.rs @@ -40,3 +40,96 @@ impl EventSender { EventStream::new(self.sender.subscribe()) } } + +#[cfg(test)] +mod tests { + use super::*; + use tokio::{ + task, + time::{timeout, Duration}, + }; + use tokio_stream::StreamExt; + + #[tokio::test] + async fn test_event_broadcast_to_listener() { + let sender = EventSender::default(); + + // Create a listener for the events + let mut listener = sender.new_listener(); + + // Broadcast an event + sender.notify("event1"); + + // Check if the listener receives the event + let received_event = listener.next().await; + assert_eq!(received_event, Some("event1")); + } + + #[tokio::test] + async fn test_event_no_listener() { + let sender = EventSender::default(); + + // Broadcast an event with no listeners + sender.notify("event2"); + + // Ensure it doesn't panic or fail when no listeners are present + // (this test passes if it runs without errors). + } + + #[tokio::test] + async fn test_multiple_listeners_receive_event() { + let sender = EventSender::default(); + + // Create two listeners + let mut listener1 = sender.new_listener(); + let mut listener2 = sender.new_listener(); + + // Broadcast an event + sender.notify("event3"); + + // Both listeners should receive the same event + let event1 = listener1.next().await; + let event2 = listener2.next().await; + + assert_eq!(event1, Some("event3")); + assert_eq!(event2, Some("event3")); + } + + #[tokio::test] + async fn test_bounded_channel_size() { + // Create a channel with size 2 + let sender = EventSender::new(2); + + // Create a listener + let mut listener = sender.new_listener(); + + // Broadcast 3 events, which exceeds the channel size + sender.notify("event4"); + sender.notify("event5"); + sender.notify("event6"); + + // Only the last two should be received due to the size limit + let received_event1 = listener.next().await; + let received_event2 = listener.next().await; + + assert_eq!(received_event1, Some("event5")); + assert_eq!(received_event2, Some("event6")); + } + + #[tokio::test] + async fn test_event_listener_timeout() { + let sender = EventSender::default(); + let mut listener = sender.new_listener(); + + // Broadcast an event asynchronously + task::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + sender.notify("delayed_event"); + }); + + // Use a timeout to ensure that the event is received within a certain time + let result = timeout(Duration::from_millis(100), listener.next()).await; + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Some("delayed_event")); + } +} From 5e0ba4104d85dcf2be9a7df91347cb84996d245f Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:11:59 +0200 Subject: [PATCH 042/239] tx-pool: migrate `ensure_max_init_code_size` to `PoolTransaction` trait (#11976) --- crates/transaction-pool/src/traits.rs | 22 ++++++++++++++++- crates/transaction-pool/src/validate/eth.rs | 26 ++++----------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 56da11fe696a..cedec56063ba 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -2,7 +2,7 @@ use crate::{ blobstore::BlobStoreError, - error::PoolResult, + error::{InvalidPoolTransactionError, PoolResult}, pool::{state::SubPool, BestTransactionFilter, TransactionEvents}, validate::ValidPoolTransaction, AllTransactionsEvents, @@ -961,6 +961,26 @@ pub trait PoolTransaction: fmt::Debug + Send + Sync + Clone { /// Returns `chain_id` fn chain_id(&self) -> Option; + + /// Ensures that the transaction's code size does not exceed the provided `max_init_code_size`. + /// + /// This is specifically relevant for contract creation transactions ([`TxKind::Create`]), + /// where the input data contains the initialization code. If the input code size exceeds + /// the configured limit, an [`InvalidPoolTransactionError::ExceedsMaxInitCodeSize`] error is + /// returned. + fn ensure_max_init_code_size( + &self, + max_init_code_size: usize, + ) -> Result<(), InvalidPoolTransactionError> { + if self.kind().is_create() && self.input().len() > max_init_code_size { + Err(InvalidPoolTransactionError::ExceedsMaxInitCodeSize( + self.size(), + max_init_code_size, + )) + } else { + Ok(()) + } + } } /// Super trait for transactions that can be converted to and from Eth transactions diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index 22744c58a798..bf7749fb85c3 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -8,7 +8,7 @@ use crate::{ }, traits::TransactionOrigin, validate::{ValidTransaction, ValidationTask, MAX_INIT_CODE_BYTE_SIZE}, - EthBlobTransactionSidecar, EthPoolTransaction, LocalTransactionConfig, PoolTransaction, + EthBlobTransactionSidecar, EthPoolTransaction, LocalTransactionConfig, TransactionValidationOutcome, TransactionValidationTaskExecutor, TransactionValidator, }; use alloy_consensus::constants::{ @@ -223,7 +223,7 @@ where // Check whether the init code size has been exceeded. if self.fork_tracker.is_shanghai_activated() { - if let Err(err) = ensure_max_init_code_size(&transaction, MAX_INIT_CODE_BYTE_SIZE) { + if let Err(err) = transaction.ensure_max_init_code_size(MAX_INIT_CODE_BYTE_SIZE) { return TransactionValidationOutcome::Invalid(transaction, err) } } @@ -711,7 +711,7 @@ impl EthTransactionValidatorBuilder { EthTransactionValidator { inner: Arc::new(inner) } } - /// Builds a the [`EthTransactionValidator`] and spawns validation tasks via the + /// Builds a [`EthTransactionValidator`] and spawns validation tasks via the /// [`TransactionValidationTaskExecutor`] /// /// The validator will spawn `additional_tasks` additional tasks for validation. @@ -783,22 +783,6 @@ impl ForkTracker { } } -/// Ensure that the code size is not greater than `max_init_code_size`. -/// `max_init_code_size` should be configurable so this will take it as an argument. -pub fn ensure_max_init_code_size( - transaction: &T, - max_init_code_size: usize, -) -> Result<(), InvalidPoolTransactionError> { - if transaction.kind().is_create() && transaction.input().len() > max_init_code_size { - Err(InvalidPoolTransactionError::ExceedsMaxInitCodeSize( - transaction.size(), - max_init_code_size, - )) - } else { - Ok(()) - } -} - /// Ensures that gas limit of the transaction exceeds the intrinsic gas of the transaction. /// /// Caution: This only checks past the Merge hardfork. @@ -833,8 +817,8 @@ pub fn ensure_intrinsic_gas( mod tests { use super::*; use crate::{ - blobstore::InMemoryBlobStore, error::PoolErrorKind, CoinbaseTipOrdering, - EthPooledTransaction, Pool, TransactionPool, + blobstore::InMemoryBlobStore, error::PoolErrorKind, traits::PoolTransaction, + CoinbaseTipOrdering, EthPooledTransaction, Pool, TransactionPool, }; use alloy_eips::eip2718::Decodable2718; use alloy_primitives::{hex, U256}; From 252cdf7f3576f76a0ffe6b4086105655b11ba420 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:35:24 +0200 Subject: [PATCH 043/239] storage: add unit tests for `StorageRevertsIter` (#11999) --- .../src/bundle_state/state_reverts.rs | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/crates/storage/provider/src/bundle_state/state_reverts.rs b/crates/storage/provider/src/bundle_state/state_reverts.rs index 09b892562fbd..3e1ba2a4b8f2 100644 --- a/crates/storage/provider/src/bundle_state/state_reverts.rs +++ b/crates/storage/provider/src/bundle_state/state_reverts.rs @@ -76,3 +76,105 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_storage_reverts_iter_empty() { + // Create empty sample data for reverts and wiped entries. + let reverts: Vec<(B256, RevertToSlot)> = vec![]; + let wiped: Vec<(B256, U256)> = vec![]; + + // Create the iterator with the empty data. + let iter = StorageRevertsIter::new(reverts, wiped); + + // Iterate and collect results into a vector for verification. + let results: Vec<_> = iter.collect(); + + // Verify that the results are empty. + assert_eq!(results, vec![]); + } + + #[test] + fn test_storage_reverts_iter_reverts_only() { + // Create sample data for only reverts. + let reverts = vec![ + (B256::from_slice(&[4; 32]), RevertToSlot::Destroyed), + (B256::from_slice(&[5; 32]), RevertToSlot::Some(U256::from(40))), + ]; + + // Create the iterator with only reverts and no wiped entries. + let iter = StorageRevertsIter::new(reverts, vec![]); + + // Iterate and collect results into a vector for verification. + let results: Vec<_> = iter.collect(); + + // Verify the output order and values. + assert_eq!( + results, + vec![ + (B256::from_slice(&[4; 32]), U256::ZERO), // Revert slot previous value + (B256::from_slice(&[5; 32]), U256::from(40)), // Only revert present. + ] + ); + } + + #[test] + fn test_storage_reverts_iter_wiped_only() { + // Create sample data for only wiped entries. + let wiped = vec![ + (B256::from_slice(&[6; 32]), U256::from(50)), + (B256::from_slice(&[7; 32]), U256::from(60)), + ]; + + // Create the iterator with only wiped entries and no reverts. + let iter = StorageRevertsIter::new(vec![], wiped); + + // Iterate and collect results into a vector for verification. + let results: Vec<_> = iter.collect(); + + // Verify the output order and values. + assert_eq!( + results, + vec![ + (B256::from_slice(&[6; 32]), U256::from(50)), // Only wiped present. + (B256::from_slice(&[7; 32]), U256::from(60)), // Only wiped present. + ] + ); + } + + #[test] + fn test_storage_reverts_iter_interleaved() { + // Create sample data for interleaved reverts and wiped entries. + let reverts = vec![ + (B256::from_slice(&[8; 32]), RevertToSlot::Some(U256::from(70))), + (B256::from_slice(&[9; 32]), RevertToSlot::Some(U256::from(80))), + // Some higher key than wiped + (B256::from_slice(&[15; 32]), RevertToSlot::Some(U256::from(90))), + ]; + + let wiped = vec![ + (B256::from_slice(&[8; 32]), U256::from(75)), // Same key as revert + (B256::from_slice(&[10; 32]), U256::from(85)), // Wiped with new key + ]; + + // Create the iterator with the sample data. + let iter = StorageRevertsIter::new(reverts, wiped); + + // Iterate and collect results into a vector for verification. + let results: Vec<_> = iter.collect(); + + // Verify the output order and values. + assert_eq!( + results, + vec![ + (B256::from_slice(&[8; 32]), U256::from(70)), // Revert takes priority. + (B256::from_slice(&[9; 32]), U256::from(80)), // Only revert present. + (B256::from_slice(&[10; 32]), U256::from(85)), // Wiped entry. + (B256::from_slice(&[15; 32]), U256::from(90)), // WGreater revert entry + ] + ); + } +} From 55d98bbc6effb0dde79b3210d0eea2a0424ebc64 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Oct 2024 15:37:39 +0200 Subject: [PATCH 044/239] fix: check failed find nodes requests before sending new ones (#11997) --- crates/net/discv4/src/config.rs | 2 +- crates/net/discv4/src/lib.rs | 135 +++++++++++++++++++++++++++----- 2 files changed, 118 insertions(+), 19 deletions(-) diff --git a/crates/net/discv4/src/config.rs b/crates/net/discv4/src/config.rs index c934f3361a16..38467304db2c 100644 --- a/crates/net/discv4/src/config.rs +++ b/crates/net/discv4/src/config.rs @@ -23,7 +23,7 @@ pub struct Discv4Config { pub udp_egress_message_buffer: usize, /// Size of the channel buffer for incoming messages. pub udp_ingress_message_buffer: usize, - /// The number of allowed failures for `FindNode` requests. Default: 5. + /// The number of allowed consecutive failures for `FindNode` requests. Default: 5. pub max_find_node_failures: u8, /// The interval to use when checking for expired nodes that need to be re-pinged. Default: /// 10min. diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index e955e45df613..779c7ee637aa 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -743,7 +743,8 @@ impl Discv4Service { trace!(target: "discv4", ?target, "Starting lookup"); let target_key = kad_key(target); - // Start a lookup context with the 16 (MAX_NODES_PER_BUCKET) closest nodes + // Start a lookup context with the 16 (MAX_NODES_PER_BUCKET) closest nodes to which we have + // a valid endpoint proof let ctx = LookupContext::new( target_key.clone(), self.kbuckets @@ -772,7 +773,10 @@ impl Discv4Service { trace!(target: "discv4", ?target, num = closest.len(), "Start lookup closest nodes"); for node in closest { - self.find_node(&node, ctx.clone()); + // here we still want to check against previous request failures and if necessary + // re-establish a new endpoint proof because it can be the case that the other node lost + // our entry and no longer has an endpoint proof on their end + self.find_node_checked(&node, ctx.clone()); } } @@ -788,6 +792,22 @@ impl Discv4Service { self.pending_find_nodes.insert(node.id, FindNodeRequest::new(ctx)); } + /// Sends a new `FindNode` packet to the node with `target` as the lookup target but checks + /// whether we should should send a new ping first to renew the endpoint proof by checking the + /// previously failed findNode requests. It could be that the node is no longer reachable or + /// lost our entry. + fn find_node_checked(&mut self, node: &NodeRecord, ctx: LookupContext) { + let max_failures = self.config.max_find_node_failures; + let needs_ping = self + .on_entry(node.id, |entry| entry.exceeds_find_node_failures(max_failures)) + .unwrap_or(true); + if needs_ping { + self.try_ping(*node, PingReason::Lookup(*node, ctx)) + } else { + self.find_node(node, ctx) + } + } + /// Notifies all listeners. /// /// Removes all listeners that are closed. @@ -860,7 +880,7 @@ impl Discv4Service { self.kbuckets.buckets_iter().fold(0, |count, bucket| count + bucket.num_connected()) } - /// Check if the peer has a bond + /// Check if the peer has an active bond. fn has_bond(&self, remote_id: PeerId, remote_ip: IpAddr) -> bool { if let Some(timestamp) = self.received_pongs.last_pong(remote_id, remote_ip) { if timestamp.elapsed() < self.config.bond_expiration { @@ -870,6 +890,19 @@ impl Discv4Service { false } + /// Applies a closure on the pending or present [`NodeEntry`]. + fn on_entry(&mut self, peer_id: PeerId, f: F) -> Option + where + F: FnOnce(&NodeEntry) -> R, + { + let key = kad_key(peer_id); + match self.kbuckets.entry(&key) { + BucketEntry::Present(entry, _) => Some(f(entry.value())), + BucketEntry::Pending(mut entry, _) => Some(f(entry.value())), + _ => None, + } + } + /// Update the entry on RE-ping. /// /// Invoked when we received the Pong to our [`PingReason::RePing`] ping. @@ -929,7 +962,7 @@ impl Discv4Service { match self.kbuckets.entry(&key) { kbucket::Entry::Present(mut entry, old_status) => { // endpoint is now proven - entry.value_mut().has_endpoint_proof = true; + entry.value_mut().establish_proof(); entry.value_mut().update_with_enr(last_enr_seq); if !old_status.is_connected() { @@ -945,7 +978,7 @@ impl Discv4Service { } kbucket::Entry::Pending(mut entry, mut status) => { // endpoint is now proven - entry.value().has_endpoint_proof = true; + entry.value().establish_proof(); entry.value().update_with_enr(last_enr_seq); if !status.is_connected() { @@ -1129,6 +1162,8 @@ impl Discv4Service { // try to send it ctx.unmark_queried(record.id); } else { + // we just received a ping from that peer so we can send a find node request + // directly self.find_node(&record, ctx); } } @@ -1419,14 +1454,28 @@ impl Discv4Service { BucketEntry::SelfEntry => { // we received our own node entry } - BucketEntry::Present(mut entry, _) => { - if entry.value_mut().has_endpoint_proof { - self.find_node(&closest, ctx.clone()); + BucketEntry::Present(entry, _) => { + if entry.value().has_endpoint_proof { + if entry + .value() + .exceeds_find_node_failures(self.config.max_find_node_failures) + { + self.try_ping(closest, PingReason::Lookup(closest, ctx.clone())) + } else { + self.find_node(&closest, ctx.clone()); + } } } BucketEntry::Pending(mut entry, _) => { if entry.value().has_endpoint_proof { - self.find_node(&closest, ctx.clone()); + if entry + .value() + .exceeds_find_node_failures(self.config.max_find_node_failures) + { + self.try_ping(closest, PingReason::Lookup(closest, ctx.clone())) + } else { + self.find_node(&closest, ctx.clone()); + } } } } @@ -1486,27 +1535,27 @@ impl Discv4Service { self.remove_node(node_id); } - self.evict_failed_neighbours(now); + self.evict_failed_find_nodes(now); } /// Handles failed responses to `FindNode` - fn evict_failed_neighbours(&mut self, now: Instant) { - let mut failed_neighbours = Vec::new(); + fn evict_failed_find_nodes(&mut self, now: Instant) { + let mut failed_find_nodes = Vec::new(); self.pending_find_nodes.retain(|node_id, find_node_request| { if now.duration_since(find_node_request.sent_at) > self.config.neighbours_expiration { if !find_node_request.answered { // node actually responded but with fewer entries than expected, but we don't // treat this as an hard error since it responded. - failed_neighbours.push(*node_id); + failed_find_nodes.push(*node_id); } return false } true }); - trace!(target: "discv4", num=%failed_neighbours.len(), "processing failed neighbours"); + trace!(target: "discv4", num=%failed_find_nodes.len(), "processing failed find nodes"); - for node_id in failed_neighbours { + for node_id in failed_find_nodes { let key = kad_key(node_id); let failures = match self.kbuckets.entry(&key) { kbucket::Entry::Present(mut entry, _) => { @@ -1523,7 +1572,7 @@ impl Discv4Service { // if the node failed to respond anything useful multiple times, remove the node from // the table, but only if there are enough other nodes in the bucket (bucket must be at // least half full) - if failures > (self.config.max_find_node_failures as usize) { + if failures > self.config.max_find_node_failures { self.soft_remove_node(node_id); } } @@ -2216,8 +2265,8 @@ struct NodeEntry { last_enr_seq: Option, /// `ForkId` if retrieved via ENR requests. fork_id: Option, - /// Counter for failed findNode requests. - find_node_failures: usize, + /// Counter for failed _consecutive_ findNode requests. + find_node_failures: u8, /// Whether the endpoint of the peer is proven. has_endpoint_proof: bool, } @@ -2244,6 +2293,17 @@ impl NodeEntry { node } + /// Marks the entry with an established proof and resets the consecutive failure counter. + fn establish_proof(&mut self) { + self.has_endpoint_proof = true; + self.find_node_failures = 0; + } + + /// Returns true if the tracked find node failures exceed the max amount + const fn exceeds_find_node_failures(&self, max_failures: u8) -> bool { + self.find_node_failures >= max_failures + } + /// Updates the last timestamp and sets the enr seq fn update_with_enr(&mut self, last_enr_seq: Option) -> Option { self.update_now(|s| std::mem::replace(&mut s.last_enr_seq, last_enr_seq)) @@ -2660,6 +2720,45 @@ mod tests { assert_eq!(ctx.inner.closest_nodes.borrow().len(), 1); } + #[tokio::test] + async fn test_reping_on_find_node_failures() { + reth_tracing::init_test_tracing(); + + let config = Discv4Config::builder().build(); + let (_discv4, mut service) = create_discv4_with_config(config).await; + + let target = PeerId::random(); + + let id = PeerId::random(); + let key = kad_key(id); + let record = NodeRecord::new("0.0.0.0:0".parse().unwrap(), id); + + let mut entry = NodeEntry::new_proven(record); + entry.find_node_failures = u8::MAX; + let _ = service.kbuckets.insert_or_update( + &key, + entry, + NodeStatus { + direction: ConnectionDirection::Incoming, + state: ConnectionState::Connected, + }, + ); + + service.lookup(target); + assert_eq!(service.pending_find_nodes.len(), 0); + assert_eq!(service.pending_pings.len(), 1); + + service.update_on_pong(record, None); + + service + .on_entry(record.id, |entry| { + // reset on pong + assert_eq!(entry.find_node_failures, 0); + assert!(entry.has_endpoint_proof); + }) + .unwrap(); + } + #[tokio::test] async fn test_service_commands() { reth_tracing::init_test_tracing(); From ab407e74447fc02d7d48fc2bbe0f6807787ec0a0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Oct 2024 15:39:41 +0200 Subject: [PATCH 045/239] chore: bump alloy 054 (#12000) --- Cargo.lock | 104 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 54 ++++++++++++++-------------- 2 files changed, 79 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e10bdb0a675..f2d134cb85f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf02dfacfc815214f9b54ff50d54900ba527a68fd73e2c5637ced3460005045" +checksum = "41ed961a48297c732a5d97ee321aa8bb5009ecadbcb077d8bec90cb54e651629" dependencies = [ "alloy-eips", "alloy-primitives", @@ -177,9 +177,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769da342b6bcd945013925ef4c40763cc82f11e002c60702dba8b444bb60e5a7" +checksum = "b69e06cf9c37be824b9d26d6d101114fdde6af0c87de2828b414c05c4b3daa71" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c698ce0ada980b17f0323e1a28c7da8a2e9abc6dff5be9ee33d1525b28ac46b6" +checksum = "dde15e14944a88bd6a57d325e9a49b75558746fe16aaccc79713ae50a6a9574c" dependencies = [ "alloy-primitives", "alloy-serde", @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1050e1d65524c030b17442b6546b564da51fdab7f71bd534b001ba65f2ebb16" +checksum = "af5979e0d5a7bf9c7eb79749121e8256e59021af611322aee56e77e20776b4b3" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da34a18446a27734473af3d77eb21c5ebbdf97ea8eb65c39c0b50916bc659023" +checksum = "204237129086ce5dc17a58025e93739b01b45313841f98fa339eb1d780511e57" dependencies = [ "alloy-consensus", "alloy-eips", @@ -269,9 +269,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439fc6a933b9f8e8b272a8cac35dbeabaf2b2eaf9590482bebedb5782153118e" +checksum = "27444ea67d360508753022807cdd0b49a95c878924c9c5f8f32668b7d7768245" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -318,9 +318,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c45dbc0e3630becef9e988b69d43339f68d67e32a854e3c855bc28bd5031895b" +checksum = "4814d141ede360bb6cd1b4b064f1aab9de391e7c4d0d4d50ac89ea4bc1e25fbd" dependencies = [ "alloy-chains", "alloy-consensus", @@ -359,9 +359,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e3961a56e10f44bfd69dd3f4b0854b90b84c612b0c43708e738933e8b47f93a" +checksum = "96ba46eb69ddf7a9925b81f15229cb74658e6eebe5dd30a5b74e2cd040380573" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -400,9 +400,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917e5504e4f8f7e39bdc322ff81589ed54c1e462240adaeb58162c2d986a5a2b" +checksum = "7fc2bd1e7403463a5f2c61e955bcc9d3072b63aa177442b0f9aa6a6d22a941e3" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -425,9 +425,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c7eb2dc6db1dd41e5e7bd2b98a38813854efc30e034afd90d1e420e7f3de2b" +checksum = "eea9bf1abdd506f985a53533f5ac01296bcd6102c5e139bbc5d40bc468d2c916" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -438,9 +438,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd468a4e3eddcd9d612cad657852de4b7475ac2080e7af9224fbf1df20ddffe0" +checksum = "ea02c25541fb19eaac4278aa5c41d2d7e0245898887e54a74bfc0f3103e99415" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -450,9 +450,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2640928d9b1d43bb1cec7a0d615e10c2b407c5bd8ff1fcbe49e6318a2b62d731" +checksum = "2382fc63fb0cf3e02818d547b80cb66cc49a31f8803d0c328402b2008bc13650" dependencies = [ "alloy-primitives", "alloy-serde", @@ -461,9 +461,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f731ad2ef8d7dd75a4d28214f4922a5b683feee1e6df35bd7b427315f94366" +checksum = "45357a642081c8ce235c0ad990c4e9279f5f18a723545076b38cfcc05cc25234" dependencies = [ "alloy-eips", "alloy-primitives", @@ -475,9 +475,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06bd0757bfb3eccde06ee3f4e378f5839fe923d40956cff586018d4427a15bb5" +checksum = "a5afe3ab1038f90faf56304aa0adf1e6a8c9844615d8f83967f932f3a70390b1" dependencies = [ "alloy-primitives", "serde", @@ -485,9 +485,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3d95c3bf03efbb7bdc1d097e2931f520aac47438b709ccd8f065a7793dd371" +checksum = "886d22d41992287a235af2f3af4299b5ced2bcafb81eb835572ad35747476946" dependencies = [ "alloy-consensus", "alloy-eips", @@ -506,9 +506,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e855b0daccf2320ba415753c3fed422abe9d3ad5d77b2d6cafcc9bcf32fe387f" +checksum = "00b034779a4850b4b03f5be5ea674a1cf7d746b2da762b34d1860ab45e48ca27" dependencies = [ "alloy-consensus", "alloy-eips", @@ -527,9 +527,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca3753b9894235f915437f908644e737d8714c686ce4e8d03afbf585b23f074" +checksum = "3246948dfa5f5060a9abe04233d741ea656ef076b12958f3242416ce9f375058" dependencies = [ "alloy-eips", "alloy-primitives", @@ -540,9 +540,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae58a997afde032cd021547c960a53eef6245f47969dd71886e9f63fb45a6048" +checksum = "4e5fb6c5c401321f802f69dcdb95b932f30f8158f6798793f914baac5995628e" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -554,9 +554,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "667e45c882fda207d4cc94c4bb35e24a23347955113dcb236a5e4e0eaddef826" +checksum = "9ad066b49c3b1b5f64cdd2399177a19926a6a15db2dbf11e2098de621f9e7480" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -566,9 +566,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c2661ca6785add8fc37aff8005439c806ffad58254c19939c6f59ac0d6596e" +checksum = "028e72eaa9703e4882344983cfe7636ce06d8cce104a78ea62fd19b46659efc4" dependencies = [ "alloy-primitives", "arbitrary", @@ -578,9 +578,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eca011160d18a7dc6d8cdc1e8dc13e2e86c908f8e41b02aa76e429d6fe7085" +checksum = "592c185d7100258c041afac51877660c7bf6213447999787197db4842f0e938e" dependencies = [ "alloy-primitives", "async-trait", @@ -592,9 +592,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c54b195a6ee5a83f32e7c697b4e6b565966737ed5a2ef9176bbbb39f720d023" +checksum = "6614f02fc1d5b079b2a4a5320018317b506fd0a6d67c1fd5542a71201724986c" dependencies = [ "alloy-consensus", "alloy-network", @@ -680,9 +680,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4a136e733f55fef0870b81e1f8f1db28e78973d1b1ae5a5df642ba39538a07" +checksum = "be77579633ebbc1266ae6fd7694f75c408beb1aeb6865d0b18f22893c265a061" dependencies = [ "alloy-json-rpc", "base64 0.22.1", @@ -700,9 +700,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a6b358a89b6d107b92d09b61a61fbc04243942182709752c796f4b29402cead" +checksum = "91fd1a5d0827939847983b46f2f79510361f901dc82f8e3c38ac7397af142c6e" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -715,9 +715,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a899c43b7f5e3bc83762dfe5128fccd9cfa99f1f03c5f26bbfb2495ae8dcd35" +checksum = "8073d1186bfeeb8fbdd1292b6f1a0731f3aed8e21e1463905abfae0b96a887a6" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -734,9 +734,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27aac1246e13c9e6fa0c784fbb0c56872c6224f78dbde388bb2213ccdf8af02" +checksum = "61f27837bb4a1d6c83a28231c94493e814882f0e9058648a97e908a5f3fc9fcf" dependencies = [ "alloy-pubsub", "alloy-transport", diff --git a/Cargo.toml b/Cargo.toml index e8f10229ed59..22a78979dfcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -432,39 +432,39 @@ alloy-rlp = "0.3.4" alloy-sol-types = "0.8.0" alloy-trie = { version = "0.7", default-features = false } -alloy-consensus = { version = "0.5.3", default-features = false } -alloy-eips = { version = "0.5.3", default-features = false } -alloy-genesis = { version = "0.5.3", default-features = false } -alloy-json-rpc = { version = "0.5.3", default-features = false } -alloy-network = { version = "0.5.3", default-features = false } -alloy-network-primitives = { version = "0.5.3", default-features = false } -alloy-node-bindings = { version = "0.5.3", default-features = false } -alloy-provider = { version = "0.5.3", features = [ +alloy-consensus = { version = "0.5.4", default-features = false } +alloy-eips = { version = "0.5.4", default-features = false } +alloy-genesis = { version = "0.5.4", default-features = false } +alloy-json-rpc = { version = "0.5.4", default-features = false } +alloy-network = { version = "0.5.4", default-features = false } +alloy-network-primitives = { version = "0.5.4", default-features = false } +alloy-node-bindings = { version = "0.5.4", default-features = false } +alloy-provider = { version = "0.5.4", features = [ "reqwest", ], default-features = false } -alloy-pubsub = { version = "0.5.3", default-features = false } -alloy-rpc-client = { version = "0.5.3", default-features = false } -alloy-rpc-types = { version = "0.5.3", features = [ +alloy-pubsub = { version = "0.5.4", default-features = false } +alloy-rpc-client = { version = "0.5.4", default-features = false } +alloy-rpc-types = { version = "0.5.4", features = [ "eth", ], default-features = false } -alloy-rpc-types-admin = { version = "0.5.3", default-features = false } -alloy-rpc-types-anvil = { version = "0.5.3", default-features = false } -alloy-rpc-types-beacon = { version = "0.5.3", default-features = false } -alloy-rpc-types-debug = { version = "0.5.3", default-features = false } -alloy-rpc-types-engine = { version = "0.5.3", default-features = false } -alloy-rpc-types-eth = { version = "0.5.3", default-features = false } -alloy-rpc-types-mev = { version = "0.5.3", default-features = false } -alloy-rpc-types-trace = { version = "0.5.3", default-features = false } -alloy-rpc-types-txpool = { version = "0.5.3", default-features = false } -alloy-serde = { version = "0.5.3", default-features = false } -alloy-signer = { version = "0.5.3", default-features = false } -alloy-signer-local = { version = "0.5.3", default-features = false } -alloy-transport = { version = "0.5.3" } -alloy-transport-http = { version = "0.5.3", features = [ +alloy-rpc-types-admin = { version = "0.5.4", default-features = false } +alloy-rpc-types-anvil = { version = "0.5.4", default-features = false } +alloy-rpc-types-beacon = { version = "0.5.4", default-features = false } +alloy-rpc-types-debug = { version = "0.5.4", default-features = false } +alloy-rpc-types-engine = { version = "0.5.4", default-features = false } +alloy-rpc-types-eth = { version = "0.5.4", default-features = false } +alloy-rpc-types-mev = { version = "0.5.4", default-features = false } +alloy-rpc-types-trace = { version = "0.5.4", default-features = false } +alloy-rpc-types-txpool = { version = "0.5.4", default-features = false } +alloy-serde = { version = "0.5.4", default-features = false } +alloy-signer = { version = "0.5.4", default-features = false } +alloy-signer-local = { version = "0.5.4", default-features = false } +alloy-transport = { version = "0.5.4" } +alloy-transport-http = { version = "0.5.4", features = [ "reqwest-rustls-tls", ], default-features = false } -alloy-transport-ipc = { version = "0.5.3", default-features = false } -alloy-transport-ws = { version = "0.5.3", default-features = false } +alloy-transport-ipc = { version = "0.5.4", default-features = false } +alloy-transport-ws = { version = "0.5.4", default-features = false } # op op-alloy-rpc-types = "0.5" From 8a40d5c6aabd8f232b9aaacce0ca474a480a9b64 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 23 Oct 2024 22:40:14 +0900 Subject: [PATCH 046/239] feat(providers): add `AtomicBlockchainProvider` (#11705) Co-authored-by: Matthias Seitz --- crates/chain-state/src/in_memory.rs | 69 +- crates/chain-state/src/lib.rs | 2 +- crates/chain-state/src/memory_overlay.rs | 349 +-- crates/engine/service/src/service.rs | 4 +- .../src/providers/blockchain_provider.rs | 1137 +--------- .../provider/src/providers/consistent.rs | 1871 +++++++++++++++++ crates/storage/provider/src/providers/mod.rs | 3 + 7 files changed, 2220 insertions(+), 1215 deletions(-) create mode 100644 crates/storage/provider/src/providers/consistent.rs diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index be33e1fd79a8..a850e66521a6 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -4,7 +4,7 @@ use crate::{ CanonStateNotification, CanonStateNotificationSender, CanonStateNotifications, ChainInfoTracker, MemoryOverlayStateProvider, }; -use alloy_eips::BlockNumHash; +use alloy_eips::{BlockHashOrNumber, BlockNumHash}; use alloy_primitives::{map::HashMap, Address, TxHash, B256}; use parking_lot::RwLock; use reth_chainspec::ChainInfo; @@ -514,7 +514,7 @@ impl CanonicalInMemoryState { historical: StateProviderBox, ) -> MemoryOverlayStateProvider { let in_memory = if let Some(state) = self.state_by_hash(hash) { - state.chain().into_iter().map(|block_state| block_state.block()).collect() + state.chain().map(|block_state| block_state.block()).collect() } else { Vec::new() }; @@ -692,10 +692,8 @@ impl BlockState { /// Returns a vector of `BlockStates` representing the entire in memory chain. /// The block state order in the output vector is newest to oldest (highest to lowest), /// including self as the first element. - pub fn chain(&self) -> Vec<&Self> { - let mut chain = vec![self]; - self.append_parent_chain(&mut chain); - chain + pub fn chain(&self) -> impl Iterator { + std::iter::successors(Some(self), |state| state.parent.as_deref()) } /// Appends the parent chain of this [`BlockState`] to the given vector. @@ -715,10 +713,59 @@ impl BlockState { /// This merges the state of all blocks that are part of the chain that the this block is /// the head of. This includes all blocks that connect back to the canonical block on disk. pub fn state_provider(&self, historical: StateProviderBox) -> MemoryOverlayStateProvider { - let in_memory = self.chain().into_iter().map(|block_state| block_state.block()).collect(); + let in_memory = self.chain().map(|block_state| block_state.block()).collect(); MemoryOverlayStateProvider::new(historical, in_memory) } + + /// Tries to find a block by [`BlockHashOrNumber`] in the chain ending at this block. + pub fn block_on_chain(&self, hash_or_num: BlockHashOrNumber) -> Option<&Self> { + self.chain().find(|block| match hash_or_num { + BlockHashOrNumber::Hash(hash) => block.hash() == hash, + BlockHashOrNumber::Number(number) => block.number() == number, + }) + } + + /// Tries to find a transaction by [`TxHash`] in the chain ending at this block. + pub fn transaction_on_chain(&self, hash: TxHash) -> Option { + self.chain().find_map(|block_state| { + block_state + .block_ref() + .block() + .body + .transactions() + .find(|tx| tx.hash() == hash) + .cloned() + }) + } + + /// Tries to find a transaction with meta by [`TxHash`] in the chain ending at this block. + pub fn transaction_meta_on_chain( + &self, + tx_hash: TxHash, + ) -> Option<(TransactionSigned, TransactionMeta)> { + self.chain().find_map(|block_state| { + block_state + .block_ref() + .block() + .body + .transactions() + .enumerate() + .find(|(_, tx)| tx.hash() == tx_hash) + .map(|(index, tx)| { + let meta = TransactionMeta { + tx_hash, + index: index as u64, + block_hash: block_state.hash(), + block_number: block_state.block_ref().block.number, + base_fee: block_state.block_ref().block.header.base_fee_per_gas, + timestamp: block_state.block_ref().block.timestamp, + excess_blob_gas: block_state.block_ref().block.excess_blob_gas, + }; + (tx.clone(), meta) + }) + }) + } } /// Represents an executed block stored in-memory. @@ -1382,7 +1429,7 @@ mod tests { let parents = single_block.parent_state_chain(); assert_eq!(parents.len(), 0); - let block_state_chain = single_block.chain(); + let block_state_chain = single_block.chain().collect::>(); assert_eq!(block_state_chain.len(), 1); assert_eq!(block_state_chain[0].block().block.number, single_block_number); assert_eq!(block_state_chain[0].block().block.hash(), single_block_hash); @@ -1393,18 +1440,18 @@ mod tests { let mut test_block_builder = TestBlockBuilder::default(); let chain = create_mock_state_chain(&mut test_block_builder, 3); - let block_state_chain = chain[2].chain(); + let block_state_chain = chain[2].chain().collect::>(); assert_eq!(block_state_chain.len(), 3); assert_eq!(block_state_chain[0].block().block.number, 3); assert_eq!(block_state_chain[1].block().block.number, 2); assert_eq!(block_state_chain[2].block().block.number, 1); - let block_state_chain = chain[1].chain(); + let block_state_chain = chain[1].chain().collect::>(); assert_eq!(block_state_chain.len(), 2); assert_eq!(block_state_chain[0].block().block.number, 2); assert_eq!(block_state_chain[1].block().block.number, 1); - let block_state_chain = chain[0].chain(); + let block_state_chain = chain[0].chain().collect::>(); assert_eq!(block_state_chain.len(), 1); assert_eq!(block_state_chain[0].block().block.number, 1); } diff --git a/crates/chain-state/src/lib.rs b/crates/chain-state/src/lib.rs index 50a103111071..bd9b43a59eae 100644 --- a/crates/chain-state/src/lib.rs +++ b/crates/chain-state/src/lib.rs @@ -22,7 +22,7 @@ pub use notifications::{ }; mod memory_overlay; -pub use memory_overlay::MemoryOverlayStateProvider; +pub use memory_overlay::{MemoryOverlayStateProvider, MemoryOverlayStateProviderRef}; #[cfg(any(test, feature = "test-utils"))] /// Common test helpers diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs index eb125dad115e..ada0faee4907 100644 --- a/crates/chain-state/src/memory_overlay.rs +++ b/crates/chain-state/src/memory_overlay.rs @@ -7,14 +7,26 @@ use alloy_primitives::{ use reth_errors::ProviderResult; use reth_primitives::{Account, Bytecode}; use reth_storage_api::{ - AccountReader, BlockHashReader, StateProofProvider, StateProvider, StateProviderBox, - StateRootProvider, StorageRootProvider, + AccountReader, BlockHashReader, StateProofProvider, StateProvider, StateRootProvider, + StorageRootProvider, }; use reth_trie::{ updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, }; use std::sync::OnceLock; +/// A state provider that stores references to in-memory blocks along with their state as well as a +/// reference of the historical state provider for fallback lookups. +#[allow(missing_debug_implementations)] +pub struct MemoryOverlayStateProviderRef<'a> { + /// Historical state provider for state lookups that are not found in in-memory blocks. + pub(crate) historical: Box, + /// The collection of executed parent blocks. Expected order is newest to oldest. + pub(crate) in_memory: Vec, + /// Lazy-loaded in-memory trie data. + pub(crate) trie_state: OnceLock, +} + /// A state provider that stores references to in-memory blocks along with their state as well as /// the historical state provider for fallback lookups. #[allow(missing_debug_implementations)] @@ -27,193 +39,200 @@ pub struct MemoryOverlayStateProvider { pub(crate) trie_state: OnceLock, } -impl MemoryOverlayStateProvider { - /// Create new memory overlay state provider. - /// - /// ## Arguments - /// - /// - `in_memory` - the collection of executed ancestor blocks in reverse. - /// - `historical` - a historical state provider for the latest ancestor block stored in the - /// database. - pub fn new(historical: Box, in_memory: Vec) -> Self { - Self { historical, in_memory, trie_state: OnceLock::new() } - } - - /// Turn this state provider into a [`StateProviderBox`] - pub fn boxed(self) -> StateProviderBox { - Box::new(self) - } - - /// Return lazy-loaded trie state aggregated from in-memory blocks. - fn trie_state(&self) -> &MemoryOverlayTrieState { - self.trie_state.get_or_init(|| { - let mut trie_state = MemoryOverlayTrieState::default(); - for block in self.in_memory.iter().rev() { - trie_state.state.extend_ref(block.hashed_state.as_ref()); - trie_state.nodes.extend_ref(block.trie.as_ref()); - } - trie_state - }) - } -} +macro_rules! impl_state_provider { + ([$($tokens:tt)*],$type:ty, $historical_type:ty) => { + impl $($tokens)* $type { + /// Create new memory overlay state provider. + /// + /// ## Arguments + /// + /// - `in_memory` - the collection of executed ancestor blocks in reverse. + /// - `historical` - a historical state provider for the latest ancestor block stored in the + /// database. + pub fn new(historical: $historical_type, in_memory: Vec) -> Self { + Self { historical, in_memory, trie_state: OnceLock::new() } + } + + /// Turn this state provider into a state provider + pub fn boxed(self) -> $historical_type { + Box::new(self) + } -impl BlockHashReader for MemoryOverlayStateProvider { - fn block_hash(&self, number: BlockNumber) -> ProviderResult> { - for block in &self.in_memory { - if block.block.number == number { - return Ok(Some(block.block.hash())) + /// Return lazy-loaded trie state aggregated from in-memory blocks. + fn trie_state(&self) -> &MemoryOverlayTrieState { + self.trie_state.get_or_init(|| { + let mut trie_state = MemoryOverlayTrieState::default(); + for block in self.in_memory.iter().rev() { + trie_state.state.extend_ref(block.hashed_state.as_ref()); + trie_state.nodes.extend_ref(block.trie.as_ref()); + } + trie_state + }) } } - self.historical.block_hash(number) - } - - fn canonical_hashes_range( - &self, - start: BlockNumber, - end: BlockNumber, - ) -> ProviderResult> { - let range = start..end; - let mut earliest_block_number = None; - let mut in_memory_hashes = Vec::new(); - for block in &self.in_memory { - if range.contains(&block.block.number) { - in_memory_hashes.insert(0, block.block.hash()); - earliest_block_number = Some(block.block.number); + impl $($tokens)* BlockHashReader for $type { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + for block in &self.in_memory { + if block.block.number == number { + return Ok(Some(block.block.hash())) + } + } + + self.historical.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + let range = start..end; + let mut earliest_block_number = None; + let mut in_memory_hashes = Vec::new(); + for block in &self.in_memory { + if range.contains(&block.block.number) { + in_memory_hashes.insert(0, block.block.hash()); + earliest_block_number = Some(block.block.number); + } + } + + let mut hashes = + self.historical.canonical_hashes_range(start, earliest_block_number.unwrap_or(end))?; + hashes.append(&mut in_memory_hashes); + Ok(hashes) } } - let mut hashes = - self.historical.canonical_hashes_range(start, earliest_block_number.unwrap_or(end))?; - hashes.append(&mut in_memory_hashes); - Ok(hashes) - } -} + impl $($tokens)* AccountReader for $type { + fn basic_account(&self, address: Address) -> ProviderResult> { + for block in &self.in_memory { + if let Some(account) = block.execution_output.account(&address) { + return Ok(account) + } + } -impl AccountReader for MemoryOverlayStateProvider { - fn basic_account(&self, address: Address) -> ProviderResult> { - for block in &self.in_memory { - if let Some(account) = block.execution_output.account(&address) { - return Ok(account) + self.historical.basic_account(address) } } - self.historical.basic_account(address) - } -} + impl $($tokens)* StateRootProvider for $type { + fn state_root(&self, state: HashedPostState) -> ProviderResult { + self.state_root_from_nodes(TrieInput::from_state(state)) + } -impl StateRootProvider for MemoryOverlayStateProvider { - fn state_root(&self, state: HashedPostState) -> ProviderResult { - self.state_root_from_nodes(TrieInput::from_state(state)) - } - - fn state_root_from_nodes(&self, mut input: TrieInput) -> ProviderResult { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); - self.historical.state_root_from_nodes(input) - } - - fn state_root_with_updates( - &self, - state: HashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - self.state_root_from_nodes_with_updates(TrieInput::from_state(state)) - } - - fn state_root_from_nodes_with_updates( - &self, - mut input: TrieInput, - ) -> ProviderResult<(B256, TrieUpdates)> { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); - self.historical.state_root_from_nodes_with_updates(input) - } -} + fn state_root_from_nodes(&self, mut input: TrieInput) -> ProviderResult { + let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); + input.prepend_cached(nodes, state); + self.historical.state_root_from_nodes(input) + } -impl StorageRootProvider for MemoryOverlayStateProvider { - // TODO: Currently this does not reuse available in-memory trie nodes. - fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { - let state = &self.trie_state().state; - let mut hashed_storage = - state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); - hashed_storage.extend(&storage); - self.historical.storage_root(address, hashed_storage) - } - - // TODO: Currently this does not reuse available in-memory trie nodes. - fn storage_proof( - &self, - address: Address, - slot: B256, - storage: HashedStorage, - ) -> ProviderResult { - let state = &self.trie_state().state; - let mut hashed_storage = - state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); - hashed_storage.extend(&storage); - self.historical.storage_proof(address, slot, hashed_storage) - } -} + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + self.state_root_from_nodes_with_updates(TrieInput::from_state(state)) + } -impl StateProofProvider for MemoryOverlayStateProvider { - fn proof( - &self, - mut input: TrieInput, - address: Address, - slots: &[B256], - ) -> ProviderResult { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); - self.historical.proof(input, address, slots) - } - - fn multiproof( - &self, - mut input: TrieInput, - targets: HashMap>, - ) -> ProviderResult { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); - self.historical.multiproof(input, targets) - } - - fn witness( - &self, - mut input: TrieInput, - target: HashedPostState, - ) -> ProviderResult> { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); - self.historical.witness(input, target) - } -} + fn state_root_from_nodes_with_updates( + &self, + mut input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); + input.prepend_cached(nodes, state); + self.historical.state_root_from_nodes_with_updates(input) + } + } + + impl $($tokens)* StorageRootProvider for $type { + // TODO: Currently this does not reuse available in-memory trie nodes. + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + let state = &self.trie_state().state; + let mut hashed_storage = + state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); + hashed_storage.extend(&storage); + self.historical.storage_root(address, hashed_storage) + } -impl StateProvider for MemoryOverlayStateProvider { - fn storage( - &self, - address: Address, - storage_key: StorageKey, - ) -> ProviderResult> { - for block in &self.in_memory { - if let Some(value) = block.execution_output.storage(&address, storage_key.into()) { - return Ok(Some(value)) + // TODO: Currently this does not reuse available in-memory trie nodes. + fn storage_proof( + &self, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> ProviderResult { + let state = &self.trie_state().state; + let mut hashed_storage = + state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); + hashed_storage.extend(&storage); + self.historical.storage_proof(address, slot, hashed_storage) } } - self.historical.storage(address, storage_key) - } + impl $($tokens)* StateProofProvider for $type { + fn proof( + &self, + mut input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); + input.prepend_cached(nodes, state); + self.historical.proof(input, address, slots) + } + + fn multiproof( + &self, + mut input: TrieInput, + targets: HashMap>, + ) -> ProviderResult { + let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); + input.prepend_cached(nodes, state); + self.historical.multiproof(input, targets) + } - fn bytecode_by_hash(&self, code_hash: B256) -> ProviderResult> { - for block in &self.in_memory { - if let Some(contract) = block.execution_output.bytecode(&code_hash) { - return Ok(Some(contract)) + fn witness( + &self, + mut input: TrieInput, + target: HashedPostState, + ) -> ProviderResult> { + let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); + input.prepend_cached(nodes, state); + self.historical.witness(input, target) } } - self.historical.bytecode_by_hash(code_hash) - } + impl $($tokens)* StateProvider for $type { + fn storage( + &self, + address: Address, + storage_key: StorageKey, + ) -> ProviderResult> { + for block in &self.in_memory { + if let Some(value) = block.execution_output.storage(&address, storage_key.into()) { + return Ok(Some(value)) + } + } + + self.historical.storage(address, storage_key) + } + + fn bytecode_by_hash(&self, code_hash: B256) -> ProviderResult> { + for block in &self.in_memory { + if let Some(contract) = block.execution_output.bytecode(&code_hash) { + return Ok(Some(contract)) + } + } + + self.historical.bytecode_by_hash(code_hash) + } + } + }; } +impl_state_provider!([], MemoryOverlayStateProvider, Box); +impl_state_provider!([<'a>], MemoryOverlayStateProviderRef<'a>, Box); + /// The collection of data necessary for trie-related operations for [`MemoryOverlayStateProvider`]. #[derive(Clone, Default, Debug)] pub(crate) struct MemoryOverlayTrieState { diff --git a/crates/engine/service/src/service.rs b/crates/engine/service/src/service.rs index 026476a8260f..198438d457f1 100644 --- a/crates/engine/service/src/service.rs +++ b/crates/engine/service/src/service.rs @@ -151,7 +151,9 @@ mod tests { use reth_exex_types::FinishedExExHeight; use reth_network_p2p::test_utils::TestFullBlockClient; use reth_primitives::SealedHeader; - use reth_provider::test_utils::create_test_provider_factory_with_chain_spec; + use reth_provider::{ + providers::BlockchainProvider2, test_utils::create_test_provider_factory_with_chain_spec, + }; use reth_prune::Pruner; use reth_tasks::TokioTaskExecutor; use std::sync::Arc; diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 13215e11a8e2..64a8a204a329 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,13 +1,15 @@ +#![allow(unused)] use crate::{ - providers::StaticFileProvider, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, - BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, - CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, - DatabaseProviderFactory, DatabaseProviderRO, EvmEnvProvider, HeaderProvider, ProviderError, + providers::{ConsistentProvider, StaticFileProvider}, + AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, + BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, + ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, DatabaseProvider, + DatabaseProviderFactory, EvmEnvProvider, FullProvider, HeaderProvider, ProviderError, ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, WithdrawalsProvider, }; -use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag, HashOrNumber}; +use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{Address, BlockHash, BlockNumber, Sealable, TxHash, TxNumber, B256, U256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ @@ -15,10 +17,10 @@ use reth_chain_state::{ MemoryOverlayStateProvider, }; use reth_chainspec::{ChainInfo, EthereumHardforks}; -use reth_db::models::BlockNumberAddress; +use reth_db::{models::BlockNumberAddress, transaction::DbTx, Database}; use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_evm::ConfigureEvmEnv; -use reth_execution_types::{BundleStateInit, ExecutionOutcome, RevertsInit}; +use reth_execution_types::ExecutionOutcome; use reth_node_types::NodeTypesWithDB; use reth_primitives::{ Account, Block, BlockWithSenders, Header, Receipt, SealedBlock, SealedBlockWithSenders, @@ -27,21 +29,17 @@ use reth_primitives::{ }; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; -use reth_storage_api::StorageChangeSetReader; +use reth_storage_api::{DBProvider, StorageChangeSetReader}; use reth_storage_errors::provider::ProviderResult; -use revm::{ - db::states::PlainStorageRevert, - primitives::{BlockEnv, CfgEnvWithHandlerCfg}, -}; +use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; use std::{ - collections::{hash_map, HashMap}, - ops::{Add, Bound, RangeBounds, RangeInclusive, Sub}, + ops::{Add, RangeBounds, RangeInclusive, Sub}, sync::Arc, time::Instant, }; use tracing::trace; -use super::ProviderNodeTypes; +use crate::providers::ProviderNodeTypes; /// The main type for interacting with the blockchain. /// @@ -50,11 +48,11 @@ use super::ProviderNodeTypes; /// type that holds an instance of the database and the blockchain tree. #[derive(Debug)] pub struct BlockchainProvider2 { - /// Provider type used to access the database. - database: ProviderFactory, + /// Provider factory used to access the database. + pub(crate) database: ProviderFactory, /// Tracks the chain info wrt forkchoice updates and in memory canonical /// state. - pub(super) canonical_in_memory_state: CanonicalInMemoryState, + pub(crate) canonical_in_memory_state: CanonicalInMemoryState, } impl Clone for BlockchainProvider2 { @@ -67,15 +65,15 @@ impl Clone for BlockchainProvider2 { } impl BlockchainProvider2 { - /// Create a new provider using only the database, fetching the latest header from - /// the database to initialize the provider. - pub fn new(database: ProviderFactory) -> ProviderResult { - let provider = database.provider()?; + /// Create a new [`BlockchainProvider2`] using only the storage, fetching the latest + /// header from the database to initialize the provider. + pub fn new(storage: ProviderFactory) -> ProviderResult { + let provider = storage.provider()?; let best = provider.chain_info()?; match provider.header_by_number(best.best_number)? { Some(header) => { drop(provider); - Ok(Self::with_latest(database, SealedHeader::new(header, best.best_hash))?) + Ok(Self::with_latest(storage, SealedHeader::new(header, best.best_hash))?) } None => Err(ProviderError::HeaderNotFound(best.best_number.into())), } @@ -86,8 +84,8 @@ impl BlockchainProvider2 { /// /// This returns a `ProviderResult` since it tries the retrieve the last finalized header from /// `database`. - pub fn with_latest(database: ProviderFactory, latest: SealedHeader) -> ProviderResult { - let provider = database.provider()?; + pub fn with_latest(storage: ProviderFactory, latest: SealedHeader) -> ProviderResult { + let provider = storage.provider()?; let finalized_header = provider .last_finalized_block_number()? .map(|num| provider.sealed_header(num)) @@ -104,7 +102,7 @@ impl BlockchainProvider2 { .transpose()? .flatten(); Ok(Self { - database, + database: storage, canonical_in_memory_state: CanonicalInMemoryState::with_head( latest, finalized_header, @@ -118,281 +116,12 @@ impl BlockchainProvider2 { self.canonical_in_memory_state.clone() } - // Helper function to convert range bounds - fn convert_range_bounds( - &self, - range: impl RangeBounds, - end_unbounded: impl FnOnce() -> T, - ) -> (T, T) - where - T: Copy + Add + Sub + From, - { - let start = match range.start_bound() { - Bound::Included(&n) => n, - Bound::Excluded(&n) => n + T::from(1u8), - Bound::Unbounded => T::from(0u8), - }; - - let end = match range.end_bound() { - Bound::Included(&n) => n, - Bound::Excluded(&n) => n - T::from(1u8), - Bound::Unbounded => end_unbounded(), - }; - - (start, end) - } - - /// Return the last N blocks of state, recreating the [`ExecutionOutcome`]. - /// - /// If the range is empty, or there are no blocks for the given range, then this returns `None`. - pub fn get_state( - &self, - range: RangeInclusive, - ) -> ProviderResult> { - if range.is_empty() { - return Ok(None) - } - let start_block_number = *range.start(); - let end_block_number = *range.end(); - - // We are not removing block meta as it is used to get block changesets. - let mut block_bodies = Vec::new(); - for block_num in range.clone() { - let block_body = self - .block_body_indices(block_num)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(block_num))?; - block_bodies.push((block_num, block_body)) - } - - // get transaction receipts - let Some(from_transaction_num) = block_bodies.first().map(|body| body.1.first_tx_num()) - else { - return Ok(None) - }; - let Some(to_transaction_num) = block_bodies.last().map(|body| body.1.last_tx_num()) else { - return Ok(None) - }; - - let mut account_changeset = Vec::new(); - for block_num in range.clone() { - let changeset = - self.account_block_changeset(block_num)?.into_iter().map(|elem| (block_num, elem)); - account_changeset.extend(changeset); - } - - let mut storage_changeset = Vec::new(); - for block_num in range { - let changeset = self.storage_changeset(block_num)?; - storage_changeset.extend(changeset); - } - - let (state, reverts) = - self.populate_bundle_state(account_changeset, storage_changeset, end_block_number)?; - - let mut receipt_iter = - self.receipts_by_tx_range(from_transaction_num..=to_transaction_num)?.into_iter(); - - let mut receipts = Vec::with_capacity(block_bodies.len()); - // loop break if we are at the end of the blocks. - for (_, block_body) in block_bodies { - let mut block_receipts = Vec::with_capacity(block_body.tx_count as usize); - for tx_num in block_body.tx_num_range() { - let receipt = receipt_iter - .next() - .ok_or_else(|| ProviderError::ReceiptNotFound(tx_num.into()))?; - block_receipts.push(Some(receipt)); - } - receipts.push(block_receipts); - } - - Ok(Some(ExecutionOutcome::new_init( - state, - reverts, - // We skip new contracts since we never delete them from the database - Vec::new(), - receipts.into(), - start_block_number, - Vec::new(), - ))) - } - - /// Populate a [`BundleStateInit`] and [`RevertsInit`] using cursors over the - /// [`reth_db::PlainAccountState`] and [`reth_db::PlainStorageState`] tables, based on the given - /// storage and account changesets. - fn populate_bundle_state( - &self, - account_changeset: Vec<(u64, AccountBeforeTx)>, - storage_changeset: Vec<(BlockNumberAddress, StorageEntry)>, - block_range_end: BlockNumber, - ) -> ProviderResult<(BundleStateInit, RevertsInit)> { - let mut state: BundleStateInit = HashMap::new(); - let mut reverts: RevertsInit = HashMap::new(); - let state_provider = self.state_by_block_number_or_tag(block_range_end.into())?; - - // add account changeset changes - for (block_number, account_before) in account_changeset.into_iter().rev() { - let AccountBeforeTx { info: old_info, address } = account_before; - match state.entry(address) { - hash_map::Entry::Vacant(entry) => { - let new_info = state_provider.basic_account(address)?; - entry.insert((old_info, new_info, HashMap::new())); - } - hash_map::Entry::Occupied(mut entry) => { - // overwrite old account state. - entry.get_mut().0 = old_info; - } - } - // insert old info into reverts. - reverts.entry(block_number).or_default().entry(address).or_default().0 = Some(old_info); - } - - // add storage changeset changes - for (block_and_address, old_storage) in storage_changeset.into_iter().rev() { - let BlockNumberAddress((block_number, address)) = block_and_address; - // get account state or insert from plain state. - let account_state = match state.entry(address) { - hash_map::Entry::Vacant(entry) => { - let present_info = state_provider.basic_account(address)?; - entry.insert((present_info, present_info, HashMap::new())) - } - hash_map::Entry::Occupied(entry) => entry.into_mut(), - }; - - // match storage. - match account_state.2.entry(old_storage.key) { - hash_map::Entry::Vacant(entry) => { - let new_storage_value = - state_provider.storage(address, old_storage.key)?.unwrap_or_default(); - entry.insert((old_storage.value, new_storage_value)); - } - hash_map::Entry::Occupied(mut entry) => { - entry.get_mut().0 = old_storage.value; - } - }; - - reverts - .entry(block_number) - .or_default() - .entry(address) - .or_default() - .1 - .push(old_storage); - } - - Ok((state, reverts)) - } - - /// Fetches a range of data from both in-memory state and persistent storage while a predicate - /// is met. - /// - /// Creates a snapshot of the in-memory chain state and database provider to prevent - /// inconsistencies. Splits the range into in-memory and storage sections, prioritizing - /// recent in-memory blocks in case of overlaps. - /// - /// * `fetch_db_range` function (`F`) provides access to the database provider, allowing the - /// user to retrieve the required items from the database using [`RangeInclusive`]. - /// * `map_block_state_item` function (`G`) provides each block of the range in the in-memory - /// state, allowing for selection or filtering for the desired data. - fn get_in_memory_or_storage_by_block_range_while( - &self, - range: impl RangeBounds, - fetch_db_range: F, - map_block_state_item: G, - mut predicate: P, - ) -> ProviderResult> - where - F: FnOnce( - &DatabaseProviderRO, - RangeInclusive, - &mut P, - ) -> ProviderResult>, - G: Fn(Arc, &mut P) -> Option, - P: FnMut(&T) -> bool, - { - // Each one provides a snapshot at the time of instantiation, but its order matters. - // - // If we acquire first the database provider, it's possible that before the in-memory chain - // snapshot is instantiated, it will flush blocks to disk. This would - // mean that our database provider would not have access to the flushed blocks (since it's - // working under an older view), while the in-memory state may have deleted them - // entirely. Resulting in gaps on the range. - let mut in_memory_chain = - self.canonical_in_memory_state.canonical_chain().collect::>(); - let db_provider = self.database_provider_ro()?; - - let (start, end) = self.convert_range_bounds(range, || { - // the first block is the highest one. - in_memory_chain - .first() - .map(|b| b.number()) - .unwrap_or_else(|| db_provider.last_block_number().unwrap_or_default()) - }); - - if start > end { - return Ok(vec![]) - } - - // Split range into storage_range and in-memory range. If the in-memory range is not - // necessary drop it early. - // - // The last block of `in_memory_chain` is the lowest block number. - let (in_memory, storage_range) = match in_memory_chain.last().as_ref().map(|b| b.number()) { - Some(lowest_memory_block) if lowest_memory_block <= end => { - let highest_memory_block = - in_memory_chain.first().as_ref().map(|b| b.number()).expect("qed"); - - // Database will for a time overlap with in-memory-chain blocks. In - // case of a re-org, it can mean that the database blocks are of a forked chain, and - // so, we should prioritize the in-memory overlapped blocks. - let in_memory_range = - lowest_memory_block.max(start)..=end.min(highest_memory_block); - - // If requested range is in the middle of the in-memory range, remove the necessary - // lowest blocks - in_memory_chain.truncate( - in_memory_chain - .len() - .saturating_sub(start.saturating_sub(lowest_memory_block) as usize), - ); - - let storage_range = - (lowest_memory_block > start).then(|| start..=lowest_memory_block - 1); - - (Some((in_memory_chain, in_memory_range)), storage_range) - } - _ => { - // Drop the in-memory chain so we don't hold blocks in memory. - drop(in_memory_chain); - - (None, Some(start..=end)) - } - }; - - let mut items = Vec::with_capacity((end - start + 1) as usize); - - if let Some(storage_range) = storage_range { - let mut db_items = fetch_db_range(&db_provider, storage_range.clone(), &mut predicate)?; - items.append(&mut db_items); - - // The predicate was not met, if the number of items differs from the expected. So, we - // return what we have. - if items.len() as u64 != storage_range.end() - storage_range.start() + 1 { - return Ok(items) - } - } - - if let Some((in_memory_chain, in_memory_range)) = in_memory { - for (num, block) in in_memory_range.zip(in_memory_chain.into_iter().rev()) { - debug_assert!(num == block.number()); - if let Some(item) = map_block_state_item(block, &mut predicate) { - items.push(item); - } else { - break - } - } - } - - Ok(items) + /// Returns a provider with a created `DbTx` inside, which allows fetching data from the + /// database using different types of providers. Example: [`HeaderProvider`] + /// [`BlockHashReader`]. This may fail if the inner read database transaction fails to open. + #[track_caller] + pub fn consistent_provider(&self) -> ProviderResult> { + ConsistentProvider::new(self.database.clone(), self.canonical_in_memory_state()) } /// This uses a given [`BlockState`] to initialize a state provider for that block. @@ -405,222 +134,14 @@ impl BlockchainProvider2 { Ok(state.state_provider(latest_historical)) } - /// Fetches data from either in-memory state or persistent storage for a range of transactions. + /// Return the last N blocks of state, recreating the [`ExecutionOutcome`]. /// - /// * `fetch_from_db`: has a [`DatabaseProviderRO`] and the storage specific range. - /// * `fetch_from_block_state`: has a [`RangeInclusive`] of elements that should be fetched from - /// [`BlockState`]. [`RangeInclusive`] is necessary to handle partial look-ups of a block. - fn get_in_memory_or_storage_by_tx_range( - &self, - range: impl RangeBounds, - fetch_from_db: S, - fetch_from_block_state: M, - ) -> ProviderResult> - where - S: FnOnce( - DatabaseProviderRO, - RangeInclusive, - ) -> ProviderResult>, - M: Fn(RangeInclusive, Arc) -> ProviderResult>, - { - let in_mem_chain = self.canonical_in_memory_state.canonical_chain().collect::>(); - let provider = self.database.provider()?; - - // Get the last block number stored in the storage which does NOT overlap with in-memory - // chain. - let last_database_block_number = in_mem_chain - .last() - .map(|b| Ok(b.anchor().number)) - .unwrap_or_else(|| provider.last_block_number())?; - - // Get the next tx number for the last block stored in the storage, which marks the start of - // the in-memory state. - let last_block_body_index = provider - .block_body_indices(last_database_block_number)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(last_database_block_number))?; - let mut in_memory_tx_num = last_block_body_index.next_tx_num(); - - let (start, end) = self.convert_range_bounds(range, || { - in_mem_chain - .iter() - .map(|b| b.block_ref().block().body.transactions.len() as u64) - .sum::() + - last_block_body_index.last_tx_num() - }); - - if start > end { - return Ok(vec![]) - } - - let mut tx_range = start..=end; - - // If the range is entirely before the first in-memory transaction number, fetch from - // storage - if *tx_range.end() < in_memory_tx_num { - return fetch_from_db(provider, tx_range); - } - - let mut items = Vec::with_capacity((tx_range.end() - tx_range.start() + 1) as usize); - - // If the range spans storage and memory, get elements from storage first. - if *tx_range.start() < in_memory_tx_num { - // Determine the range that needs to be fetched from storage. - let db_range = *tx_range.start()..=in_memory_tx_num.saturating_sub(1); - - // Set the remaining transaction range for in-memory - tx_range = in_memory_tx_num..=*tx_range.end(); - - items.extend(fetch_from_db(provider, db_range)?); - } - - // Iterate from the lowest block to the highest in-memory chain - for block_state in in_mem_chain.into_iter().rev() { - let block_tx_count = block_state.block_ref().block().body.transactions.len(); - let remaining = (tx_range.end() - tx_range.start() + 1) as usize; - - // If the transaction range start is equal or higher than the next block first - // transaction, advance - if *tx_range.start() >= in_memory_tx_num + block_tx_count as u64 { - in_memory_tx_num += block_tx_count as u64; - continue - } - - // This should only be more than 0 once, in case of a partial range inside a block. - let skip = (tx_range.start() - in_memory_tx_num) as usize; - - items.extend(fetch_from_block_state( - skip..=skip + (remaining.min(block_tx_count - skip) - 1), - block_state, - )?); - - in_memory_tx_num += block_tx_count as u64; - - // Break if the range has been fully processed - if in_memory_tx_num > *tx_range.end() { - break - } - - // Set updated range - tx_range = in_memory_tx_num..=*tx_range.end(); - } - - Ok(items) - } - - /// Fetches data from either in-memory state or persistent storage by transaction - /// [`HashOrNumber`]. - fn get_in_memory_or_storage_by_tx( - &self, - id: HashOrNumber, - fetch_from_db: S, - fetch_from_block_state: M, - ) -> ProviderResult> - where - S: FnOnce(DatabaseProviderRO) -> ProviderResult>, - M: Fn(usize, TxNumber, Arc) -> ProviderResult>, - { - // Order of instantiation matters. More information on: - // `get_in_memory_or_storage_by_block_range_while`. - let in_mem_chain = self.canonical_in_memory_state.canonical_chain().collect::>(); - let provider = self.database.provider()?; - - // Get the last block number stored in the database which does NOT overlap with in-memory - // chain. - let last_database_block_number = in_mem_chain - .last() - .map(|b| Ok(b.anchor().number)) - .unwrap_or_else(|| provider.last_block_number())?; - - // Get the next tx number for the last block stored in the database and consider it the - // first tx number of the in-memory state - let last_block_body_index = provider - .block_body_indices(last_database_block_number)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(last_database_block_number))?; - let mut in_memory_tx_num = last_block_body_index.next_tx_num(); - - // If the transaction number is less than the first in-memory transaction number, make a - // database lookup - if let HashOrNumber::Number(id) = id { - if id < in_memory_tx_num { - return fetch_from_db(provider) - } - } - - // Iterate from the lowest block to the highest - for block_state in in_mem_chain.into_iter().rev() { - let executed_block = block_state.block_ref(); - let block = executed_block.block(); - - for tx_index in 0..block.body.transactions.len() { - match id { - HashOrNumber::Hash(tx_hash) => { - if tx_hash == block.body.transactions[tx_index].hash() { - return fetch_from_block_state(tx_index, in_memory_tx_num, block_state) - } - } - HashOrNumber::Number(id) => { - if id == in_memory_tx_num { - return fetch_from_block_state(tx_index, in_memory_tx_num, block_state) - } - } - } - - in_memory_tx_num += 1; - } - } - - // Not found in-memory, so check database. - if let HashOrNumber::Hash(_) = id { - return fetch_from_db(provider) - } - - Ok(None) - } - - /// Fetches data from either in-memory state or persistent storage by [`BlockHashOrNumber`]. - fn get_in_memory_or_storage_by_block( + /// If the range is empty, or there are no blocks for the given range, then this returns `None`. + pub fn get_state( &self, - id: BlockHashOrNumber, - fetch_from_db: S, - fetch_from_block_state: M, - ) -> ProviderResult - where - S: FnOnce(DatabaseProviderRO) -> ProviderResult, - M: Fn(Arc) -> ProviderResult, - { - let block_state = match id { - BlockHashOrNumber::Hash(block_hash) => { - self.canonical_in_memory_state.state_by_hash(block_hash) - } - BlockHashOrNumber::Number(block_number) => { - self.canonical_in_memory_state.state_by_number(block_number) - } - }; - - if let Some(block_state) = block_state { - return fetch_from_block_state(block_state) - } - fetch_from_db(self.database_provider_ro()?) - } -} - -impl BlockchainProvider2 { - /// Ensures that the given block number is canonical (synced) - /// - /// This is a helper for guarding the `HistoricalStateProvider` against block numbers that are - /// out of range and would lead to invalid results, mainly during initial sync. - /// - /// Verifying the `block_number` would be expensive since we need to lookup sync table - /// Instead, we ensure that the `block_number` is within the range of the - /// [`Self::best_block_number`] which is updated when a block is synced. - #[inline] - fn ensure_canonical_block(&self, block_number: BlockNumber) -> ProviderResult<()> { - let latest = self.best_block_number()?; - if block_number > latest { - Err(ProviderError::HeaderNotFound(block_number.into())) - } else { - Ok(()) - } + range: RangeInclusive, + ) -> ProviderResult> { + self.consistent_provider()?.get_state(range) } } @@ -646,78 +167,34 @@ impl StaticFileProviderFactory for BlockchainProvider2 impl HeaderProvider for BlockchainProvider2 { fn header(&self, block_hash: &BlockHash) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - (*block_hash).into(), - |db_provider| db_provider.header(block_hash), - |block_state| Ok(Some(block_state.block_ref().block().header.header().clone())), - ) + self.consistent_provider()?.header(block_hash) } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - num.into(), - |db_provider| db_provider.header_by_number(num), - |block_state| Ok(Some(block_state.block_ref().block().header.header().clone())), - ) + self.consistent_provider()?.header_by_number(num) } fn header_td(&self, hash: &BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(*hash)? { - self.header_td_by_number(num) - } else { - Ok(None) - } + self.consistent_provider()?.header_td(hash) } fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - let number = if self.canonical_in_memory_state.hash_by_number(number).is_some() { - // If the block exists in memory, we should return a TD for it. - // - // The canonical in memory state should only store post-merge blocks. Post-merge blocks - // have zero difficulty. This means we can use the total difficulty for the last - // finalized block number if present (so that we are not affected by reorgs), if not the - // last number in the database will be used. - if let Some(last_finalized_num_hash) = - self.canonical_in_memory_state.get_finalized_num_hash() - { - last_finalized_num_hash.number - } else { - self.last_block_number()? - } - } else { - // Otherwise, return what we have on disk for the input block - number - }; - self.database.header_td_by_number(number) + self.consistent_provider()?.header_td_by_number(number) } fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { - self.get_in_memory_or_storage_by_block_range_while( - range, - |db_provider, range, _| db_provider.headers_range(range), - |block_state, _| Some(block_state.block_ref().block().header.header().clone()), - |_| true, - ) + self.consistent_provider()?.headers_range(range) } fn sealed_header(&self, number: BlockNumber) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - number.into(), - |db_provider| db_provider.sealed_header(number), - |block_state| Ok(Some(block_state.block_ref().block().header.clone())), - ) + self.consistent_provider()?.sealed_header(number) } fn sealed_headers_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block_range_while( - range, - |db_provider, range, _| db_provider.sealed_headers_range(range), - |block_state, _| Some(block_state.block_ref().block().header.clone()), - |_| true, - ) + self.consistent_provider()?.sealed_headers_range(range) } fn sealed_headers_while( @@ -725,25 +202,13 @@ impl HeaderProvider for BlockchainProvider2 { range: impl RangeBounds, predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block_range_while( - range, - |db_provider, range, predicate| db_provider.sealed_headers_while(range, predicate), - |block_state, predicate| { - let header = &block_state.block_ref().block().header; - predicate(header).then(|| header.clone()) - }, - predicate, - ) + self.consistent_provider()?.sealed_headers_while(range, predicate) } } impl BlockHashReader for BlockchainProvider2 { fn block_hash(&self, number: u64) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - number.into(), - |db_provider| db_provider.block_hash(number), - |block_state| Ok(Some(block_state.hash())), - ) + self.consistent_provider()?.block_hash(number) } fn canonical_hashes_range( @@ -751,15 +216,7 @@ impl BlockHashReader for BlockchainProvider2 { start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block_range_while( - start..end, - |db_provider, inclusive_range, _| { - db_provider - .canonical_hashes_range(*inclusive_range.start(), *inclusive_range.end() + 1) - }, - |block_state, _| Some(block_state.hash()), - |_| true, - ) + self.consistent_provider()?.canonical_hashes_range(start, end) } } @@ -777,11 +234,7 @@ impl BlockNumReader for BlockchainProvider2 { } fn block_number(&self, hash: B256) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - hash.into(), - |db_provider| db_provider.block_number(hash), - |block_state| Ok(Some(block_state.number())), - ) + self.consistent_provider()?.block_number(hash) } } @@ -801,28 +254,11 @@ impl BlockIdReader for BlockchainProvider2 { impl BlockReader for BlockchainProvider2 { fn find_block_by_hash(&self, hash: B256, source: BlockSource) -> ProviderResult> { - match source { - BlockSource::Any | BlockSource::Canonical => { - // Note: it's fine to return the unsealed block because the caller already has - // the hash - self.get_in_memory_or_storage_by_block( - hash.into(), - |db_provider| db_provider.find_block_by_hash(hash, source), - |block_state| Ok(Some(block_state.block_ref().block().clone().unseal())), - ) - } - BlockSource::Pending => { - Ok(self.canonical_in_memory_state.pending_block().map(|block| block.unseal())) - } - } + self.consistent_provider()?.find_block_by_hash(hash, source) } fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - id, - |db_provider| db_provider.block(id), - |block_state| Ok(Some(block_state.block_ref().block().clone().unseal())), - ) + self.consistent_provider()?.block(id) } fn pending_block(&self) -> ProviderResult> { @@ -838,51 +274,14 @@ impl BlockReader for BlockchainProvider2 { } fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - self.get_in_memory_or_storage_by_block( - id, - |db_provider| db_provider.ommers(id), - |block_state| { - if self.chain_spec().final_paris_total_difficulty(block_state.number()).is_some() { - return Ok(Some(Vec::new())) - } - - Ok(Some(block_state.block_ref().block().body.ommers.clone())) - }, - ) + self.consistent_provider()?.ommers(id) } fn block_body_indices( &self, number: BlockNumber, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - number.into(), - |db_provider| db_provider.block_body_indices(number), - |block_state| { - // Find the last block indices on database - let last_storage_block_number = block_state.anchor().number; - let mut stored_indices = self - .database - .block_body_indices(last_storage_block_number)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(last_storage_block_number))?; - - // Prepare our block indices - stored_indices.first_tx_num = stored_indices.next_tx_num(); - stored_indices.tx_count = 0; - - // Iterate from the lowest block in memory until our target block - for state in block_state.chain().into_iter().rev() { - let block_tx_count = state.block_ref().block.body.transactions.len() as u64; - if state.block_ref().block().number == number { - stored_indices.tx_count = block_tx_count; - } else { - stored_indices.first_tx_num += block_tx_count; - } - } - - Ok(Some(stored_indices)) - }, - ) + self.consistent_provider()?.block_body_indices(number) } /// Returns the block with senders with matching number or hash from database. @@ -896,11 +295,7 @@ impl BlockReader for BlockchainProvider2 { id: BlockHashOrNumber, transaction_kind: TransactionVariant, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - id, - |db_provider| db_provider.block_with_senders(id, transaction_kind), - |block_state| Ok(Some(block_state.block_with_senders())), - ) + self.consistent_provider()?.block_with_senders(id, transaction_kind) } fn sealed_block_with_senders( @@ -908,259 +303,116 @@ impl BlockReader for BlockchainProvider2 { id: BlockHashOrNumber, transaction_kind: TransactionVariant, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block( - id, - |db_provider| db_provider.sealed_block_with_senders(id, transaction_kind), - |block_state| Ok(Some(block_state.sealed_block_with_senders())), - ) + self.consistent_provider()?.sealed_block_with_senders(id, transaction_kind) } fn block_range(&self, range: RangeInclusive) -> ProviderResult> { - self.get_in_memory_or_storage_by_block_range_while( - range, - |db_provider, range, _| db_provider.block_range(range), - |block_state, _| Some(block_state.block_ref().block().clone().unseal()), - |_| true, - ) + self.consistent_provider()?.block_range(range) } fn block_with_senders_range( &self, range: RangeInclusive, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block_range_while( - range, - |db_provider, range, _| db_provider.block_with_senders_range(range), - |block_state, _| Some(block_state.block_with_senders()), - |_| true, - ) + self.consistent_provider()?.block_with_senders_range(range) } fn sealed_block_with_senders_range( &self, range: RangeInclusive, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_block_range_while( - range, - |db_provider, range, _| db_provider.sealed_block_with_senders_range(range), - |block_state, _| Some(block_state.sealed_block_with_senders()), - |_| true, - ) + self.consistent_provider()?.sealed_block_with_senders_range(range) } } impl TransactionsProvider for BlockchainProvider2 { fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx( - tx_hash.into(), - |db_provider| db_provider.transaction_id(tx_hash), - |_, tx_number, _| Ok(Some(tx_number)), - ) + self.consistent_provider()?.transaction_id(tx_hash) } fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx( - id.into(), - |provider| provider.transaction_by_id(id), - |tx_index, _, block_state| { - Ok(block_state.block_ref().block().body.transactions.get(tx_index).cloned()) - }, - ) + self.consistent_provider()?.transaction_by_id(id) } fn transaction_by_id_no_hash( &self, id: TxNumber, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx( - id.into(), - |provider| provider.transaction_by_id_no_hash(id), - |tx_index, _, block_state| { - Ok(block_state - .block_ref() - .block() - .body - .transactions - .get(tx_index) - .cloned() - .map(Into::into)) - }, - ) + self.consistent_provider()?.transaction_by_id_no_hash(id) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { - if let Some(tx) = self.canonical_in_memory_state.transaction_by_hash(hash) { - return Ok(Some(tx)) - } - - self.database.transaction_by_hash(hash) + self.consistent_provider()?.transaction_by_hash(hash) } fn transaction_by_hash_with_meta( &self, tx_hash: TxHash, ) -> ProviderResult> { - if let Some((tx, meta)) = - self.canonical_in_memory_state.transaction_by_hash_with_meta(tx_hash) - { - return Ok(Some((tx, meta))) - } - - self.database.transaction_by_hash_with_meta(tx_hash) + self.consistent_provider()?.transaction_by_hash_with_meta(tx_hash) } fn transaction_block(&self, id: TxNumber) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx( - id.into(), - |provider| provider.transaction_block(id), - |_, _, block_state| Ok(Some(block_state.block_ref().block().number)), - ) + self.consistent_provider()?.transaction_block(id) } fn transactions_by_block( &self, id: BlockHashOrNumber, ) -> ProviderResult>> { - self.get_in_memory_or_storage_by_block( - id, - |provider| provider.transactions_by_block(id), - |block_state| Ok(Some(block_state.block_ref().block().body.transactions.clone())), - ) + self.consistent_provider()?.transactions_by_block(id) } fn transactions_by_block_range( &self, range: impl RangeBounds, ) -> ProviderResult>> { - self.get_in_memory_or_storage_by_block_range_while( - range, - |db_provider, range, _| db_provider.transactions_by_block_range(range), - |block_state, _| Some(block_state.block_ref().block().body.transactions.clone()), - |_| true, - ) + self.consistent_provider()?.transactions_by_block_range(range) } fn transactions_by_tx_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx_range( - range, - |db_provider, db_range| db_provider.transactions_by_tx_range(db_range), - |index_range, block_state| { - Ok(block_state.block_ref().block().body.transactions[index_range] - .iter() - .cloned() - .map(Into::into) - .collect()) - }, - ) + self.consistent_provider()?.transactions_by_tx_range(range) } fn senders_by_tx_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx_range( - range, - |db_provider, db_range| db_provider.senders_by_tx_range(db_range), - |index_range, block_state| Ok(block_state.block_ref().senders[index_range].to_vec()), - ) + self.consistent_provider()?.senders_by_tx_range(range) } fn transaction_sender(&self, id: TxNumber) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx( - id.into(), - |provider| provider.transaction_sender(id), - |tx_index, _, block_state| Ok(block_state.block_ref().senders.get(tx_index).copied()), - ) + self.consistent_provider()?.transaction_sender(id) } } impl ReceiptProvider for BlockchainProvider2 { fn receipt(&self, id: TxNumber) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx( - id.into(), - |provider| provider.receipt(id), - |tx_index, _, block_state| { - Ok(block_state.executed_block_receipts().get(tx_index).cloned()) - }, - ) + self.consistent_provider()?.receipt(id) } fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult> { - for block_state in self.canonical_in_memory_state.canonical_chain() { - let executed_block = block_state.block_ref(); - let block = executed_block.block(); - let receipts = block_state.executed_block_receipts(); - - // assuming 1:1 correspondence between transactions and receipts - debug_assert_eq!( - block.body.transactions.len(), - receipts.len(), - "Mismatch between transaction and receipt count" - ); - - if let Some(tx_index) = block.body.transactions.iter().position(|tx| tx.hash() == hash) - { - // safe to use tx_index for receipts due to 1:1 correspondence - return Ok(receipts.get(tx_index).cloned()); - } - } - - self.database.receipt_by_hash(hash) + self.consistent_provider()?.receipt_by_hash(hash) } fn receipts_by_block(&self, block: BlockHashOrNumber) -> ProviderResult>> { - self.get_in_memory_or_storage_by_block( - block, - |db_provider| db_provider.receipts_by_block(block), - |block_state| Ok(Some(block_state.executed_block_receipts())), - ) + self.consistent_provider()?.receipts_by_block(block) } fn receipts_by_tx_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.get_in_memory_or_storage_by_tx_range( - range, - |db_provider, db_range| db_provider.receipts_by_tx_range(db_range), - |index_range, block_state| { - Ok(block_state.executed_block_receipts().drain(index_range).collect()) - }, - ) + self.consistent_provider()?.receipts_by_tx_range(range) } } impl ReceiptProviderIdExt for BlockchainProvider2 { fn receipts_by_block_id(&self, block: BlockId) -> ProviderResult>> { - match block { - BlockId::Hash(rpc_block_hash) => { - let mut receipts = self.receipts_by_block(rpc_block_hash.block_hash.into())?; - if receipts.is_none() && !rpc_block_hash.require_canonical.unwrap_or(false) { - let block_state = self - .canonical_in_memory_state - .state_by_hash(rpc_block_hash.block_hash) - .ok_or(ProviderError::StateForHashNotFound(rpc_block_hash.block_hash))?; - receipts = Some(block_state.executed_block_receipts()); - } - Ok(receipts) - } - BlockId::Number(num_tag) => match num_tag { - BlockNumberOrTag::Pending => Ok(self - .canonical_in_memory_state - .pending_state() - .map(|block_state| block_state.executed_block_receipts())), - _ => { - if let Some(num) = self.convert_block_number(num_tag)? { - self.receipts_by_block(num.into()) - } else { - Ok(None) - } - } - }, - } + self.consistent_provider()?.receipts_by_block_id(block) } } @@ -1170,47 +422,25 @@ impl WithdrawalsProvider for BlockchainProvider2 { id: BlockHashOrNumber, timestamp: u64, ) -> ProviderResult> { - if !self.chain_spec().is_shanghai_active_at_timestamp(timestamp) { - return Ok(None) - } - - self.get_in_memory_or_storage_by_block( - id, - |db_provider| db_provider.withdrawals_by_block(id, timestamp), - |block_state| Ok(block_state.block_ref().block().body.withdrawals.clone()), - ) + self.consistent_provider()?.withdrawals_by_block(id, timestamp) } fn latest_withdrawal(&self) -> ProviderResult> { - let best_block_num = self.best_block_number()?; - - self.get_in_memory_or_storage_by_block( - best_block_num.into(), - |db_provider| db_provider.latest_withdrawal(), - |block_state| { - Ok(block_state - .block_ref() - .block() - .body - .withdrawals - .clone() - .and_then(|mut w| w.pop())) - }, - ) + self.consistent_provider()?.latest_withdrawal() } } impl StageCheckpointReader for BlockchainProvider2 { fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult> { - self.database.provider()?.get_stage_checkpoint(id) + self.consistent_provider()?.get_stage_checkpoint(id) } fn get_stage_checkpoint_progress(&self, id: StageId) -> ProviderResult>> { - self.database.provider()?.get_stage_checkpoint_progress(id) + self.consistent_provider()?.get_stage_checkpoint_progress(id) } fn get_all_checkpoints(&self) -> ProviderResult> { - self.database.provider()?.get_all_checkpoints() + self.consistent_provider()?.get_all_checkpoints() } } @@ -1225,9 +455,7 @@ impl EvmEnvProvider for BlockchainProvider2 { where EvmConfig: ConfigureEvmEnv
, { - let hash = self.convert_number(at)?.ok_or(ProviderError::HeaderNotFound(at))?; - let header = self.header(&hash)?.ok_or(ProviderError::HeaderNotFound(at))?; - self.fill_env_with_header(cfg, block_env, &header, evm_config) + self.consistent_provider()?.fill_env_at(cfg, block_env, at, evm_config) } fn fill_env_with_header( @@ -1240,11 +468,7 @@ impl EvmEnvProvider for BlockchainProvider2 { where EvmConfig: ConfigureEvmEnv
, { - let total_difficulty = self - .header_td_by_number(header.number)? - .ok_or_else(|| ProviderError::HeaderNotFound(header.number.into()))?; - evm_config.fill_cfg_and_block_env(cfg, block_env, header, total_difficulty); - Ok(()) + self.consistent_provider()?.fill_env_with_header(cfg, block_env, header, evm_config) } fn fill_cfg_env_at( @@ -1256,9 +480,7 @@ impl EvmEnvProvider for BlockchainProvider2 { where EvmConfig: ConfigureEvmEnv
, { - let hash = self.convert_number(at)?.ok_or(ProviderError::HeaderNotFound(at))?; - let header = self.header(&hash)?.ok_or(ProviderError::HeaderNotFound(at))?; - self.fill_cfg_env_with_header(cfg, &header, evm_config) + self.consistent_provider()?.fill_cfg_env_at(cfg, at, evm_config) } fn fill_cfg_env_with_header( @@ -1270,11 +492,7 @@ impl EvmEnvProvider for BlockchainProvider2 { where EvmConfig: ConfigureEvmEnv
, { - let total_difficulty = self - .header_td_by_number(header.number)? - .ok_or_else(|| ProviderError::HeaderNotFound(header.number.into()))?; - evm_config.fill_cfg_env(cfg, header, total_difficulty); - Ok(()) + self.consistent_provider()?.fill_cfg_env_with_header(cfg, header, evm_config) } } @@ -1283,11 +501,11 @@ impl PruneCheckpointReader for BlockchainProvider2 { &self, segment: PruneSegment, ) -> ProviderResult> { - self.database.provider()?.get_prune_checkpoint(segment) + self.consistent_provider()?.get_prune_checkpoint(segment) } fn get_prune_checkpoints(&self) -> ProviderResult> { - self.database.provider()?.get_prune_checkpoints() + self.consistent_provider()?.get_prune_checkpoints() } } @@ -1318,8 +536,9 @@ impl StateProviderFactory for BlockchainProvider2 { block_number: BlockNumber, ) -> ProviderResult { trace!(target: "providers::blockchain", ?block_number, "Getting history by block number"); - self.ensure_canonical_block(block_number)?; - let hash = self + let provider = self.consistent_provider()?; + provider.ensure_canonical_block(block_number)?; + let hash = provider .block_hash(block_number)? .ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?; self.history_by_block_hash(hash) @@ -1328,14 +547,11 @@ impl StateProviderFactory for BlockchainProvider2 { fn history_by_block_hash(&self, block_hash: BlockHash) -> ProviderResult { trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); - self.get_in_memory_or_storage_by_block( + self.consistent_provider()?.get_in_memory_or_storage_by_block( block_hash.into(), - |_| { - // TODO(joshie): port history_by_block_hash to DatabaseProvider and use db_provider - self.database.history_by_block_hash(block_hash) - }, + |_| self.database.history_by_block_hash(block_hash), |block_state| { - let state_provider = self.block_state_provider(&block_state)?; + let state_provider = self.block_state_provider(block_state)?; Ok(Box::new(state_provider)) }, ) @@ -1444,105 +660,35 @@ where } } -impl BlockReaderIdExt for BlockchainProvider2 +impl BlockReaderIdExt for BlockchainProvider2 where Self: BlockReader + ReceiptProviderIdExt, { fn block_by_id(&self, id: BlockId) -> ProviderResult> { - match id { - BlockId::Number(num) => self.block_by_number_or_tag(num), - BlockId::Hash(hash) => { - // TODO: should we only apply this for the RPCs that are listed in EIP-1898? - // so not at the provider level? - // if we decide to do this at a higher level, then we can make this an automatic - // trait impl - if Some(true) == hash.require_canonical { - // check the database, canonical blocks are only stored in the database - self.find_block_by_hash(hash.block_hash, BlockSource::Canonical) - } else { - self.block_by_hash(hash.block_hash) - } - } - } + self.consistent_provider()?.block_by_id(id) } fn header_by_number_or_tag(&self, id: BlockNumberOrTag) -> ProviderResult> { - Ok(match id { - BlockNumberOrTag::Latest => { - Some(self.canonical_in_memory_state.get_canonical_head().unseal()) - } - BlockNumberOrTag::Finalized => { - self.canonical_in_memory_state.get_finalized_header().map(|h| h.unseal()) - } - BlockNumberOrTag::Safe => { - self.canonical_in_memory_state.get_safe_header().map(|h| h.unseal()) - } - BlockNumberOrTag::Earliest => self.header_by_number(0)?, - BlockNumberOrTag::Pending => self.canonical_in_memory_state.pending_header(), - - BlockNumberOrTag::Number(num) => self.header_by_number(num)?, - }) + self.consistent_provider()?.header_by_number_or_tag(id) } fn sealed_header_by_number_or_tag( &self, id: BlockNumberOrTag, ) -> ProviderResult> { - match id { - BlockNumberOrTag::Latest => { - Ok(Some(self.canonical_in_memory_state.get_canonical_head())) - } - BlockNumberOrTag::Finalized => { - Ok(self.canonical_in_memory_state.get_finalized_header()) - } - BlockNumberOrTag::Safe => Ok(self.canonical_in_memory_state.get_safe_header()), - BlockNumberOrTag::Earliest => self.header_by_number(0)?.map_or_else( - || Ok(None), - |h| { - let sealed = h.seal_slow(); - let (header, seal) = sealed.into_parts(); - Ok(Some(SealedHeader::new(header, seal))) - }, - ), - BlockNumberOrTag::Pending => Ok(self.canonical_in_memory_state.pending_sealed_header()), - BlockNumberOrTag::Number(num) => self.header_by_number(num)?.map_or_else( - || Ok(None), - |h| { - let sealed = h.seal_slow(); - let (header, seal) = sealed.into_parts(); - Ok(Some(SealedHeader::new(header, seal))) - }, - ), - } + self.consistent_provider()?.sealed_header_by_number_or_tag(id) } fn sealed_header_by_id(&self, id: BlockId) -> ProviderResult> { - Ok(match id { - BlockId::Number(num) => self.sealed_header_by_number_or_tag(num)?, - BlockId::Hash(hash) => self.header(&hash.block_hash)?.map(|h| { - let sealed = h.seal_slow(); - let (header, seal) = sealed.into_parts(); - SealedHeader::new(header, seal) - }), - }) + self.consistent_provider()?.sealed_header_by_id(id) } fn header_by_id(&self, id: BlockId) -> ProviderResult> { - Ok(match id { - BlockId::Number(num) => self.header_by_number_or_tag(num)?, - BlockId::Hash(hash) => self.header(&hash.block_hash)?, - }) + self.consistent_provider()?.header_by_id(id) } fn ommers_by_id(&self, id: BlockId) -> ProviderResult>> { - match id { - BlockId::Number(num) => self.ommers_by_number_or_tag(num), - BlockId::Hash(hash) => { - // TODO: EIP-1898 question, see above - // here it is not handled - self.ommers(BlockHashOrNumber::Hash(hash.block_hash)) - } - } + self.consistent_provider()?.ommers_by_id(id) } } @@ -1569,49 +715,7 @@ impl StorageChangeSetReader for BlockchainProvider2 { &self, block_number: BlockNumber, ) -> ProviderResult> { - if let Some(state) = self.canonical_in_memory_state.state_by_number(block_number) { - let changesets = state - .block() - .execution_output - .bundle - .reverts - .clone() - .into_plain_state_reverts() - .storage - .into_iter() - .flatten() - .flat_map(|revert: PlainStorageRevert| { - revert.storage_revert.into_iter().map(move |(key, value)| { - ( - BlockNumberAddress((block_number, revert.address)), - StorageEntry { key: key.into(), value: value.to_previous_value() }, - ) - }) - }) - .collect(); - Ok(changesets) - } else { - // Perform checks on whether or not changesets exist for the block. - let provider = self.database.provider()?; - - // No prune checkpoint means history should exist and we should `unwrap_or(true)` - let storage_history_exists = provider - .get_prune_checkpoint(PruneSegment::StorageHistory)? - .and_then(|checkpoint| { - // return true if the block number is ahead of the prune checkpoint. - // - // The checkpoint stores the highest pruned block number, so we should make - // sure the block_number is strictly greater. - checkpoint.block_number.map(|checkpoint| block_number > checkpoint) - }) - .unwrap_or(true); - - if !storage_history_exists { - return Err(ProviderError::StateAtBlockPruned(block_number)) - } - - provider.storage_changeset(block_number) - } + self.consistent_provider()?.storage_changeset(block_number) } } @@ -1620,50 +724,14 @@ impl ChangeSetReader for BlockchainProvider2 { &self, block_number: BlockNumber, ) -> ProviderResult> { - if let Some(state) = self.canonical_in_memory_state.state_by_number(block_number) { - let changesets = state - .block_ref() - .execution_output - .bundle - .reverts - .clone() - .into_plain_state_reverts() - .accounts - .into_iter() - .flatten() - .map(|(address, info)| AccountBeforeTx { address, info: info.map(Into::into) }) - .collect(); - Ok(changesets) - } else { - // Perform checks on whether or not changesets exist for the block. - let provider = self.database.provider()?; - // No prune checkpoint means history should exist and we should `unwrap_or(true)` - let account_history_exists = provider - .get_prune_checkpoint(PruneSegment::AccountHistory)? - .and_then(|checkpoint| { - // return true if the block number is ahead of the prune checkpoint. - // - // The checkpoint stores the highest pruned block number, so we should make - // sure the block_number is strictly greater. - checkpoint.block_number.map(|checkpoint| block_number > checkpoint) - }) - .unwrap_or(true); - - if !account_history_exists { - return Err(ProviderError::StateAtBlockPruned(block_number)) - } - - provider.account_block_changeset(block_number) - } + self.consistent_provider()?.account_block_changeset(block_number) } } impl AccountReader for BlockchainProvider2 { /// Get basic account information. fn basic_account(&self, address: Address) -> ProviderResult> { - // use latest state provider - let state_provider = self.latest()?; - state_provider.basic_account(address) + self.consistent_provider()?.basic_account(address) } } @@ -1678,12 +746,7 @@ impl StateReader for BlockchainProvider2 { /// because the tree thread is responsible for modifying the [`CanonicalInMemoryState`] in the /// first place. fn get_state(&self, block: BlockNumber) -> ProviderResult> { - if let Some(state) = self.canonical_in_memory_state.state_by_number(block) { - let state = state.block_ref().execution_outcome().clone(); - Ok(Some(state)) - } else { - self.get_state(block..=block) - } + StateReader::get_state(&self.consistent_provider()?, block) } } diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs new file mode 100644 index 000000000000..d6847fa1b8f6 --- /dev/null +++ b/crates/storage/provider/src/providers/consistent.rs @@ -0,0 +1,1871 @@ +use super::{DatabaseProviderRO, ProviderFactory, ProviderNodeTypes}; +use crate::{ + providers::StaticFileProvider, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, + BlockReader, BlockReaderIdExt, BlockSource, ChainSpecProvider, ChangeSetReader, EvmEnvProvider, + HeaderProvider, ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, + StageCheckpointReader, StateReader, StaticFileProviderFactory, TransactionVariant, + TransactionsProvider, WithdrawalsProvider, +}; +use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag, HashOrNumber}; +use alloy_primitives::{Address, BlockHash, BlockNumber, Sealable, TxHash, TxNumber, B256, U256}; +use reth_chain_state::{BlockState, CanonicalInMemoryState, MemoryOverlayStateProviderRef}; +use reth_chainspec::{ChainInfo, EthereumHardforks}; +use reth_db::models::BlockNumberAddress; +use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; +use reth_evm::ConfigureEvmEnv; +use reth_execution_types::{BundleStateInit, ExecutionOutcome, RevertsInit}; +use reth_primitives::{ + Account, Block, BlockWithSenders, Header, Receipt, SealedBlock, SealedBlockWithSenders, + SealedHeader, StorageEntry, TransactionMeta, TransactionSigned, TransactionSignedNoHash, + Withdrawal, Withdrawals, +}; +use reth_prune_types::{PruneCheckpoint, PruneSegment}; +use reth_stages_types::{StageCheckpoint, StageId}; +use reth_storage_api::{DatabaseProviderFactory, StateProvider, StorageChangeSetReader}; +use reth_storage_errors::provider::ProviderResult; +use revm::{ + db::states::PlainStorageRevert, + primitives::{BlockEnv, CfgEnvWithHandlerCfg}, +}; +use std::{ + collections::{hash_map, HashMap}, + ops::{Add, Bound, RangeBounds, RangeInclusive, Sub}, + sync::Arc, +}; +use tracing::trace; + +/// Type that interacts with a snapshot view of the blockchain (storage and in-memory) at time of +/// instantiation, EXCEPT for pending, safe and finalized block which might change while holding +/// this provider. +/// +/// CAUTION: Avoid holding this provider for too long or the inner database transaction will +/// time-out. +#[derive(Debug)] +pub struct ConsistentProvider { + /// Storage provider. + storage_provider: as DatabaseProviderFactory>::Provider, + /// Head block at time of [`Self`] creation + head_block: Option>, + /// In-memory canonical state. This is not a snapshot, and can change! Use with caution. + canonical_in_memory_state: CanonicalInMemoryState, +} + +impl ConsistentProvider { + /// Create a new provider using [`ProviderFactory`] and [`CanonicalInMemoryState`], + /// + /// Underneath it will take a snapshot by fetching [`CanonicalInMemoryState::head_state`] and + /// [`ProviderFactory::database_provider_ro`] effectively maintaining one single snapshotted + /// view of memory and database. + pub fn new( + storage_provider_factory: ProviderFactory, + state: CanonicalInMemoryState, + ) -> ProviderResult { + // Each one provides a snapshot at the time of instantiation, but its order matters. + // + // If we acquire first the database provider, it's possible that before the in-memory chain + // snapshot is instantiated, it will flush blocks to disk. This would + // mean that our database provider would not have access to the flushed blocks (since it's + // working under an older view), while the in-memory state may have deleted them + // entirely. Resulting in gaps on the range. + let head_block = state.head_state(); + let storage_provider = storage_provider_factory.database_provider_ro()?; + Ok(Self { storage_provider, head_block, canonical_in_memory_state: state }) + } + + // Helper function to convert range bounds + fn convert_range_bounds( + &self, + range: impl RangeBounds, + end_unbounded: impl FnOnce() -> T, + ) -> (T, T) + where + T: Copy + Add + Sub + From, + { + let start = match range.start_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n + T::from(1u8), + Bound::Unbounded => T::from(0u8), + }; + + let end = match range.end_bound() { + Bound::Included(&n) => n, + Bound::Excluded(&n) => n - T::from(1u8), + Bound::Unbounded => end_unbounded(), + }; + + (start, end) + } + + /// Storage provider for latest block + fn latest_ref<'a>(&'a self) -> ProviderResult> { + trace!(target: "providers::blockchain", "Getting latest block state provider"); + + // use latest state provider if the head state exists + if let Some(state) = &self.head_block { + trace!(target: "providers::blockchain", "Using head state for latest state provider"); + Ok(self.block_state_provider_ref(state)?.boxed()) + } else { + trace!(target: "providers::blockchain", "Using database state for latest state provider"); + self.storage_provider.latest() + } + } + + fn history_by_block_hash_ref<'a>( + &'a self, + block_hash: BlockHash, + ) -> ProviderResult> { + trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); + + self.get_in_memory_or_storage_by_block( + block_hash.into(), + |_| self.storage_provider.history_by_block_hash(block_hash), + |block_state| { + let state_provider = self.block_state_provider_ref(block_state)?; + Ok(Box::new(state_provider)) + }, + ) + } + + /// Returns a state provider indexed by the given block number or tag. + fn state_by_block_number_ref<'a>( + &'a self, + number: BlockNumber, + ) -> ProviderResult> { + let hash = + self.block_hash(number)?.ok_or_else(|| ProviderError::HeaderNotFound(number.into()))?; + self.history_by_block_hash_ref(hash) + } + + /// Return the last N blocks of state, recreating the [`ExecutionOutcome`]. + /// + /// If the range is empty, or there are no blocks for the given range, then this returns `None`. + pub fn get_state( + &self, + range: RangeInclusive, + ) -> ProviderResult> { + if range.is_empty() { + return Ok(None) + } + let start_block_number = *range.start(); + let end_block_number = *range.end(); + + // We are not removing block meta as it is used to get block changesets. + let mut block_bodies = Vec::new(); + for block_num in range.clone() { + let block_body = self + .block_body_indices(block_num)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(block_num))?; + block_bodies.push((block_num, block_body)) + } + + // get transaction receipts + let Some(from_transaction_num) = block_bodies.first().map(|body| body.1.first_tx_num()) + else { + return Ok(None) + }; + let Some(to_transaction_num) = block_bodies.last().map(|body| body.1.last_tx_num()) else { + return Ok(None) + }; + + let mut account_changeset = Vec::new(); + for block_num in range.clone() { + let changeset = + self.account_block_changeset(block_num)?.into_iter().map(|elem| (block_num, elem)); + account_changeset.extend(changeset); + } + + let mut storage_changeset = Vec::new(); + for block_num in range { + let changeset = self.storage_changeset(block_num)?; + storage_changeset.extend(changeset); + } + + let (state, reverts) = + self.populate_bundle_state(account_changeset, storage_changeset, end_block_number)?; + + let mut receipt_iter = + self.receipts_by_tx_range(from_transaction_num..=to_transaction_num)?.into_iter(); + + let mut receipts = Vec::with_capacity(block_bodies.len()); + // loop break if we are at the end of the blocks. + for (_, block_body) in block_bodies { + let mut block_receipts = Vec::with_capacity(block_body.tx_count as usize); + for tx_num in block_body.tx_num_range() { + let receipt = receipt_iter + .next() + .ok_or_else(|| ProviderError::ReceiptNotFound(tx_num.into()))?; + block_receipts.push(Some(receipt)); + } + receipts.push(block_receipts); + } + + Ok(Some(ExecutionOutcome::new_init( + state, + reverts, + // We skip new contracts since we never delete them from the database + Vec::new(), + receipts.into(), + start_block_number, + Vec::new(), + ))) + } + + /// Populate a [`BundleStateInit`] and [`RevertsInit`] using cursors over the + /// [`reth_db::PlainAccountState`] and [`reth_db::PlainStorageState`] tables, based on the given + /// storage and account changesets. + fn populate_bundle_state( + &self, + account_changeset: Vec<(u64, AccountBeforeTx)>, + storage_changeset: Vec<(BlockNumberAddress, StorageEntry)>, + block_range_end: BlockNumber, + ) -> ProviderResult<(BundleStateInit, RevertsInit)> { + let mut state: BundleStateInit = HashMap::new(); + let mut reverts: RevertsInit = HashMap::new(); + let state_provider = self.state_by_block_number_ref(block_range_end)?; + + // add account changeset changes + for (block_number, account_before) in account_changeset.into_iter().rev() { + let AccountBeforeTx { info: old_info, address } = account_before; + match state.entry(address) { + hash_map::Entry::Vacant(entry) => { + let new_info = state_provider.basic_account(address)?; + entry.insert((old_info, new_info, HashMap::new())); + } + hash_map::Entry::Occupied(mut entry) => { + // overwrite old account state. + entry.get_mut().0 = old_info; + } + } + // insert old info into reverts. + reverts.entry(block_number).or_default().entry(address).or_default().0 = Some(old_info); + } + + // add storage changeset changes + for (block_and_address, old_storage) in storage_changeset.into_iter().rev() { + let BlockNumberAddress((block_number, address)) = block_and_address; + // get account state or insert from plain state. + let account_state = match state.entry(address) { + hash_map::Entry::Vacant(entry) => { + let present_info = state_provider.basic_account(address)?; + entry.insert((present_info, present_info, HashMap::new())) + } + hash_map::Entry::Occupied(entry) => entry.into_mut(), + }; + + // match storage. + match account_state.2.entry(old_storage.key) { + hash_map::Entry::Vacant(entry) => { + let new_storage_value = + state_provider.storage(address, old_storage.key)?.unwrap_or_default(); + entry.insert((old_storage.value, new_storage_value)); + } + hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().0 = old_storage.value; + } + }; + + reverts + .entry(block_number) + .or_default() + .entry(address) + .or_default() + .1 + .push(old_storage); + } + + Ok((state, reverts)) + } + + /// Fetches a range of data from both in-memory state and persistent storage while a predicate + /// is met. + /// + /// Creates a snapshot of the in-memory chain state and database provider to prevent + /// inconsistencies. Splits the range into in-memory and storage sections, prioritizing + /// recent in-memory blocks in case of overlaps. + /// + /// * `fetch_db_range` function (`F`) provides access to the database provider, allowing the + /// user to retrieve the required items from the database using [`RangeInclusive`]. + /// * `map_block_state_item` function (`G`) provides each block of the range in the in-memory + /// state, allowing for selection or filtering for the desired data. + fn get_in_memory_or_storage_by_block_range_while( + &self, + range: impl RangeBounds, + fetch_db_range: F, + map_block_state_item: G, + mut predicate: P, + ) -> ProviderResult> + where + F: FnOnce( + &DatabaseProviderRO, + RangeInclusive, + &mut P, + ) -> ProviderResult>, + G: Fn(&BlockState, &mut P) -> Option, + P: FnMut(&T) -> bool, + { + // Each one provides a snapshot at the time of instantiation, but its order matters. + // + // If we acquire first the database provider, it's possible that before the in-memory chain + // snapshot is instantiated, it will flush blocks to disk. This would + // mean that our database provider would not have access to the flushed blocks (since it's + // working under an older view), while the in-memory state may have deleted them + // entirely. Resulting in gaps on the range. + let mut in_memory_chain = + self.head_block.as_ref().map(|b| b.chain().collect::>()).unwrap_or_default(); + let db_provider = &self.storage_provider; + + let (start, end) = self.convert_range_bounds(range, || { + // the first block is the highest one. + in_memory_chain + .first() + .map(|b| b.number()) + .unwrap_or_else(|| db_provider.last_block_number().unwrap_or_default()) + }); + + if start > end { + return Ok(vec![]) + } + + // Split range into storage_range and in-memory range. If the in-memory range is not + // necessary drop it early. + // + // The last block of `in_memory_chain` is the lowest block number. + let (in_memory, storage_range) = match in_memory_chain.last().as_ref().map(|b| b.number()) { + Some(lowest_memory_block) if lowest_memory_block <= end => { + let highest_memory_block = + in_memory_chain.first().as_ref().map(|b| b.number()).expect("qed"); + + // Database will for a time overlap with in-memory-chain blocks. In + // case of a re-org, it can mean that the database blocks are of a forked chain, and + // so, we should prioritize the in-memory overlapped blocks. + let in_memory_range = + lowest_memory_block.max(start)..=end.min(highest_memory_block); + + // If requested range is in the middle of the in-memory range, remove the necessary + // lowest blocks + in_memory_chain.truncate( + in_memory_chain + .len() + .saturating_sub(start.saturating_sub(lowest_memory_block) as usize), + ); + + let storage_range = + (lowest_memory_block > start).then(|| start..=lowest_memory_block - 1); + + (Some((in_memory_chain, in_memory_range)), storage_range) + } + _ => { + // Drop the in-memory chain so we don't hold blocks in memory. + drop(in_memory_chain); + + (None, Some(start..=end)) + } + }; + + let mut items = Vec::with_capacity((end - start + 1) as usize); + + if let Some(storage_range) = storage_range { + let mut db_items = fetch_db_range(db_provider, storage_range.clone(), &mut predicate)?; + items.append(&mut db_items); + + // The predicate was not met, if the number of items differs from the expected. So, we + // return what we have. + if items.len() as u64 != storage_range.end() - storage_range.start() + 1 { + return Ok(items) + } + } + + if let Some((in_memory_chain, in_memory_range)) = in_memory { + for (num, block) in in_memory_range.zip(in_memory_chain.into_iter().rev()) { + debug_assert!(num == block.number()); + if let Some(item) = map_block_state_item(block, &mut predicate) { + items.push(item); + } else { + break + } + } + } + + Ok(items) + } + + /// This uses a given [`BlockState`] to initialize a state provider for that block. + fn block_state_provider_ref( + &self, + state: &BlockState, + ) -> ProviderResult> { + let anchor_hash = state.anchor().hash; + let latest_historical = self.history_by_block_hash_ref(anchor_hash)?; + let in_memory = state.chain().map(|block_state| block_state.block()).collect(); + Ok(MemoryOverlayStateProviderRef::new(latest_historical, in_memory)) + } + + /// Fetches data from either in-memory state or persistent storage for a range of transactions. + /// + /// * `fetch_from_db`: has a `DatabaseProviderRO` and the storage specific range. + /// * `fetch_from_block_state`: has a [`RangeInclusive`] of elements that should be fetched from + /// [`BlockState`]. [`RangeInclusive`] is necessary to handle partial look-ups of a block. + fn get_in_memory_or_storage_by_tx_range( + &self, + range: impl RangeBounds, + fetch_from_db: S, + fetch_from_block_state: M, + ) -> ProviderResult> + where + S: FnOnce( + &DatabaseProviderRO, + RangeInclusive, + ) -> ProviderResult>, + M: Fn(RangeInclusive, &BlockState) -> ProviderResult>, + { + let in_mem_chain = self.head_block.iter().flat_map(|b| b.chain()).collect::>(); + let provider = &self.storage_provider; + + // Get the last block number stored in the storage which does NOT overlap with in-memory + // chain. + let last_database_block_number = in_mem_chain + .last() + .map(|b| Ok(b.anchor().number)) + .unwrap_or_else(|| provider.last_block_number())?; + + // Get the next tx number for the last block stored in the storage, which marks the start of + // the in-memory state. + let last_block_body_index = provider + .block_body_indices(last_database_block_number)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(last_database_block_number))?; + let mut in_memory_tx_num = last_block_body_index.next_tx_num(); + + let (start, end) = self.convert_range_bounds(range, || { + in_mem_chain + .iter() + .map(|b| b.block_ref().block().body.transactions.len() as u64) + .sum::() + + last_block_body_index.last_tx_num() + }); + + if start > end { + return Ok(vec![]) + } + + let mut tx_range = start..=end; + + // If the range is entirely before the first in-memory transaction number, fetch from + // storage + if *tx_range.end() < in_memory_tx_num { + return fetch_from_db(provider, tx_range); + } + + let mut items = Vec::with_capacity((tx_range.end() - tx_range.start() + 1) as usize); + + // If the range spans storage and memory, get elements from storage first. + if *tx_range.start() < in_memory_tx_num { + // Determine the range that needs to be fetched from storage. + let db_range = *tx_range.start()..=in_memory_tx_num.saturating_sub(1); + + // Set the remaining transaction range for in-memory + tx_range = in_memory_tx_num..=*tx_range.end(); + + items.extend(fetch_from_db(provider, db_range)?); + } + + // Iterate from the lowest block to the highest in-memory chain + for block_state in in_mem_chain.iter().rev() { + let block_tx_count = block_state.block_ref().block().body.transactions.len(); + let remaining = (tx_range.end() - tx_range.start() + 1) as usize; + + // If the transaction range start is equal or higher than the next block first + // transaction, advance + if *tx_range.start() >= in_memory_tx_num + block_tx_count as u64 { + in_memory_tx_num += block_tx_count as u64; + continue + } + + // This should only be more than 0 once, in case of a partial range inside a block. + let skip = (tx_range.start() - in_memory_tx_num) as usize; + + items.extend(fetch_from_block_state( + skip..=skip + (remaining.min(block_tx_count - skip) - 1), + block_state, + )?); + + in_memory_tx_num += block_tx_count as u64; + + // Break if the range has been fully processed + if in_memory_tx_num > *tx_range.end() { + break + } + + // Set updated range + tx_range = in_memory_tx_num..=*tx_range.end(); + } + + Ok(items) + } + + /// Fetches data from either in-memory state or persistent storage by transaction + /// [`HashOrNumber`]. + fn get_in_memory_or_storage_by_tx( + &self, + id: HashOrNumber, + fetch_from_db: S, + fetch_from_block_state: M, + ) -> ProviderResult> + where + S: FnOnce(&DatabaseProviderRO) -> ProviderResult>, + M: Fn(usize, TxNumber, &BlockState) -> ProviderResult>, + { + let in_mem_chain = self.head_block.iter().flat_map(|b| b.chain()).collect::>(); + let provider = &self.storage_provider; + + // Get the last block number stored in the database which does NOT overlap with in-memory + // chain. + let last_database_block_number = in_mem_chain + .last() + .map(|b| Ok(b.anchor().number)) + .unwrap_or_else(|| provider.last_block_number())?; + + // Get the next tx number for the last block stored in the database and consider it the + // first tx number of the in-memory state + let last_block_body_index = provider + .block_body_indices(last_database_block_number)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(last_database_block_number))?; + let mut in_memory_tx_num = last_block_body_index.next_tx_num(); + + // If the transaction number is less than the first in-memory transaction number, make a + // database lookup + if let HashOrNumber::Number(id) = id { + if id < in_memory_tx_num { + return fetch_from_db(provider) + } + } + + // Iterate from the lowest block to the highest + for block_state in in_mem_chain.iter().rev() { + let executed_block = block_state.block_ref(); + let block = executed_block.block(); + + for tx_index in 0..block.body.transactions.len() { + match id { + HashOrNumber::Hash(tx_hash) => { + if tx_hash == block.body.transactions[tx_index].hash() { + return fetch_from_block_state(tx_index, in_memory_tx_num, block_state) + } + } + HashOrNumber::Number(id) => { + if id == in_memory_tx_num { + return fetch_from_block_state(tx_index, in_memory_tx_num, block_state) + } + } + } + + in_memory_tx_num += 1; + } + } + + // Not found in-memory, so check database. + if let HashOrNumber::Hash(_) = id { + return fetch_from_db(provider) + } + + Ok(None) + } + + /// Fetches data from either in-memory state or persistent storage by [`BlockHashOrNumber`]. + pub(crate) fn get_in_memory_or_storage_by_block( + &self, + id: BlockHashOrNumber, + fetch_from_db: S, + fetch_from_block_state: M, + ) -> ProviderResult + where + S: FnOnce(&DatabaseProviderRO) -> ProviderResult, + M: Fn(&BlockState) -> ProviderResult, + { + if let Some(Some(block_state)) = self.head_block.as_ref().map(|b| b.block_on_chain(id)) { + return fetch_from_block_state(block_state) + } + fetch_from_db(&self.storage_provider) + } +} + +impl ConsistentProvider { + /// Ensures that the given block number is canonical (synced) + /// + /// This is a helper for guarding the `HistoricalStateProvider` against block numbers that are + /// out of range and would lead to invalid results, mainly during initial sync. + /// + /// Verifying the `block_number` would be expensive since we need to lookup sync table + /// Instead, we ensure that the `block_number` is within the range of the + /// [`Self::best_block_number`] which is updated when a block is synced. + #[inline] + pub(crate) fn ensure_canonical_block(&self, block_number: BlockNumber) -> ProviderResult<()> { + let latest = self.best_block_number()?; + if block_number > latest { + Err(ProviderError::HeaderNotFound(block_number.into())) + } else { + Ok(()) + } + } +} + +impl StaticFileProviderFactory for ConsistentProvider { + fn static_file_provider(&self) -> StaticFileProvider { + self.storage_provider.static_file_provider() + } +} + +impl HeaderProvider for ConsistentProvider { + fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + (*block_hash).into(), + |db_provider| db_provider.header(block_hash), + |block_state| Ok(Some(block_state.block_ref().block().header.header().clone())), + ) + } + + fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + num.into(), + |db_provider| db_provider.header_by_number(num), + |block_state| Ok(Some(block_state.block_ref().block().header.header().clone())), + ) + } + + fn header_td(&self, hash: &BlockHash) -> ProviderResult> { + if let Some(num) = self.block_number(*hash)? { + self.header_td_by_number(num) + } else { + Ok(None) + } + } + + fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { + let number = if self.head_block.as_ref().map(|b| b.block_on_chain(number.into())).is_some() + { + // If the block exists in memory, we should return a TD for it. + // + // The canonical in memory state should only store post-merge blocks. Post-merge blocks + // have zero difficulty. This means we can use the total difficulty for the last + // finalized block number if present (so that we are not affected by reorgs), if not the + // last number in the database will be used. + if let Some(last_finalized_num_hash) = + self.canonical_in_memory_state.get_finalized_num_hash() + { + last_finalized_num_hash.number + } else { + self.last_block_number()? + } + } else { + // Otherwise, return what we have on disk for the input block + number + }; + self.storage_provider.header_td_by_number(number) + } + + fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, _| db_provider.headers_range(range), + |block_state, _| Some(block_state.block_ref().block().header.header().clone()), + |_| true, + ) + } + + fn sealed_header(&self, number: BlockNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + number.into(), + |db_provider| db_provider.sealed_header(number), + |block_state| Ok(Some(block_state.block_ref().block().header.clone())), + ) + } + + fn sealed_headers_range( + &self, + range: impl RangeBounds, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, _| db_provider.sealed_headers_range(range), + |block_state, _| Some(block_state.block_ref().block().header.clone()), + |_| true, + ) + } + + fn sealed_headers_while( + &self, + range: impl RangeBounds, + predicate: impl FnMut(&SealedHeader) -> bool, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, predicate| db_provider.sealed_headers_while(range, predicate), + |block_state, predicate| { + let header = &block_state.block_ref().block().header; + predicate(header).then(|| header.clone()) + }, + predicate, + ) + } +} + +impl BlockHashReader for ConsistentProvider { + fn block_hash(&self, number: u64) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + number.into(), + |db_provider| db_provider.block_hash(number), + |block_state| Ok(Some(block_state.hash())), + ) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block_range_while( + start..end, + |db_provider, inclusive_range, _| { + db_provider + .canonical_hashes_range(*inclusive_range.start(), *inclusive_range.end() + 1) + }, + |block_state, _| Some(block_state.hash()), + |_| true, + ) + } +} + +impl BlockNumReader for ConsistentProvider { + fn chain_info(&self) -> ProviderResult { + let best_number = self.best_block_number()?; + Ok(ChainInfo { best_hash: self.block_hash(best_number)?.unwrap_or_default(), best_number }) + } + + fn best_block_number(&self) -> ProviderResult { + self.head_block.as_ref().map(|b| Ok(b.number())).unwrap_or_else(|| self.last_block_number()) + } + + fn last_block_number(&self) -> ProviderResult { + self.storage_provider.last_block_number() + } + + fn block_number(&self, hash: B256) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + hash.into(), + |db_provider| db_provider.block_number(hash), + |block_state| Ok(Some(block_state.number())), + ) + } +} + +impl BlockIdReader for ConsistentProvider { + fn pending_block_num_hash(&self) -> ProviderResult> { + Ok(self.canonical_in_memory_state.pending_block_num_hash()) + } + + fn safe_block_num_hash(&self) -> ProviderResult> { + Ok(self.canonical_in_memory_state.get_safe_num_hash()) + } + + fn finalized_block_num_hash(&self) -> ProviderResult> { + Ok(self.canonical_in_memory_state.get_finalized_num_hash()) + } +} + +impl BlockReader for ConsistentProvider { + fn find_block_by_hash(&self, hash: B256, source: BlockSource) -> ProviderResult> { + match source { + BlockSource::Any | BlockSource::Canonical => { + // Note: it's fine to return the unsealed block because the caller already has + // the hash + self.get_in_memory_or_storage_by_block( + hash.into(), + |db_provider| db_provider.find_block_by_hash(hash, source), + |block_state| Ok(Some(block_state.block_ref().block().clone().unseal())), + ) + } + BlockSource::Pending => { + Ok(self.canonical_in_memory_state.pending_block().map(|block| block.unseal())) + } + } + } + + fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + id, + |db_provider| db_provider.block(id), + |block_state| Ok(Some(block_state.block_ref().block().clone().unseal())), + ) + } + + fn pending_block(&self) -> ProviderResult> { + Ok(self.canonical_in_memory_state.pending_block()) + } + + fn pending_block_with_senders(&self) -> ProviderResult> { + Ok(self.canonical_in_memory_state.pending_block_with_senders()) + } + + fn pending_block_and_receipts(&self) -> ProviderResult)>> { + Ok(self.canonical_in_memory_state.pending_block_and_receipts()) + } + + fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { + self.get_in_memory_or_storage_by_block( + id, + |db_provider| db_provider.ommers(id), + |block_state| { + if self.chain_spec().final_paris_total_difficulty(block_state.number()).is_some() { + return Ok(Some(Vec::new())) + } + + Ok(Some(block_state.block_ref().block().body.ommers.clone())) + }, + ) + } + + fn block_body_indices( + &self, + number: BlockNumber, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + number.into(), + |db_provider| db_provider.block_body_indices(number), + |block_state| { + // Find the last block indices on database + let last_storage_block_number = block_state.anchor().number; + let mut stored_indices = self + .storage_provider + .block_body_indices(last_storage_block_number)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(last_storage_block_number))?; + + // Prepare our block indices + stored_indices.first_tx_num = stored_indices.next_tx_num(); + stored_indices.tx_count = 0; + + // Iterate from the lowest block in memory until our target block + for state in block_state.chain().collect::>().into_iter().rev() { + let block_tx_count = state.block_ref().block.body.transactions.len() as u64; + if state.block_ref().block().number == number { + stored_indices.tx_count = block_tx_count; + } else { + stored_indices.first_tx_num += block_tx_count; + } + } + + Ok(Some(stored_indices)) + }, + ) + } + + /// Returns the block with senders with matching number or hash from database. + /// + /// **NOTE: If [`TransactionVariant::NoHash`] is provided then the transactions have invalid + /// hashes, since they would need to be calculated on the spot, and we want fast querying.** + /// + /// Returns `None` if block is not found. + fn block_with_senders( + &self, + id: BlockHashOrNumber, + transaction_kind: TransactionVariant, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + id, + |db_provider| db_provider.block_with_senders(id, transaction_kind), + |block_state| Ok(Some(block_state.block_with_senders())), + ) + } + + fn sealed_block_with_senders( + &self, + id: BlockHashOrNumber, + transaction_kind: TransactionVariant, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block( + id, + |db_provider| db_provider.sealed_block_with_senders(id, transaction_kind), + |block_state| Ok(Some(block_state.sealed_block_with_senders())), + ) + } + + fn block_range(&self, range: RangeInclusive) -> ProviderResult> { + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, _| db_provider.block_range(range), + |block_state, _| Some(block_state.block_ref().block().clone().unseal()), + |_| true, + ) + } + + fn block_with_senders_range( + &self, + range: RangeInclusive, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, _| db_provider.block_with_senders_range(range), + |block_state, _| Some(block_state.block_with_senders()), + |_| true, + ) + } + + fn sealed_block_with_senders_range( + &self, + range: RangeInclusive, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, _| db_provider.sealed_block_with_senders_range(range), + |block_state, _| Some(block_state.sealed_block_with_senders()), + |_| true, + ) + } +} + +impl TransactionsProvider for ConsistentProvider { + fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + tx_hash.into(), + |db_provider| db_provider.transaction_id(tx_hash), + |_, tx_number, _| Ok(Some(tx_number)), + ) + } + + fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + id.into(), + |provider| provider.transaction_by_id(id), + |tx_index, _, block_state| { + Ok(block_state.block_ref().block().body.transactions.get(tx_index).cloned()) + }, + ) + } + + fn transaction_by_id_no_hash( + &self, + id: TxNumber, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + id.into(), + |provider| provider.transaction_by_id_no_hash(id), + |tx_index, _, block_state| { + Ok(block_state + .block_ref() + .block() + .body + .transactions + .get(tx_index) + .cloned() + .map(Into::into)) + }, + ) + } + + fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { + if let Some(tx) = self.head_block.as_ref().and_then(|b| b.transaction_on_chain(hash)) { + return Ok(Some(tx)) + } + + self.storage_provider.transaction_by_hash(hash) + } + + fn transaction_by_hash_with_meta( + &self, + tx_hash: TxHash, + ) -> ProviderResult> { + if let Some((tx, meta)) = + self.head_block.as_ref().and_then(|b| b.transaction_meta_on_chain(tx_hash)) + { + return Ok(Some((tx, meta))) + } + + self.storage_provider.transaction_by_hash_with_meta(tx_hash) + } + + fn transaction_block(&self, id: TxNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + id.into(), + |provider| provider.transaction_block(id), + |_, _, block_state| Ok(Some(block_state.block_ref().block().number)), + ) + } + + fn transactions_by_block( + &self, + id: BlockHashOrNumber, + ) -> ProviderResult>> { + self.get_in_memory_or_storage_by_block( + id, + |provider| provider.transactions_by_block(id), + |block_state| Ok(Some(block_state.block_ref().block().body.transactions.clone())), + ) + } + + fn transactions_by_block_range( + &self, + range: impl RangeBounds, + ) -> ProviderResult>> { + self.get_in_memory_or_storage_by_block_range_while( + range, + |db_provider, range, _| db_provider.transactions_by_block_range(range), + |block_state, _| Some(block_state.block_ref().block().body.transactions.clone()), + |_| true, + ) + } + + fn transactions_by_tx_range( + &self, + range: impl RangeBounds, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx_range( + range, + |db_provider, db_range| db_provider.transactions_by_tx_range(db_range), + |index_range, block_state| { + Ok(block_state.block_ref().block().body.transactions[index_range] + .iter() + .cloned() + .map(Into::into) + .collect()) + }, + ) + } + + fn senders_by_tx_range( + &self, + range: impl RangeBounds, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx_range( + range, + |db_provider, db_range| db_provider.senders_by_tx_range(db_range), + |index_range, block_state| Ok(block_state.block_ref().senders[index_range].to_vec()), + ) + } + + fn transaction_sender(&self, id: TxNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + id.into(), + |provider| provider.transaction_sender(id), + |tx_index, _, block_state| Ok(block_state.block_ref().senders.get(tx_index).copied()), + ) + } +} + +impl ReceiptProvider for ConsistentProvider { + fn receipt(&self, id: TxNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + id.into(), + |provider| provider.receipt(id), + |tx_index, _, block_state| { + Ok(block_state.executed_block_receipts().get(tx_index).cloned()) + }, + ) + } + + fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult> { + for block_state in self.head_block.iter().flat_map(|b| b.chain()) { + let executed_block = block_state.block_ref(); + let block = executed_block.block(); + let receipts = block_state.executed_block_receipts(); + + // assuming 1:1 correspondence between transactions and receipts + debug_assert_eq!( + block.body.transactions.len(), + receipts.len(), + "Mismatch between transaction and receipt count" + ); + + if let Some(tx_index) = block.body.transactions.iter().position(|tx| tx.hash() == hash) + { + // safe to use tx_index for receipts due to 1:1 correspondence + return Ok(receipts.get(tx_index).cloned()); + } + } + + self.storage_provider.receipt_by_hash(hash) + } + + fn receipts_by_block(&self, block: BlockHashOrNumber) -> ProviderResult>> { + self.get_in_memory_or_storage_by_block( + block, + |db_provider| db_provider.receipts_by_block(block), + |block_state| Ok(Some(block_state.executed_block_receipts())), + ) + } + + fn receipts_by_tx_range( + &self, + range: impl RangeBounds, + ) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx_range( + range, + |db_provider, db_range| db_provider.receipts_by_tx_range(db_range), + |index_range, block_state| { + Ok(block_state.executed_block_receipts().drain(index_range).collect()) + }, + ) + } +} + +impl ReceiptProviderIdExt for ConsistentProvider { + fn receipts_by_block_id(&self, block: BlockId) -> ProviderResult>> { + match block { + BlockId::Hash(rpc_block_hash) => { + let mut receipts = self.receipts_by_block(rpc_block_hash.block_hash.into())?; + if receipts.is_none() && !rpc_block_hash.require_canonical.unwrap_or(false) { + if let Some(state) = self + .head_block + .as_ref() + .and_then(|b| b.block_on_chain(rpc_block_hash.block_hash.into())) + { + receipts = Some(state.executed_block_receipts()); + } + } + Ok(receipts) + } + BlockId::Number(num_tag) => match num_tag { + BlockNumberOrTag::Pending => Ok(self + .canonical_in_memory_state + .pending_state() + .map(|block_state| block_state.executed_block_receipts())), + _ => { + if let Some(num) = self.convert_block_number(num_tag)? { + self.receipts_by_block(num.into()) + } else { + Ok(None) + } + } + }, + } + } +} + +impl WithdrawalsProvider for ConsistentProvider { + fn withdrawals_by_block( + &self, + id: BlockHashOrNumber, + timestamp: u64, + ) -> ProviderResult> { + if !self.chain_spec().is_shanghai_active_at_timestamp(timestamp) { + return Ok(None) + } + + self.get_in_memory_or_storage_by_block( + id, + |db_provider| db_provider.withdrawals_by_block(id, timestamp), + |block_state| Ok(block_state.block_ref().block().body.withdrawals.clone()), + ) + } + + fn latest_withdrawal(&self) -> ProviderResult> { + let best_block_num = self.best_block_number()?; + + self.get_in_memory_or_storage_by_block( + best_block_num.into(), + |db_provider| db_provider.latest_withdrawal(), + |block_state| { + Ok(block_state + .block_ref() + .block() + .body + .withdrawals + .clone() + .and_then(|mut w| w.pop())) + }, + ) + } +} + +impl StageCheckpointReader for ConsistentProvider { + fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult> { + self.storage_provider.get_stage_checkpoint(id) + } + + fn get_stage_checkpoint_progress(&self, id: StageId) -> ProviderResult>> { + self.storage_provider.get_stage_checkpoint_progress(id) + } + + fn get_all_checkpoints(&self) -> ProviderResult> { + self.storage_provider.get_all_checkpoints() + } +} + +impl EvmEnvProvider for ConsistentProvider { + fn fill_env_at( + &self, + cfg: &mut CfgEnvWithHandlerCfg, + block_env: &mut BlockEnv, + at: BlockHashOrNumber, + evm_config: EvmConfig, + ) -> ProviderResult<()> + where + EvmConfig: ConfigureEvmEnv
, + { + let hash = self.convert_number(at)?.ok_or(ProviderError::HeaderNotFound(at))?; + let header = self.header(&hash)?.ok_or(ProviderError::HeaderNotFound(at))?; + self.fill_env_with_header(cfg, block_env, &header, evm_config) + } + + fn fill_env_with_header( + &self, + cfg: &mut CfgEnvWithHandlerCfg, + block_env: &mut BlockEnv, + header: &Header, + evm_config: EvmConfig, + ) -> ProviderResult<()> + where + EvmConfig: ConfigureEvmEnv
, + { + let total_difficulty = self + .header_td_by_number(header.number)? + .ok_or_else(|| ProviderError::HeaderNotFound(header.number.into()))?; + evm_config.fill_cfg_and_block_env(cfg, block_env, header, total_difficulty); + Ok(()) + } + + fn fill_cfg_env_at( + &self, + cfg: &mut CfgEnvWithHandlerCfg, + at: BlockHashOrNumber, + evm_config: EvmConfig, + ) -> ProviderResult<()> + where + EvmConfig: ConfigureEvmEnv
, + { + let hash = self.convert_number(at)?.ok_or(ProviderError::HeaderNotFound(at))?; + let header = self.header(&hash)?.ok_or(ProviderError::HeaderNotFound(at))?; + self.fill_cfg_env_with_header(cfg, &header, evm_config) + } + + fn fill_cfg_env_with_header( + &self, + cfg: &mut CfgEnvWithHandlerCfg, + header: &Header, + evm_config: EvmConfig, + ) -> ProviderResult<()> + where + EvmConfig: ConfigureEvmEnv
, + { + let total_difficulty = self + .header_td_by_number(header.number)? + .ok_or_else(|| ProviderError::HeaderNotFound(header.number.into()))?; + evm_config.fill_cfg_env(cfg, header, total_difficulty); + Ok(()) + } +} + +impl PruneCheckpointReader for ConsistentProvider { + fn get_prune_checkpoint( + &self, + segment: PruneSegment, + ) -> ProviderResult> { + self.storage_provider.get_prune_checkpoint(segment) + } + + fn get_prune_checkpoints(&self) -> ProviderResult> { + self.storage_provider.get_prune_checkpoints() + } +} + +impl ChainSpecProvider for ConsistentProvider { + type ChainSpec = N::ChainSpec; + + fn chain_spec(&self) -> Arc { + ChainSpecProvider::chain_spec(&self.storage_provider) + } +} + +impl BlockReaderIdExt for ConsistentProvider { + fn block_by_id(&self, id: BlockId) -> ProviderResult> { + match id { + BlockId::Number(num) => self.block_by_number_or_tag(num), + BlockId::Hash(hash) => { + // TODO: should we only apply this for the RPCs that are listed in EIP-1898? + // so not at the provider level? + // if we decide to do this at a higher level, then we can make this an automatic + // trait impl + if Some(true) == hash.require_canonical { + // check the database, canonical blocks are only stored in the database + self.find_block_by_hash(hash.block_hash, BlockSource::Canonical) + } else { + self.block_by_hash(hash.block_hash) + } + } + } + } + + fn header_by_number_or_tag(&self, id: BlockNumberOrTag) -> ProviderResult> { + Ok(match id { + BlockNumberOrTag::Latest => { + Some(self.canonical_in_memory_state.get_canonical_head().unseal()) + } + BlockNumberOrTag::Finalized => { + self.canonical_in_memory_state.get_finalized_header().map(|h| h.unseal()) + } + BlockNumberOrTag::Safe => { + self.canonical_in_memory_state.get_safe_header().map(|h| h.unseal()) + } + BlockNumberOrTag::Earliest => self.header_by_number(0)?, + BlockNumberOrTag::Pending => self.canonical_in_memory_state.pending_header(), + + BlockNumberOrTag::Number(num) => self.header_by_number(num)?, + }) + } + + fn sealed_header_by_number_or_tag( + &self, + id: BlockNumberOrTag, + ) -> ProviderResult> { + match id { + BlockNumberOrTag::Latest => { + Ok(Some(self.canonical_in_memory_state.get_canonical_head())) + } + BlockNumberOrTag::Finalized => { + Ok(self.canonical_in_memory_state.get_finalized_header()) + } + BlockNumberOrTag::Safe => Ok(self.canonical_in_memory_state.get_safe_header()), + BlockNumberOrTag::Earliest => self.header_by_number(0)?.map_or_else( + || Ok(None), + |h| { + let sealed = h.seal_slow(); + let (header, seal) = sealed.into_parts(); + Ok(Some(SealedHeader::new(header, seal))) + }, + ), + BlockNumberOrTag::Pending => Ok(self.canonical_in_memory_state.pending_sealed_header()), + BlockNumberOrTag::Number(num) => self.header_by_number(num)?.map_or_else( + || Ok(None), + |h| { + let sealed = h.seal_slow(); + let (header, seal) = sealed.into_parts(); + Ok(Some(SealedHeader::new(header, seal))) + }, + ), + } + } + + fn sealed_header_by_id(&self, id: BlockId) -> ProviderResult> { + Ok(match id { + BlockId::Number(num) => self.sealed_header_by_number_or_tag(num)?, + BlockId::Hash(hash) => self.header(&hash.block_hash)?.map(|h| { + let sealed = h.seal_slow(); + let (header, seal) = sealed.into_parts(); + SealedHeader::new(header, seal) + }), + }) + } + + fn header_by_id(&self, id: BlockId) -> ProviderResult> { + Ok(match id { + BlockId::Number(num) => self.header_by_number_or_tag(num)?, + BlockId::Hash(hash) => self.header(&hash.block_hash)?, + }) + } + + fn ommers_by_id(&self, id: BlockId) -> ProviderResult>> { + match id { + BlockId::Number(num) => self.ommers_by_number_or_tag(num), + BlockId::Hash(hash) => { + // TODO: EIP-1898 question, see above + // here it is not handled + self.ommers(BlockHashOrNumber::Hash(hash.block_hash)) + } + } + } +} + +impl StorageChangeSetReader for ConsistentProvider { + fn storage_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult> { + if let Some(state) = + self.head_block.as_ref().and_then(|b| b.block_on_chain(block_number.into())) + { + let changesets = state + .block() + .execution_output + .bundle + .reverts + .clone() + .into_plain_state_reverts() + .storage + .into_iter() + .flatten() + .flat_map(|revert: PlainStorageRevert| { + revert.storage_revert.into_iter().map(move |(key, value)| { + ( + BlockNumberAddress((block_number, revert.address)), + StorageEntry { key: key.into(), value: value.to_previous_value() }, + ) + }) + }) + .collect(); + Ok(changesets) + } else { + // Perform checks on whether or not changesets exist for the block. + + // No prune checkpoint means history should exist and we should `unwrap_or(true)` + let storage_history_exists = self + .storage_provider + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| { + // return true if the block number is ahead of the prune checkpoint. + // + // The checkpoint stores the highest pruned block number, so we should make + // sure the block_number is strictly greater. + checkpoint.block_number.map(|checkpoint| block_number > checkpoint) + }) + .unwrap_or(true); + + if !storage_history_exists { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + self.storage_provider.storage_changeset(block_number) + } + } +} + +impl ChangeSetReader for ConsistentProvider { + fn account_block_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult> { + if let Some(state) = + self.head_block.as_ref().and_then(|b| b.block_on_chain(block_number.into())) + { + let changesets = state + .block_ref() + .execution_output + .bundle + .reverts + .clone() + .into_plain_state_reverts() + .accounts + .into_iter() + .flatten() + .map(|(address, info)| AccountBeforeTx { address, info: info.map(Into::into) }) + .collect(); + Ok(changesets) + } else { + // Perform checks on whether or not changesets exist for the block. + + // No prune checkpoint means history should exist and we should `unwrap_or(true)` + let account_history_exists = self + .storage_provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| { + // return true if the block number is ahead of the prune checkpoint. + // + // The checkpoint stores the highest pruned block number, so we should make + // sure the block_number is strictly greater. + checkpoint.block_number.map(|checkpoint| block_number > checkpoint) + }) + .unwrap_or(true); + + if !account_history_exists { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + self.storage_provider.account_block_changeset(block_number) + } + } +} + +impl AccountReader for ConsistentProvider { + /// Get basic account information. + fn basic_account(&self, address: Address) -> ProviderResult> { + // use latest state provider + let state_provider = self.latest_ref()?; + state_provider.basic_account(address) + } +} + +impl StateReader for ConsistentProvider { + /// Re-constructs the [`ExecutionOutcome`] from in-memory and database state, if necessary. + /// + /// If data for the block does not exist, this will return [`None`]. + /// + /// NOTE: This cannot be called safely in a loop outside of the blockchain tree thread. This is + /// because the [`CanonicalInMemoryState`] could change during a reorg, causing results to be + /// inconsistent. Currently this can safely be called within the blockchain tree thread, + /// because the tree thread is responsible for modifying the [`CanonicalInMemoryState`] in the + /// first place. + fn get_state(&self, block: BlockNumber) -> ProviderResult> { + if let Some(state) = self.head_block.as_ref().and_then(|b| b.block_on_chain(block.into())) { + let state = state.block_ref().execution_outcome().clone(); + Ok(Some(state)) + } else { + Self::get_state(self, block..=block) + } + } +} + +#[cfg(test)] +mod tests { + use crate::{ + providers::blockchain_provider::BlockchainProvider2, + test_utils::create_test_provider_factory, BlockWriter, + }; + use alloy_eips::BlockHashOrNumber; + use alloy_primitives::B256; + use itertools::Itertools; + use rand::Rng; + use reth_chain_state::{ExecutedBlock, NewCanonicalChain}; + use reth_db::models::AccountBeforeTx; + use reth_execution_types::ExecutionOutcome; + use reth_primitives::SealedBlock; + use reth_storage_api::{BlockReader, BlockSource, ChangeSetReader}; + use reth_testing_utils::generators::{ + self, random_block_range, random_changeset_range, random_eoa_accounts, BlockRangeParams, + }; + use revm::db::BundleState; + use std::{ + ops::{Bound, Range, RangeBounds}, + sync::Arc, + }; + + const TEST_BLOCKS_COUNT: usize = 5; + + fn random_blocks( + rng: &mut impl Rng, + database_blocks: usize, + in_memory_blocks: usize, + requests_count: Option>, + withdrawals_count: Option>, + tx_count: impl RangeBounds, + ) -> (Vec, Vec) { + let block_range = (database_blocks + in_memory_blocks - 1) as u64; + + let tx_start = match tx_count.start_bound() { + Bound::Included(&n) | Bound::Excluded(&n) => n, + Bound::Unbounded => u8::MIN, + }; + let tx_end = match tx_count.end_bound() { + Bound::Included(&n) | Bound::Excluded(&n) => n + 1, + Bound::Unbounded => u8::MAX, + }; + + let blocks = random_block_range( + rng, + 0..=block_range, + BlockRangeParams { + parent: Some(B256::ZERO), + tx_count: tx_start..tx_end, + requests_count, + withdrawals_count, + }, + ); + let (database_blocks, in_memory_blocks) = blocks.split_at(database_blocks); + (database_blocks.to_vec(), in_memory_blocks.to_vec()) + } + + #[test] + fn test_block_reader_find_block_by_hash() -> eyre::Result<()> { + // Initialize random number generator and provider factory + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + + // Generate 10 random blocks and split into database and in-memory blocks + let blocks = random_block_range( + &mut rng, + 0..=10, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, + ); + let (database_blocks, in_memory_blocks) = blocks.split_at(5); + + // Insert first 5 blocks into the database + let provider_rw = factory.provider_rw()?; + for block in database_blocks { + provider_rw.insert_historical_block( + block.clone().seal_with_senders().expect("failed to seal block with senders"), + )?; + } + provider_rw.commit()?; + + // Create a new provider + let provider = BlockchainProvider2::new(factory)?; + let consistent_provider = provider.consistent_provider()?; + + // Useful blocks + let first_db_block = database_blocks.first().unwrap(); + let first_in_mem_block = in_memory_blocks.first().unwrap(); + let last_in_mem_block = in_memory_blocks.last().unwrap(); + + // No block in memory before setting in memory state + assert_eq!( + consistent_provider.find_block_by_hash(first_in_mem_block.hash(), BlockSource::Any)?, + None + ); + assert_eq!( + consistent_provider + .find_block_by_hash(first_in_mem_block.hash(), BlockSource::Canonical)?, + None + ); + // No pending block in memory + assert_eq!( + consistent_provider + .find_block_by_hash(first_in_mem_block.hash(), BlockSource::Pending)?, + None + ); + + // Insert first block into the in-memory state + let in_memory_block_senders = + first_in_mem_block.senders().expect("failed to recover senders"); + let chain = NewCanonicalChain::Commit { + new: vec![ExecutedBlock::new( + Arc::new(first_in_mem_block.clone()), + Arc::new(in_memory_block_senders), + Default::default(), + Default::default(), + Default::default(), + )], + }; + consistent_provider.canonical_in_memory_state.update_chain(chain); + let consistent_provider = provider.consistent_provider()?; + + // Now the block should be found in memory + assert_eq!( + consistent_provider.find_block_by_hash(first_in_mem_block.hash(), BlockSource::Any)?, + Some(first_in_mem_block.clone().into()) + ); + assert_eq!( + consistent_provider + .find_block_by_hash(first_in_mem_block.hash(), BlockSource::Canonical)?, + Some(first_in_mem_block.clone().into()) + ); + + // Find the first block in database by hash + assert_eq!( + consistent_provider.find_block_by_hash(first_db_block.hash(), BlockSource::Any)?, + Some(first_db_block.clone().into()) + ); + assert_eq!( + consistent_provider + .find_block_by_hash(first_db_block.hash(), BlockSource::Canonical)?, + Some(first_db_block.clone().into()) + ); + + // No pending block in database + assert_eq!( + consistent_provider.find_block_by_hash(first_db_block.hash(), BlockSource::Pending)?, + None + ); + + // Insert the last block into the pending state + provider.canonical_in_memory_state.set_pending_block(ExecutedBlock { + block: Arc::new(last_in_mem_block.clone()), + senders: Default::default(), + execution_output: Default::default(), + hashed_state: Default::default(), + trie: Default::default(), + }); + + // Now the last block should be found in memory + assert_eq!( + consistent_provider + .find_block_by_hash(last_in_mem_block.hash(), BlockSource::Pending)?, + Some(last_in_mem_block.clone().into()) + ); + + Ok(()) + } + + #[test] + fn test_block_reader_block() -> eyre::Result<()> { + // Initialize random number generator and provider factory + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + + // Generate 10 random blocks and split into database and in-memory blocks + let blocks = random_block_range( + &mut rng, + 0..=10, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, + ); + let (database_blocks, in_memory_blocks) = blocks.split_at(5); + + // Insert first 5 blocks into the database + let provider_rw = factory.provider_rw()?; + for block in database_blocks { + provider_rw.insert_historical_block( + block.clone().seal_with_senders().expect("failed to seal block with senders"), + )?; + } + provider_rw.commit()?; + + // Create a new provider + let provider = BlockchainProvider2::new(factory)?; + let consistent_provider = provider.consistent_provider()?; + + // First in memory block + let first_in_mem_block = in_memory_blocks.first().unwrap(); + // First database block + let first_db_block = database_blocks.first().unwrap(); + + // First in memory block should not be found yet as not integrated to the in-memory state + assert_eq!( + consistent_provider.block(BlockHashOrNumber::Hash(first_in_mem_block.hash()))?, + None + ); + assert_eq!( + consistent_provider.block(BlockHashOrNumber::Number(first_in_mem_block.number))?, + None + ); + + // Insert first block into the in-memory state + let in_memory_block_senders = + first_in_mem_block.senders().expect("failed to recover senders"); + let chain = NewCanonicalChain::Commit { + new: vec![ExecutedBlock::new( + Arc::new(first_in_mem_block.clone()), + Arc::new(in_memory_block_senders), + Default::default(), + Default::default(), + Default::default(), + )], + }; + consistent_provider.canonical_in_memory_state.update_chain(chain); + + let consistent_provider = provider.consistent_provider()?; + + // First in memory block should be found + assert_eq!( + consistent_provider.block(BlockHashOrNumber::Hash(first_in_mem_block.hash()))?, + Some(first_in_mem_block.clone().into()) + ); + assert_eq!( + consistent_provider.block(BlockHashOrNumber::Number(first_in_mem_block.number))?, + Some(first_in_mem_block.clone().into()) + ); + + // First database block should be found + assert_eq!( + consistent_provider.block(BlockHashOrNumber::Hash(first_db_block.hash()))?, + Some(first_db_block.clone().into()) + ); + assert_eq!( + consistent_provider.block(BlockHashOrNumber::Number(first_db_block.number))?, + Some(first_db_block.clone().into()) + ); + + Ok(()) + } + + #[test] + fn test_changeset_reader() -> eyre::Result<()> { + let mut rng = generators::rng(); + + let (database_blocks, in_memory_blocks) = + random_blocks(&mut rng, TEST_BLOCKS_COUNT, 1, None, None, 0..1); + + let first_database_block = database_blocks.first().map(|block| block.number).unwrap(); + let last_database_block = database_blocks.last().map(|block| block.number).unwrap(); + let first_in_memory_block = in_memory_blocks.first().map(|block| block.number).unwrap(); + + let accounts = random_eoa_accounts(&mut rng, 2); + + let (database_changesets, database_state) = random_changeset_range( + &mut rng, + &database_blocks, + accounts.into_iter().map(|(address, account)| (address, (account, Vec::new()))), + 0..0, + 0..0, + ); + let (in_memory_changesets, in_memory_state) = random_changeset_range( + &mut rng, + &in_memory_blocks, + database_state + .iter() + .map(|(address, (account, storage))| (*address, (*account, storage.clone()))), + 0..0, + 0..0, + ); + + let factory = create_test_provider_factory(); + + let provider_rw = factory.provider_rw()?; + provider_rw.append_blocks_with_state( + database_blocks + .into_iter() + .map(|b| b.seal_with_senders().expect("failed to seal block with senders")) + .collect(), + ExecutionOutcome { + bundle: BundleState::new( + database_state.into_iter().map(|(address, (account, _))| { + (address, None, Some(account.into()), Default::default()) + }), + database_changesets + .iter() + .map(|block_changesets| { + block_changesets.iter().map(|(address, account, _)| { + (*address, Some(Some((*account).into())), []) + }) + }) + .collect::>(), + Vec::new(), + ), + first_block: first_database_block, + ..Default::default() + }, + Default::default(), + Default::default(), + )?; + provider_rw.commit()?; + + let provider = BlockchainProvider2::new(factory)?; + + let in_memory_changesets = in_memory_changesets.into_iter().next().unwrap(); + let chain = NewCanonicalChain::Commit { + new: vec![in_memory_blocks + .first() + .map(|block| { + let senders = block.senders().expect("failed to recover senders"); + ExecutedBlock::new( + Arc::new(block.clone()), + Arc::new(senders), + Arc::new(ExecutionOutcome { + bundle: BundleState::new( + in_memory_state.into_iter().map(|(address, (account, _))| { + (address, None, Some(account.into()), Default::default()) + }), + [in_memory_changesets.iter().map(|(address, account, _)| { + (*address, Some(Some((*account).into())), Vec::new()) + })], + [], + ), + first_block: first_in_memory_block, + ..Default::default() + }), + Default::default(), + Default::default(), + ) + }) + .unwrap()], + }; + provider.canonical_in_memory_state.update_chain(chain); + + let consistent_provider = provider.consistent_provider()?; + + assert_eq!( + consistent_provider.account_block_changeset(last_database_block).unwrap(), + database_changesets + .into_iter() + .last() + .unwrap() + .into_iter() + .sorted_by_key(|(address, _, _)| *address) + .map(|(address, account, _)| AccountBeforeTx { address, info: Some(account) }) + .collect::>() + ); + assert_eq!( + consistent_provider.account_block_changeset(first_in_memory_block).unwrap(), + in_memory_changesets + .into_iter() + .sorted_by_key(|(address, _, _)| *address) + .map(|(address, account, _)| AccountBeforeTx { address, info: Some(account) }) + .collect::>() + ); + + Ok(()) + } +} diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index a67ebf89ba62..c81ef05d2eaa 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -61,6 +61,9 @@ pub use consistent_view::{ConsistentDbView, ConsistentViewError}; mod blockchain_provider; pub use blockchain_provider::BlockchainProvider2; +mod consistent; +pub use consistent::ConsistentProvider; + /// Helper trait keeping common requirements of providers for [`NodeTypesWithDB`]. pub trait ProviderNodeTypes: NodeTypesWithDB {} From e98a050dc7ce868d079587d8f417b51d80a36cc8 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 23 Oct 2024 15:27:52 +0100 Subject: [PATCH 047/239] fix(trie): account for existing nodes when revealing a node (#11836) --- crates/trie/sparse/src/errors.rs | 10 + crates/trie/sparse/src/trie.rs | 301 +++++++++++++++++++++++++++---- 2 files changed, 280 insertions(+), 31 deletions(-) diff --git a/crates/trie/sparse/src/errors.rs b/crates/trie/sparse/src/errors.rs index f60d1736c06f..506b206fdd79 100644 --- a/crates/trie/sparse/src/errors.rs +++ b/crates/trie/sparse/src/errors.rs @@ -4,6 +4,8 @@ use alloy_primitives::{Bytes, B256}; use reth_trie::Nibbles; use thiserror::Error; +use crate::SparseNode; + /// Result type with [`SparseStateTrieError`] as error. pub type SparseStateTrieResult = Result; @@ -43,6 +45,14 @@ pub enum SparseTrieError { /// Node hash hash: B256, }, + /// Encountered unexpected node at path when revealing. + #[error("encountered an invalid node at path {path:?} when revealing: {node:?}")] + Reveal { + /// Path to the node. + path: Nibbles, + /// Node that was at the path when revealing. + node: Box, + }, /// RLP error. #[error(transparent)] Rlp(#[from] alloy_rlp::Error), diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 39deb50e7a60..fae1141ec1c0 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -142,21 +142,55 @@ impl RevealedSparseTrie { stack_ptr += 1; } } - self.nodes - .insert(path, SparseNode::Branch { state_mask: branch.state_mask, hash: None }); - } - TrieNode::Extension(ext) => { - let mut child_path = path.clone(); - child_path.extend_from_slice_unchecked(&ext.key); - self.reveal_node_or_hash(child_path, &ext.child)?; - self.nodes.insert(path, SparseNode::Extension { key: ext.key, hash: None }); - } - TrieNode::Leaf(leaf) => { - let mut full = path.clone(); - full.extend_from_slice_unchecked(&leaf.key); - self.values.insert(full, leaf.value); - self.nodes.insert(path, SparseNode::new_leaf(leaf.key)); + + match self.nodes.get(&path) { + // Blinded and non-existent nodes can be replaced. + Some(SparseNode::Hash(_)) | None => { + self.nodes.insert( + path, + SparseNode::Branch { state_mask: branch.state_mask, hash: None }, + ); + } + // Branch node already exists, or an extension node was placed where a + // branch node was before. + Some(SparseNode::Branch { .. } | SparseNode::Extension { .. }) => {} + // All other node types can't be handled. + Some(node @ (SparseNode::Empty | SparseNode::Leaf { .. })) => { + return Err(SparseTrieError::Reveal { path, node: Box::new(node.clone()) }) + } + } } + TrieNode::Extension(ext) => match self.nodes.get(&path) { + Some(SparseNode::Hash(_)) | None => { + let mut child_path = path.clone(); + child_path.extend_from_slice_unchecked(&ext.key); + self.reveal_node_or_hash(child_path, &ext.child)?; + self.nodes.insert(path, SparseNode::Extension { key: ext.key, hash: None }); + } + // Extension node already exists, or an extension node was placed where a branch + // node was before. + Some(SparseNode::Extension { .. } | SparseNode::Branch { .. }) => {} + // All other node types can't be handled. + Some(node @ (SparseNode::Empty | SparseNode::Leaf { .. })) => { + return Err(SparseTrieError::Reveal { path, node: Box::new(node.clone()) }) + } + }, + TrieNode::Leaf(leaf) => match self.nodes.get(&path) { + Some(SparseNode::Hash(_)) | None => { + let mut full = path.clone(); + full.extend_from_slice_unchecked(&leaf.key); + self.values.insert(full, leaf.value); + self.nodes.insert(path, SparseNode::new_leaf(leaf.key)); + } + // Left node already exists. + Some(SparseNode::Leaf { .. }) => {} + // All other node types can't be handled. + Some( + node @ (SparseNode::Empty | + SparseNode::Extension { .. } | + SparseNode::Branch { .. }), + ) => return Err(SparseTrieError::Reveal { path, node: Box::new(node.clone()) }), + }, } Ok(()) @@ -164,8 +198,18 @@ impl RevealedSparseTrie { fn reveal_node_or_hash(&mut self, path: Nibbles, child: &[u8]) -> SparseTrieResult<()> { if child.len() == B256::len_bytes() + 1 { - // TODO: revise insert to not overwrite existing entries - self.nodes.insert(path, SparseNode::Hash(B256::from_slice(&child[1..]))); + let hash = B256::from_slice(&child[1..]); + match self.nodes.get(&path) { + // Hash node with a different hash can't be handled. + Some(node @ SparseNode::Hash(previous_hash)) if previous_hash != &hash => { + return Err(SparseTrieError::Reveal { path, node: Box::new(node.clone()) }) + } + None => { + self.nodes.insert(path, SparseNode::Hash(hash)); + } + // All other node types mean that it has already been revealed. + Some(_) => {} + } return Ok(()) } @@ -273,6 +317,10 @@ impl RevealedSparseTrie { // there is no node at the path. When a leaf node is a blinded `Hash`, it will have an entry // in `nodes`, but not in the `values`. + // If the path wasn't present in `values`, we still need to walk the trie and ensure that + // there is no node at the path. When a leaf node is a blinded `Hash`, it will have an entry + // in `nodes`, but not in the `values`. + let mut removed_nodes = self.take_nodes_for_path(path)?; debug!(target: "trie::sparse", ?path, ?removed_nodes, "Removed nodes for path"); // Pop the first node from the stack which is the leaf node we want to remove. @@ -1159,36 +1207,32 @@ mod tests { // Empty pretty_assertions::assert_eq!( sparse.nodes.clone().into_iter().collect::>(), - BTreeMap::from_iter([(Nibbles::default(), SparseNode::Empty),]) + BTreeMap::from_iter([(Nibbles::default(), SparseNode::Empty)]) ); } #[test] fn sparse_trie_remove_leaf_blinded() { - let mut sparse = RevealedSparseTrie::default(); - let leaf = LeafNode::new( Nibbles::default(), alloy_rlp::encode_fixed_size(&U256::from(1)).to_vec(), ); + let branch = TrieNode::Branch(BranchNode::new( + vec![ + RlpNode::word_rlp(&B256::repeat_byte(1)), + RlpNode::from_raw_rlp(&alloy_rlp::encode(leaf.clone())).unwrap(), + ], + TrieMask::new(0b11), + )); + + let mut sparse = RevealedSparseTrie::from_root(branch.clone()).unwrap(); // Reveal a branch node and one of its children // // Branch (Mask = 11) // ├── 0 -> Hash (Path = 0) // └── 1 -> Leaf (Path = 1) - sparse - .reveal_node( - Nibbles::default(), - TrieNode::Branch(BranchNode::new( - vec![ - RlpNode::word_rlp(&B256::repeat_byte(1)), - RlpNode::from_raw_rlp(&alloy_rlp::encode(leaf.clone())).unwrap(), - ], - TrieMask::new(0b11), - )), - ) - .unwrap(); + sparse.reveal_node(Nibbles::default(), branch).unwrap(); sparse.reveal_node(Nibbles::from_nibbles([0x1]), TrieNode::Leaf(leaf)).unwrap(); // Removing a blinded leaf should result in an error @@ -1279,4 +1323,199 @@ mod tests { ) )| { test(updates.into_iter().collect()) }); } + + /// We have three leaves that share the same prefix: 0x00, 0x01 and 0x02. Hash builder trie has + /// only nodes 0x00 and 0x01, and we have proofs for them. Node B is new and inserted in the + /// sparse trie first. + /// + /// 1. Reveal the hash builder proof to leaf 0x00 in the sparse trie. + /// 2. Insert leaf 0x01 into the sparse trie. + /// 3. Reveal the hash builder proof to leaf 0x02 in the sparse trie. + /// + /// The hash builder proof to the leaf 0x02 didn't have the leaf 0x01 at the corresponding + /// nibble of the branch node, so we need to adjust the branch node instead of fully + /// replacing it. + #[test] + fn sparse_trie_reveal_node_1() { + let key1 = || Nibbles::from_nibbles_unchecked([0x00]); + let key2 = || Nibbles::from_nibbles_unchecked([0x01]); + let key3 = || Nibbles::from_nibbles_unchecked([0x02]); + let value = || alloy_rlp::encode_fixed_size(&B256::repeat_byte(1)); + + // Generate the proof for the root node and initialize the sparse trie with it + let (_, proof_nodes) = hash_builder_root_with_proofs( + [(key1(), value()), (key3(), value())], + [Nibbles::default()], + ); + let mut sparse = RevealedSparseTrie::from_root( + TrieNode::decode(&mut &proof_nodes.nodes_sorted()[0].1[..]).unwrap(), + ) + .unwrap(); + + // Generate the proof for the first key and reveal it in the sparse trie + let (_, proof_nodes) = + hash_builder_root_with_proofs([(key1(), value()), (key3(), value())], [key1()]); + for (path, node) in proof_nodes.nodes_sorted() { + sparse.reveal_node(path, TrieNode::decode(&mut &node[..]).unwrap()).unwrap(); + } + + // Check that the branch node exists with only two nibbles set + assert_eq!( + sparse.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b101.into())) + ); + + // Insert the leaf for the second key + sparse.update_leaf(key2(), value().to_vec()).unwrap(); + + // Check that the branch node was updated and another nibble was set + assert_eq!( + sparse.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b111.into())) + ); + + // Generate the proof for the third key and reveal it in the sparse trie + let (_, proof_nodes_3) = + hash_builder_root_with_proofs([(key1(), value()), (key3(), value())], [key3()]); + for (path, node) in proof_nodes_3.nodes_sorted() { + sparse.reveal_node(path, TrieNode::decode(&mut &node[..]).unwrap()).unwrap(); + } + + // Check that nothing changed in the branch node + assert_eq!( + sparse.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b111.into())) + ); + + // Generate the nodes for the full trie with all three key using the hash builder, and + // compare them to the sparse trie + let (_, proof_nodes) = hash_builder_root_with_proofs( + [(key1(), value()), (key2(), value()), (key3(), value())], + [key1(), key2(), key3()], + ); + + assert_eq_sparse_trie_proof_nodes(&sparse, proof_nodes); + } + + /// We have three leaves: 0x0000, 0x0101, and 0x0102. Hash builder trie has all nodes, and we + /// have proofs for them. + /// + /// 1. Reveal the hash builder proof to leaf 0x00 in the sparse trie. + /// 2. Remove leaf 0x00 from the sparse trie (that will remove the branch node and create an + /// extension node with the key 0x0000). + /// 3. Reveal the hash builder proof to leaf 0x0101 in the sparse trie. + /// + /// The hash builder proof to the leaf 0x0101 had a branch node in the path, but we turned it + /// into an extension node, so it should ignore this node. + #[test] + fn sparse_trie_reveal_node_2() { + let key1 = || Nibbles::from_nibbles_unchecked([0x00, 0x00]); + let key2 = || Nibbles::from_nibbles_unchecked([0x01, 0x01]); + let key3 = || Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let value = || alloy_rlp::encode_fixed_size(&B256::repeat_byte(1)); + + // Generate the proof for the root node and initialize the sparse trie with it + let (_, proof_nodes) = hash_builder_root_with_proofs( + [(key1(), value()), (key2(), value()), (key3(), value())], + [Nibbles::default()], + ); + let mut sparse = RevealedSparseTrie::from_root( + TrieNode::decode(&mut &proof_nodes.nodes_sorted()[0].1[..]).unwrap(), + ) + .unwrap(); + + // Generate the proof for the children of the root branch node and reveal it in the sparse + // trie + let (_, proof_nodes) = hash_builder_root_with_proofs( + [(key1(), value()), (key2(), value()), (key3(), value())], + [key1(), Nibbles::from_nibbles_unchecked([0x01])], + ); + for (path, node) in proof_nodes.nodes_sorted() { + sparse.reveal_node(path, TrieNode::decode(&mut &node[..]).unwrap()).unwrap(); + } + + // Check that the branch node exists + assert_eq!( + sparse.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b11.into())) + ); + + // Remove the leaf for the first key + sparse.remove_leaf(&key1()).unwrap(); + + // Check that the branch node was turned into an extension node + assert_eq!( + sparse.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_ext(Nibbles::from_nibbles_unchecked([0x01]))) + ); + + // Generate the proof for the third key and reveal it in the sparse trie + let (_, proof_nodes) = hash_builder_root_with_proofs( + [(key1(), value()), (key2(), value()), (key3(), value())], + [key2()], + ); + for (path, node) in proof_nodes.nodes_sorted() { + sparse.reveal_node(path, TrieNode::decode(&mut &node[..]).unwrap()).unwrap(); + } + + // Check that nothing changed in the extension node + assert_eq!( + sparse.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_ext(Nibbles::from_nibbles_unchecked([0x01]))) + ); + } + + /// We have two leaves that share the same prefix: 0x0001 and 0x0002, and a leaf with a + /// different prefix: 0x0100. Hash builder trie has only the first two leaves, and we have + /// proofs for them. + /// + /// 1. Insert the leaf 0x0100 into the sparse trie, and check that the root extensino node was + /// turned into a branch node. + /// 2. Reveal the leaf 0x0001 in the sparse trie, and check that the root branch node wasn't + /// overwritten with the extension node from the proof. + #[test] + fn sparse_trie_reveal_node_3() { + let key1 = || Nibbles::from_nibbles_unchecked([0x00, 0x01]); + let key2 = || Nibbles::from_nibbles_unchecked([0x00, 0x02]); + let key3 = || Nibbles::from_nibbles_unchecked([0x01, 0x00]); + let value = || alloy_rlp::encode_fixed_size(&B256::repeat_byte(1)); + + // Generate the proof for the root node and initialize the sparse trie with it + let (_, proof_nodes) = hash_builder_root_with_proofs( + [(key1(), value()), (key2(), value())], + [Nibbles::default()], + ); + let mut sparse = RevealedSparseTrie::from_root( + TrieNode::decode(&mut &proof_nodes.nodes_sorted()[0].1[..]).unwrap(), + ) + .unwrap(); + + // Check that the root extension node exists + assert_matches!( + sparse.nodes.get(&Nibbles::default()), + Some(SparseNode::Extension { key, hash: None }) if *key == Nibbles::from_nibbles([0x00]) + ); + + // Insert the leaf with a different prefix + sparse.update_leaf(key3(), value().to_vec()).unwrap(); + + // Check that the extension node was turned into a branch node + assert_matches!( + sparse.nodes.get(&Nibbles::default()), + Some(SparseNode::Branch { state_mask, hash: None }) if *state_mask == TrieMask::new(0b11) + ); + + // Generate the proof for the first key and reveal it in the sparse trie + let (_, proof_nodes) = + hash_builder_root_with_proofs([(key1(), value()), (key2(), value())], [key1()]); + for (path, node) in proof_nodes.nodes_sorted() { + sparse.reveal_node(path, TrieNode::decode(&mut &node[..]).unwrap()).unwrap(); + } + + // Check that the branch node wasn't overwritten by the extension node in the proof + assert_matches!( + sparse.nodes.get(&Nibbles::default()), + Some(SparseNode::Branch { state_mask, hash: None }) if *state_mask == TrieMask::new(0b11) + ); + } } From fa30a4f758e5f166e73c99bc25fbff34962ea3be Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 23 Oct 2024 16:29:32 +0200 Subject: [PATCH 048/239] feat: add osaka hardfork (#11984) --- crates/chainspec/src/spec.rs | 8 ++++++++ crates/ethereum-forks/src/hardfork/ethereum.rs | 2 ++ crates/ethereum-forks/src/hardforks/ethereum.rs | 5 +++++ crates/ethereum/cli/src/chainspec.rs | 4 +++- crates/ethereum/evm/src/config.rs | 4 +++- 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index a7f45727dd8e..0e38d866b0d4 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -617,6 +617,7 @@ impl From for ChainSpec { (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), + (EthereumHardfork::Osaka.boxed(), genesis.config.osaka_time), ]; let mut time_hardforks = time_hardfork_opts @@ -864,6 +865,13 @@ impl ChainSpecBuilder { self } + /// Enable Osaka at genesis. + pub fn osaka_activated(mut self) -> Self { + self = self.prague_activated(); + self.hardforks.insert(EthereumHardfork::Osaka, ForkCondition::Timestamp(0)); + self + } + /// Build the resulting [`ChainSpec`]. /// /// # Panics diff --git a/crates/ethereum-forks/src/hardfork/ethereum.rs b/crates/ethereum-forks/src/hardfork/ethereum.rs index 3d85b54a9608..4e13b001786c 100644 --- a/crates/ethereum-forks/src/hardfork/ethereum.rs +++ b/crates/ethereum-forks/src/hardfork/ethereum.rs @@ -49,6 +49,8 @@ hardfork!( Cancun, /// Prague: Prague, + /// Osaka: + Osaka, } ); diff --git a/crates/ethereum-forks/src/hardforks/ethereum.rs b/crates/ethereum-forks/src/hardforks/ethereum.rs index 3069367158fa..086d2d3b46ec 100644 --- a/crates/ethereum-forks/src/hardforks/ethereum.rs +++ b/crates/ethereum-forks/src/hardforks/ethereum.rs @@ -21,6 +21,11 @@ pub trait EthereumHardforks: Hardforks { self.is_fork_active_at_timestamp(EthereumHardfork::Prague, timestamp) } + /// Convenience method to check if [`EthereumHardfork::Osaka`] is active at a given timestamp. + fn is_osaka_active_at_timestamp(&self, timestamp: u64) -> bool { + self.is_fork_active_at_timestamp(EthereumHardfork::Osaka, timestamp) + } + /// Convenience method to check if [`EthereumHardfork::Byzantium`] is active at a given block /// number. fn is_byzantium_active_at_block(&self, block_number: u64) -> bool { diff --git a/crates/ethereum/cli/src/chainspec.rs b/crates/ethereum/cli/src/chainspec.rs index cbcce9f69f61..a60d70179421 100644 --- a/crates/ethereum/cli/src/chainspec.rs +++ b/crates/ethereum/cli/src/chainspec.rs @@ -89,7 +89,8 @@ mod tests { "terminalTotalDifficulty": 0, "shanghaiTime": 0, "cancunTime": 0, - "pragueTime": 0 + "pragueTime": 0, + "osakaTime": 0 } }"#; @@ -97,5 +98,6 @@ mod tests { assert!(spec.is_shanghai_active_at_timestamp(0)); assert!(spec.is_cancun_active_at_timestamp(0)); assert!(spec.is_prague_active_at_timestamp(0)); + assert!(spec.is_osaka_active_at_timestamp(0)); } } diff --git a/crates/ethereum/evm/src/config.rs b/crates/ethereum/evm/src/config.rs index e5253307b33b..9d6b6d8796ce 100644 --- a/crates/ethereum/evm/src/config.rs +++ b/crates/ethereum/evm/src/config.rs @@ -11,7 +11,9 @@ pub fn revm_spec_by_timestamp_after_merge( chain_spec: &ChainSpec, timestamp: u64, ) -> revm_primitives::SpecId { - if chain_spec.is_prague_active_at_timestamp(timestamp) { + if chain_spec.is_osaka_active_at_timestamp(timestamp) { + revm_primitives::OSAKA + } else if chain_spec.is_prague_active_at_timestamp(timestamp) { revm_primitives::PRAGUE } else if chain_spec.is_cancun_active_at_timestamp(timestamp) { revm_primitives::CANCUN From b7167a9ddc8377a235f6d1d50e4581127f2a8bf0 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:30:57 +0200 Subject: [PATCH 049/239] test(tx-pool): add unit test for `GetPooledTransactionLimit` (#11975) --- crates/transaction-pool/src/traits.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index cedec56063ba..00cda8e1cbe5 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1594,4 +1594,27 @@ mod tests { assert_eq!(pooled_tx.blob_sidecar, EthBlobTransactionSidecar::None); assert_eq!(pooled_tx.cost, U256::from(100) + U256::from(10 * 1000)); } + + #[test] + fn test_pooled_transaction_limit() { + // No limit should never exceed + let limit_none = GetPooledTransactionLimit::None; + // Any size should return false + assert!(!limit_none.exceeds(1000)); + + // Size limit of 2MB (2 * 1024 * 1024 bytes) + let size_limit_2mb = GetPooledTransactionLimit::ResponseSizeSoftLimit(2 * 1024 * 1024); + + // Test with size below the limit + // 1MB is below 2MB, should return false + assert!(!size_limit_2mb.exceeds(1024 * 1024)); + + // Test with size exactly at the limit + // 2MB equals the limit, should return false + assert!(!size_limit_2mb.exceeds(2 * 1024 * 1024)); + + // Test with size exceeding the limit + // 3MB is above the 2MB limit, should return true + assert!(size_limit_2mb.exceeds(3 * 1024 * 1024)); + } } From 889a7e0b981e4a36394f6b4e554483daaabfeb25 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Wed, 23 Oct 2024 16:31:22 +0200 Subject: [PATCH 050/239] primitive-traits: use alloy `_DURATION` constants (#11960) --- Cargo.lock | 2 ++ crates/node/core/Cargo.toml | 15 ++++----------- crates/node/core/src/args/payload_builder.rs | 3 ++- crates/payload/basic/Cargo.toml | 1 + crates/payload/basic/src/lib.rs | 4 ++-- crates/primitives-traits/src/constants/mod.rs | 10 ---------- 6 files changed, 11 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2d134cb85f7..92a9d6887d61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6370,6 +6370,7 @@ name = "reth-basic-payload-builder" version = "1.1.0" dependencies = [ "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "futures-core", @@ -7911,6 +7912,7 @@ name = "reth-node-core" version = "1.1.0" dependencies = [ "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "clap", diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 73c552f4d7a8..0c9672d17779 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -40,6 +40,7 @@ reth-stages-types.workspace = true alloy-primitives.workspace = true alloy-rpc-types-engine = { workspace = true, features = ["jwt"] } alloy-consensus.workspace = true +alloy-eips.workspace = true # misc eyre.workspace = true @@ -76,18 +77,10 @@ proptest.workspace = true tokio.workspace = true [features] -optimism = [ - "reth-primitives/optimism", - "reth-db/optimism" -] +optimism = ["reth-primitives/optimism", "reth-db/optimism"] # Features for vergen to generate correct env vars -jemalloc = [ - "reth-cli-util/jemalloc" -] -asm-keccak = [ - "reth-primitives/asm-keccak", - "alloy-primitives/asm-keccak" -] +jemalloc = ["reth-cli-util/jemalloc"] +asm-keccak = ["reth-primitives/asm-keccak", "alloy-primitives/asm-keccak"] [build-dependencies] vergen = { version = "8.0.0", features = ["build", "cargo", "git", "gitcl"] } diff --git a/crates/node/core/src/args/payload_builder.rs b/crates/node/core/src/args/payload_builder.rs index 4a18fd5b0b7d..dceb10726ff2 100644 --- a/crates/node/core/src/args/payload_builder.rs +++ b/crates/node/core/src/args/payload_builder.rs @@ -1,11 +1,12 @@ use crate::{cli::config::PayloadBuilderConfig, version::default_extradata}; use alloy_consensus::constants::MAXIMUM_EXTRA_DATA_SIZE; +use alloy_eips::merge::SLOT_DURATION; use clap::{ builder::{RangedU64ValueParser, TypedValueParser}, Arg, Args, Command, }; use reth_cli_util::{parse_duration_from_secs, parse_duration_from_secs_or_ms}; -use reth_primitives::constants::{ETHEREUM_BLOCK_GAS_LIMIT, SLOT_DURATION}; +use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use std::{borrow::Cow, ffi::OsStr, time::Duration}; /// Parameters for configuring the Payload Builder diff --git a/crates/payload/basic/Cargo.toml b/crates/payload/basic/Cargo.toml index 9047768892a9..88ab99272dbd 100644 --- a/crates/payload/basic/Cargo.toml +++ b/crates/payload/basic/Cargo.toml @@ -27,6 +27,7 @@ alloy-rlp.workspace = true alloy-primitives.workspace = true revm.workspace = true alloy-consensus.workspace = true +alloy-eips.workspace = true # async tokio = { workspace = true, features = ["sync", "time"] } diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index fcc8be9a88ed..4274d451e43f 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -10,6 +10,7 @@ use crate::metrics::PayloadBuilderMetrics; use alloy_consensus::constants::EMPTY_WITHDRAWALS; +use alloy_eips::merge::SLOT_DURATION; use alloy_primitives::{Bytes, B256, U256}; use futures_core::ready; use futures_util::FutureExt; @@ -22,8 +23,7 @@ use reth_payload_primitives::{ BuiltPayload, PayloadBuilderAttributes, PayloadBuilderError, PayloadKind, }; use reth_primitives::{ - constants::{RETH_CLIENT_VERSION, SLOT_DURATION}, - proofs, BlockNumberOrTag, SealedBlock, Withdrawals, + constants::RETH_CLIENT_VERSION, proofs, BlockNumberOrTag, SealedBlock, Withdrawals, }; use reth_provider::{ BlockReaderIdExt, BlockSource, CanonStateNotification, ProviderError, StateProviderFactory, diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index 2874c596a8a4..a4091a4a9d9c 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -1,7 +1,6 @@ //! Ethereum protocol-related constants use alloy_primitives::{address, b256, Address, B256, U256}; -use core::time::Duration; /// Gas units, for example [`GIGAGAS`]. pub mod gas_units; @@ -13,15 +12,6 @@ pub const RETH_CLIENT_VERSION: &str = concat!("reth/v", env!("CARGO_PKG_VERSION" /// An EPOCH is a series of 32 slots. pub const EPOCH_SLOTS: u64 = 32; -/// The duration of a slot in seconds. -/// -/// This is the time period of 12 seconds in which a randomly chosen validator has time to propose a -/// block. -pub const SLOT_DURATION: Duration = Duration::from_secs(12); - -/// An EPOCH is a series of 32 slots (~6.4min). -pub const EPOCH_DURATION: Duration = Duration::from_secs(12 * EPOCH_SLOTS); - /// The default block nonce in the beacon consensus pub const BEACON_NONCE: u64 = 0u64; From 89eb73f3d23f6d6771e472e6588633e8cb284943 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 23 Oct 2024 16:44:37 +0200 Subject: [PATCH 051/239] refactor: replace extra fields with `ExecutionPayloadSidecar` in engine (#11901) --- Cargo.lock | 2 - .../src/commands/debug_cmd/replay_engine.rs | 6 +- crates/consensus/beacon/Cargo.toml | 13 ++- crates/consensus/beacon/src/engine/handle.rs | 15 +--- crates/consensus/beacon/src/engine/message.rs | 12 +-- crates/consensus/beacon/src/engine/mod.rs | 50 +++++------ .../consensus/beacon/src/engine/test_utils.rs | 10 +-- crates/engine/local/src/miner.rs | 9 +- crates/engine/tree/src/tree/mod.rs | 34 +++----- crates/engine/util/src/engine_store.rs | 17 ++-- crates/engine/util/src/reorg.rs | 85 ++++++++----------- crates/engine/util/src/skip_new_payload.rs | 16 +--- crates/payload/validator/Cargo.toml | 1 - crates/payload/validator/src/lib.rs | 21 ++--- crates/rpc/rpc-engine-api/src/engine_api.rs | 44 +++++++--- crates/rpc/rpc-engine-api/tests/it/payload.rs | 19 +++-- .../rpc-types-compat/src/engine/payload.rs | 40 ++++----- 17 files changed, 179 insertions(+), 215 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92a9d6887d61..350e6a77ae55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6394,7 +6394,6 @@ dependencies = [ name = "reth-beacon-consensus" version = "1.1.0" dependencies = [ - "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", @@ -8356,7 +8355,6 @@ dependencies = [ name = "reth-payload-validator" version = "1.1.0" dependencies = [ - "alloy-eips", "alloy-rpc-types", "reth-chainspec", "reth-primitives", diff --git a/bin/reth/src/commands/debug_cmd/replay_engine.rs b/bin/reth/src/commands/debug_cmd/replay_engine.rs index de497cbe0070..e7b3de6b6c13 100644 --- a/bin/reth/src/commands/debug_cmd/replay_engine.rs +++ b/bin/reth/src/commands/debug_cmd/replay_engine.rs @@ -170,10 +170,8 @@ impl> Command { beacon_engine_handle.fork_choice_updated(state, payload_attrs).await?; debug!(target: "reth::cli", ?response, "Received for forkchoice updated"); } - StoredEngineApiMessage::NewPayload { payload, cancun_fields } => { - // todo: prague (last arg) - let response = - beacon_engine_handle.new_payload(payload, cancun_fields, None).await?; + StoredEngineApiMessage::NewPayload { payload, sidecar } => { + let response = beacon_engine_handle.new_payload(payload, sidecar).await?; debug!(target: "reth::cli", ?response, "Received for new payload"); } }; diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index dd1e339319b6..b141bd34edbc 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -31,7 +31,6 @@ reth-node-types.workspace = true reth-chainspec = { workspace = true, optional = true } # ethereum -alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rpc-types-engine.workspace = true @@ -78,10 +77,10 @@ assert_matches.workspace = true [features] optimism = [ - "reth-chainspec", - "reth-primitives/optimism", - "reth-provider/optimism", - "reth-blockchain-tree/optimism", - "reth-db/optimism", - "reth-db-api/optimism" + "reth-chainspec", + "reth-primitives/optimism", + "reth-provider/optimism", + "reth-blockchain-tree/optimism", + "reth-db/optimism", + "reth-db-api/optimism", ] diff --git a/crates/consensus/beacon/src/engine/handle.rs b/crates/consensus/beacon/src/engine/handle.rs index bb5c4dee174c..4aafc6e07c1c 100644 --- a/crates/consensus/beacon/src/engine/handle.rs +++ b/crates/consensus/beacon/src/engine/handle.rs @@ -4,9 +4,8 @@ use crate::{ engine::message::OnForkChoiceUpdated, BeaconConsensusEngineEvent, BeaconEngineMessage, BeaconForkChoiceUpdateError, BeaconOnNewPayloadError, }; -use alloy_eips::eip7685::Requests; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, + ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, }; use futures::TryFutureExt; use reth_engine_primitives::EngineTypes; @@ -47,18 +46,10 @@ where pub async fn new_payload( &self, payload: ExecutionPayload, - cancun_fields: Option, - execution_requests: Option, + sidecar: ExecutionPayloadSidecar, ) -> Result { let (tx, rx) = oneshot::channel(); - // HACK(onbjerg): We should have a pectra payload fields struct, this is just a temporary - // workaround. - let _ = self.to_engine.send(BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - }); + let _ = self.to_engine.send(BeaconEngineMessage::NewPayload { payload, sidecar, tx }); rx.await.map_err(|_| BeaconOnNewPayloadError::EngineUnavailable)? } diff --git a/crates/consensus/beacon/src/engine/message.rs b/crates/consensus/beacon/src/engine/message.rs index 56328f03db0b..e33decbd848f 100644 --- a/crates/consensus/beacon/src/engine/message.rs +++ b/crates/consensus/beacon/src/engine/message.rs @@ -1,7 +1,6 @@ use crate::engine::{error::BeaconOnNewPayloadError, forkchoice::ForkchoiceStatus}; -use alloy_eips::eip7685::Requests; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionPayload, ForkChoiceUpdateResult, ForkchoiceState, + ExecutionPayload, ExecutionPayloadSidecar, ForkChoiceUpdateResult, ForkchoiceState, ForkchoiceUpdateError, ForkchoiceUpdated, PayloadId, PayloadStatus, PayloadStatusEnum, }; use futures::{future::Either, FutureExt}; @@ -145,12 +144,9 @@ pub enum BeaconEngineMessage { NewPayload { /// The execution payload received by Engine API. payload: ExecutionPayload, - /// The cancun-related newPayload fields, if any. - cancun_fields: Option, - // HACK(onbjerg): We should have a pectra payload fields struct, this is just a temporary - // workaround. - /// The pectra EIP-7685 execution requests. - execution_requests: Option, + /// The execution payload sidecar with additional version-specific fields received by + /// engine API. + sidecar: ExecutionPayloadSidecar, /// The sender for returning payload status result. tx: oneshot::Sender>, }, diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index cff648b2843b..5af1e26accac 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1,7 +1,6 @@ -use alloy_eips::eip7685::Requests; use alloy_primitives::{BlockNumber, B256}; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionPayload, ForkchoiceState, PayloadStatus, PayloadStatusEnum, + ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError, }; use futures::{stream::BoxStream, Future, StreamExt}; @@ -1081,14 +1080,11 @@ where /// /// This returns a [`PayloadStatus`] that represents the outcome of a processed new payload and /// returns an error if an internal error occurred. - #[instrument(level = "trace", skip(self, payload, cancun_fields), fields(block_hash = ?payload.block_hash(), block_number = %payload.block_number(), is_pipeline_idle = %self.sync.is_pipeline_idle()), target = "consensus::engine")] + #[instrument(level = "trace", skip(self, payload, sidecar), fields(block_hash = ?payload.block_hash(), block_number = %payload.block_number(), is_pipeline_idle = %self.sync.is_pipeline_idle()), target = "consensus::engine")] fn on_new_payload( &mut self, payload: ExecutionPayload, - cancun_fields: Option, - // HACK(onbjerg): We should have a pectra payload fields struct, this is just a temporary - // workaround. - execution_requests: Option, + sidecar: ExecutionPayloadSidecar, ) -> Result, BeaconOnNewPayloadError> { self.metrics.new_payload_messages.increment(1); @@ -1118,11 +1114,7 @@ where // // This validation **MUST** be instantly run in all cases even during active sync process. let parent_hash = payload.parent_hash(); - let block = match self.payload_validator.ensure_well_formed_payload( - payload, - cancun_fields.into(), - execution_requests, - ) { + let block = match self.payload_validator.ensure_well_formed_payload(payload, sidecar) { Ok(block) => block, Err(error) => { error!(target: "consensus::engine", %error, "Invalid payload"); @@ -1867,13 +1859,8 @@ where BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx } => { this.on_forkchoice_updated(state, payload_attrs, tx); } - BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - } => { - match this.on_new_payload(payload, cancun_fields, execution_requests) { + BeaconEngineMessage::NewPayload { payload, sidecar, tx } => { + match this.on_new_payload(payload, sidecar) { Ok(Either::Right(block)) => { this.set_blockchain_tree_action( BlockchainTreeAction::InsertNewPayload { block, tx }, @@ -2061,7 +2048,12 @@ mod tests { assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); // consensus engine is still idle because no FCUs were received - let _ = env.send_new_payload(block_to_payload_v1(SealedBlock::default()), None).await; + let _ = env + .send_new_payload( + block_to_payload_v1(SealedBlock::default()), + ExecutionPayloadSidecar::none(), + ) + .await; assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); @@ -2626,7 +2618,7 @@ mod tests { 0, BlockParams { ommers_count: Some(0), ..Default::default() }, )), - None, + ExecutionPayloadSidecar::none(), ) .await; @@ -2641,7 +2633,7 @@ mod tests { 1, BlockParams { ommers_count: Some(0), ..Default::default() }, )), - None, + ExecutionPayloadSidecar::none(), ) .await; @@ -2719,7 +2711,10 @@ mod tests { // Send new payload let result = env - .send_new_payload_retry_on_syncing(block_to_payload_v1(block2.clone()), None) + .send_new_payload_retry_on_syncing( + block_to_payload_v1(block2.clone()), + ExecutionPayloadSidecar::none(), + ) .await .unwrap(); @@ -2854,7 +2849,9 @@ mod tests { 2, BlockParams { parent: Some(parent), ommers_count: Some(0), ..Default::default() }, ); - let res = env.send_new_payload(block_to_payload_v1(block), None).await; + let res = env + .send_new_payload(block_to_payload_v1(block), ExecutionPayloadSidecar::none()) + .await; let expected_result = PayloadStatus::from_status(PayloadStatusEnum::Syncing); assert_matches!(res, Ok(result) => assert_eq!(result, expected_result)); @@ -2924,7 +2921,10 @@ mod tests { // Send new payload let result = env - .send_new_payload_retry_on_syncing(block_to_payload_v1(block2.clone()), None) + .send_new_payload_retry_on_syncing( + block_to_payload_v1(block2.clone()), + ExecutionPayloadSidecar::none(), + ) .await .unwrap(); diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 7e9e1ec6b26d..912f0a871bf2 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -6,7 +6,7 @@ use crate::{ }; use alloy_primitives::{BlockNumber, Sealable, B256}; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionPayload, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, + ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, }; use reth_blockchain_tree::{ config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree, @@ -68,9 +68,9 @@ impl TestEnv { pub async fn send_new_payload>( &self, payload: T, - cancun_fields: Option, + sidecar: ExecutionPayloadSidecar, ) -> Result { - self.engine_handle.new_payload(payload.into(), cancun_fields, None).await + self.engine_handle.new_payload(payload.into(), sidecar).await } /// Sends the `ExecutionPayload` message to the consensus engine and retries if the engine @@ -78,11 +78,11 @@ impl TestEnv { pub async fn send_new_payload_retry_on_syncing>( &self, payload: T, - cancun_fields: Option, + sidecar: ExecutionPayloadSidecar, ) -> Result { let payload: ExecutionPayload = payload.into(); loop { - let result = self.send_new_payload(payload.clone(), cancun_fields.clone()).await?; + let result = self.send_new_payload(payload.clone(), sidecar.clone()).await?; if !result.is_syncing() { return Ok(result) } diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index 552cbd047764..706ddc43de3f 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -1,7 +1,7 @@ //! Contains the implementation of the mining mode for the local engine. use alloy_primitives::{TxHash, B256}; -use alloy_rpc_types_engine::{CancunPayloadFields, ForkchoiceState}; +use alloy_rpc_types_engine::{CancunPayloadFields, ExecutionPayloadSidecar, ForkchoiceState}; use eyre::OptionExt; use futures_util::{stream::Fuse, StreamExt}; use reth_beacon_consensus::BeaconEngineMessage; @@ -221,9 +221,10 @@ where let (tx, rx) = oneshot::channel(); self.to_engine.send(BeaconEngineMessage::NewPayload { payload: block_to_payload(payload.block().clone()), - cancun_fields, - // todo: prague - execution_requests: None, + // todo: prague support + sidecar: cancun_fields + .map(ExecutionPayloadSidecar::v3) + .unwrap_or_else(ExecutionPayloadSidecar::none), tx, })?; diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 021b3149ad5d..555cf89164f9 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -10,7 +10,7 @@ use alloy_primitives::{ BlockNumber, B256, U256, }; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionPayload, ForkchoiceState, PayloadStatus, PayloadStatusEnum, + ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError, }; use reth_beacon_consensus::{ @@ -70,7 +70,6 @@ use crate::{ engine::{EngineApiKind, EngineApiRequest}, tree::metrics::EngineApiMetrics, }; -use alloy_eips::eip7685::Requests; pub use config::TreeConfig; pub use invalid_block_hook::{InvalidBlockHooks, NoopInvalidBlockHook}; pub use persistence_state::PersistenceState; @@ -722,8 +721,7 @@ where fn on_new_payload( &mut self, payload: ExecutionPayload, - cancun_fields: Option, - execution_requests: Option, + sidecar: ExecutionPayloadSidecar, ) -> Result, InsertBlockFatalError> { trace!(target: "engine::tree", "invoked new payload"); self.metrics.engine.new_payload_messages.increment(1); @@ -754,11 +752,7 @@ where // // This validation **MUST** be instantly run in all cases even during active sync process. let parent_hash = payload.parent_hash(); - let block = match self.payload_validator.ensure_well_formed_payload( - payload, - cancun_fields.into(), - execution_requests, - ) { + let block = match self.payload_validator.ensure_well_formed_payload(payload, sidecar) { Ok(block) => block, Err(error) => { error!(target: "engine::tree", %error, "Invalid payload"); @@ -1241,14 +1235,8 @@ where error!(target: "engine::tree", "Failed to send event: {err:?}"); } } - BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - } => { - let output = - self.on_new_payload(payload, cancun_fields, execution_requests); + BeaconEngineMessage::NewPayload { payload, sidecar, tx } => { + let output = self.on_new_payload(payload, sidecar); if let Err(err) = tx.send(output.map(|o| o.outcome).map_err(|e| { reth_beacon_consensus::BeaconOnNewPayloadError::Internal( Box::new(e), @@ -2585,6 +2573,7 @@ mod tests { use crate::persistence::PersistenceAction; use alloy_primitives::{Bytes, Sealable}; use alloy_rlp::Decodable; + use alloy_rpc_types_engine::{CancunPayloadFields, ExecutionPayloadSidecar}; use assert_matches::assert_matches; use reth_beacon_consensus::{EthBeaconConsensus, ForkchoiceStatus}; use reth_chain_state::{test_utils::TestBlockBuilder, BlockState}; @@ -2862,11 +2851,10 @@ mod tests { self.tree .on_new_payload( payload.into(), - Some(CancunPayloadFields { + ExecutionPayloadSidecar::v3(CancunPayloadFields { parent_beacon_block_root: block.parent_beacon_block_root.unwrap(), versioned_hashes: vec![], }), - None, ) .unwrap(); } @@ -3129,7 +3117,10 @@ mod tests { let mut test_harness = TestHarness::new(HOLESKY.clone()); - let outcome = test_harness.tree.on_new_payload(payload.into(), None, None).unwrap(); + let outcome = test_harness + .tree + .on_new_payload(payload.into(), ExecutionPayloadSidecar::none()) + .unwrap(); assert!(outcome.outcome.is_syncing()); // ensure block is buffered @@ -3173,8 +3164,7 @@ mod tests { .on_engine_message(FromEngine::Request( BeaconEngineMessage::NewPayload { payload: payload.clone().into(), - cancun_fields: None, - execution_requests: None, + sidecar: ExecutionPayloadSidecar::none(), tx, } .into(), diff --git a/crates/engine/util/src/engine_store.rs b/crates/engine/util/src/engine_store.rs index de193bf3bbe0..85c5e126fa44 100644 --- a/crates/engine/util/src/engine_store.rs +++ b/crates/engine/util/src/engine_store.rs @@ -1,6 +1,6 @@ //! Stores engine API messages to disk for later inspection and replay. -use alloy_rpc_types_engine::{CancunPayloadFields, ExecutionPayload, ForkchoiceState}; +use alloy_rpc_types_engine::{ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState}; use futures::{Stream, StreamExt}; use reth_beacon_consensus::BeaconEngineMessage; use reth_engine_primitives::EngineTypes; @@ -30,8 +30,9 @@ pub enum StoredEngineApiMessage { NewPayload { /// The [`ExecutionPayload`] sent in the persisted call. payload: ExecutionPayload, - /// The Cancun-specific fields sent in the persisted call, if any. - cancun_fields: Option, + /// The execution payload sidecar with additional version-specific fields received by + /// engine API. + sidecar: ExecutionPayloadSidecar, }, } @@ -73,20 +74,14 @@ impl EngineMessageStore { })?, )?; } - // todo(onbjerg): execution requests - BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests: _, - tx: _tx, - } => { + BeaconEngineMessage::NewPayload { payload, sidecar, tx: _tx } => { let filename = format!("{}-new_payload-{}.json", timestamp, payload.block_hash()); fs::write( self.path.join(filename), serde_json::to_vec( &StoredEngineApiMessage::::NewPayload { payload: payload.clone(), - cancun_fields: cancun_fields.clone(), + sidecar: sidecar.clone(), }, )?, )?; diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index 85216e32fad0..d109fb9e94ae 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -1,10 +1,9 @@ //! Stream wrapper that simulates reorgs. use alloy_consensus::Transaction; -use alloy_eips::eip7685::Requests; use alloy_primitives::U256; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionPayload, ForkchoiceState, PayloadStatus, + CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState, PayloadStatus, }; use futures::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt}; use itertools::Either; @@ -150,12 +149,7 @@ where let next = ready!(this.stream.poll_next_unpin(cx)); let item = match (next, &this.last_forkchoice_state) { ( - Some(BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - }), + Some(BeaconEngineMessage::NewPayload { payload, sidecar, tx }), Some(last_forkchoice_state), ) if this.forkchoice_states_forwarded > this.frequency && // Only enter reorg state if new payload attaches to current head. @@ -170,29 +164,26 @@ where // forkchoice state. We will rely on CL to reorg us back to canonical chain. // TODO: This is an expensive blocking operation, ideally it's spawned as a task // so that the stream could yield the control back. - let (reorg_payload, reorg_cancun_fields, reorg_execution_requests) = - match create_reorg_head( - this.provider, - this.evm_config, - this.payload_validator, - *this.depth, - payload.clone(), - cancun_fields.clone(), - execution_requests.clone(), - ) { - Ok(result) => result, - Err(error) => { - error!(target: "engine::stream::reorg", %error, "Error attempting to create reorg head"); - // Forward the payload and attempt to create reorg on top of - // the next one - return Poll::Ready(Some(BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - })) - } - }; + let (reorg_payload, reorg_sidecar) = match create_reorg_head( + this.provider, + this.evm_config, + this.payload_validator, + *this.depth, + payload.clone(), + sidecar.clone(), + ) { + Ok(result) => result, + Err(error) => { + error!(target: "engine::stream::reorg", %error, "Error attempting to create reorg head"); + // Forward the payload and attempt to create reorg on top of + // the next one + return Poll::Ready(Some(BeaconEngineMessage::NewPayload { + payload, + sidecar, + tx, + })) + } + }; let reorg_forkchoice_state = ForkchoiceState { finalized_block_hash: last_forkchoice_state.finalized_block_hash, safe_block_hash: last_forkchoice_state.safe_block_hash, @@ -208,17 +199,11 @@ where let queue = VecDeque::from([ // Current payload - BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - }, + BeaconEngineMessage::NewPayload { payload, sidecar, tx }, // Reorg payload BeaconEngineMessage::NewPayload { payload: reorg_payload, - cancun_fields: reorg_cancun_fields, - execution_requests: reorg_execution_requests, + sidecar: reorg_sidecar, tx: reorg_payload_tx, }, // Reorg forkchoice state @@ -252,9 +237,8 @@ fn create_reorg_head( payload_validator: &ExecutionPayloadValidator, mut depth: usize, next_payload: ExecutionPayload, - next_cancun_fields: Option, - next_execution_requests: Option, -) -> RethResult<(ExecutionPayload, Option, Option)> + next_sidecar: ExecutionPayloadSidecar, +) -> RethResult<(ExecutionPayload, ExecutionPayloadSidecar)> where Provider: BlockReader + StateProviderFactory, Evm: ConfigureEvm
, @@ -264,11 +248,7 @@ where // Ensure next payload is valid. let next_block = payload_validator - .ensure_well_formed_payload( - next_payload, - next_cancun_fields.into(), - next_execution_requests, - ) + .ensure_well_formed_payload(next_payload, next_sidecar) .map_err(RethError::msg)?; // Fetch reorg target block depending on its depth and its parent. @@ -439,11 +419,16 @@ where Ok(( block_to_payload(reorg_block), + // todo(onbjerg): how do we support execution requests? reorg_target .header .parent_beacon_block_root - .map(|root| CancunPayloadFields { parent_beacon_block_root: root, versioned_hashes }), - // todo(prague) - None, + .map(|root| { + ExecutionPayloadSidecar::v3(CancunPayloadFields { + parent_beacon_block_root: root, + versioned_hashes, + }) + }) + .unwrap_or_else(ExecutionPayloadSidecar::none), )) } diff --git a/crates/engine/util/src/skip_new_payload.rs b/crates/engine/util/src/skip_new_payload.rs index 47c48282eef6..16f2e98197c9 100644 --- a/crates/engine/util/src/skip_new_payload.rs +++ b/crates/engine/util/src/skip_new_payload.rs @@ -41,19 +41,14 @@ where loop { let next = ready!(this.stream.poll_next_unpin(cx)); let item = match next { - Some(BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - }) => { + Some(BeaconEngineMessage::NewPayload { payload, sidecar, tx }) => { if this.skipped < this.threshold { *this.skipped += 1; tracing::warn!( target: "engine::stream::skip_new_payload", block_number = payload.block_number(), block_hash = %payload.block_hash(), - ?cancun_fields, + ?sidecar, threshold=this.threshold, skipped=this.skipped, "Skipping new payload" ); @@ -61,12 +56,7 @@ where continue } *this.skipped = 0; - Some(BeaconEngineMessage::NewPayload { - payload, - cancun_fields, - execution_requests, - tx, - }) + Some(BeaconEngineMessage::NewPayload { payload, sidecar, tx }) } next => next, }; diff --git a/crates/payload/validator/Cargo.toml b/crates/payload/validator/Cargo.toml index 619b99f28de2..2662b987f889 100644 --- a/crates/payload/validator/Cargo.toml +++ b/crates/payload/validator/Cargo.toml @@ -18,5 +18,4 @@ reth-primitives.workspace = true reth-rpc-types-compat.workspace = true # alloy -alloy-eips.workspace = true alloy-rpc-types = { workspace = true, features = ["engine"] } diff --git a/crates/payload/validator/src/lib.rs b/crates/payload/validator/src/lib.rs index 3ec7b206a5b1..9952815fd982 100644 --- a/crates/payload/validator/src/lib.rs +++ b/crates/payload/validator/src/lib.rs @@ -8,8 +8,9 @@ #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] -use alloy_eips::eip7685::Requests; -use alloy_rpc_types::engine::{ExecutionPayload, MaybeCancunPayloadFields, PayloadError}; +use alloy_rpc_types::engine::{ + ExecutionPayload, ExecutionPayloadSidecar, MaybeCancunPayloadFields, PayloadError, +}; use reth_chainspec::EthereumHardforks; use reth_primitives::SealedBlock; use reth_rpc_types_compat::engine::payload::try_into_block; @@ -112,15 +113,12 @@ impl ExecutionPayloadValidator { pub fn ensure_well_formed_payload( &self, payload: ExecutionPayload, - cancun_fields: MaybeCancunPayloadFields, - execution_requests: Option, + sidecar: ExecutionPayloadSidecar, ) -> Result { let expected_hash = payload.block_hash(); // First parse the block - let sealed_block = - try_into_block(payload, cancun_fields.parent_beacon_block_root(), execution_requests)? - .seal_slow(); + let sealed_block = try_into_block(payload, &sidecar)?.seal_slow(); // Ensure the hash included in the payload matches the block hash if expected_hash != sealed_block.hash() { @@ -139,7 +137,7 @@ impl ExecutionPayloadValidator { // cancun active but excess blob gas not present return Err(PayloadError::PostCancunBlockWithoutExcessBlobGas) } - if cancun_fields.as_ref().is_none() { + if sidecar.cancun().is_none() { // cancun active but cancun fields not present return Err(PayloadError::PostCancunWithoutCancunFields) } @@ -156,7 +154,7 @@ impl ExecutionPayloadValidator { // cancun not active but excess blob gas present return Err(PayloadError::PreCancunBlockWithExcessBlobGas) } - if cancun_fields.as_ref().is_some() { + if sidecar.cancun().is_some() { // cancun not active but cancun fields present return Err(PayloadError::PreCancunWithCancunFields) } @@ -175,7 +173,10 @@ impl ExecutionPayloadValidator { } // EIP-4844 checks - self.ensure_matching_blob_versioned_hashes(&sealed_block, &cancun_fields)?; + self.ensure_matching_blob_versioned_hashes( + &sealed_block, + &sidecar.cancun().cloned().into(), + )?; Ok(sealed_block) } diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index ca055a77ea10..eb280408ecdb 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -5,8 +5,8 @@ use alloy_eips::{eip4844::BlobAndProofV1, eip7685::Requests}; use alloy_primitives::{BlockHash, BlockNumber, B256, U64}; use alloy_rpc_types_engine::{ CancunPayloadFields, ClientVersionV1, ExecutionPayload, ExecutionPayloadBodiesV1, - ExecutionPayloadInputV2, ExecutionPayloadV1, ExecutionPayloadV3, ForkchoiceState, - ForkchoiceUpdated, PayloadId, PayloadStatus, TransitionConfiguration, + ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1, ExecutionPayloadV3, + ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus, TransitionConfiguration, }; use async_trait::async_trait; use jsonrpsee_core::RpcResult; @@ -140,7 +140,11 @@ where self.inner .validator .validate_version_specific_fields(EngineApiMessageVersion::V1, payload_or_attrs)?; - Ok(self.inner.beacon_consensus.new_payload(payload, None, None).await?) + Ok(self + .inner + .beacon_consensus + .new_payload(payload, ExecutionPayloadSidecar::none()) + .await?) } /// See also @@ -156,7 +160,11 @@ where self.inner .validator .validate_version_specific_fields(EngineApiMessageVersion::V2, payload_or_attrs)?; - Ok(self.inner.beacon_consensus.new_payload(payload, None, None).await?) + Ok(self + .inner + .beacon_consensus + .new_payload(payload, ExecutionPayloadSidecar::none()) + .await?) } /// See also @@ -176,9 +184,17 @@ where .validator .validate_version_specific_fields(EngineApiMessageVersion::V3, payload_or_attrs)?; - let cancun_fields = CancunPayloadFields { versioned_hashes, parent_beacon_block_root }; - - Ok(self.inner.beacon_consensus.new_payload(payload, Some(cancun_fields), None).await?) + Ok(self + .inner + .beacon_consensus + .new_payload( + payload, + ExecutionPayloadSidecar::v3(CancunPayloadFields { + versioned_hashes, + parent_beacon_block_root, + }), + ) + .await?) } /// See also @@ -187,8 +203,6 @@ where payload: ExecutionPayloadV3, versioned_hashes: Vec, parent_beacon_block_root: B256, - // TODO(onbjerg): Figure out why we even get these here, since we'll check the requests - // from execution against the requests root in the header. execution_requests: Requests, ) -> EngineApiResult { let payload = ExecutionPayload::from(payload); @@ -201,14 +215,16 @@ where .validator .validate_version_specific_fields(EngineApiMessageVersion::V4, payload_or_attrs)?; - let cancun_fields = CancunPayloadFields { versioned_hashes, parent_beacon_block_root }; - - // HACK(onbjerg): We should have a pectra payload fields struct, this is just a temporary - // workaround. Ok(self .inner .beacon_consensus - .new_payload(payload, Some(cancun_fields), Some(execution_requests)) + .new_payload( + payload, + ExecutionPayloadSidecar::v4( + CancunPayloadFields { versioned_hashes, parent_beacon_block_root }, + execution_requests, + ), + ) .await?) } diff --git a/crates/rpc/rpc-engine-api/tests/it/payload.rs b/crates/rpc/rpc-engine-api/tests/it/payload.rs index 007a62db045b..febbc291e35e 100644 --- a/crates/rpc/rpc-engine-api/tests/it/payload.rs +++ b/crates/rpc/rpc-engine-api/tests/it/payload.rs @@ -3,7 +3,8 @@ use alloy_primitives::{Bytes, Sealable, U256}; use alloy_rlp::{Decodable, Error as RlpError}; use alloy_rpc_types_engine::{ - ExecutionPayload, ExecutionPayloadBodyV1, ExecutionPayloadV1, PayloadError, + ExecutionPayload, ExecutionPayloadBodyV1, ExecutionPayloadSidecar, ExecutionPayloadV1, + PayloadError, }; use assert_matches::assert_matches; use reth_primitives::{proofs, Block, SealedBlock, SealedHeader, TransactionSigned, Withdrawals}; @@ -75,7 +76,10 @@ fn payload_validation() { b }); - assert_matches!(try_into_sealed_block(block_with_valid_extra_data, None, None), Ok(_)); + assert_matches!( + try_into_sealed_block(block_with_valid_extra_data, &ExecutionPayloadSidecar::none()), + Ok(_) + ); // Invalid extra data let block_with_invalid_extra_data = Bytes::from_static(&[0; 33]); @@ -84,7 +88,7 @@ fn payload_validation() { b }); assert_matches!( - try_into_sealed_block(invalid_extra_data_block, None, None), + try_into_sealed_block(invalid_extra_data_block, &ExecutionPayloadSidecar::none()), Err(PayloadError::ExtraData(data)) if data == block_with_invalid_extra_data ); @@ -94,7 +98,7 @@ fn payload_validation() { b }); assert_matches!( - try_into_sealed_block(block_with_zero_base_fee, None, None), + try_into_sealed_block(block_with_zero_base_fee, &ExecutionPayloadSidecar::none()), Err(PayloadError::BaseFee(val)) if val.is_zero() ); @@ -113,7 +117,7 @@ fn payload_validation() { b }); assert_matches!( - try_into_sealed_block(block_with_ommers.clone(), None, None), + try_into_sealed_block(block_with_ommers.clone(), &ExecutionPayloadSidecar::none()), Err(PayloadError::BlockHash { consensus, .. }) if consensus == block_with_ommers.block_hash() ); @@ -124,7 +128,7 @@ fn payload_validation() { b }); assert_matches!( - try_into_sealed_block(block_with_difficulty.clone(), None, None), + try_into_sealed_block(block_with_difficulty.clone(), &ExecutionPayloadSidecar::none()), Err(PayloadError::BlockHash { consensus, .. }) if consensus == block_with_difficulty.block_hash() ); @@ -134,9 +138,8 @@ fn payload_validation() { b }); assert_matches!( - try_into_sealed_block(block_with_nonce.clone(), None, None), + try_into_sealed_block(block_with_nonce.clone(), &ExecutionPayloadSidecar::none()), Err(PayloadError::BlockHash { consensus, .. }) if consensus == block_with_nonce.block_hash() - ); // Valid block diff --git a/crates/rpc/rpc-types-compat/src/engine/payload.rs b/crates/rpc/rpc-types-compat/src/engine/payload.rs index b63b7453aeb0..b4c45a617812 100644 --- a/crates/rpc/rpc-types-compat/src/engine/payload.rs +++ b/crates/rpc/rpc-types-compat/src/engine/payload.rs @@ -9,7 +9,8 @@ use alloy_eips::{ use alloy_primitives::{B256, U256}; use alloy_rpc_types_engine::{ payload::{ExecutionPayloadBodyV1, ExecutionPayloadFieldV2, ExecutionPayloadInputV2}, - ExecutionPayload, ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, PayloadError, + ExecutionPayload, ExecutionPayloadSidecar, ExecutionPayloadV1, ExecutionPayloadV2, + ExecutionPayloadV3, PayloadError, }; use reth_primitives::{ proofs::{self}, @@ -248,17 +249,18 @@ pub fn convert_block_to_payload_input_v2(value: SealedBlock) -> ExecutionPayload } } -/// Tries to create a new block (without a block hash) from the given payload and optional parent -/// beacon block root. +/// Tries to create a new unsealed block from the given payload and payload sidecar. +/// /// Performs additional validation of `extra_data` and `base_fee_per_gas` fields. /// -/// NOTE: The log bloom is assumed to be validated during serialization. +/// # Note +/// +/// The log bloom is assumed to be validated during serialization. /// /// See pub fn try_into_block( value: ExecutionPayload, - parent_beacon_block_root: Option, - execution_requests: Option, + sidecar: &ExecutionPayloadSidecar, ) -> Result { let mut base_payload = match value { ExecutionPayload::V1(payload) => try_payload_v1_to_block(payload)?, @@ -266,29 +268,30 @@ pub fn try_into_block( ExecutionPayload::V3(payload) => try_payload_v3_to_block(payload)?, }; - base_payload.header.parent_beacon_block_root = parent_beacon_block_root; - base_payload.header.requests_hash = execution_requests.map(|reqs| reqs.requests_hash()); + base_payload.header.parent_beacon_block_root = sidecar.parent_beacon_block_root(); + base_payload.header.requests_hash = sidecar.requests().map(Requests::requests_hash); Ok(base_payload) } -/// Tries to create a new block from the given payload and optional parent beacon block root. -/// -/// NOTE: Empty ommers, nonce and difficulty values are validated upon computing block hash and -/// comparing the value with `payload.block_hash`. +/// Tries to create a sealed new block from the given payload and payload sidecar. /// /// Uses [`try_into_block`] to convert from the [`ExecutionPayload`] to [`Block`] and seals the /// block with its hash. /// /// Uses [`validate_block_hash`] to validate the payload block hash and ultimately return the /// [`SealedBlock`]. +/// +/// # Note +/// +/// Empty ommers, nonce, difficulty, and execution request values are validated upon computing block +/// hash and comparing the value with `payload.block_hash`. pub fn try_into_sealed_block( payload: ExecutionPayload, - parent_beacon_block_root: Option, - execution_requests: Option, + sidecar: &ExecutionPayloadSidecar, ) -> Result { let block_hash = payload.block_hash(); - let base_payload = try_into_block(payload, parent_beacon_block_root, execution_requests)?; + let base_payload = try_into_block(payload, sidecar)?; // validate block hash and return validate_block_hash(block_hash, base_payload) @@ -356,8 +359,8 @@ mod tests { }; use alloy_primitives::{b256, hex, Bytes, U256}; use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionPayload, ExecutionPayloadV1, ExecutionPayloadV2, - ExecutionPayloadV3, + CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ExecutionPayloadV1, + ExecutionPayloadV2, ExecutionPayloadV3, }; #[test] @@ -575,8 +578,7 @@ mod tests { let cancun_fields = CancunPayloadFields { parent_beacon_block_root, versioned_hashes }; // convert into block - let block = - try_into_block(payload, Some(cancun_fields.parent_beacon_block_root), None).unwrap(); + let block = try_into_block(payload, &ExecutionPayloadSidecar::v3(cancun_fields)).unwrap(); // Ensure the actual hash is calculated if we set the fields to what they should be validate_block_hash(block_hash_with_blob_fee_fields, block).unwrap(); From bf612bee50cada67624791baa7707f260acf1be4 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Wed, 23 Oct 2024 17:03:25 +0200 Subject: [PATCH 052/239] chore(hive): update expected failures (#12006) --- .github/assets/hive/expected_failures.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml index d4b3d2bcbd3c..ec7bd0549001 100644 --- a/.github/assets/hive/expected_failures.yaml +++ b/.github/assets/hive/expected_failures.yaml @@ -41,8 +41,6 @@ engine-withdrawals: - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) -# https://github.com/paradigmxyz/reth/issues/8305 -# https://github.com/paradigmxyz/reth/issues/6217 engine-api: [] # https://github.com/paradigmxyz/reth/issues/8305 @@ -58,6 +56,4 @@ engine-cancun: - Invalid NewPayload, Incomplete VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) - Invalid NewPayload, Extra VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) -# https://github.com/paradigmxyz/reth/issues/8579 -sync: - - sync reth -> reth +sync: [] From f3853e71b36314a370ace6bf3ac699531e635be7 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 23 Oct 2024 16:22:59 +0100 Subject: [PATCH 053/239] test(trie): get sparse trie nodes at depth (#12007) --- crates/trie/sparse/src/trie.rs | 86 ++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index fae1141ec1c0..4d195cbf34cf 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -569,6 +569,17 @@ impl RevealedSparseTrie { /// Update hashes of the nodes that are located at a level deeper than or equal to the provided /// depth. Root node has a level of 0. pub fn update_rlp_node_level(&mut self, depth: usize) { + let targets = self.get_nodes_at_depth(depth); + let mut prefix_set = self.prefix_set.clone().freeze(); + for target in targets { + self.rlp_node(target, &mut prefix_set); + } + } + + /// Returns a list of paths to the nodes that are located at the provided depth when counting + /// from the root node. If there's a leaf at a depth less than the provided depth, it will be + /// included in the result. + fn get_nodes_at_depth(&self, depth: usize) -> HashSet { let mut paths = Vec::from([(Nibbles::default(), 0)]); let mut targets = HashSet::::default(); @@ -602,10 +613,7 @@ impl RevealedSparseTrie { } } - let mut prefix_set = self.prefix_set.clone().freeze(); - for target in targets { - self.rlp_node(target, &mut prefix_set); - } + targets } fn rlp_node(&mut self, path: Nibbles, prefix_set: &mut PrefixSet) -> RlpNode { @@ -1518,4 +1526,74 @@ mod tests { Some(SparseNode::Branch { state_mask, hash: None }) if *state_mask == TrieMask::new(0b11) ); } + + #[test] + fn sparse_trie_get_nodes_at_depth() { + let mut sparse = RevealedSparseTrie::default(); + + let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); + + // Extension (Key = 5) – Level 0 + // └── Branch (Mask = 1011) – Level 1 + // ├── 0 -> Extension (Key = 23) – Level 2 + // │ └── Branch (Mask = 0101) – Level 3 + // │ ├── 1 -> Leaf (Key = 1, Path = 50231) – Level 4 + // │ └── 3 -> Leaf (Key = 3, Path = 50233) – Level 4 + // ├── 2 -> Leaf (Key = 013, Path = 52013) – Level 2 + // └── 3 -> Branch (Mask = 0101) – Level 2 + // ├── 1 -> Leaf (Key = 3102, Path = 53102) – Level 3 + // └── 3 -> Branch (Mask = 1010) – Level 3 + // ├── 0 -> Leaf (Key = 3302, Path = 53302) – Level 4 + // └── 2 -> Leaf (Key = 3320, Path = 53320) – Level 4 + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone()) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone()) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone()) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone()) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone()) + .unwrap(); + sparse.update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value).unwrap(); + + assert_eq!(sparse.get_nodes_at_depth(0), HashSet::from([Nibbles::default()])); + assert_eq!( + sparse.get_nodes_at_depth(1), + HashSet::from([Nibbles::from_nibbles_unchecked([0x5])]) + ); + assert_eq!( + sparse.get_nodes_at_depth(2), + HashSet::from([ + Nibbles::from_nibbles_unchecked([0x5, 0x0]), + Nibbles::from_nibbles_unchecked([0x5, 0x2]), + Nibbles::from_nibbles_unchecked([0x5, 0x3]) + ]) + ); + assert_eq!( + sparse.get_nodes_at_depth(3), + HashSet::from([ + Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3]), + Nibbles::from_nibbles_unchecked([0x5, 0x2]), + Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x1]), + Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x3]) + ]) + ); + assert_eq!( + sparse.get_nodes_at_depth(4), + HashSet::from([ + Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3, 0x1]), + Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3, 0x3]), + Nibbles::from_nibbles_unchecked([0x5, 0x2]), + Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x1]), + Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x3, 0x0]), + Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x3, 0x2]) + ]) + ); + } } From b73261936ee0995818030d696ad1c8778ecd449a Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 23 Oct 2024 16:59:22 +0100 Subject: [PATCH 054/239] chore(trie): prefix set doc comment clarification (#12010) --- crates/trie/trie/src/prefix_set.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/trie/trie/src/prefix_set.rs b/crates/trie/trie/src/prefix_set.rs index da912fbbdad9..0cf16f939d7b 100644 --- a/crates/trie/trie/src/prefix_set.rs +++ b/crates/trie/trie/src/prefix_set.rs @@ -168,8 +168,7 @@ pub struct PrefixSet { } impl PrefixSet { - /// Returns `true` if any of the keys in the set has the given prefix or - /// if the given prefix is a prefix of any key in the set. + /// Returns `true` if any of the keys in the set has the given prefix #[inline] pub fn contains(&mut self, prefix: &[u8]) -> bool { if self.all { From 57a21fcb9efc274675ea00a644ebbfa627940c6d Mon Sep 17 00:00:00 2001 From: Julian Meyer Date: Wed, 23 Oct 2024 09:22:51 -0700 Subject: [PATCH 055/239] chore: increase max proof window (#12001) --- crates/rpc/rpc-server-types/src/constants.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-server-types/src/constants.rs b/crates/rpc/rpc-server-types/src/constants.rs index 0bc441819323..126bc722fd52 100644 --- a/crates/rpc/rpc-server-types/src/constants.rs +++ b/crates/rpc/rpc-server-types/src/constants.rs @@ -51,9 +51,9 @@ pub const DEFAULT_MAX_SIMULATE_BLOCKS: u64 = 256; /// The default eth historical proof window. pub const DEFAULT_ETH_PROOF_WINDOW: u64 = 0; -/// Maximum eth historical proof window. Equivalent to roughly one and a half months of data on a 12 -/// second block time, and a week on a 2 second block time. -pub const MAX_ETH_PROOF_WINDOW: u64 = 7 * 24 * 60 * 60 / 2; +/// Maximum eth historical proof window. Equivalent to roughly 6 months of data on a 12 +/// second block time, and a month on a 2 second block time. +pub const MAX_ETH_PROOF_WINDOW: u64 = 28 * 24 * 60 * 60 / 2; /// GPO specific constants pub mod gas_oracle { From d6f5a89a277a9490137d634b21c4cca4e728015f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Oct 2024 20:40:57 +0200 Subject: [PATCH 056/239] test: tests for empty block bodies (#12013) --- crates/net/eth-wire-types/src/blocks.rs | 9 +++++++++ crates/net/eth-wire-types/src/message.rs | 16 +++++++++++++++- crates/primitives/src/block.rs | 9 +++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/crates/net/eth-wire-types/src/blocks.rs b/crates/net/eth-wire-types/src/blocks.rs index d60c63fc1f6e..878b4573f2b6 100644 --- a/crates/net/eth-wire-types/src/blocks.rs +++ b/crates/net/eth-wire-types/src/blocks.rs @@ -497,4 +497,13 @@ mod tests { let result = RequestPair::decode(&mut &data[..]).unwrap(); assert_eq!(result, expected); } + + #[test] + fn empty_block_bodies_rlp() { + let body = BlockBodies::default(); + let mut buf = Vec::new(); + body.encode(&mut buf); + let decoded = BlockBodies::decode(&mut buf.as_slice()).unwrap(); + assert_eq!(body, decoded); + } } diff --git a/crates/net/eth-wire-types/src/message.rs b/crates/net/eth-wire-types/src/message.rs index 9ef8e6c71474..4afcb34e13bf 100644 --- a/crates/net/eth-wire-types/src/message.rs +++ b/crates/net/eth-wire-types/src/message.rs @@ -494,7 +494,8 @@ where mod tests { use super::MessageError; use crate::{ - message::RequestPair, EthMessage, EthMessageID, GetNodeData, NodeData, ProtocolMessage, + message::RequestPair, EthMessage, EthMessageID, EthVersion, GetNodeData, NodeData, + ProtocolMessage, }; use alloy_primitives::hex; use alloy_rlp::{Decodable, Encodable, Error}; @@ -566,4 +567,17 @@ mod tests { let result = RequestPair::>::decode(&mut &*raw_pair); assert!(matches!(result, Err(Error::UnexpectedLength))); } + + #[test] + fn empty_block_bodies_protocol() { + let empty_block_bodies = ProtocolMessage::from(EthMessage::BlockBodies(RequestPair { + request_id: 0, + message: Default::default(), + })); + let mut buf = Vec::new(); + empty_block_bodies.encode(&mut buf); + let decoded = + ProtocolMessage::decode_message(EthVersion::Eth68, &mut buf.as_slice()).unwrap(); + assert_eq!(empty_block_bodies, decoded); + } } diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index 717b0446beae..a06979300acd 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -1090,4 +1090,13 @@ mod tests { let block = block.seal_slow(); assert_eq!(sealed, block.hash()); } + + #[test] + fn empty_block_rlp() { + let body = BlockBody::default(); + let mut buf = Vec::new(); + body.encode(&mut buf); + let decoded = BlockBody::decode(&mut buf.as_slice()).unwrap(); + assert_eq!(body, decoded); + } } From 2fb63b04911affa850ada5a336953942f4f11b0f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Oct 2024 21:40:29 +0200 Subject: [PATCH 057/239] chore: dont log if nothing to evict (#12015) --- crates/net/discv4/src/lib.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index 779c7ee637aa..a99906bdf09b 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -1513,11 +1513,12 @@ impl Discv4Service { true }); - trace!(target: "discv4", num=%failed_pings.len(), "evicting nodes due to failed pong"); - - // remove nodes that failed to pong - for node_id in failed_pings { - self.remove_node(node_id); + if !failed_pings.is_empty() { + // remove nodes that failed to pong + trace!(target: "discv4", num=%failed_pings.len(), "evicting nodes due to failed pong"); + for node_id in failed_pings { + self.remove_node(node_id); + } } let mut failed_lookups = Vec::new(); @@ -1528,11 +1529,13 @@ impl Discv4Service { } true }); - trace!(target: "discv4", num=%failed_lookups.len(), "evicting nodes due to failed lookup"); - // remove nodes that failed the e2e lookup process, so we can restart it - for node_id in failed_lookups { - self.remove_node(node_id); + if !failed_lookups.is_empty() { + // remove nodes that failed the e2e lookup process, so we can restart it + trace!(target: "discv4", num=%failed_lookups.len(), "evicting nodes due to failed lookup"); + for node_id in failed_lookups { + self.remove_node(node_id); + } } self.evict_failed_find_nodes(now); @@ -1553,6 +1556,10 @@ impl Discv4Service { true }); + if failed_find_nodes.is_empty() { + return + } + trace!(target: "discv4", num=%failed_find_nodes.len(), "processing failed find nodes"); for node_id in failed_find_nodes { From 565e4b400d6e3a4125efdcd50bb5a2c2d60f65ca Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 01:15:15 +0200 Subject: [PATCH 058/239] refactor(primitive-traits): use alloy `ETHEREUM_BLOCK_GAS_LIMIT` constant (#12019) --- Cargo.lock | 1 + crates/chainspec/src/spec.rs | 6 ++--- crates/node/core/src/args/payload_builder.rs | 3 +-- crates/node/core/src/args/txpool.rs | 3 ++- crates/optimism/chainspec/Cargo.toml | 22 ++++++++++--------- crates/optimism/chainspec/src/op.rs | 2 +- crates/optimism/chainspec/src/op_sepolia.rs | 2 +- crates/primitives-traits/src/constants/mod.rs | 3 --- crates/rpc/rpc-server-types/src/constants.rs | 5 ++--- crates/rpc/rpc/src/eth/helpers/state.rs | 2 +- crates/rpc/rpc/src/eth/helpers/transaction.rs | 2 +- crates/transaction-pool/src/config.rs | 3 ++- crates/transaction-pool/src/noop.rs | 4 ++-- crates/transaction-pool/src/pool/txpool.rs | 5 ++--- crates/transaction-pool/tests/it/evict.rs | 3 ++- 15 files changed, 32 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 350e6a77ae55..b66da618106d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8060,6 +8060,7 @@ version = "1.1.0" dependencies = [ "alloy-chains", "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", "derive_more 1.0.0", diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index 0e38d866b0d4..bebf7ca26028 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -9,6 +9,7 @@ use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256}; use derive_more::From; use alloy_consensus::constants::{DEV_GENESIS_HASH, MAINNET_GENESIS_HASH}; +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use reth_ethereum_forks::{ ChainHardforks, DisplayHardforks, EthereumHardfork, EthereumHardforks, ForkCondition, ForkFilter, ForkFilterKey, ForkHash, ForkId, Hardfork, Hardforks, Head, DEV_HARDFORKS, @@ -18,10 +19,7 @@ use reth_network_peers::{ sepolia_nodes, NodeRecord, }; use reth_primitives_traits::{ - constants::{ - EIP1559_INITIAL_BASE_FEE, ETHEREUM_BLOCK_GAS_LIMIT, HOLESKY_GENESIS_HASH, - SEPOLIA_GENESIS_HASH, - }, + constants::{EIP1559_INITIAL_BASE_FEE, HOLESKY_GENESIS_HASH, SEPOLIA_GENESIS_HASH}, Header, SealedHeader, }; use reth_trie_common::root::state_root_ref_unhashed; diff --git a/crates/node/core/src/args/payload_builder.rs b/crates/node/core/src/args/payload_builder.rs index dceb10726ff2..524a93195de1 100644 --- a/crates/node/core/src/args/payload_builder.rs +++ b/crates/node/core/src/args/payload_builder.rs @@ -1,12 +1,11 @@ use crate::{cli::config::PayloadBuilderConfig, version::default_extradata}; use alloy_consensus::constants::MAXIMUM_EXTRA_DATA_SIZE; -use alloy_eips::merge::SLOT_DURATION; +use alloy_eips::{eip1559::ETHEREUM_BLOCK_GAS_LIMIT, merge::SLOT_DURATION}; use clap::{ builder::{RangedU64ValueParser, TypedValueParser}, Arg, Args, Command, }; use reth_cli_util::{parse_duration_from_secs, parse_duration_from_secs_or_ms}; -use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use std::{borrow::Cow, ffi::OsStr, time::Duration}; /// Parameters for configuring the Payload Builder diff --git a/crates/node/core/src/args/txpool.rs b/crates/node/core/src/args/txpool.rs index 63f6c566ca2b..282313555f76 100644 --- a/crates/node/core/src/args/txpool.rs +++ b/crates/node/core/src/args/txpool.rs @@ -1,9 +1,10 @@ //! Transaction pool arguments use crate::cli::config::RethTransactionPoolConfig; +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::Address; use clap::Args; -use reth_primitives::constants::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; +use reth_primitives::constants::MIN_PROTOCOL_BASE_FEE; use reth_transaction_pool::{ blobstore::disk::DEFAULT_MAX_CACHED_BLOBS, pool::{NEW_TX_LISTENER_BUFFER_SIZE, PENDING_TX_LISTENER_BUFFER_SIZE}, diff --git a/crates/optimism/chainspec/Cargo.toml b/crates/optimism/chainspec/Cargo.toml index 6b068dabbf0c..4e573ce29946 100644 --- a/crates/optimism/chainspec/Cargo.toml +++ b/crates/optimism/chainspec/Cargo.toml @@ -26,6 +26,7 @@ alloy-chains.workspace = true alloy-genesis.workspace = true alloy-primitives.workspace = true alloy-consensus.workspace = true +alloy-eips.workspace = true # op op-alloy-rpc-types.workspace = true @@ -45,14 +46,15 @@ op-alloy-rpc-types.workspace = true [features] default = ["std"] std = [ - "alloy-chains/std", - "alloy-genesis/std", - "alloy-primitives/std", - "op-alloy-rpc-types/std", - "reth-chainspec/std", - "reth-ethereum-forks/std", - "reth-primitives-traits/std", - "reth-optimism-forks/std", - "alloy-consensus/std", - "once_cell/std" + "alloy-chains/std", + "alloy-genesis/std", + "alloy-primitives/std", + "alloy-eips/std", + "op-alloy-rpc-types/std", + "reth-chainspec/std", + "reth-ethereum-forks/std", + "reth-primitives-traits/std", + "reth-optimism-forks/std", + "alloy-consensus/std", + "once_cell/std", ] diff --git a/crates/optimism/chainspec/src/op.rs b/crates/optimism/chainspec/src/op.rs index 8c0da5320f94..5afb236cd33e 100644 --- a/crates/optimism/chainspec/src/op.rs +++ b/crates/optimism/chainspec/src/op.rs @@ -3,11 +3,11 @@ use alloc::{sync::Arc, vec}; use alloy_chains::Chain; +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::{b256, U256}; use reth_chainspec::{once_cell_set, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; use reth_ethereum_forks::EthereumHardfork; use reth_optimism_forks::OptimismHardfork; -use reth_primitives_traits::constants::ETHEREUM_BLOCK_GAS_LIMIT; use crate::{LazyLock, OpChainSpec}; diff --git a/crates/optimism/chainspec/src/op_sepolia.rs b/crates/optimism/chainspec/src/op_sepolia.rs index d3243ebd5346..31c9eda6bddd 100644 --- a/crates/optimism/chainspec/src/op_sepolia.rs +++ b/crates/optimism/chainspec/src/op_sepolia.rs @@ -3,11 +3,11 @@ use alloc::{sync::Arc, vec}; use alloy_chains::{Chain, NamedChain}; +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::{b256, U256}; use reth_chainspec::{once_cell_set, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; use reth_ethereum_forks::EthereumHardfork; use reth_optimism_forks::OptimismHardfork; -use reth_primitives_traits::constants::ETHEREUM_BLOCK_GAS_LIMIT; use crate::{LazyLock, OpChainSpec}; diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index a4091a4a9d9c..377f66cf0040 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -15,9 +15,6 @@ pub const EPOCH_SLOTS: u64 = 32; /// The default block nonce in the beacon consensus pub const BEACON_NONCE: u64 = 0u64; -/// The default Ethereum block gas limit. -pub const ETHEREUM_BLOCK_GAS_LIMIT: u64 = 30_000_000; - /// The minimum tx fee below which the txpool will reject the transaction. /// /// Configured to `7` WEI which is the lowest possible value of base fee under mainnet EIP-1559 diff --git a/crates/rpc/rpc-server-types/src/constants.rs b/crates/rpc/rpc-server-types/src/constants.rs index 126bc722fd52..48019745a34f 100644 --- a/crates/rpc/rpc-server-types/src/constants.rs +++ b/crates/rpc/rpc-server-types/src/constants.rs @@ -80,9 +80,8 @@ pub mod gas_oracle { /// The default gas limit for `eth_call` and adjacent calls. /// - /// This is different from the default to regular 30M block gas limit - /// [`ETHEREUM_BLOCK_GAS_LIMIT`](reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT) to allow - /// for more complex calls. + /// This is different from the default to regular 30M block gas limit `ETHEREUM_BLOCK_GAS_LIMIT` + /// to allow for more complex calls. pub const RPC_DEFAULT_GAS_CAP: u64 = 50_000_000; /// Allowed error ratio for gas estimation diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 8a35842798bc..429a10333d1b 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -45,11 +45,11 @@ where #[cfg(test)] mod tests { use super::*; + use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::{Address, StorageKey, StorageValue, U256}; use reth_chainspec::MAINNET; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; - use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}; use reth_rpc_eth_api::helpers::EthState; use reth_rpc_eth_types::{ diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 8bd9997f6e8a..24a13cb8062c 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -55,11 +55,11 @@ where #[cfg(test)] mod tests { + use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::{hex_literal::hex, Bytes}; use reth_chainspec::ChainSpecProvider; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; - use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_provider::test_utils::NoopProvider; use reth_rpc_eth_api::helpers::EthTransactions; use reth_rpc_eth_types::{ diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 30703f888c38..8fe49f47652d 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -3,8 +3,9 @@ use crate::{ PoolSize, TransactionOrigin, }; use alloy_consensus::constants::EIP4844_TX_TYPE_ID; +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::Address; -use reth_primitives::constants::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; +use reth_primitives::constants::MIN_PROTOCOL_BASE_FEE; use std::{collections::HashSet, ops::Mul}; /// Guarantees max transactions for one sender, compatible with geth/erigon diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 4464ae1fc8a5..11c5e7eea29b 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -16,10 +16,10 @@ use crate::{ PooledTransactionsElement, PropagatedTransactions, TransactionEvents, TransactionOrigin, TransactionPool, TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction, }; -use alloy_eips::eip4844::BlobAndProofV1; +use alloy_eips::{eip1559::ETHEREUM_BLOCK_GAS_LIMIT, eip4844::BlobAndProofV1}; use alloy_primitives::{Address, TxHash, B256, U256}; use reth_eth_wire_types::HandleMempoolData; -use reth_primitives::{constants::ETHEREUM_BLOCK_GAS_LIMIT, BlobTransactionSidecar}; +use reth_primitives::BlobTransactionSidecar; use std::{collections::HashSet, marker::PhantomData, sync::Arc}; use tokio::sync::{mpsc, mpsc::Receiver}; diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 03e69c39067c..a3f192992d1a 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -22,10 +22,9 @@ use alloy_consensus::constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::{Address, TxHash, B256}; -use reth_primitives::constants::{ - eip4844::BLOB_TX_MIN_BLOB_GASPRICE, ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE, -}; +use reth_primitives::constants::{eip4844::BLOB_TX_MIN_BLOB_GASPRICE, MIN_PROTOCOL_BASE_FEE}; use rustc_hash::FxHashMap; use smallvec::SmallVec; use std::{ diff --git a/crates/transaction-pool/tests/it/evict.rs b/crates/transaction-pool/tests/it/evict.rs index c7438c9964e9..c1d0bbaa6421 100644 --- a/crates/transaction-pool/tests/it/evict.rs +++ b/crates/transaction-pool/tests/it/evict.rs @@ -1,8 +1,9 @@ //! Transaction pool eviction tests. +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use alloy_primitives::{Address, B256}; use rand::distributions::Uniform; -use reth_primitives::constants::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; +use reth_primitives::constants::MIN_PROTOCOL_BASE_FEE; use reth_transaction_pool::{ error::PoolErrorKind, test_utils::{ From 044e2d6aea9fbc214563a45ff432c2ef01b42206 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 01:15:45 +0200 Subject: [PATCH 059/239] refactor(primitive-traits): use alloy `EPOCH_SLOTS` constant (#12018) --- Cargo.lock | 1 + crates/consensus/beacon/Cargo.toml | 1 + crates/consensus/beacon/src/engine/mod.rs | 5 ++--- crates/primitives-traits/src/constants/mod.rs | 3 --- crates/storage/provider/src/providers/state/historical.rs | 3 ++- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b66da618106d..155cb3417bd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6394,6 +6394,7 @@ dependencies = [ name = "reth-beacon-consensus" version = "1.1.0" dependencies = [ + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml index b141bd34edbc..1abc09b2a44b 100644 --- a/crates/consensus/beacon/Cargo.toml +++ b/crates/consensus/beacon/Cargo.toml @@ -33,6 +33,7 @@ reth-chainspec = { workspace = true, optional = true } # ethereum alloy-primitives.workspace = true alloy-rpc-types-engine.workspace = true +alloy-eips.workspace = true # async tokio = { workspace = true, features = ["sync"] } diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 5af1e26accac..2363b9078408 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1,3 +1,4 @@ +use alloy_eips::merge::EPOCH_SLOTS; use alloy_primitives::{BlockNumber, B256}; use alloy_rpc_types_engine::{ ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState, PayloadStatus, PayloadStatusEnum, @@ -19,9 +20,7 @@ use reth_node_types::NodeTypesWithEngine; use reth_payload_builder::PayloadBuilderHandle; use reth_payload_primitives::{PayloadAttributes, PayloadBuilder, PayloadBuilderAttributes}; use reth_payload_validator::ExecutionPayloadValidator; -use reth_primitives::{ - constants::EPOCH_SLOTS, BlockNumHash, Head, Header, SealedBlock, SealedHeader, -}; +use reth_primitives::{BlockNumHash, Head, Header, SealedBlock, SealedHeader}; use reth_provider::{ providers::ProviderNodeTypes, BlockIdReader, BlockReader, BlockSource, CanonChainTracker, ChainSpecProvider, ProviderError, StageCheckpointReader, diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index 377f66cf0040..86bd9349585e 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -9,9 +9,6 @@ pub use gas_units::{GIGAGAS, KILOGAS, MEGAGAS}; /// The client version: `reth/v{major}.{minor}.{patch}` pub const RETH_CLIENT_VERSION: &str = concat!("reth/v", env!("CARGO_PKG_VERSION")); -/// An EPOCH is a series of 32 slots. -pub const EPOCH_SLOTS: u64 = 32; - /// The default block nonce in the beacon consensus pub const BEACON_NONCE: u64 = 0u64; diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 640041e0801f..56a1d057e704 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -2,6 +2,7 @@ use crate::{ providers::{state::macros::delegate_provider_impls, StaticFileProvider}, AccountReader, BlockHashReader, ProviderError, StateProvider, StateRootProvider, }; +use alloy_eips::merge::EPOCH_SLOTS; use alloy_primitives::{ map::{HashMap, HashSet}, Address, BlockNumber, Bytes, StorageKey, StorageValue, B256, @@ -13,7 +14,7 @@ use reth_db_api::{ table::Table, transaction::DbTx, }; -use reth_primitives::{constants::EPOCH_SLOTS, Account, Bytecode, StaticFileSegment}; +use reth_primitives::{Account, Bytecode, StaticFileSegment}; use reth_storage_api::{StateProofProvider, StorageRootProvider}; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ From f2195026ccbc1d74298362901f4edede0d363023 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 02:10:00 +0200 Subject: [PATCH 060/239] test: more unit tests for `TreeState` (#11687) --- crates/blockchain-tree/Cargo.toml | 1 + crates/blockchain-tree/src/state.rs | 300 ++++++++++++++++++++++++++++ 2 files changed, 301 insertions(+) diff --git a/crates/blockchain-tree/Cargo.toml b/crates/blockchain-tree/Cargo.toml index aa8fab16fa54..3fa6de2b402c 100644 --- a/crates/blockchain-tree/Cargo.toml +++ b/crates/blockchain-tree/Cargo.toml @@ -57,6 +57,7 @@ reth-consensus = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true reth-revm.workspace = true reth-evm-ethereum.workspace = true +reth-execution-types.workspace = true parking_lot.workspace = true assert_matches.workspace = true alloy-genesis.workspace = true diff --git a/crates/blockchain-tree/src/state.rs b/crates/blockchain-tree/src/state.rs index b76db9e6a9ce..ca8af6f9b581 100644 --- a/crates/blockchain-tree/src/state.rs +++ b/crates/blockchain-tree/src/state.rs @@ -61,6 +61,7 @@ impl TreeState { pub(crate) fn block_by_hash(&self, block_hash: BlockHash) -> Option<&SealedBlock> { self.block_with_senders_by_hash(block_hash).map(|block| &block.block) } + /// Returns the block with matching hash from any side-chain. /// /// Caution: This will not return blocks from the canonical chain. @@ -128,3 +129,302 @@ impl From for SidechainId { Self(value) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::canonical_chain::CanonicalChain; + use alloy_primitives::B256; + use reth_execution_types::Chain; + use reth_provider::ExecutionOutcome; + + #[test] + fn test_tree_state_initialization() { + // Set up some dummy data for initialization + let last_finalized_block_number = 10u64; + let last_canonical_hashes = vec![(9u64, B256::random()), (10u64, B256::random())]; + let buffer_limit = 5; + + // Initialize the tree state + let tree_state = TreeState::new( + last_finalized_block_number, + last_canonical_hashes.clone(), + buffer_limit, + ); + + // Verify the tree state after initialization + assert_eq!(tree_state.block_chain_id_generator, 0); + assert_eq!(tree_state.block_indices().last_finalized_block(), last_finalized_block_number); + assert_eq!( + *tree_state.block_indices.canonical_chain().inner(), + *CanonicalChain::new(last_canonical_hashes.into_iter().collect()).inner() + ); + assert!(tree_state.chains.is_empty()); + assert!(tree_state.buffered_blocks.lru.is_empty()); + } + + #[test] + fn test_tree_state_next_id() { + // Initialize the tree state + let mut tree_state = TreeState::new(0, vec![], 5); + + // Generate a few sidechain IDs + let first_id = tree_state.next_id(); + let second_id = tree_state.next_id(); + + // Verify the generated sidechain IDs and the updated generator state + assert_eq!(first_id, SidechainId(0)); + assert_eq!(second_id, SidechainId(1)); + assert_eq!(tree_state.block_chain_id_generator, 2); + } + + #[test] + fn test_tree_state_insert_chain() { + // Initialize tree state + let mut tree_state = TreeState::new(0, vec![], 5); + + // Create a chain with two blocks + let block = SealedBlockWithSenders::default(); + let block1_hash = B256::random(); + let block2_hash = B256::random(); + + let mut block1 = block.clone(); + let mut block2 = block; + + block1.block.header.set_hash(block1_hash); + block1.block.header.set_block_number(9); + block2.block.header.set_hash(block2_hash); + block2.block.header.set_block_number(10); + + let chain = AppendableChain::new(Chain::new( + [block1, block2], + Default::default(), + Default::default(), + )); + + // Insert the chain into the TreeState + let chain_id = tree_state.insert_chain(chain).unwrap(); + + // Verify the chain ID and that it was added to the chains collection + assert_eq!(chain_id, SidechainId(0)); + assert!(tree_state.chains.contains_key(&chain_id)); + + // Ensure that the block indices are updated + assert_eq!( + tree_state.block_indices.get_side_chain_id(&block1_hash).unwrap(), + SidechainId(0) + ); + assert_eq!( + tree_state.block_indices.get_side_chain_id(&block2_hash).unwrap(), + SidechainId(0) + ); + + // Ensure that the block chain ID generator was updated + assert_eq!(tree_state.block_chain_id_generator, 1); + + // Create an empty chain + let chain_empty = AppendableChain::new(Chain::default()); + + // Insert the empty chain into the tree state + let chain_id = tree_state.insert_chain(chain_empty); + + // Ensure that the empty chain was not inserted + assert!(chain_id.is_none()); + + // Nothing should have changed and no new chain should have been added + assert!(tree_state.chains.contains_key(&SidechainId(0))); + assert!(!tree_state.chains.contains_key(&SidechainId(1))); + assert_eq!( + tree_state.block_indices.get_side_chain_id(&block1_hash).unwrap(), + SidechainId(0) + ); + assert_eq!( + tree_state.block_indices.get_side_chain_id(&block2_hash).unwrap(), + SidechainId(0) + ); + assert_eq!(tree_state.block_chain_id_generator, 1); + } + + #[test] + fn test_block_by_hash_side_chain() { + // Initialize a tree state with some dummy data + let mut tree_state = TreeState::new(0, vec![], 5); + + // Create two side-chain blocks with random hashes + let block1_hash = B256::random(); + let block2_hash = B256::random(); + + let mut block1 = SealedBlockWithSenders::default(); + let mut block2 = SealedBlockWithSenders::default(); + + block1.block.header.set_hash(block1_hash); + block1.block.header.set_block_number(9); + block2.block.header.set_hash(block2_hash); + block2.block.header.set_block_number(10); + + // Create an chain with these blocks + let chain = AppendableChain::new(Chain::new( + vec![block1.clone(), block2.clone()], + Default::default(), + Default::default(), + )); + + // Insert the side chain into the TreeState + tree_state.insert_chain(chain).unwrap(); + + // Retrieve the blocks by their hashes + let retrieved_block1 = tree_state.block_by_hash(block1_hash); + assert_eq!(*retrieved_block1.unwrap(), block1.block); + + let retrieved_block2 = tree_state.block_by_hash(block2_hash); + assert_eq!(*retrieved_block2.unwrap(), block2.block); + + // Test block_by_hash with a random hash that doesn't exist + let non_existent_hash = B256::random(); + let result = tree_state.block_by_hash(non_existent_hash); + + // Ensure that no block is found + assert!(result.is_none()); + } + + #[test] + fn test_block_with_senders_by_hash() { + // Initialize a tree state with some dummy data + let mut tree_state = TreeState::new(0, vec![], 5); + + // Create two side-chain blocks with random hashes + let block1_hash = B256::random(); + let block2_hash = B256::random(); + + let mut block1 = SealedBlockWithSenders::default(); + let mut block2 = SealedBlockWithSenders::default(); + + block1.block.header.set_hash(block1_hash); + block1.block.header.set_block_number(9); + block2.block.header.set_hash(block2_hash); + block2.block.header.set_block_number(10); + + // Create a chain with these blocks + let chain = AppendableChain::new(Chain::new( + vec![block1.clone(), block2.clone()], + Default::default(), + Default::default(), + )); + + // Insert the side chain into the TreeState + tree_state.insert_chain(chain).unwrap(); + + // Test to retrieve the blocks with senders by their hashes + let retrieved_block1 = tree_state.block_with_senders_by_hash(block1_hash); + assert_eq!(*retrieved_block1.unwrap(), block1); + + let retrieved_block2 = tree_state.block_with_senders_by_hash(block2_hash); + assert_eq!(*retrieved_block2.unwrap(), block2); + + // Test block_with_senders_by_hash with a random hash that doesn't exist + let non_existent_hash = B256::random(); + let result = tree_state.block_with_senders_by_hash(non_existent_hash); + + // Ensure that no block is found + assert!(result.is_none()); + } + + #[test] + fn test_get_buffered_block() { + // Initialize a tree state with some dummy data + let mut tree_state = TreeState::new(0, vec![], 5); + + // Create a block with a random hash and add it to the buffer + let block_hash = B256::random(); + let mut block = SealedBlockWithSenders::default(); + block.block.header.set_hash(block_hash); + + // Add the block to the buffered blocks in the TreeState + tree_state.buffered_blocks.insert_block(block.clone()); + + // Test get_buffered_block to retrieve the block by its hash + let retrieved_block = tree_state.get_buffered_block(&block_hash); + assert_eq!(*retrieved_block.unwrap(), block); + + // Test get_buffered_block with a non-existent hash + let non_existent_hash = B256::random(); + let result = tree_state.get_buffered_block(&non_existent_hash); + + // Ensure that no block is found + assert!(result.is_none()); + } + + #[test] + fn test_lowest_buffered_ancestor() { + // Initialize a tree state with some dummy data + let mut tree_state = TreeState::new(0, vec![], 5); + + // Create blocks with random hashes and set up parent-child relationships + let ancestor_hash = B256::random(); + let descendant_hash = B256::random(); + + let mut ancestor_block = SealedBlockWithSenders::default(); + let mut descendant_block = SealedBlockWithSenders::default(); + + ancestor_block.block.header.set_hash(ancestor_hash); + descendant_block.block.header.set_hash(descendant_hash); + descendant_block.block.header.set_parent_hash(ancestor_hash); + + // Insert the blocks into the buffer + tree_state.buffered_blocks.insert_block(ancestor_block.clone()); + tree_state.buffered_blocks.insert_block(descendant_block.clone()); + + // Test lowest_buffered_ancestor for the descendant block + let lowest_ancestor = tree_state.lowest_buffered_ancestor(&descendant_hash); + assert!(lowest_ancestor.is_some()); + assert_eq!(lowest_ancestor.unwrap().block.header.hash(), ancestor_hash); + + // Test lowest_buffered_ancestor with a non-existent hash + let non_existent_hash = B256::random(); + let result = tree_state.lowest_buffered_ancestor(&non_existent_hash); + + // Ensure that no ancestor is found + assert!(result.is_none()); + } + + #[test] + fn test_receipts_by_block_hash() { + // Initialize a tree state with some dummy data + let mut tree_state = TreeState::new(0, vec![], 5); + + // Create a block with a random hash and receipts + let block_hash = B256::random(); + let receipt1 = Receipt::default(); + let receipt2 = Receipt::default(); + + let mut block = SealedBlockWithSenders::default(); + block.block.header.set_hash(block_hash); + + let receipts = vec![receipt1, receipt2]; + + // Create a chain with the block and its receipts + let chain = AppendableChain::new(Chain::new( + vec![block.clone()], + ExecutionOutcome { receipts: receipts.clone().into(), ..Default::default() }, + Default::default(), + )); + + // Insert the chain into the TreeState + tree_state.insert_chain(chain).unwrap(); + + // Test receipts_by_block_hash for the inserted block + let retrieved_receipts = tree_state.receipts_by_block_hash(block_hash); + assert!(retrieved_receipts.is_some()); + + // Check if the correct receipts are returned + let receipts_ref: Vec<&Receipt> = receipts.iter().collect(); + assert_eq!(retrieved_receipts.unwrap(), receipts_ref); + + // Test receipts_by_block_hash with a non-existent block hash + let non_existent_hash = B256::random(); + let result = tree_state.receipts_by_block_hash(non_existent_hash); + + // Ensure that no receipts are found + assert!(result.is_none()); + } +} From 40935321e3127d02c11552a253e93fcb2a02bc37 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 24 Oct 2024 02:28:16 +0200 Subject: [PATCH 061/239] chore(cli): engine cli options conflict with legacy (#11993) --- bin/reth/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index f424163a24fe..e146912c06f9 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -33,11 +33,11 @@ pub struct EngineArgs { pub legacy: bool, /// Configure persistence threshold for engine experimental. - #[arg(long = "engine.persistence-threshold", requires = "experimental", default_value_t = DEFAULT_PERSISTENCE_THRESHOLD)] + #[arg(long = "engine.persistence-threshold", conflicts_with = "legacy", default_value_t = DEFAULT_PERSISTENCE_THRESHOLD)] pub persistence_threshold: u64, /// Configure the target number of blocks to keep in memory. - #[arg(long = "engine.memory-block-buffer-target", requires = "experimental", default_value_t = DEFAULT_MEMORY_BLOCK_BUFFER_TARGET)] + #[arg(long = "engine.memory-block-buffer-target", conflicts_with = "legacy", default_value_t = DEFAULT_MEMORY_BLOCK_BUFFER_TARGET)] pub memory_block_buffer_target: u64, } From 082f2cd2356262d4ffd34ea9565bd4c1aba1a5f4 Mon Sep 17 00:00:00 2001 From: caglarkaya Date: Thu, 24 Oct 2024 04:19:39 +0300 Subject: [PATCH 062/239] refactor: use op-alloy deposit signature (#12016) --- Cargo.lock | 1 - crates/optimism/chainspec/src/lib.rs | 8 +-- crates/optimism/evm/src/execute.rs | 5 +- crates/primitives/Cargo.toml | 52 ++++++++----------- crates/primitives/src/transaction/mod.rs | 9 ++-- .../primitives/src/transaction/signature.rs | 5 +- 6 files changed, 31 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 155cb3417bd7..63f730b1b661 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8394,7 +8394,6 @@ dependencies = [ "reth-chainspec", "reth-codecs", "reth-ethereum-forks", - "reth-optimism-chainspec", "reth-primitives-traits", "reth-static-file-types", "reth-testing-utils", diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 98c6589d1ceb..83c499de5257 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -20,7 +20,7 @@ mod op_sepolia; use alloc::{vec, vec::Vec}; use alloy_chains::Chain; use alloy_genesis::Genesis; -use alloy_primitives::{Parity, Signature, B256, U256}; +use alloy_primitives::{B256, U256}; pub use base::BASE_MAINNET; pub use base_sepolia::BASE_SEPOLIA; use core::fmt::Display; @@ -178,12 +178,6 @@ pub struct OpChainSpec { pub inner: ChainSpec, } -/// Returns the signature for the optimism deposit transactions, which don't include a -/// signature. -pub fn optimism_deposit_tx_signature() -> Signature { - Signature::new(U256::ZERO, U256::ZERO, Parity::Parity(false)) -} - impl EthChainSpec for OpChainSpec { fn chain(&self) -> alloy_chains::Chain { self.inner.chain() diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 748e57e6b33b..1cd92409847e 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -297,9 +297,10 @@ mod tests { use crate::OpChainSpec; use alloy_consensus::TxEip1559; use alloy_primitives::{b256, Address, StorageKey, StorageValue}; + use op_alloy_consensus::TxDeposit; use reth_chainspec::MIN_TRANSACTION_GAS; use reth_evm::execute::{BasicBlockExecutorProvider, BatchExecutor, BlockExecutorProvider}; - use reth_optimism_chainspec::{optimism_deposit_tx_signature, OpChainSpecBuilder}; + use reth_optimism_chainspec::OpChainSpecBuilder; use reth_primitives::{Account, Block, BlockBody, Signature, Transaction, TransactionSigned}; use reth_revm::{ database::StateProviderDatabase, test_utils::StateProviderTest, L1_BLOCK_CONTRACT, @@ -465,7 +466,7 @@ mod tests { gas_limit: MIN_TRANSACTION_GAS, ..Default::default() }), - optimism_deposit_tx_signature(), + TxDeposit::signature(), ); let provider = executor_provider(chain_spec); diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 566a114bebf9..107c218c7589 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -20,9 +20,6 @@ reth-trie-common.workspace = true revm-primitives = { workspace = true, features = ["serde"] } reth-codecs = { workspace = true, optional = true } -# op-reth -reth-optimism-chainspec = { workspace = true, optional = true } - # ethereum alloy-consensus.workspace = true alloy-primitives = { workspace = true, features = ["rand", "rlp"] } @@ -34,14 +31,15 @@ alloy-eips = { workspace = true, features = ["serde"] } # optimism op-alloy-rpc-types = { workspace = true, optional = true } op-alloy-consensus = { workspace = true, features = [ - "arbitrary", + "arbitrary", + "serde", ], optional = true } # crypto secp256k1 = { workspace = true, features = [ - "global-context", - "recovery", - "rand", + "global-context", + "recovery", + "rand", ], optional = true } k256.workspace = true # for eip-4844 @@ -83,9 +81,9 @@ test-fuzz.workspace = true criterion.workspace = true pprof = { workspace = true, features = [ - "flamegraph", - "frame-pointer", - "criterion", + "flamegraph", + "frame-pointer", + "criterion", ] } [features] @@ -101,13 +99,10 @@ std = [ "once_cell/std", "revm-primitives/std", "secp256k1?/std", - "serde/std" + "serde/std", ] reth-codec = ["dep:reth-codecs", "dep:zstd", "dep:modular-bitfield", "std"] -asm-keccak = [ - "alloy-primitives/asm-keccak", - "revm-primitives/asm-keccak" -] +asm-keccak = ["alloy-primitives/asm-keccak", "revm-primitives/asm-keccak"] arbitrary = [ "dep:arbitrary", "alloy-eips/arbitrary", @@ -124,38 +119,37 @@ arbitrary = [ "alloy-rpc-types?/arbitrary", "alloy-serde?/arbitrary", "op-alloy-consensus?/arbitrary", - "op-alloy-rpc-types?/arbitrary" + "op-alloy-rpc-types?/arbitrary", ] secp256k1 = ["dep:secp256k1"] c-kzg = [ - "dep:c-kzg", - "alloy-consensus/kzg", - "alloy-eips/kzg", - "revm-primitives/c-kzg", + "dep:c-kzg", + "alloy-consensus/kzg", + "alloy-eips/kzg", + "revm-primitives/c-kzg", ] optimism = [ - "dep:op-alloy-consensus", - "dep:reth-optimism-chainspec", - "reth-codecs?/optimism", - "revm-primitives/optimism", + "dep:op-alloy-consensus", + "reth-codecs?/optimism", + "revm-primitives/optimism", ] alloy-compat = [ - "dep:alloy-rpc-types", - "dep:alloy-serde", - "dep:op-alloy-rpc-types", + "dep:alloy-rpc-types", + "dep:alloy-serde", + "dep:op-alloy-rpc-types", ] test-utils = [ "reth-primitives-traits/test-utils", "reth-chainspec/test-utils", "reth-codecs?/test-utils", - "reth-trie-common/test-utils" + "reth-trie-common/test-utils", ] serde-bincode-compat = [ "alloy-consensus/serde-bincode-compat", "op-alloy-consensus?/serde-bincode-compat", "reth-primitives-traits/serde-bincode-compat", "serde_with", - "alloy-eips/serde-bincode-compat" + "alloy-eips/serde-bincode-compat", ] [[bench]] diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 1d410da1ea8f..7798433d05dd 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -56,8 +56,6 @@ mod variant; #[cfg(feature = "optimism")] use op_alloy_consensus::TxDeposit; #[cfg(feature = "optimism")] -use reth_optimism_chainspec::optimism_deposit_tx_signature; -#[cfg(feature = "optimism")] pub use tx_type::DEPOSIT_TX_TYPE_ID; #[cfg(any(test, feature = "reth-codec"))] use tx_type::{ @@ -955,7 +953,7 @@ impl TransactionSignedNoHash { // transactions with an empty signature // // NOTE: this is very hacky and only relevant for op-mainnet pre bedrock - if self.is_legacy() && self.signature == optimism_deposit_tx_signature() { + if self.is_legacy() && self.signature == TxDeposit::signature() { return Some(Address::ZERO) } } @@ -1530,7 +1528,7 @@ impl Decodable2718 for TransactionSigned { #[cfg(feature = "optimism")] TxType::Deposit => Ok(Self::from_transaction_and_signature( Transaction::Deposit(TxDeposit::decode(buf)?), - optimism_deposit_tx_signature(), + TxDeposit::signature(), )), } } @@ -1575,8 +1573,7 @@ impl<'a> arbitrary::Arbitrary<'a> for TransactionSigned { } #[cfg(feature = "optimism")] - let signature = - if transaction.is_deposit() { optimism_deposit_tx_signature() } else { signature }; + let signature = if transaction.is_deposit() { TxDeposit::signature() } else { signature }; Ok(Self::from_transaction_and_signature(transaction, signature)) } diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index 39c0f92fda88..5bfdab8e68e9 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -4,9 +4,6 @@ use alloy_rlp::{Decodable, Error as RlpError}; pub use alloy_primitives::Signature; -#[cfg(feature = "optimism")] -use reth_optimism_chainspec::optimism_deposit_tx_signature; - /// The order of the secp256k1 curve, divided by two. Signatures that should be checked according /// to EIP-2 should have an S value less than or equal to this. /// @@ -82,7 +79,7 @@ pub fn legacy_parity(signature: &Signature, chain_id: Option) -> Parity { // transactions with an empty signature // // NOTE: this is very hacky and only relevant for op-mainnet pre bedrock - if *signature == optimism_deposit_tx_signature() { + if *signature == op_alloy_consensus::TxDeposit::signature() { return Parity::Parity(false) } Parity::NonEip155(signature.v().y_parity()) From 7a06298cf70fec4338cb5014f12a6a561d7962f2 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Thu, 24 Oct 2024 06:46:07 +0200 Subject: [PATCH 063/239] chore(rpc): make `TransactionCompat::fill` stateful (#11732) --- Cargo.lock | 1 + crates/optimism/rpc/Cargo.toml | 2 +- crates/optimism/rpc/src/eth/mod.rs | 19 +++++++++-- crates/optimism/rpc/src/eth/transaction.rs | 30 ++++++++++++----- crates/optimism/rpc/src/lib.rs | 2 +- crates/rpc/rpc-builder/src/eth.rs | 4 ++- crates/rpc/rpc-builder/src/lib.rs | 9 +++-- crates/rpc/rpc-eth-api/src/core.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/block.rs | 3 +- crates/rpc/rpc-eth-api/src/helpers/call.rs | 3 +- .../rpc-eth-api/src/helpers/transaction.rs | 8 +++-- crates/rpc/rpc-eth-api/src/types.rs | 7 ++++ crates/rpc/rpc-eth-types/src/simulate.rs | 4 ++- crates/rpc/rpc-eth-types/src/transaction.rs | 6 ++-- crates/rpc/rpc-types-compat/Cargo.toml | 3 ++ crates/rpc/rpc-types-compat/src/block.rs | 8 +++-- .../rpc-types-compat/src/transaction/mod.rs | 32 ++++++++++++++---- crates/rpc/rpc/src/eth/core.rs | 13 ++++++-- crates/rpc/rpc/src/eth/filter.rs | 21 +++++++----- crates/rpc/rpc/src/eth/helpers/types.rs | 6 +++- crates/rpc/rpc/src/eth/pubsub.rs | 33 ++++++++++++------- crates/rpc/rpc/src/txpool.rs | 28 ++++++++-------- 22 files changed, 168 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63f730b1b661..f749672fe9d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8877,6 +8877,7 @@ dependencies = [ "alloy-serde", "reth-primitives", "reth-trie-common", + "serde", "serde_json", ] diff --git a/crates/optimism/rpc/Cargo.toml b/crates/optimism/rpc/Cargo.toml index dc0f96c4012d..17ebec7ff741 100644 --- a/crates/optimism/rpc/Cargo.toml +++ b/crates/optimism/rpc/Cargo.toml @@ -56,7 +56,7 @@ serde_json.workspace = true # misc thiserror.workspace = true tracing.workspace = true -derive_more.workspace = true +derive_more = { workspace = true, features = ["constructor", "deref"] } [dev-dependencies] reth-optimism-chainspec.workspace = true diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index d65dd8edd1d5..04774a4651c0 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -39,7 +39,7 @@ use reth_tasks::{ }; use reth_transaction_pool::TransactionPool; -use crate::{OpEthApiError, OpTxBuilder, SequencerClient}; +use crate::{OpEthApiError, SequencerClient}; /// Adapter for [`EthApiInner`], which holds all the data required to serve core `eth_` API. pub type EthApiNodeBackend = EthApiInner< @@ -59,7 +59,7 @@ pub type EthApiNodeBackend = EthApiInner< /// /// This type implements the [`FullEthApi`](reth_rpc_eth_api::helpers::FullEthApi) by implemented /// all the `Eth` helper traits and prerequisite traits. -#[derive(Clone, Deref)] +#[derive(Deref)] pub struct OpEthApi { /// Gateway to node's core components. #[deref] @@ -102,7 +102,11 @@ where { type Error = OpEthApiError; type NetworkTypes = Optimism; - type TransactionCompat = OpTxBuilder; + type TransactionCompat = Self; + + fn tx_resp_builder(&self) -> &Self::TransactionCompat { + self + } } impl EthApiSpec for OpEthApi @@ -249,3 +253,12 @@ impl fmt::Debug for OpEthApi { f.debug_struct("OpEthApi").finish_non_exhaustive() } } + +impl Clone for OpEthApi +where + N: FullNodeComponents, +{ + fn clone(&self) -> Self { + Self { inner: self.inner.clone(), sequencer_client: self.sequencer_client.clone() } + } +} diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index b7575c24416a..4ac2d7e6b74f 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -6,7 +6,7 @@ use alloy_rpc_types::TransactionInfo; use op_alloy_rpc_types::Transaction; use reth_node_api::FullNodeComponents; use reth_primitives::TransactionSignedEcRecovered; -use reth_provider::{BlockReaderIdExt, TransactionsProvider}; +use reth_provider::{BlockReaderIdExt, ReceiptProvider, TransactionsProvider}; use reth_rpc::eth::EthTxBuilder; use reth_rpc_eth_api::{ helpers::{EthSigner, EthTransactions, LoadTransaction, SpawnBlocking}, @@ -88,22 +88,34 @@ where } } -/// Builds OP transaction response type. -#[derive(Clone, Debug, Copy)] -pub struct OpTxBuilder; - -impl TransactionCompat for OpTxBuilder { +impl TransactionCompat for OpEthApi +where + N: FullNodeComponents, +{ type Transaction = Transaction; - fn fill(tx: TransactionSignedEcRecovered, tx_info: TransactionInfo) -> Self::Transaction { + fn fill( + &self, + tx: TransactionSignedEcRecovered, + tx_info: TransactionInfo, + ) -> Self::Transaction { let signed_tx = tx.clone().into_signed(); + let hash = tx.hash; - let mut inner = EthTxBuilder::fill(tx, tx_info).inner; + let mut inner = EthTxBuilder.fill(tx, tx_info).inner; if signed_tx.is_deposit() { inner.gas_price = Some(signed_tx.max_fee_per_gas()) } + let deposit_receipt_version = self + .inner + .provider() + .receipt_by_hash(hash) + .ok() // todo: change sig to return result + .flatten() + .and_then(|receipt| receipt.deposit_receipt_version); + Transaction { inner, source_hash: signed_tx.source_hash(), @@ -111,7 +123,7 @@ impl TransactionCompat for OpTxBuilder { // only include is_system_tx if true: is_system_tx: (signed_tx.is_deposit() && signed_tx.is_system_transaction()) .then_some(true), - deposit_receipt_version: None, // todo: how to fill this field? + deposit_receipt_version, } } diff --git a/crates/optimism/rpc/src/lib.rs b/crates/optimism/rpc/src/lib.rs index e3fef7adb5b7..0ff1451d05b7 100644 --- a/crates/optimism/rpc/src/lib.rs +++ b/crates/optimism/rpc/src/lib.rs @@ -15,5 +15,5 @@ pub mod eth; pub mod sequencer; pub use error::{OpEthApiError, OptimismInvalidTransactionError, SequencerClientError}; -pub use eth::{transaction::OpTxBuilder, OpEthApi, OpReceiptBuilder}; +pub use eth::{OpEthApi, OpReceiptBuilder}; pub use sequencer::SequencerClient; diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs index 613652678a20..40acecfedf33 100644 --- a/crates/rpc/rpc-builder/src/eth.rs +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -22,7 +22,7 @@ pub struct EthHandlers { /// Polling based filter handler available on all transports pub filter: EthFilter, /// Handler for subscriptions only available for transports that support it (ws, ipc) - pub pubsub: EthPubSub, + pub pubsub: EthPubSub, } impl EthHandlers @@ -94,6 +94,7 @@ where ctx.cache.clone(), ctx.config.filter_config(), Box::new(ctx.executor.clone()), + api.tx_resp_builder().clone(), ); let pubsub = EthPubSub::with_spawner( @@ -102,6 +103,7 @@ where ctx.events.clone(), ctx.network.clone(), Box::new(ctx.executor.clone()), + api.tx_resp_builder().clone(), ); Self { api, cache: ctx.cache, filter, pubsub } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index cd93aeb620e3..72b53efe674c 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1199,9 +1199,12 @@ where .into_rpc() .into(), RethRpcModule::Web3 => Web3Api::new(self.network.clone()).into_rpc().into(), - RethRpcModule::Txpool => { - TxPoolApi::<_, EthApi>::new(self.pool.clone()).into_rpc().into() - } + RethRpcModule::Txpool => TxPoolApi::new( + self.pool.clone(), + self.eth.api.tx_resp_builder().clone(), + ) + .into_rpc() + .into(), RethRpcModule::Rpc => RPCApi::new( namespaces .iter() diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index 20edf96d810d..66bc5a44d2db 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -502,7 +502,7 @@ where trace!(target: "rpc::eth", ?hash, "Serving eth_getTransactionByHash"); Ok(EthTransactions::transaction_by_hash(self, hash) .await? - .map(|tx| tx.into_transaction::())) + .map(|tx| tx.into_transaction(self.tx_resp_builder()))) } /// Handler for: `eth_getRawTransactionByBlockHashAndIndex` diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 9bf35d850af6..da5f275ef0cf 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -63,11 +63,12 @@ pub trait EthBlocks: LoadBlock { .map_err(Self::Error::from_eth_err)?; } - let block = from_block::( + let block = from_block( (*block).clone().unseal(), total_difficulty.unwrap_or_default(), full.into(), Some(block_hash), + self.tx_resp_builder(), ) .map_err(Self::Error::from_eth_err)?; Ok(Some(block)) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 0acf6646294f..1510233c5059 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -191,7 +191,7 @@ pub trait EthCall: Call + LoadPendingBlock { results.push((env.tx.caller, res.result)); } - let block = simulate::build_block::( + let block = simulate::build_block( results, transactions, &block_env, @@ -199,6 +199,7 @@ pub trait EthCall: Call + LoadPendingBlock { total_difficulty, return_full_transactions, &db, + this.tx_resp_builder(), )?; parent_hash = block.inner.header.hash; diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index d29787d7a23b..0d16a5c9145b 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -209,9 +209,10 @@ pub trait EthTransactions: LoadTransaction { index: Some(index as u64), }; - return Ok(Some(from_recovered_with_block_context::( + return Ok(Some(from_recovered_with_block_context( tx.clone().with_signer(*signer), tx_info, + self.tx_resp_builder(), ))) } } @@ -237,7 +238,7 @@ pub trait EthTransactions: LoadTransaction { LoadState::pool(self).get_transaction_by_sender_and_nonce(sender, nonce) { let transaction = tx.transaction.clone().into_consensus(); - return Ok(Some(from_recovered::(transaction.into()))); + return Ok(Some(from_recovered(transaction.into(), self.tx_resp_builder()))); } } @@ -288,9 +289,10 @@ pub trait EthTransactions: LoadTransaction { base_fee: base_fee_per_gas.map(u128::from), index: Some(index as u64), }; - from_recovered_with_block_context::( + from_recovered_with_block_context( tx.clone().with_signer(*signer), tx_info, + self.tx_resp_builder(), ) }) }) diff --git a/crates/rpc/rpc-eth-api/src/types.rs b/crates/rpc/rpc-eth-api/src/types.rs index 9ddc23ea32e5..653730ed3c99 100644 --- a/crates/rpc/rpc-eth-api/src/types.rs +++ b/crates/rpc/rpc-eth-api/src/types.rs @@ -23,12 +23,19 @@ pub trait EthApiTypes: Send + Sync + Clone { type NetworkTypes: Network; /// Conversion methods for transaction RPC type. type TransactionCompat: Send + Sync + Clone + fmt::Debug; + + /// Returns reference to transaction response builder. + fn tx_resp_builder(&self) -> &Self::TransactionCompat; } impl EthApiTypes for () { type Error = EthApiError; type NetworkTypes = AnyNetwork; type TransactionCompat = (); + + fn tx_resp_builder(&self) -> &Self::TransactionCompat { + self + } } /// Adapter for network specific transaction type. diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index 77db511e6251..1d443861d4f6 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -172,6 +172,7 @@ where } /// Handles outputs of the calls execution and builds a [`SimulatedBlock`]. +#[expect(clippy::too_many_arguments)] pub fn build_block( results: Vec<(Address, ExecutionResult)>, transactions: Vec, @@ -180,6 +181,7 @@ pub fn build_block( total_difficulty: U256, full_transactions: bool, db: &CacheDB>>, + tx_resp_builder: &T, ) -> Result>, EthApiError> { let mut calls: Vec = Vec::with_capacity(results.len()); let mut senders = Vec::with_capacity(results.len()); @@ -304,6 +306,6 @@ pub fn build_block( let txs_kind = if full_transactions { BlockTransactionsKind::Full } else { BlockTransactionsKind::Hashes }; - let block = from_block::(block, total_difficulty, txs_kind, None)?; + let block = from_block(block, total_difficulty, txs_kind, None, tx_resp_builder)?; Ok(SimulatedBlock { inner: block, calls }) } diff --git a/crates/rpc/rpc-eth-types/src/transaction.rs b/crates/rpc/rpc-eth-types/src/transaction.rs index c3ca1b503aef..7d2237a1b7fb 100644 --- a/crates/rpc/rpc-eth-types/src/transaction.rs +++ b/crates/rpc/rpc-eth-types/src/transaction.rs @@ -41,9 +41,9 @@ impl TransactionSource { } /// Conversion into network specific transaction type. - pub fn into_transaction(self) -> T::Transaction { + pub fn into_transaction(self, resp_builder: &T) -> T::Transaction { match self { - Self::Pool(tx) => from_recovered::(tx), + Self::Pool(tx) => from_recovered(tx, resp_builder), Self::Block { transaction, index, block_hash, block_number, base_fee } => { let tx_info = TransactionInfo { hash: Some(transaction.hash()), @@ -53,7 +53,7 @@ impl TransactionSource { base_fee: base_fee.map(u128::from), }; - from_recovered_with_block_context::(transaction, tx_info) + from_recovered_with_block_context(transaction, tx_info, resp_builder) } } } diff --git a/crates/rpc/rpc-types-compat/Cargo.toml b/crates/rpc/rpc-types-compat/Cargo.toml index 8e436f0d3934..7d5eac9dbb91 100644 --- a/crates/rpc/rpc-types-compat/Cargo.toml +++ b/crates/rpc/rpc-types-compat/Cargo.toml @@ -26,5 +26,8 @@ alloy-serde.workspace = true alloy-rpc-types-engine.workspace = true alloy-consensus.workspace = true +# io +serde.workspace = true + [dev-dependencies] serde_json.workspace = true \ No newline at end of file diff --git a/crates/rpc/rpc-types-compat/src/block.rs b/crates/rpc/rpc-types-compat/src/block.rs index a650a69c1c1f..8cddc8c44977 100644 --- a/crates/rpc/rpc-types-compat/src/block.rs +++ b/crates/rpc/rpc-types-compat/src/block.rs @@ -20,12 +20,15 @@ pub fn from_block( total_difficulty: U256, kind: BlockTransactionsKind, block_hash: Option, + tx_resp_builder: &T, ) -> Result, BlockError> { match kind { BlockTransactionsKind::Hashes => { Ok(from_block_with_tx_hashes::(block, total_difficulty, block_hash)) } - BlockTransactionsKind::Full => from_block_full::(block, total_difficulty, block_hash), + BlockTransactionsKind::Full => { + from_block_full::(block, total_difficulty, block_hash, tx_resp_builder) + } } } @@ -60,6 +63,7 @@ pub fn from_block_full( mut block: BlockWithSenders, total_difficulty: U256, block_hash: Option, + tx_resp_builder: &T, ) -> Result, BlockError> { let block_hash = block_hash.unwrap_or_else(|| block.block.header.hash_slow()); let block_number = block.block.number; @@ -83,7 +87,7 @@ pub fn from_block_full( index: Some(idx as u64), }; - from_recovered_with_block_context::(signed_tx_ec_recovered, tx_info) + from_recovered_with_block_context::(signed_tx_ec_recovered, tx_info, tx_resp_builder) }) .collect::>(); diff --git a/crates/rpc/rpc-types-compat/src/transaction/mod.rs b/crates/rpc/rpc-types-compat/src/transaction/mod.rs index 7ffd48cb1f72..f8b46454dc23 100644 --- a/crates/rpc/rpc-types-compat/src/transaction/mod.rs +++ b/crates/rpc/rpc-types-compat/src/transaction/mod.rs @@ -2,6 +2,7 @@ mod signature; pub use signature::*; + use std::fmt; use alloy_consensus::Transaction as _; @@ -11,6 +12,7 @@ use alloy_rpc_types::{ }; use alloy_serde::WithOtherFields; use reth_primitives::{TransactionSigned, TransactionSignedEcRecovered, TxType}; +use serde::{Deserialize, Serialize}; /// Create a new rpc transaction result for a mined transaction, using the given block hash, /// number, and tx index fields to populate the corresponding fields in the rpc result. @@ -20,21 +22,33 @@ use reth_primitives::{TransactionSigned, TransactionSignedEcRecovered, TxType}; pub fn from_recovered_with_block_context( tx: TransactionSignedEcRecovered, tx_info: TransactionInfo, + resp_builder: &T, ) -> T::Transaction { - T::fill(tx, tx_info) + resp_builder.fill(tx, tx_info) } /// Create a new rpc transaction result for a _pending_ signed transaction, setting block /// environment related fields to `None`. -pub fn from_recovered(tx: TransactionSignedEcRecovered) -> T::Transaction { - T::fill(tx, TransactionInfo::default()) +pub fn from_recovered( + tx: TransactionSignedEcRecovered, + resp_builder: &T, +) -> T::Transaction { + resp_builder.fill(tx, TransactionInfo::default()) } /// Builds RPC transaction w.r.t. network. pub trait TransactionCompat: Send + Sync + Unpin + Clone + fmt::Debug { /// RPC transaction response type. - type Transaction: Send + Clone + Default + fmt::Debug; - + type Transaction: Serialize + + for<'de> Deserialize<'de> + + Send + + Sync + + Unpin + + Clone + + Default + + fmt::Debug; + + /// /// Formats gas price and max fee per gas for RPC transaction response w.r.t. network specific /// transaction type. fn gas_price(signed_tx: &TransactionSigned, base_fee: Option) -> GasPrice { @@ -63,7 +77,7 @@ pub trait TransactionCompat: Send + Sync + Unpin + Clone + fmt::Debug { /// Create a new rpc transaction result for a _pending_ signed transaction, setting block /// environment related fields to `None`. - fn fill(tx: TransactionSignedEcRecovered, tx_inf: TransactionInfo) -> Self::Transaction; + fn fill(&self, tx: TransactionSignedEcRecovered, tx_inf: TransactionInfo) -> Self::Transaction; /// Truncates the input of a transaction to only the first 4 bytes. // todo: remove in favour of using constructor on `TransactionResponse` or similar @@ -80,7 +94,11 @@ impl TransactionCompat for () { // `alloy_network::AnyNetwork` type Transaction = WithOtherFields; - fn fill(_tx: TransactionSignedEcRecovered, _tx_info: TransactionInfo) -> Self::Transaction { + fn fill( + &self, + _tx: TransactionSignedEcRecovered, + _tx_info: TransactionInfo, + ) -> Self::Transaction { WithOtherFields::default() } diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index 5c7fbbd00239..21787873e966 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -36,12 +36,15 @@ use crate::eth::EthTxBuilder; #[derive(Deref)] pub struct EthApi { /// All nested fields bundled together. + #[deref] pub(super) inner: Arc>, + /// Transaction RPC response builder. + pub tx_resp_builder: EthTxBuilder, } impl Clone for EthApi { fn clone(&self) -> Self { - Self { inner: self.inner.clone() } + Self { inner: self.inner.clone(), tx_resp_builder: EthTxBuilder } } } @@ -81,7 +84,7 @@ where proof_permits, ); - Self { inner: Arc::new(inner) } + Self { inner: Arc::new(inner), tx_resp_builder: EthTxBuilder } } } @@ -119,7 +122,7 @@ where ctx.config.proof_permits, ); - Self { inner: Arc::new(inner) } + Self { inner: Arc::new(inner), tx_resp_builder: EthTxBuilder } } } @@ -131,6 +134,10 @@ where // todo: replace with alloy_network::Ethereum type NetworkTypes = AnyNetwork; type TransactionCompat = EthTxBuilder; + + fn tx_resp_builder(&self) -> &Self::TransactionCompat { + &self.tx_resp_builder + } } impl std::fmt::Debug diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index b136861c7961..24058da1734c 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -4,7 +4,6 @@ use std::{ collections::HashMap, fmt, iter::StepBy, - marker::PhantomData, ops::RangeInclusive, sync::Arc, time::{Duration, Instant}, @@ -44,7 +43,7 @@ pub struct EthFilter { /// All nested fields bundled together inner: Arc>>, /// Assembles response data w.r.t. network. - _tx_resp_builder: PhantomData, + tx_resp_builder: Eth::TransactionCompat, } impl Clone for EthFilter @@ -52,7 +51,7 @@ where Eth: EthApiTypes, { fn clone(&self) -> Self { - Self { inner: self.inner.clone(), _tx_resp_builder: PhantomData } + Self { inner: self.inner.clone(), tx_resp_builder: self.tx_resp_builder.clone() } } } @@ -76,6 +75,7 @@ where eth_cache: EthStateCache, config: EthFilterConfig, task_spawner: Box, + tx_resp_builder: Eth::TransactionCompat, ) -> Self { let EthFilterConfig { max_blocks_per_filter, max_logs_per_response, stale_filter_ttl } = config; @@ -93,7 +93,7 @@ where max_logs_per_response: max_logs_per_response.unwrap_or(usize::MAX), }; - let eth_filter = Self { inner: Arc::new(inner), _tx_resp_builder: PhantomData }; + let eth_filter = Self { inner: Arc::new(inner), tx_resp_builder }; let this = eth_filter.clone(); eth_filter.inner.task_spawner.spawn_critical( @@ -278,7 +278,7 @@ where PendingTransactionFilterKind::Full => { let stream = self.inner.pool.new_pending_pool_transactions_listener(); let full_txs_receiver = - FullTransactionsReceiver::<_, Eth::TransactionCompat>::new(stream); + FullTransactionsReceiver::new(stream, self.tx_resp_builder.clone()); FilterKind::PendingTransaction(PendingTransactionKind::FullTransaction(Arc::new( full_txs_receiver, ))) @@ -603,7 +603,7 @@ impl PendingTransactionsReceiver { #[derive(Debug, Clone)] struct FullTransactionsReceiver { txs_stream: Arc>>, - _tx_resp_builder: PhantomData, + tx_resp_builder: TxCompat, } impl FullTransactionsReceiver @@ -612,8 +612,8 @@ where TxCompat: TransactionCompat, { /// Creates a new `FullTransactionsReceiver` encapsulating the provided transaction stream. - fn new(stream: NewSubpoolTransactionStream) -> Self { - Self { txs_stream: Arc::new(Mutex::new(stream)), _tx_resp_builder: PhantomData } + fn new(stream: NewSubpoolTransactionStream, tx_resp_builder: TxCompat) -> Self { + Self { txs_stream: Arc::new(Mutex::new(stream)), tx_resp_builder } } /// Returns all new pending transactions received since the last poll. @@ -625,7 +625,10 @@ where let mut prepared_stream = self.txs_stream.lock().await; while let Ok(tx) = prepared_stream.try_recv() { - pending_txs.push(from_recovered::(tx.transaction.to_recovered_transaction())) + pending_txs.push(from_recovered( + tx.transaction.to_recovered_transaction(), + &self.tx_resp_builder, + )) } FilterChanges::Transactions(pending_txs) } diff --git a/crates/rpc/rpc/src/eth/helpers/types.rs b/crates/rpc/rpc/src/eth/helpers/types.rs index ab7b1a268e06..848bcdc365a5 100644 --- a/crates/rpc/rpc/src/eth/helpers/types.rs +++ b/crates/rpc/rpc/src/eth/helpers/types.rs @@ -21,7 +21,11 @@ where { type Transaction = ::TransactionResponse; - fn fill(tx: TransactionSignedEcRecovered, tx_info: TransactionInfo) -> Self::Transaction { + fn fill( + &self, + tx: TransactionSignedEcRecovered, + tx_info: TransactionInfo, + ) -> Self::Transaction { let signer = tx.signer(); let signed_tx = tx.into_signed(); diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 7bd1fd03d3b9..ac962610ef8a 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -1,6 +1,6 @@ //! `eth_` `PubSub` RPC handler implementation -use std::{marker::PhantomData, sync::Arc}; +use std::sync::Arc; use alloy_primitives::TxHash; use alloy_rpc_types::{ @@ -17,7 +17,7 @@ use jsonrpsee::{ }; use reth_network_api::NetworkInfo; use reth_provider::{BlockReader, CanonStateSubscriptions, EvmEnvProvider}; -use reth_rpc_eth_api::{pubsub::EthPubSubApiServer, FullEthApiTypes, RpcTransaction}; +use reth_rpc_eth_api::{pubsub::EthPubSubApiServer, TransactionCompat}; use reth_rpc_eth_types::logs_utils; use reth_rpc_server_types::result::{internal_rpc_err, invalid_params_rpc_err}; use reth_rpc_types_compat::transaction::from_recovered; @@ -38,7 +38,7 @@ pub struct EthPubSub { inner: Arc>, /// The type that's used to spawn subscription tasks. subscription_task_spawner: Box, - _tx_resp_builder: PhantomData, + tx_resp_builder: Eth, } // === impl EthPubSub === @@ -47,13 +47,20 @@ impl EthPubSub Self { + pub fn new( + provider: Provider, + pool: Pool, + chain_events: Events, + network: Network, + tx_resp_builder: Eth, + ) -> Self { Self::with_spawner( provider, pool, chain_events, network, Box::::default(), + tx_resp_builder, ) } @@ -64,21 +71,22 @@ impl EthPubSub, + tx_resp_builder: Eth, ) -> Self { let inner = EthPubSubInner { provider, pool, chain_events, network }; - Self { inner: Arc::new(inner), subscription_task_spawner, _tx_resp_builder: PhantomData } + Self { inner: Arc::new(inner), subscription_task_spawner, tx_resp_builder } } } #[async_trait::async_trait] -impl EthPubSubApiServer> +impl EthPubSubApiServer for EthPubSub where Provider: BlockReader + EvmEnvProvider + Clone + 'static, Pool: TransactionPool + 'static, Events: CanonStateSubscriptions + Clone + 'static, Network: NetworkInfo + Clone + 'static, - Eth: FullEthApiTypes + 'static, + Eth: TransactionCompat + 'static, { /// Handler for `eth_subscribe` async fn subscribe( @@ -89,8 +97,9 @@ where ) -> jsonrpsee::core::SubscriptionResult { let sink = pending.accept().await?; let pubsub = self.inner.clone(); + let resp_builder = self.tx_resp_builder.clone(); self.subscription_task_spawner.spawn(Box::pin(async move { - let _ = handle_accepted::<_, _, _, _, Eth>(pubsub, sink, kind, params).await; + let _ = handle_accepted(pubsub, sink, kind, params, resp_builder).await; })); Ok(()) @@ -103,13 +112,14 @@ async fn handle_accepted( accepted_sink: SubscriptionSink, kind: SubscriptionKind, params: Option, + tx_resp_builder: Eth, ) -> Result<(), ErrorObject<'static>> where Provider: BlockReader + EvmEnvProvider + Clone + 'static, Pool: TransactionPool + 'static, Events: CanonStateSubscriptions + Clone + 'static, Network: NetworkInfo + Clone + 'static, - Eth: FullEthApiTypes, + Eth: TransactionCompat, { match kind { SubscriptionKind::NewHeads => { @@ -140,10 +150,9 @@ where Params::Bool(true) => { // full transaction objects requested let stream = pubsub.full_pending_transaction_stream().map(|tx| { - EthSubscriptionResult::FullTransaction(Box::new(from_recovered::< - Eth::TransactionCompat, - >( + EthSubscriptionResult::FullTransaction(Box::new(from_recovered( tx.transaction.to_recovered_transaction(), + &tx_resp_builder, ))) }); return pipe_from_stream(accepted_sink, stream).await diff --git a/crates/rpc/rpc/src/txpool.rs b/crates/rpc/rpc/src/txpool.rs index 47aaac0bbfd5..d03e10ca75a8 100644 --- a/crates/rpc/rpc/src/txpool.rs +++ b/crates/rpc/rpc/src/txpool.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, marker::PhantomData}; +use std::collections::BTreeMap; use alloy_consensus::Transaction; use alloy_primitives::Address; @@ -9,7 +9,6 @@ use async_trait::async_trait; use jsonrpsee::core::RpcResult as Result; use reth_primitives::TransactionSignedEcRecovered; use reth_rpc_api::TxPoolApiServer; -use reth_rpc_eth_api::{FullEthApiTypes, RpcTransaction}; use reth_rpc_types_compat::{transaction::from_recovered, TransactionCompat}; use reth_transaction_pool::{AllPoolTransactions, PoolTransaction, TransactionPool}; use tracing::trace; @@ -21,33 +20,34 @@ use tracing::trace; pub struct TxPoolApi { /// An interface to interact with the pool pool: Pool, - _tx_resp_builder: PhantomData, + tx_resp_builder: Eth, } impl TxPoolApi { /// Creates a new instance of `TxpoolApi`. - pub const fn new(pool: Pool) -> Self { - Self { pool, _tx_resp_builder: PhantomData } + pub const fn new(pool: Pool, tx_resp_builder: Eth) -> Self { + Self { pool, tx_resp_builder } } } impl TxPoolApi where Pool: TransactionPool + 'static, - Eth: FullEthApiTypes, + Eth: TransactionCompat, { - fn content(&self) -> TxpoolContent> { + fn content(&self) -> TxpoolContent { #[inline] fn insert( tx: &Tx, content: &mut BTreeMap>, + resp_builder: &RpcTxB, ) where Tx: PoolTransaction>, RpcTxB: TransactionCompat, { content.entry(tx.sender()).or_default().insert( tx.nonce().to_string(), - from_recovered::(tx.clone().into_consensus().into()), + from_recovered(tx.clone().into_consensus().into(), resp_builder), ); } @@ -55,10 +55,10 @@ where let mut content = TxpoolContent { pending: BTreeMap::new(), queued: BTreeMap::new() }; for pending in pending { - insert::<_, Eth::TransactionCompat>(&pending.transaction, &mut content.pending); + insert::<_, Eth>(&pending.transaction, &mut content.pending, &self.tx_resp_builder); } for queued in queued { - insert::<_, Eth::TransactionCompat>(&queued.transaction, &mut content.queued); + insert::<_, Eth>(&queued.transaction, &mut content.queued, &self.tx_resp_builder); } content @@ -66,10 +66,10 @@ where } #[async_trait] -impl TxPoolApiServer> for TxPoolApi +impl TxPoolApiServer for TxPoolApi where Pool: TransactionPool + 'static, - Eth: FullEthApiTypes + 'static, + Eth: TransactionCompat + 'static, { /// Returns the number of transactions currently pending for inclusion in the next block(s), as /// well as the ones that are being scheduled for future execution only. @@ -131,7 +131,7 @@ where async fn txpool_content_from( &self, from: Address, - ) -> Result>> { + ) -> Result> { trace!(target: "rpc::eth", ?from, "Serving txpool_contentFrom"); Ok(self.content().remove_from(&from)) } @@ -141,7 +141,7 @@ where /// /// See [here](https://geth.ethereum.org/docs/rpc/ns-txpool#txpool_content) for more details /// Handler for `txpool_content` - async fn txpool_content(&self) -> Result>> { + async fn txpool_content(&self) -> Result> { trace!(target: "rpc::eth", "Serving txpool_content"); Ok(self.content()) } From d7f08cd8762dbe81c7c83655f7f94d5ce628daa3 Mon Sep 17 00:00:00 2001 From: Hai | RISE <150876604+hai-rise@users.noreply.github.com> Date: Thu, 24 Oct 2024 12:09:14 +0700 Subject: [PATCH 064/239] chore: remove some clones (#12008) --- crates/evm/src/system_calls/mod.rs | 4 ++-- crates/optimism/payload/src/builder.rs | 14 ++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index d71dcfedabb2..daaf1d1414f7 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -171,7 +171,7 @@ where DB::Error: Display, { let result_and_state = eip2935::transact_blockhashes_contract_call( - &self.evm_config.clone(), + &self.evm_config, &self.chain_spec, timestamp, block_number, @@ -226,7 +226,7 @@ where DB::Error: Display, { let result_and_state = eip4788::transact_beacon_root_contract_call( - &self.evm_config.clone(), + &self.evm_config, &self.chain_spec, timestamp, block_number, diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 3ed00c49aec5..a05690490209 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -105,13 +105,7 @@ where args: BuildArguments, ) -> Result, PayloadBuilderError> { let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); - optimism_payload( - self.evm_config.clone(), - args, - cfg_env, - block_env, - self.compute_pending_block, - ) + optimism_payload(&self.evm_config, args, cfg_env, block_env, self.compute_pending_block) } fn on_missing_payload( @@ -140,7 +134,7 @@ where best_payload: None, }; let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); - optimism_payload(self.evm_config.clone(), args, cfg_env, block_env, false)? + optimism_payload(&self.evm_config, args, cfg_env, block_env, false)? .into_payload() .ok_or_else(|| PayloadBuilderError::MissingPayload) } @@ -156,7 +150,7 @@ where /// a result indicating success with the payload or an error in case of failure. #[inline] pub(crate) fn optimism_payload( - evm_config: EvmConfig, + evm_config: &EvmConfig, args: BuildArguments, initialized_cfg: CfgEnvWithHandlerCfg, initialized_block_env: BlockEnv, @@ -430,7 +424,7 @@ where &mut db, &chain_spec, attributes.payload_attributes.timestamp, - attributes.clone().payload_attributes.withdrawals, + attributes.payload_attributes.withdrawals.clone(), )?; // merge all transitions into bundle state, this would apply the withdrawal balance changes From fcca8b1523fa7c1917754389fd08c9b21eb57987 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Thu, 24 Oct 2024 13:26:25 +0800 Subject: [PATCH 065/239] refactor: BlockchainTestCase::run rm repetitive convert ForkSpec to ChainSpec (#11896) Co-authored-by: Matthias Seitz --- testing/ef-tests/src/cases/blockchain_test.rs | 14 ++++++-------- testing/ef-tests/src/models.rs | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index d29aafa82120..7d80ec6c47fa 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -6,6 +6,7 @@ use crate::{ }; use alloy_rlp::Decodable; use rayon::iter::{ParallelBridge, ParallelIterator}; +use reth_chainspec::ChainSpec; use reth_primitives::{BlockBody, SealedBlock, StaticFileSegment}; use reth_provider::{ providers::StaticFileWriter, test_utils::create_test_provider_factory_with_chain_spec, @@ -83,11 +84,10 @@ impl Case for BlockchainTestCase { .par_bridge() .try_for_each(|case| { // Create a new test database and initialize a provider for the test case. - let provider = create_test_provider_factory_with_chain_spec(Arc::new( - case.network.clone().into(), - )) - .database_provider_rw() - .unwrap(); + let chain_spec: Arc = Arc::new(case.network.into()); + let provider = create_test_provider_factory_with_chain_spec(chain_spec.clone()) + .database_provider_rw() + .unwrap(); // Insert initial test state into the provider. provider.insert_historical_block( @@ -127,9 +127,7 @@ impl Case for BlockchainTestCase { // Execute the execution stage using the EVM processor factory for the test case // network. let _ = ExecutionStage::new_with_executor( - reth_evm_ethereum::execute::EthExecutorProvider::ethereum(Arc::new( - case.network.clone().into(), - )), + reth_evm_ethereum::execute::EthExecutorProvider::ethereum(chain_spec), ) .execute( &provider, diff --git a/testing/ef-tests/src/models.rs b/testing/ef-tests/src/models.rs index 30e5e5bb20c2..b5dc073c1da3 100644 --- a/testing/ef-tests/src/models.rs +++ b/testing/ef-tests/src/models.rs @@ -257,7 +257,7 @@ impl Account { } /// Fork specification. -#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Deserialize)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Deserialize)] pub enum ForkSpec { /// Frontier Frontier, From e04d1b4b4a3a6ab6e4f784ad129a88054bab460e Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Thu, 24 Oct 2024 08:04:57 +0200 Subject: [PATCH 066/239] perf(net): P2P sink, revert pull/11658 (#11712) --- crates/net/eth-wire/src/p2pstream.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 9882e39787e2..76075838bc76 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -614,25 +614,24 @@ where /// Returns `Poll::Ready(Ok(()))` when no buffered items remain. fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let mut this = self.project(); - loop { - match ready!(this.inner.as_mut().poll_flush(cx)) { - Err(err) => { - trace!(target: "net::p2p", - %err, - "error flushing p2p stream" - ); - return Poll::Ready(Err(err.into())) - } - Ok(()) => { + let poll_res = loop { + match this.inner.as_mut().poll_ready(cx) { + Poll::Pending => break Poll::Pending, + Poll::Ready(Err(err)) => break Poll::Ready(Err(err.into())), + Poll::Ready(Ok(())) => { let Some(message) = this.outgoing_messages.pop_front() else { - return Poll::Ready(Ok(())) + break Poll::Ready(Ok(())) }; if let Err(err) = this.inner.as_mut().start_send(message) { - return Poll::Ready(Err(err.into())) + break Poll::Ready(Err(err.into())) } } } - } + }; + + ready!(this.inner.as_mut().poll_flush(cx))?; + + poll_res } fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { From 0df3148357cbfb9b811fd04da6a69fca24e59be2 Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Thu, 24 Oct 2024 13:29:07 +0700 Subject: [PATCH 067/239] feat(payload): introduce payload freezing for predetermined blocks (#11790) --- crates/payload/basic/src/lib.rs | 74 ++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index 4274d451e43f..b8eab3c0fea0 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -178,7 +178,7 @@ where deadline, // ticks immediately interval: tokio::time::interval(self.config.interval), - best_payload: None, + best_payload: PayloadState::Missing, pending_block: None, cached_reads, payload_task_guard: self.payload_task_guard.clone(), @@ -324,8 +324,8 @@ where deadline: Pin>, /// The interval at which the job should build a new payload after the last. interval: Interval, - /// The best payload so far. - best_payload: Option, + /// The best payload so far and its state. + best_payload: PayloadState, /// Receiver for the block that is currently being built. pending_block: Option>, /// Restricts how many generator tasks can be executed at once. @@ -362,7 +362,7 @@ where let _cancel = cancel.clone(); let guard = self.payload_task_guard.clone(); let payload_config = self.config.clone(); - let best_payload = self.best_payload.clone(); + let best_payload = self.best_payload.payload().cloned(); self.metrics.inc_initiated_payload_builds(); let cached_reads = self.cached_reads.take().unwrap_or_default(); let builder = self.builder.clone(); @@ -407,8 +407,9 @@ where // check if the interval is reached while this.interval.poll_tick(cx).is_ready() { - // start a new job if there is no pending block and we haven't reached the deadline - if this.pending_block.is_none() { + // start a new job if there is no pending block, we haven't reached the deadline, + // and the payload isn't frozen + if this.pending_block.is_none() && !this.best_payload.is_frozen() { this.spawn_build_job(); } } @@ -420,7 +421,11 @@ where BuildOutcome::Better { payload, cached_reads } => { this.cached_reads = Some(cached_reads); debug!(target: "payload_builder", value = %payload.fees(), "built better payload"); - this.best_payload = Some(payload); + this.best_payload = PayloadState::Best(payload); + } + BuildOutcome::Freeze(payload) => { + debug!(target: "payload_builder", "payload frozen, no further building will occur"); + this.best_payload = PayloadState::Frozen(payload); } BuildOutcome::Aborted { fees, cached_reads } => { this.cached_reads = Some(cached_reads); @@ -459,17 +464,18 @@ where type BuiltPayload = Builder::BuiltPayload; fn best_payload(&self) -> Result { - if let Some(ref payload) = self.best_payload { - return Ok(payload.clone()) + if let Some(payload) = self.best_payload.payload() { + Ok(payload.clone()) + } else { + // No payload has been built yet, but we need to return something that the CL then + // can deliver, so we need to return an empty payload. + // + // Note: it is assumed that this is unlikely to happen, as the payload job is + // started right away and the first full block should have been + // built by the time CL is requesting the payload. + self.metrics.inc_requested_empty_payload(); + self.builder.build_empty_payload(&self.client, self.config.clone()) } - // No payload has been built yet, but we need to return something that the CL then can - // deliver, so we need to return an empty payload. - // - // Note: it is assumed that this is unlikely to happen, as the payload job is started right - // away and the first full block should have been built by the time CL is requesting the - // payload. - self.metrics.inc_requested_empty_payload(); - self.builder.build_empty_payload(&self.client, self.config.clone()) } fn payload_attributes(&self) -> Result { @@ -480,8 +486,7 @@ where &mut self, kind: PayloadKind, ) -> (Self::ResolvePayloadFuture, KeepPayloadJobAlive) { - let best_payload = self.best_payload.take(); - + let best_payload = self.best_payload.payload().cloned(); if best_payload.is_none() && self.pending_block.is_none() { // ensure we have a job scheduled if we don't have a best payload yet and none is active self.spawn_build_job(); @@ -545,6 +550,34 @@ where } } +/// Represents the current state of a payload being built. +#[derive(Debug, Clone)] +pub enum PayloadState

{ + /// No payload has been built yet. + Missing, + /// The best payload built so far, which may still be improved upon. + Best(P), + /// The payload is frozen and no further building should occur. + /// + /// Contains the final payload `P` that should be used. + Frozen(P), +} + +impl

PayloadState

{ + /// Checks if the payload is frozen. + pub const fn is_frozen(&self) -> bool { + matches!(self, Self::Frozen(_)) + } + + /// Returns the payload if it exists (either Best or Frozen). + pub const fn payload(&self) -> Option<&P> { + match self { + Self::Missing => None, + Self::Best(p) | Self::Frozen(p) => Some(p), + } + } +} + /// The future that returns the best payload to be served to the consensus layer. /// /// This returns the payload that's supposed to be sent to the CL. @@ -725,6 +758,9 @@ pub enum BuildOutcome { }, /// Build job was cancelled Cancelled, + + /// The payload is final and no further building should occur + Freeze(Payload), } impl BuildOutcome { From 84a30b0404e73d11d16208cc5898afff1b9f4f03 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:53:15 +0200 Subject: [PATCH 068/239] primitive-traits: use alloy `INITIAL_BASE_FEE` constant (#12022) --- Cargo.lock | 1 + crates/blockchain-tree/src/blockchain_tree.rs | 8 ++++---- crates/chain-state/src/test_utils.rs | 9 ++++----- crates/chainspec/src/spec.rs | 6 +++--- crates/consensus/common/Cargo.toml | 1 + crates/consensus/common/src/validation.rs | 2 +- crates/ethereum/evm/src/lib.rs | 4 ++-- crates/primitives-traits/src/constants/mod.rs | 3 --- 8 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f749672fe9d8..e261297ae51c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6741,6 +6741,7 @@ name = "reth-consensus-common" version = "1.1.0" dependencies = [ "alloy-consensus", + "alloy-eips", "alloy-primitives", "mockall", "rand 0.8.5", diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 95c0361f31f5..4468d82052c8 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -1375,6 +1375,7 @@ where mod tests { use super::*; use alloy_consensus::{TxEip1559, EMPTY_ROOT_HASH}; + use alloy_eips::eip1559::INITIAL_BASE_FEE; use alloy_genesis::{Genesis, GenesisAccount}; use alloy_primitives::{keccak256, Address, Sealable, B256}; use assert_matches::assert_matches; @@ -1386,7 +1387,6 @@ mod tests { use reth_evm::test_utils::MockExecutorProvider; use reth_evm_ethereum::execute::EthExecutorProvider; use reth_primitives::{ - constants::EIP1559_INITIAL_BASE_FEE, proofs::{calculate_receipt_root, calculate_transaction_root}, revm_primitives::AccountInfo, Account, BlockBody, Header, Signature, Transaction, TransactionSigned, @@ -1560,7 +1560,7 @@ mod tests { provider_rw.commit().unwrap(); } - let single_tx_cost = U256::from(EIP1559_INITIAL_BASE_FEE * MIN_TRANSACTION_GAS); + let single_tx_cost = U256::from(INITIAL_BASE_FEE * MIN_TRANSACTION_GAS); let mock_tx = |nonce: u64| -> TransactionSignedEcRecovered { TransactionSigned::from_transaction_and_signature( Transaction::Eip1559(TxEip1559 { @@ -1568,7 +1568,7 @@ mod tests { nonce, gas_limit: MIN_TRANSACTION_GAS, to: Address::ZERO.into(), - max_fee_per_gas: EIP1559_INITIAL_BASE_FEE as u128, + max_fee_per_gas: INITIAL_BASE_FEE as u128, ..Default::default() }), Signature::test_signature(), @@ -1605,7 +1605,7 @@ mod tests { gas_used: body.len() as u64 * MIN_TRANSACTION_GAS, gas_limit: chain_spec.max_gas_limit, mix_hash: B256::random(), - base_fee_per_gas: Some(EIP1559_INITIAL_BASE_FEE), + base_fee_per_gas: Some(INITIAL_BASE_FEE), transactions_root, receipts_root, state_root: state_root_unhashed(HashMap::from([( diff --git a/crates/chain-state/src/test_utils.rs b/crates/chain-state/src/test_utils.rs index f1648ab6bff4..564df9fe341a 100644 --- a/crates/chain-state/src/test_utils.rs +++ b/crates/chain-state/src/test_utils.rs @@ -3,7 +3,7 @@ use crate::{ CanonStateSubscriptions, }; use alloy_consensus::{Transaction as _, TxEip1559, EMPTY_ROOT_HASH}; -use alloy_eips::eip7685::Requests; +use alloy_eips::{eip1559::INITIAL_BASE_FEE, eip7685::Requests}; use alloy_primitives::{Address, BlockNumber, Sealable, B256, U256}; use alloy_signer::SignerSync; use alloy_signer_local::PrivateKeySigner; @@ -11,7 +11,6 @@ use rand::{thread_rng, Rng}; use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS}; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{ - constants::EIP1559_INITIAL_BASE_FEE, proofs::{calculate_receipt_root, calculate_transaction_root, calculate_withdrawals_root}, BlockBody, Header, Receipt, Receipts, SealedBlock, SealedBlockWithSenders, SealedHeader, Transaction, TransactionSigned, TransactionSignedEcRecovered, @@ -75,7 +74,7 @@ impl TestBlockBuilder { /// Gas cost of a single transaction generated by the block builder. pub fn single_tx_cost() -> U256 { - U256::from(EIP1559_INITIAL_BASE_FEE * MIN_TRANSACTION_GAS) + U256::from(INITIAL_BASE_FEE * MIN_TRANSACTION_GAS) } /// Generates a random [`SealedBlockWithSenders`]. @@ -92,7 +91,7 @@ impl TestBlockBuilder { nonce, gas_limit: MIN_TRANSACTION_GAS, to: Address::random().into(), - max_fee_per_gas: EIP1559_INITIAL_BASE_FEE as u128, + max_fee_per_gas: INITIAL_BASE_FEE as u128, max_priority_fee_per_gas: 1, ..Default::default() }); @@ -136,7 +135,7 @@ impl TestBlockBuilder { gas_used: transactions.len() as u64 * MIN_TRANSACTION_GAS, gas_limit: self.chain_spec.max_gas_limit, mix_hash: B256::random(), - base_fee_per_gas: Some(EIP1559_INITIAL_BASE_FEE), + base_fee_per_gas: Some(INITIAL_BASE_FEE), transactions_root: calculate_transaction_root(&transactions), receipts_root: calculate_receipt_root(&receipts), beneficiary: Address::random(), diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index bebf7ca26028..b1a23f1fa62e 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -3,7 +3,7 @@ pub use alloy_eips::eip1559::BaseFeeParams; use alloc::{boxed::Box, sync::Arc, vec::Vec}; use alloy_chains::{Chain, NamedChain}; use alloy_consensus::constants::EMPTY_WITHDRAWALS; -use alloy_eips::eip7685::EMPTY_REQUESTS_HASH; +use alloy_eips::{eip1559::INITIAL_BASE_FEE, eip7685::EMPTY_REQUESTS_HASH}; use alloy_genesis::Genesis; use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256}; use derive_more::From; @@ -19,7 +19,7 @@ use reth_network_peers::{ sepolia_nodes, NodeRecord, }; use reth_primitives_traits::{ - constants::{EIP1559_INITIAL_BASE_FEE, HOLESKY_GENESIS_HASH, SEPOLIA_GENESIS_HASH}, + constants::{HOLESKY_GENESIS_HASH, SEPOLIA_GENESIS_HASH}, Header, SealedHeader, }; use reth_trie_common::root::state_root_ref_unhashed; @@ -314,7 +314,7 @@ impl ChainSpec { pub fn initial_base_fee(&self) -> Option { // If the base fee is set in the genesis block, we use that instead of the default. let genesis_base_fee = - self.genesis.base_fee_per_gas.map(|fee| fee as u64).unwrap_or(EIP1559_INITIAL_BASE_FEE); + self.genesis.base_fee_per_gas.map(|fee| fee as u64).unwrap_or(INITIAL_BASE_FEE); // If London is activated at genesis, we set the initial base fee as per EIP-1559. self.hardforks.fork(EthereumHardfork::London).active_at_block(0).then_some(genesis_base_fee) diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml index eaae1301b46a..c83312577e9e 100644 --- a/crates/consensus/common/Cargo.toml +++ b/crates/consensus/common/Cargo.toml @@ -20,6 +20,7 @@ reth-consensus.workspace = true alloy-primitives.workspace = true revm-primitives.workspace = true alloy-consensus.workspace = true +alloy-eips.workspace = true [dev-dependencies] reth-storage-api.workspace = true diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index dabb8c3c34d2..1070bbdbc0f5 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -204,7 +204,7 @@ pub fn validate_against_parent_eip1559_base_fee Date: Thu, 24 Oct 2024 11:56:05 +0200 Subject: [PATCH 069/239] primitive-traits: rm `ALLOWED_FUTURE_BLOCK_TIME_SECONDS` constant (#12028) --- crates/primitives-traits/src/constants/mod.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index 19decfb2d135..6bd21eb82279 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -87,15 +87,6 @@ pub const OP_SYSTEM_TX_TO_ADDR: Address = address!("4200000000000000000000000000 /// the database. pub const BEACON_CONSENSUS_REORG_UNWIND_DEPTH: u64 = 3; -/// Max seconds from current time allowed for blocks, before they're considered future blocks. -/// -/// This is only used when checking whether or not the timestamp for pre-merge blocks is in the -/// future. -/// -/// See: -/// -pub const ALLOWED_FUTURE_BLOCK_TIME_SECONDS: u64 = 15; - #[cfg(test)] mod tests { use super::*; From 8bfb7f9ce9b527dbbfc94327519abbafb666a1d8 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:58:50 +0200 Subject: [PATCH 070/239] primitive-traits: use alloy `BEACON_NONCE` constant (#12029) --- crates/ethereum/payload/src/lib.rs | 4 ++-- crates/optimism/payload/src/builder.rs | 2 +- crates/primitives-traits/src/constants/mod.rs | 3 --- crates/rpc/rpc-eth-api/src/helpers/pending_block.rs | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 7f94acf723cf..951a909b91ca 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -10,7 +10,7 @@ #![allow(clippy::useless_let_if_seq)] use alloy_consensus::EMPTY_OMMER_ROOT_HASH; -use alloy_eips::eip7685::Requests; +use alloy_eips::{eip7685::Requests, merge::BEACON_NONCE}; use alloy_primitives::U256; use reth_basic_payload_builder::{ commit_withdrawals, is_better_payload, BuildArguments, BuildOutcome, PayloadBuilder, @@ -25,7 +25,7 @@ use reth_execution_types::ExecutionOutcome; use reth_payload_builder::{EthBuiltPayload, EthPayloadBuilderAttributes}; use reth_payload_primitives::{PayloadBuilderAttributes, PayloadBuilderError}; use reth_primitives::{ - constants::{eip4844::MAX_DATA_GAS_PER_BLOCK, BEACON_NONCE}, + constants::eip4844::MAX_DATA_GAS_PER_BLOCK, proofs::{self}, revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg}, Block, BlockBody, EthereumHardforks, Header, Receipt, diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index a05690490209..c85abfad7c4e 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use alloy_consensus::EMPTY_OMMER_ROOT_HASH; +use alloy_eips::merge::BEACON_NONCE; use alloy_primitives::U256; use reth_basic_payload_builder::*; use reth_chain_state::ExecutedBlock; @@ -14,7 +15,6 @@ use reth_optimism_consensus::calculate_receipt_root_no_memo_optimism; use reth_optimism_forks::OptimismHardfork; use reth_payload_primitives::{PayloadBuilderAttributes, PayloadBuilderError}; use reth_primitives::{ - constants::BEACON_NONCE, proofs, revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg}, Block, BlockBody, Header, Receipt, TxType, diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index 6bd21eb82279..c11a62963996 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -9,9 +9,6 @@ pub use gas_units::{GIGAGAS, KILOGAS, MEGAGAS}; /// The client version: `reth/v{major}.{minor}.{patch}` pub const RETH_CLIENT_VERSION: &str = concat!("reth/v", env!("CARGO_PKG_VERSION")); -/// The default block nonce in the beacon consensus -pub const BEACON_NONCE: u64 = 0u64; - /// The minimum tx fee below which the txpool will reject the transaction. /// /// Configured to `7` WEI which is the lowest possible value of base fee under mainnet EIP-1559 diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 407ddf1874ab..872f17ee9107 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use crate::{EthApiTypes, FromEthApiError, FromEvmError}; use alloy_consensus::EMPTY_OMMER_ROOT_HASH; -use alloy_eips::eip7685::EMPTY_REQUESTS_HASH; +use alloy_eips::{eip7685::EMPTY_REQUESTS_HASH, merge::BEACON_NONCE}; use alloy_primitives::{BlockNumber, B256, U256}; use alloy_rpc_types::BlockNumberOrTag; use futures::Future; @@ -17,7 +17,7 @@ use reth_evm::{ }; use reth_execution_types::ExecutionOutcome; use reth_primitives::{ - constants::{eip4844::MAX_DATA_GAS_PER_BLOCK, BEACON_NONCE}, + constants::eip4844::MAX_DATA_GAS_PER_BLOCK, proofs::calculate_transaction_root, revm_primitives::{ BlockEnv, CfgEnv, CfgEnvWithHandlerCfg, EVMError, Env, ExecutionResult, InvalidTransaction, From ebd1ed9fb822377d816c798585c0911be45a2b32 Mon Sep 17 00:00:00 2001 From: Deil Urba Date: Thu, 24 Oct 2024 11:49:14 +0100 Subject: [PATCH 071/239] feat(exex): notifications trait (#11972) --- crates/exex/exex/src/context.rs | 2 +- crates/exex/exex/src/notifications.rs | 64 ++++++++++++++++++--------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/crates/exex/exex/src/context.rs b/crates/exex/exex/src/context.rs index 9af12e260a72..c4b4f351baa7 100644 --- a/crates/exex/exex/src/context.rs +++ b/crates/exex/exex/src/context.rs @@ -7,7 +7,7 @@ use reth_primitives::Head; use reth_tasks::TaskExecutor; use tokio::sync::mpsc::UnboundedSender; -use crate::{ExExEvent, ExExNotifications}; +use crate::{ExExEvent, ExExNotifications, ExExNotificationsStream}; /// Captures the context that an `ExEx` has access to. pub struct ExExContext { diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index d0c94d34f644..90a0ee230a4d 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -21,6 +21,40 @@ pub struct ExExNotifications { inner: ExExNotificationsInner, } +/// A trait, that represents a stream of [`ExExNotification`]s. The stream will emit notifications +/// for all blocks. If the stream is configured with a head via [`ExExNotifications::set_with_head`] +/// or [`ExExNotifications::with_head`], it will run backfill jobs to catch up to the node head. +pub trait ExExNotificationsStream: Stream> + Unpin { + /// Sets [`ExExNotificationsStream`] to a stream of [`ExExNotification`]s without a head. + /// + /// It's a no-op if the stream has already been configured without a head. + /// + /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. + fn set_without_head(&mut self); + + /// Sets [`ExExNotificationsStream`] to a stream of [`ExExNotification`]s with the provided + /// head. + /// + /// It's a no-op if the stream has already been configured with a head. + /// + /// See the documentation of [`ExExNotificationsWithHead`] for more details. + fn set_with_head(&mut self, exex_head: ExExHead); + + /// Returns a new [`ExExNotificationsStream`] without a head. + /// + /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. + fn without_head(self) -> Self + where + Self: Sized; + + /// Returns a new [`ExExNotificationsStream`] with the provided head. + /// + /// See the documentation of [`ExExNotificationsWithHead`] for more details. + fn with_head(self, exex_head: ExExHead) -> Self + where + Self: Sized; +} + #[derive(Debug)] enum ExExNotificationsInner { /// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. @@ -52,13 +86,14 @@ impl ExExNotifications { )), } } +} - /// Sets [`ExExNotifications`] to a stream of [`ExExNotification`]s without a head. - /// - /// It's a no-op if the stream has already been configured without a head. - /// - /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. - pub fn set_without_head(&mut self) { +impl ExExNotificationsStream for ExExNotifications +where + P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, + E: BlockExecutorProvider + Clone + Unpin + 'static, +{ + fn set_without_head(&mut self) { let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); self.inner = ExExNotificationsInner::WithoutHead(match current { ExExNotificationsInner::WithoutHead(notifications) => notifications, @@ -73,20 +108,12 @@ impl ExExNotifications { }); } - /// Returns a new [`ExExNotifications`] without a head. - /// - /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. - pub fn without_head(mut self) -> Self { + fn without_head(mut self) -> Self { self.set_without_head(); self } - /// Sets [`ExExNotifications`] to a stream of [`ExExNotification`]s with the provided head. - /// - /// It's a no-op if the stream has already been configured with a head. - /// - /// See the documentation of [`ExExNotificationsWithHead`] for more details. - pub fn set_with_head(&mut self, exex_head: ExExHead) { + fn set_with_head(&mut self, exex_head: ExExHead) { let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); self.inner = ExExNotificationsInner::WithHead(match current { ExExNotificationsInner::WithoutHead(notifications) => { @@ -104,10 +131,7 @@ impl ExExNotifications { }); } - /// Returns a new [`ExExNotifications`] with the provided head. - /// - /// See the documentation of [`ExExNotificationsWithHead`] for more details. - pub fn with_head(mut self, exex_head: ExExHead) -> Self { + fn with_head(mut self, exex_head: ExExHead) -> Self { self.set_with_head(exex_head); self } From d50da7fcd65da3c37ad77b7b5f645f12ffc055b6 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 12:50:43 +0200 Subject: [PATCH 072/239] primitives-traits: use alloy `SEPOLIA_GENESIS_HASH` constant (#12024) --- crates/chainspec/src/spec.rs | 7 ++----- crates/primitives-traits/src/constants/mod.rs | 4 ---- crates/primitives/src/lib.rs | 2 +- crates/storage/db-common/src/init.rs | 4 ++-- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index b1a23f1fa62e..02f4b5ca983e 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -8,7 +8,7 @@ use alloy_genesis::Genesis; use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256}; use derive_more::From; -use alloy_consensus::constants::{DEV_GENESIS_HASH, MAINNET_GENESIS_HASH}; +use alloy_consensus::constants::{DEV_GENESIS_HASH, MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH}; use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; use reth_ethereum_forks::{ ChainHardforks, DisplayHardforks, EthereumHardfork, EthereumHardforks, ForkCondition, @@ -18,10 +18,7 @@ use reth_network_peers::{ base_nodes, base_testnet_nodes, holesky_nodes, mainnet_nodes, op_nodes, op_testnet_nodes, sepolia_nodes, NodeRecord, }; -use reth_primitives_traits::{ - constants::{HOLESKY_GENESIS_HASH, SEPOLIA_GENESIS_HASH}, - Header, SealedHeader, -}; +use reth_primitives_traits::{constants::HOLESKY_GENESIS_HASH, Header, SealedHeader}; use reth_trie_common::root::state_root_ref_unhashed; use crate::{constants::MAINNET_DEPOSIT_CONTRACT, once_cell_set, EthChainSpec, LazyLock, OnceLock}; diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index c11a62963996..c5ee6bc3e83c 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -60,10 +60,6 @@ pub const OP_SEPOLIA_EIP1559_DEFAULT_ELASTICITY_MULTIPLIER: u128 = 6; /// [transaction costs](https://community.optimism.io/docs/developers/build/differences/#transaction-costs) doc. pub const BASE_SEPOLIA_EIP1559_DEFAULT_ELASTICITY_MULTIPLIER: u128 = 10; -/// Sepolia genesis hash: `0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9` -pub const SEPOLIA_GENESIS_HASH: B256 = - b256!("25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9"); - /// Holesky genesis hash: `0xb5f7f912443c940f21fd611f12828d75b534364ed9e95ca4e307729a4661bde4` pub const HOLESKY_GENESIS_HASH: B256 = b256!("b5f7f912443c940f21fd611f12828d75b534364ed9e95ca4e307729a4661bde4"); diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 7a9a6bf457fb..4e3f1d3bd249 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -39,7 +39,7 @@ pub use block::{ }; #[cfg(feature = "reth-codec")] pub use compression::*; -pub use constants::{HOLESKY_GENESIS_HASH, SEPOLIA_GENESIS_HASH}; +pub use constants::HOLESKY_GENESIS_HASH; pub use receipt::{ gas_spent_by_transactions, Receipt, ReceiptWithBloom, ReceiptWithBloomRef, Receipts, }; diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index f0695421ec5c..014751733e6b 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -581,7 +581,7 @@ struct GenesisAccountWithAddress { #[cfg(test)] mod tests { use super::*; - use alloy_consensus::constants::MAINNET_GENESIS_HASH; + use alloy_consensus::constants::{MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH}; use alloy_genesis::Genesis; use reth_chainspec::{Chain, ChainSpec, HOLESKY, MAINNET, SEPOLIA}; use reth_db::DatabaseEnv; @@ -592,7 +592,7 @@ mod tests { transaction::DbTx, Database, }; - use reth_primitives::{HOLESKY_GENESIS_HASH, SEPOLIA_GENESIS_HASH}; + use reth_primitives::HOLESKY_GENESIS_HASH; use reth_primitives_traits::IntegerList; use reth_provider::{ test_utils::{create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB}, From 2d83f2048937a1ddd9f9646bd513a3e52364af78 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Oct 2024 13:12:13 +0200 Subject: [PATCH 073/239] chore: reuse alloy-primitives logs bloom (#12031) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crates/primitives-traits/src/lib.rs | 2 +- crates/primitives-traits/src/log.rs | 15 --------------- crates/primitives/src/receipt.rs | 4 ++-- crates/rpc/rpc-eth-types/src/simulate.rs | 5 +++-- crates/trie/sparse/src/trie.rs | 14 ++++++++++---- 7 files changed, 19 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e261297ae51c..ab521843740b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f35429a652765189c1c5092870d8360ee7b7769b09b06d89ebaefd34676446" +checksum = "c71738eb20c42c5fb149571e76536a0f309d142f3957c28791662b96baf77a3d" dependencies = [ "alloy-rlp", "arbitrary", diff --git a/Cargo.toml b/Cargo.toml index 22a78979dfcb..e23efdeb3156 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -427,7 +427,7 @@ revm-primitives = { version = "13.0.0", features = [ # eth alloy-chains = "0.1.32" alloy-dyn-abi = "0.8.0" -alloy-primitives = { version = "0.8.7", default-features = false } +alloy-primitives = { version = "0.8.9", default-features = false } alloy-rlp = "0.3.4" alloy-sol-types = "0.8.0" alloy-trie = { version = "0.7", default-features = false } diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index a77669ec367e..57d1119b0351 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -39,7 +39,7 @@ mod error; pub use error::{GotExpected, GotExpectedBoxed}; mod log; -pub use log::{logs_bloom, Log, LogData}; +pub use alloy_primitives::{logs_bloom, Log, LogData}; mod storage; pub use storage::StorageEntry; diff --git a/crates/primitives-traits/src/log.rs b/crates/primitives-traits/src/log.rs index 6e6b4733518a..0b445aeeba9e 100644 --- a/crates/primitives-traits/src/log.rs +++ b/crates/primitives-traits/src/log.rs @@ -1,18 +1,3 @@ -use alloy_primitives::Bloom; -pub use alloy_primitives::{Log, LogData}; - -/// Calculate receipt logs bloom. -pub fn logs_bloom<'a>(logs: impl IntoIterator) -> Bloom { - let mut bloom = Bloom::ZERO; - for log in logs { - bloom.m3_2048(log.address.as_slice()); - for topic in log.topics() { - bloom.m3_2048(topic.as_slice()); - } - } - bloom -} - #[cfg(test)] mod tests { use alloy_primitives::{Address, Bytes, Log as AlloyLog, B256}; diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index b117f8d96156..bb6c0841b8c8 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -1,6 +1,6 @@ #[cfg(feature = "reth-codec")] use crate::compression::{RECEIPT_COMPRESSOR, RECEIPT_DECOMPRESSOR}; -use crate::{logs_bloom, TxType}; +use crate::TxType; use alloc::{vec, vec::Vec}; use alloy_consensus::constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, @@ -49,7 +49,7 @@ impl Receipt { /// Calculates [`Log`]'s bloom filter. this is slow operation and [`ReceiptWithBloom`] can /// be used to cache this value. pub fn bloom_slow(&self) -> Bloom { - logs_bloom(self.logs.iter()) + alloy_primitives::logs_bloom(self.logs.iter()) } /// Calculates the bloom filter for the receipt and returns the [`ReceiptWithBloom`] container diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index 1d443861d4f6..4249c78fe6ac 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -9,7 +9,6 @@ use alloy_rpc_types::{ use alloy_rpc_types_eth::transaction::TransactionRequest; use jsonrpsee_types::ErrorObject; use reth_primitives::{ - logs_bloom, proofs::{calculate_receipt_root, calculate_transaction_root}, BlockBody, BlockWithSenders, Receipt, Signature, Transaction, TransactionSigned, TransactionSignedNoHash, @@ -290,7 +289,9 @@ pub fn build_block( receipts_root: calculate_receipt_root(&receipts), transactions_root: calculate_transaction_root(&transactions), state_root, - logs_bloom: logs_bloom(receipts.iter().flat_map(|r| r.receipt.logs.iter())), + logs_bloom: alloy_primitives::logs_bloom( + receipts.iter().flat_map(|r| r.receipt.logs.iter()), + ), mix_hash: block_env.prevrandao.unwrap_or_default(), ..Default::default() }; diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 4d195cbf34cf..8d65378f614e 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -1257,19 +1257,24 @@ mod tests { // to test the sparse trie updates. const KEY_NIBBLES_LEN: usize = 3; - fn test(updates: Vec>>) { + fn test(updates: I) + where + I: IntoIterator, + T: IntoIterator)> + Clone, + { let mut rng = generators::rng(); let mut state = BTreeMap::default(); let mut sparse = RevealedSparseTrie::default(); for update in updates { - let keys_to_delete_len = update.len() / 2; - + let mut count = 0; // Insert state updates into the sparse trie and calculate the root for (key, value) in update.clone() { sparse.update_leaf(key, value).unwrap(); + count += 1; } + let keys_to_delete_len = count / 2; let sparse_root = sparse.root(); // Insert state updates into the hash builder and calculate the root @@ -1329,7 +1334,8 @@ mod tests { ), 1..100, ) - )| { test(updates.into_iter().collect()) }); + )| { + test(updates) }); } /// We have three leaves that share the same prefix: 0x00, 0x01 and 0x02. Hash builder trie has From 5e4da59b3a83c869647ad38b6f679f7594c11938 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Thu, 24 Oct 2024 15:12:34 +0400 Subject: [PATCH 074/239] feat: Add more complex E2E test (#12005) --- Cargo.lock | 5 + crates/e2e-test-utils/Cargo.toml | 2 + crates/e2e-test-utils/src/engine_api.rs | 51 +++++++--- crates/e2e-test-utils/src/lib.rs | 107 +++++++++++++++++--- crates/e2e-test-utils/src/node.rs | 18 +++- crates/e2e-test-utils/src/traits.rs | 15 ++- crates/ethereum/node/Cargo.toml | 3 + crates/ethereum/node/tests/e2e/p2p.rs | 126 +++++++++++++++++++++++- 8 files changed, 298 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab521843740b..3449e381f98f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7020,6 +7020,7 @@ dependencies = [ "reth", "reth-chainspec", "reth-db", + "reth-engine-local", "reth-network-peers", "reth-node-builder", "reth-payload-builder", @@ -7033,6 +7034,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "url", ] [[package]] @@ -7966,8 +7968,11 @@ dependencies = [ "alloy-consensus", "alloy-genesis", "alloy-primitives", + "alloy-provider", + "alloy-signer", "eyre", "futures", + "rand 0.8.5", "reth", "reth-auto-seal-consensus", "reth-basic-payload-builder", diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index 9fa3e2b60ab1..04f031daa580 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -23,9 +23,11 @@ reth-node-builder = { workspace = true, features = ["test-utils"] } reth-tokio-util.workspace = true reth-stages-types.workspace = true reth-network-peers.workspace = true +reth-engine-local.workspace = true # rpc jsonrpsee.workspace = true +url.workspace = true # ethereum alloy-primitives.workspace = true diff --git a/crates/e2e-test-utils/src/engine_api.rs b/crates/e2e-test-utils/src/engine_api.rs index f4aa8fdf5ff0..5027b2620a6d 100644 --- a/crates/e2e-test-utils/src/engine_api.rs +++ b/crates/e2e-test-utils/src/engine_api.rs @@ -12,19 +12,22 @@ use reth::{ types::engine::{ForkchoiceState, PayloadStatusEnum}, }, }; +use reth_chainspec::EthereumHardforks; +use reth_node_builder::BuiltPayload; use reth_payload_builder::PayloadId; use reth_rpc_layer::AuthClientService; -use std::marker::PhantomData; +use std::{marker::PhantomData, sync::Arc}; /// Helper for engine api operations #[derive(Debug)] -pub struct EngineApiTestContext { +pub struct EngineApiTestContext { + pub chain_spec: Arc, pub canonical_stream: CanonStateNotificationStream, pub engine_api_client: HttpClient>, pub _marker: PhantomData, } -impl EngineApiTestContext { +impl EngineApiTestContext { /// Retrieves a v3 payload from the engine api pub async fn get_payload_v3( &self, @@ -51,18 +54,40 @@ impl EngineApiTestContext { ) -> eyre::Result where E::ExecutionPayloadEnvelopeV3: From + PayloadEnvelopeExt, + E::ExecutionPayloadEnvelopeV4: From + PayloadEnvelopeExt, { - // setup payload for submission - let envelope_v3: ::ExecutionPayloadEnvelopeV3 = payload.into(); - // submit payload to engine api - let submission = EngineApiClient::::new_payload_v3( - &self.engine_api_client, - envelope_v3.execution_payload(), - versioned_hashes, - payload_builder_attributes.parent_beacon_block_root().unwrap(), - ) - .await?; + let submission = if self + .chain_spec + .is_prague_active_at_timestamp(payload_builder_attributes.timestamp()) + { + let requests = payload + .executed_block() + .unwrap() + .execution_outcome() + .requests + .first() + .unwrap() + .clone(); + let envelope: ::ExecutionPayloadEnvelopeV4 = payload.into(); + EngineApiClient::::new_payload_v4( + &self.engine_api_client, + envelope.execution_payload(), + versioned_hashes, + payload_builder_attributes.parent_beacon_block_root().unwrap(), + requests, + ) + .await? + } else { + let envelope: ::ExecutionPayloadEnvelopeV3 = payload.into(); + EngineApiClient::::new_payload_v3( + &self.engine_api_client, + envelope.execution_payload(), + versioned_hashes, + payload_builder_attributes.parent_beacon_block_root().unwrap(), + ) + .await? + }; assert_eq!(submission.status, expected_status); diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 48e56910e6c0..f5ee1e5e6696 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -11,11 +11,13 @@ use reth::{ }; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; +use reth_engine_local::LocalPayloadAttributesBuilder; use reth_node_builder::{ - components::NodeComponentsBuilder, rpc::RethRpcAddOns, FullNodeTypesAdapter, Node, NodeAdapter, - NodeComponents, NodeTypesWithDBAdapter, NodeTypesWithEngine, RethFullAdapter, + components::NodeComponentsBuilder, rpc::RethRpcAddOns, EngineNodeLauncher, + FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypesWithDBAdapter, + NodeTypesWithEngine, PayloadAttributesBuilder, PayloadTypes, }; -use reth_provider::providers::BlockchainProvider; +use reth_provider::providers::{BlockchainProvider, BlockchainProvider2}; use tracing::{span, Level}; use wallet::Wallet; @@ -102,21 +104,102 @@ where Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into()))) } +/// Creates the initial setup with `num_nodes` started and interconnected. +pub async fn setup_engine( + num_nodes: usize, + chain_spec: Arc, + is_dev: bool, +) -> eyre::Result<( + Vec>>>, + TaskManager, + Wallet, +)> +where + N: Default + + Node>>> + + NodeTypesWithEngine, + N::ComponentsBuilder: NodeComponentsBuilder< + TmpNodeAdapter>>, + Components: NodeComponents< + TmpNodeAdapter>>, + Network: PeersHandleProvider, + >, + >, + N::AddOns: RethRpcAddOns>>>, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Engine as PayloadTypes>::PayloadAttributes, + >, +{ + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let network_config = NetworkArgs { + discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, + ..NetworkArgs::default() + }; + + // Create nodes and peer them + let mut nodes: Vec> = Vec::with_capacity(num_nodes); + + for idx in 0..num_nodes { + let node_config = NodeConfig::new(chain_spec.clone()) + .with_network(network_config.clone()) + .with_unused_ports() + .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()) + .set_dev(is_dev); + + let span = span!(Level::INFO, "node", idx); + let _enter = span.enter(); + let node = N::default(); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) + .testing_node(exec.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(node.add_ons()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + Default::default(), + ); + builder.launch_with(launcher) + }) + .await?; + + let mut node = NodeTestContext::new(node).await?; + + // Connect each node in a chain. + if let Some(previous_node) = nodes.last_mut() { + previous_node.connect(&mut node).await; + } + + // Connect last node with the first if there are more than two + if idx + 1 == num_nodes && num_nodes > 2 { + if let Some(first_node) = nodes.first_mut() { + node.connect(first_node).await; + } + } + + nodes.push(node); + } + + Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into()))) +} + // Type aliases type TmpDB = Arc>; -type TmpNodeAdapter = FullNodeTypesAdapter< - NodeTypesWithDBAdapter, - BlockchainProvider>, ->; +type TmpNodeAdapter>> = + FullNodeTypesAdapter, Provider>; /// Type alias for a `NodeAdapter` -pub type Adapter = NodeAdapter< - RethFullAdapter, - <>>::ComponentsBuilder as NodeComponentsBuilder< - RethFullAdapter, +pub type Adapter>> = NodeAdapter< + TmpNodeAdapter, + <>>::ComponentsBuilder as NodeComponentsBuilder< + TmpNodeAdapter, >>::Components, >; /// Type alias for a type of `NodeHelper` -pub type NodeHelperType = NodeTestContext, AO>; +pub type NodeHelperType>> = + NodeTestContext, AO>; diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 776a437a58e9..f40072018044 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -18,9 +18,10 @@ use reth::{ }, }; use reth_chainspec::EthereumHardforks; -use reth_node_builder::{rpc::RethRpcAddOns, NodeTypesWithEngine}; +use reth_node_builder::{rpc::RethRpcAddOns, NodeTypes, NodeTypesWithEngine}; use reth_stages_types::StageId; use tokio_stream::StreamExt; +use url::Url; use crate::{ engine_api::EngineApiTestContext, network::NetworkTestContext, payload::PayloadTestContext, @@ -41,7 +42,10 @@ where /// Context for testing network functionalities. pub network: NetworkTestContext, /// Context for testing the Engine API. - pub engine_api: EngineApiTestContext<::Engine>, + pub engine_api: EngineApiTestContext< + ::Engine, + ::ChainSpec, + >, /// Context for testing RPC features. pub rpc: RpcTestContext, } @@ -63,6 +67,7 @@ where payload: PayloadTestContext::new(builder).await?, network: NetworkTestContext::new(node.network.clone()), engine_api: EngineApiTestContext { + chain_spec: node.chain_spec(), engine_api_client: node.auth_server_handle().http_client(), canonical_stream: node.provider.canonical_state_stream(), _marker: PhantomData::, @@ -89,6 +94,7 @@ where ) -> eyre::Result> where Engine::ExecutionPayloadEnvelopeV3: From + PayloadEnvelopeExt, + Engine::ExecutionPayloadEnvelopeV4: From + PayloadEnvelopeExt, AddOns::EthApi: EthApiSpec + EthTransactions + TraceExt + FullEthApiTypes, { let mut chain = Vec::with_capacity(length as usize); @@ -137,6 +143,8 @@ where where ::ExecutionPayloadEnvelopeV3: From + PayloadEnvelopeExt, + ::ExecutionPayloadEnvelopeV4: + From + PayloadEnvelopeExt, { let (payload, eth_attr) = self.new_payload(attributes_generator).await?; @@ -236,4 +244,10 @@ where } Ok(()) } + + /// Returns the RPC URL. + pub fn rpc_url(&self) -> Url { + let addr = self.inner.rpc_server_handle().http_local_addr().unwrap(); + format!("http://{}", addr).parse().unwrap() + } } diff --git a/crates/e2e-test-utils/src/traits.rs b/crates/e2e-test-utils/src/traits.rs index 6786492140b0..a70bbf7afb76 100644 --- a/crates/e2e-test-utils/src/traits.rs +++ b/crates/e2e-test-utils/src/traits.rs @@ -1,4 +1,5 @@ -use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelopeV3; +use alloy_rpc_types::engine::ExecutionPayloadEnvelopeV4; +use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4}; use reth::rpc::types::engine::{ExecutionPayloadEnvelopeV3, ExecutionPayloadV3}; /// The execution payload envelope type. @@ -13,8 +14,20 @@ impl PayloadEnvelopeExt for OpExecutionPayloadEnvelopeV3 { } } +impl PayloadEnvelopeExt for OpExecutionPayloadEnvelopeV4 { + fn execution_payload(&self) -> ExecutionPayloadV3 { + self.execution_payload.clone() + } +} + impl PayloadEnvelopeExt for ExecutionPayloadEnvelopeV3 { fn execution_payload(&self) -> ExecutionPayloadV3 { self.execution_payload.clone() } } + +impl PayloadEnvelopeExt for ExecutionPayloadEnvelopeV4 { + fn execution_payload(&self) -> ExecutionPayloadV3 { + self.execution_payload.clone() + } +} diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index 11555cdc4a53..e7784637a062 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -52,6 +52,9 @@ alloy-genesis.workspace = true tokio.workspace = true serde_json.workspace = true alloy-consensus.workspace = true +alloy-provider.workspace = true +rand.workspace = true +alloy-signer.workspace = true [features] default = [] diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index a40c1b3f4b4e..0fae23a08571 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -1,7 +1,19 @@ use crate::utils::eth_payload_attributes; +use alloy_consensus::TxType; +use alloy_primitives::bytes; +use alloy_provider::{ + network::{ + Ethereum, EthereumWallet, NetworkWallet, TransactionBuilder, TransactionBuilder7702, + }, + Provider, ProviderBuilder, SendableTx, +}; +use alloy_signer::SignerSync; +use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; +use reth::rpc::types::TransactionRequest; use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::{setup, transaction::TransactionTestContext}; +use reth_e2e_test_utils::{setup, setup_engine, transaction::TransactionTestContext}; use reth_node_ethereum::EthereumNode; +use revm::primitives::{AccessListItem, Authorization}; use std::sync::Arc; #[tokio::test] @@ -45,3 +57,115 @@ async fn can_sync() -> eyre::Result<()> { Ok(()) } + +#[tokio::test] +async fn e2e_test_send_transactions() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let seed: [u8; 32] = rand::thread_rng().gen(); + let mut rng = StdRng::from_seed(seed); + println!("Seed: {:?}", seed); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .prague_activated() + .build(), + ); + + let (mut nodes, _tasks, wallet) = + setup_engine::(2, chain_spec.clone(), false).await?; + let mut node = nodes.pop().unwrap(); + let signers = wallet.gen(); + let provider = ProviderBuilder::new().with_recommended_fillers().on_http(node.rpc_url()); + + // simple contract which writes to storage on any call + let dummy_bytecode = bytes!("6080604052348015600f57600080fd5b50602880601d6000396000f3fe4360a09081523360c0526040608081905260e08152902080805500fea164736f6c6343000810000a"); + let mut call_destinations = signers.iter().map(|s| s.address()).collect::>(); + + // Produce 100 random blocks with random transactions + for _ in 0..100 { + let tx_count = rng.gen_range(1..20); + + let mut pending = vec![]; + for _ in 0..tx_count { + let signer = signers.choose(&mut rng).unwrap(); + let tx_type = TxType::try_from(rng.gen_range(0..=4)).unwrap(); + + let mut tx = TransactionRequest::default().with_from(signer.address()); + + let should_create = + rng.gen::() && tx_type != TxType::Eip4844 && tx_type != TxType::Eip7702; + if should_create { + tx = tx.into_create().with_input(dummy_bytecode.clone()); + } else { + tx = tx.with_to(*call_destinations.choose(&mut rng).unwrap()).with_input( + (0..rng.gen_range(0..10000)).map(|_| rng.gen()).collect::>(), + ); + } + + if matches!(tx_type, TxType::Legacy | TxType::Eip2930) { + tx = tx.with_gas_price(provider.get_gas_price().await?); + } + + if rng.gen::() || tx_type == TxType::Eip2930 { + tx = tx.with_access_list( + vec![AccessListItem { + address: *call_destinations.choose(&mut rng).unwrap(), + storage_keys: (0..rng.gen_range(0..100)).map(|_| rng.gen()).collect(), + }] + .into(), + ); + } + + if tx_type == TxType::Eip7702 { + let signer = signers.choose(&mut rng).unwrap(); + let auth = Authorization { + chain_id: provider.get_chain_id().await?, + address: *call_destinations.choose(&mut rng).unwrap(), + nonce: provider.get_transaction_count(signer.address()).await?, + }; + let sig = signer.sign_hash_sync(&auth.signature_hash())?; + tx = tx.with_authorization_list(vec![auth.into_signed(sig)]) + } + + let SendableTx::Builder(tx) = provider.fill(tx).await? else { unreachable!() }; + let tx = + NetworkWallet::::sign_request(&EthereumWallet::new(signer.clone()), tx) + .await?; + + pending.push(provider.send_tx_envelope(tx).await?); + } + + let (payload, _) = node.advance_block(vec![], eth_payload_attributes).await?; + assert!(payload.block().raw_transactions().len() == tx_count); + + for pending in pending { + let receipt = pending.get_receipt().await?; + if let Some(address) = receipt.contract_address { + call_destinations.push(address); + } + } + } + + let second_node = nodes.pop().unwrap(); + let second_provider = + ProviderBuilder::new().with_recommended_fillers().on_http(second_node.rpc_url()); + + assert_eq!(second_provider.get_block_number().await?, 0); + + let head = provider.get_block_by_number(Default::default(), false).await?.unwrap().header.hash; + second_node.engine_api.update_forkchoice(head, head).await?; + + let start = std::time::Instant::now(); + + while provider.get_block_number().await? != second_provider.get_block_number().await? { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + assert!(start.elapsed() <= std::time::Duration::from_secs(10), "timed out"); + } + + Ok(()) +} From 8b317f206f89e5618e0a6ce77a4e999c34597b51 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Oct 2024 14:36:52 +0200 Subject: [PATCH 075/239] fix: exclude withdrawals from uncle block (#12038) --- crates/rpc/rpc-types-compat/src/block.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-types-compat/src/block.rs b/crates/rpc/rpc-types-compat/src/block.rs index 8cddc8c44977..a954e05e4f65 100644 --- a/crates/rpc/rpc-types-compat/src/block.rs +++ b/crates/rpc/rpc-types-compat/src/block.rs @@ -183,14 +183,14 @@ fn from_block_with_transactions( /// an Uncle from its header. pub fn uncle_block_from_header(header: PrimitiveHeader) -> Block { let hash = header.hash_slow(); - let rpc_header = from_primitive_with_hash(SealedHeader::new(header.clone(), hash)); let uncle_block = PrimitiveBlock { header, ..Default::default() }; let size = Some(U256::from(uncle_block.length())); + let rpc_header = from_primitive_with_hash(SealedHeader::new(uncle_block.header, hash)); Block { uncles: vec![], header: rpc_header, transactions: BlockTransactions::Uncle, - withdrawals: Some(vec![]), + withdrawals: None, size, } } From bea6e278dfdb052ce7bbcec5d42349221aa04b84 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:29:24 +0200 Subject: [PATCH 076/239] primitive-traits: use alloy `MIN_PROTOCOL_BASE_FEE` constant (#12037) --- crates/node/core/src/args/txpool.rs | 3 +-- crates/primitives-traits/src/constants/mod.rs | 26 +------------------ crates/transaction-pool/src/config.rs | 3 +-- crates/transaction-pool/src/pool/txpool.rs | 4 +-- crates/transaction-pool/src/test_utils/gen.rs | 6 ++--- .../transaction-pool/src/test_utils/mock.rs | 5 ++-- crates/transaction-pool/tests/it/evict.rs | 3 +-- 7 files changed, 10 insertions(+), 40 deletions(-) diff --git a/crates/node/core/src/args/txpool.rs b/crates/node/core/src/args/txpool.rs index 282313555f76..538315101adf 100644 --- a/crates/node/core/src/args/txpool.rs +++ b/crates/node/core/src/args/txpool.rs @@ -1,10 +1,9 @@ //! Transaction pool arguments use crate::cli::config::RethTransactionPoolConfig; -use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; +use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; use alloy_primitives::Address; use clap::Args; -use reth_primitives::constants::MIN_PROTOCOL_BASE_FEE; use reth_transaction_pool::{ blobstore::disk::DEFAULT_MAX_CACHED_BLOBS, pool::{NEW_TX_LISTENER_BUFFER_SIZE, PENDING_TX_LISTENER_BUFFER_SIZE}, diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index c5ee6bc3e83c..6ed88ce592cc 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -1,6 +1,6 @@ //! Ethereum protocol-related constants -use alloy_primitives::{address, b256, Address, B256, U256}; +use alloy_primitives::{address, b256, Address, B256}; /// Gas units, for example [`GIGAGAS`]. pub mod gas_units; @@ -9,20 +9,6 @@ pub use gas_units::{GIGAGAS, KILOGAS, MEGAGAS}; /// The client version: `reth/v{major}.{minor}.{patch}` pub const RETH_CLIENT_VERSION: &str = concat!("reth/v", env!("CARGO_PKG_VERSION")); -/// The minimum tx fee below which the txpool will reject the transaction. -/// -/// Configured to `7` WEI which is the lowest possible value of base fee under mainnet EIP-1559 -/// parameters. `BASE_FEE_MAX_CHANGE_DENOMINATOR` -/// is `8`, or 12.5%. Once the base fee has dropped to `7` WEI it cannot decrease further because -/// 12.5% of 7 is less than 1. -/// -/// Note that min base fee under different 1559 parameterizations may differ, but there's no -/// significant harm in leaving this setting as is. -pub const MIN_PROTOCOL_BASE_FEE: u64 = 7; - -/// Same as [`MIN_PROTOCOL_BASE_FEE`] but as a U256. -pub const MIN_PROTOCOL_BASE_FEE_U256: U256 = U256::from_limbs([7u64, 0, 0, 0]); - /// Base fee max change denominator as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) pub const EIP1559_DEFAULT_BASE_FEE_MAX_CHANGE_DENOMINATOR: u64 = 8; @@ -79,13 +65,3 @@ pub const OP_SYSTEM_TX_TO_ADDR: Address = address!("4200000000000000000000000000 /// Unwind depth of `3` blocks significantly reduces the chance that the reorged block is kept in /// the database. pub const BEACON_CONSENSUS_REORG_UNWIND_DEPTH: u64 = 3; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn min_protocol_sanity() { - assert_eq!(MIN_PROTOCOL_BASE_FEE_U256.to::(), MIN_PROTOCOL_BASE_FEE); - } -} diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 8fe49f47652d..d4518846258a 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -3,9 +3,8 @@ use crate::{ PoolSize, TransactionOrigin, }; use alloy_consensus::constants::EIP4844_TX_TYPE_ID; -use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; +use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; use alloy_primitives::Address; -use reth_primitives::constants::MIN_PROTOCOL_BASE_FEE; use std::{collections::HashSet, ops::Mul}; /// Guarantees max transactions for one sender, compatible with geth/erigon diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index a3f192992d1a..42de860db794 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -22,9 +22,9 @@ use alloy_consensus::constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; -use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; +use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; use alloy_primitives::{Address, TxHash, B256}; -use reth_primitives::constants::{eip4844::BLOB_TX_MIN_BLOB_GASPRICE, MIN_PROTOCOL_BASE_FEE}; +use reth_primitives::constants::eip4844::BLOB_TX_MIN_BLOB_GASPRICE; use rustc_hash::FxHashMap; use smallvec::SmallVec; use std::{ diff --git a/crates/transaction-pool/src/test_utils/gen.rs b/crates/transaction-pool/src/test_utils/gen.rs index d51bf80270de..858098ec91ad 100644 --- a/crates/transaction-pool/src/test_utils/gen.rs +++ b/crates/transaction-pool/src/test_utils/gen.rs @@ -1,12 +1,10 @@ use crate::EthPooledTransaction; use alloy_consensus::{TxEip1559, TxEip4844, TxLegacy}; -use alloy_eips::{eip2718::Encodable2718, eip2930::AccessList}; +use alloy_eips::{eip1559::MIN_PROTOCOL_BASE_FEE, eip2718::Encodable2718, eip2930::AccessList}; use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; use rand::Rng; use reth_chainspec::MAINNET; -use reth_primitives::{ - constants::MIN_PROTOCOL_BASE_FEE, sign_message, Transaction, TransactionSigned, -}; +use reth_primitives::{sign_message, Transaction, TransactionSigned}; /// A generator for transactions for testing purposes. #[derive(Debug)] diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 474cf5cc8f83..99b0caaf48a3 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -11,7 +11,7 @@ use alloy_consensus::{ constants::{EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, LEGACY_TX_TYPE_ID}, TxEip1559, TxEip2930, TxEip4844, TxLegacy, }; -use alloy_eips::eip2930::AccessList; +use alloy_eips::{eip1559::MIN_PROTOCOL_BASE_FEE, eip2930::AccessList}; use alloy_primitives::{Address, Bytes, ChainId, TxHash, TxKind, B256, U256}; use paste::paste; use rand::{ @@ -19,8 +19,7 @@ use rand::{ prelude::Distribution, }; use reth_primitives::{ - constants::{eip4844::DATA_GAS_PER_BLOB, MIN_PROTOCOL_BASE_FEE}, - transaction::TryFromRecoveredTransactionError, + constants::eip4844::DATA_GAS_PER_BLOB, transaction::TryFromRecoveredTransactionError, BlobTransactionSidecar, BlobTransactionValidationError, PooledTransactionsElementEcRecovered, Signature, Transaction, TransactionSigned, TransactionSignedEcRecovered, TxType, }; diff --git a/crates/transaction-pool/tests/it/evict.rs b/crates/transaction-pool/tests/it/evict.rs index c1d0bbaa6421..fea50962fd9e 100644 --- a/crates/transaction-pool/tests/it/evict.rs +++ b/crates/transaction-pool/tests/it/evict.rs @@ -1,9 +1,8 @@ //! Transaction pool eviction tests. -use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT; +use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; use alloy_primitives::{Address, B256}; use rand::distributions::Uniform; -use reth_primitives::constants::MIN_PROTOCOL_BASE_FEE; use reth_transaction_pool::{ error::PoolErrorKind, test_utils::{ From 11149d295e195fcdb480fe33e72c8c5c72cf900c Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:13:05 +0200 Subject: [PATCH 077/239] feat(payload builder): transaction pool filter (#10542) Co-authored-by: Matthias Seitz --- Cargo.lock | 2 +- crates/transaction-pool/src/pool/best.rs | 2 +- crates/transaction-pool/src/traits.rs | 30 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3449e381f98f..c3003c991fbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11913,4 +11913,4 @@ checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", -] +] \ No newline at end of file diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 52f25a9db8dc..77cd35d8a6be 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -208,7 +208,7 @@ pub struct BestTransactionFilter { impl BestTransactionFilter { /// Create a new [`BestTransactionFilter`] with the given predicate. - pub(crate) const fn new(best: I, predicate: P) -> Self { + pub const fn new(best: I, predicate: P) -> Self { Self { best, predicate } } } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 00cda8e1cbe5..eb9d35ad56cf 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -814,6 +814,36 @@ impl BestTransactions for std::iter::Empty { fn set_skip_blobs(&mut self, _skip_blobs: bool) {} } +/// A filter that allows to check if a transaction satisfies a set of conditions +pub trait TransactionFilter { + /// The type of the transaction to check. + type Transaction; + + /// Returns true if the transaction satisfies the conditions. + fn is_valid(&self, transaction: &Self::Transaction) -> bool; +} + +/// A no-op implementation of [`TransactionFilter`] which +/// marks all transactions as valid. +#[derive(Debug, Clone)] +pub struct NoopTransactionFilter(std::marker::PhantomData); + +// We can't derive Default because this forces T to be +// Default as well, which isn't necessary. +impl Default for NoopTransactionFilter { + fn default() -> Self { + Self(std::marker::PhantomData) + } +} + +impl TransactionFilter for NoopTransactionFilter { + type Transaction = T; + + fn is_valid(&self, _transaction: &Self::Transaction) -> bool { + true + } +} + /// A Helper type that bundles the best transactions attributes together. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct BestTransactionsAttributes { From 53c547663796edc1891bf4c6c5a8b4c0f65597cb Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:20:11 +0200 Subject: [PATCH 078/239] chore(primitive-traits): rm `EIP1559_` constants (#12036) --- crates/primitives-traits/src/constants/mod.rs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index 6ed88ce592cc..f3dd28e19295 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -9,43 +9,9 @@ pub use gas_units::{GIGAGAS, KILOGAS, MEGAGAS}; /// The client version: `reth/v{major}.{minor}.{patch}` pub const RETH_CLIENT_VERSION: &str = concat!("reth/v", env!("CARGO_PKG_VERSION")); -/// Base fee max change denominator as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) -pub const EIP1559_DEFAULT_BASE_FEE_MAX_CHANGE_DENOMINATOR: u64 = 8; - -/// Elasticity multiplier as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) -pub const EIP1559_DEFAULT_ELASTICITY_MULTIPLIER: u64 = 2; - /// Minimum gas limit allowed for transactions. pub const MINIMUM_GAS_LIMIT: u64 = 5000; -/// Base fee max change denominator for Optimism Mainnet as defined in the Optimism -/// [transaction costs](https://community.optimism.io/docs/developers/build/differences/#transaction-costs) doc. -pub const OP_MAINNET_EIP1559_DEFAULT_BASE_FEE_MAX_CHANGE_DENOMINATOR: u128 = 50; - -/// Base fee max change denominator for Optimism Mainnet as defined in the Optimism Canyon -/// hardfork. -pub const OP_MAINNET_EIP1559_BASE_FEE_MAX_CHANGE_DENOMINATOR_CANYON: u128 = 250; - -/// Base fee max change denominator for Optimism Mainnet as defined in the Optimism -/// [transaction costs](https://community.optimism.io/docs/developers/build/differences/#transaction-costs) doc. -pub const OP_MAINNET_EIP1559_DEFAULT_ELASTICITY_MULTIPLIER: u128 = 6; - -/// Base fee max change denominator for Optimism Sepolia as defined in the Optimism -/// [transaction costs](https://community.optimism.io/docs/developers/build/differences/#transaction-costs) doc. -pub const OP_SEPOLIA_EIP1559_DEFAULT_BASE_FEE_MAX_CHANGE_DENOMINATOR: u128 = 50; - -/// Base fee max change denominator for Optimism Sepolia as defined in the Optimism Canyon -/// hardfork. -pub const OP_SEPOLIA_EIP1559_BASE_FEE_MAX_CHANGE_DENOMINATOR_CANYON: u128 = 250; - -/// Base fee max change denominator for Optimism Sepolia as defined in the Optimism -/// [transaction costs](https://community.optimism.io/docs/developers/build/differences/#transaction-costs) doc. -pub const OP_SEPOLIA_EIP1559_DEFAULT_ELASTICITY_MULTIPLIER: u128 = 6; - -/// Base fee max change denominator for Base Sepolia as defined in the Optimism -/// [transaction costs](https://community.optimism.io/docs/developers/build/differences/#transaction-costs) doc. -pub const BASE_SEPOLIA_EIP1559_DEFAULT_ELASTICITY_MULTIPLIER: u128 = 10; - /// Holesky genesis hash: `0xb5f7f912443c940f21fd611f12828d75b534364ed9e95ca4e307729a4661bde4` pub const HOLESKY_GENESIS_HASH: B256 = b256!("b5f7f912443c940f21fd611f12828d75b534364ed9e95ca4e307729a4661bde4"); From e74f185d95df7a18734899826890dbd12811cb36 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Oct 2024 15:28:25 +0200 Subject: [PATCH 079/239] feat: add BestTransactions::filter_transactions (#12041) --- crates/transaction-pool/src/traits.rs | 13 +++++++++++++ crates/transaction-pool/tests/it/best.rs | 10 ++++++++++ crates/transaction-pool/tests/it/main.rs | 2 ++ 3 files changed, 25 insertions(+) create mode 100644 crates/transaction-pool/tests/it/best.rs diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index eb9d35ad56cf..fa5e22ec0aa3 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -761,6 +761,19 @@ pub trait BestTransactions: Iterator + Send { /// /// If set to true, no blob transactions will be returned. fn set_skip_blobs(&mut self, skip_blobs: bool); + + /// Creates an iterator which uses a closure to determine whether a transaction should be + /// returned by the iterator. + /// + /// All items the closure returns false for are marked as invalid via [`Self::mark_invalid`] and + /// descendant transactions will be skipped. + fn filter_transactions

(self, predicate: P) -> BestTransactionFilter + where + P: FnMut(&Self::Item) -> bool, + Self: Sized, + { + BestTransactionFilter::new(self, predicate) + } } impl BestTransactions for Box diff --git a/crates/transaction-pool/tests/it/best.rs b/crates/transaction-pool/tests/it/best.rs new file mode 100644 index 000000000000..cd7a93eaedb9 --- /dev/null +++ b/crates/transaction-pool/tests/it/best.rs @@ -0,0 +1,10 @@ +//! Best transaction and filter testing + +use reth_transaction_pool::{noop::NoopTransactionPool, BestTransactions, TransactionPool}; + +#[test] +fn test_best_transactions() { + let noop = NoopTransactionPool::default(); + let mut best = noop.best_transactions().filter_transactions(|_| true); + assert!(best.next().is_none()); +} diff --git a/crates/transaction-pool/tests/it/main.rs b/crates/transaction-pool/tests/it/main.rs index ead33a328ddf..7db2b14c9530 100644 --- a/crates/transaction-pool/tests/it/main.rs +++ b/crates/transaction-pool/tests/it/main.rs @@ -9,4 +9,6 @@ mod listeners; #[cfg(feature = "test-utils")] mod pending; +mod best; + const fn main() {} From 2fba3c027b7396db868b162e16635197cd584822 Mon Sep 17 00:00:00 2001 From: Deil Urba Date: Thu, 24 Oct 2024 14:32:44 +0100 Subject: [PATCH 080/239] feat: `ExExContext`'s dynamic config (#11983) --- crates/chainspec/src/api.rs | 7 ++- crates/exex/exex/src/context.rs | 21 ++++++- crates/exex/exex/src/dyn_context.rs | 86 +++++++++++++++++++++++++++ crates/exex/exex/src/lib.rs | 3 + crates/exex/exex/src/notifications.rs | 8 ++- crates/node/types/Cargo.toml | 2 +- crates/optimism/chainspec/src/lib.rs | 7 +-- 7 files changed, 121 insertions(+), 13 deletions(-) create mode 100644 crates/exex/exex/src/dyn_context.rs diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index f7061ff18fe7..3751789cac8c 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -4,7 +4,8 @@ use alloy_chains::Chain; use alloy_eips::eip1559::BaseFeeParams; use alloy_genesis::Genesis; use alloy_primitives::B256; -use core::fmt::{Debug, Display}; +use core::fmt::Debug; +use reth_ethereum_forks::DisplayHardforks; use reth_network_peers::NodeRecord; use reth_primitives_traits::Header; @@ -38,7 +39,7 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { fn prune_delete_limit(&self) -> usize; /// Returns a string representation of the hardforks. - fn display_hardforks(&self) -> impl Display; + fn display_hardforks(&self) -> DisplayHardforks; /// The genesis header. fn genesis_header(&self) -> &Header; @@ -83,7 +84,7 @@ impl EthChainSpec for ChainSpec { self.prune_delete_limit } - fn display_hardforks(&self) -> impl Display { + fn display_hardforks(&self) -> DisplayHardforks { self.display_hardforks() } diff --git a/crates/exex/exex/src/context.rs b/crates/exex/exex/src/context.rs index c4b4f351baa7..70972f9be7f7 100644 --- a/crates/exex/exex/src/context.rs +++ b/crates/exex/exex/src/context.rs @@ -7,7 +7,7 @@ use reth_primitives::Head; use reth_tasks::TaskExecutor; use tokio::sync::mpsc::UnboundedSender; -use crate::{ExExEvent, ExExNotifications, ExExNotificationsStream}; +use crate::{ExExContextDyn, ExExEvent, ExExNotifications, ExExNotificationsStream}; /// Captures the context that an `ExEx` has access to. pub struct ExExContext { @@ -55,7 +55,24 @@ where } } -impl ExExContext { +impl ExExContext +where + Node: FullNodeComponents, + Node::Provider: Debug, + Node::Executor: Debug, +{ + /// Returns dynamic version of the context + pub fn into_dyn(self) -> ExExContextDyn { + ExExContextDyn::from(self) + } +} + +impl ExExContext +where + Node: FullNodeComponents, + Node::Provider: Debug, + Node::Executor: Debug, +{ /// Returns the transaction pool of the node. pub fn pool(&self) -> &Node::Pool { self.components.pool() diff --git a/crates/exex/exex/src/dyn_context.rs b/crates/exex/exex/src/dyn_context.rs new file mode 100644 index 000000000000..19d5a0e5062e --- /dev/null +++ b/crates/exex/exex/src/dyn_context.rs @@ -0,0 +1,86 @@ +//! Mirrored version of [`ExExContext`](`crate::ExExContext`) +//! without generic abstraction over [Node](`reth_node_api::FullNodeComponents`) + +use std::{fmt::Debug, sync::Arc}; + +use reth_chainspec::{EthChainSpec, Head}; +use reth_node_api::FullNodeComponents; +use reth_node_core::node_config::NodeConfig; +use tokio::sync::mpsc; + +use crate::{ExExContext, ExExEvent, ExExNotificationsStream}; + +// TODO(0xurb) - add `node` after abstractions +/// Captures the context that an `ExEx` has access to. +pub struct ExExContextDyn { + /// The current head of the blockchain at launch. + pub head: Head, + /// The config of the node + pub config: NodeConfig>, + /// The loaded node config + pub reth_config: reth_config::Config, + /// Channel used to send [`ExExEvent`]s to the rest of the node. + /// + /// # Important + /// + /// The exex should emit a `FinishedHeight` whenever a processed block is safe to prune. + /// Additionally, the exex can pre-emptively emit a `FinishedHeight` event to specify what + /// blocks to receive notifications for. + pub events: mpsc::UnboundedSender, + /// Channel to receive [`ExExNotification`](crate::ExExNotification)s. + /// + /// # Important + /// + /// Once an [`ExExNotification`](crate::ExExNotification) is sent over the channel, it is + /// considered delivered by the node. + pub notifications: Box, +} + +impl Debug for ExExContextDyn { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExExContext") + .field("head", &self.head) + .field("config", &self.config) + .field("reth_config", &self.reth_config) + .field("events", &self.events) + .field("notifications", &self.notifications) + .finish() + } +} + +impl From> for ExExContextDyn +where + Node: FullNodeComponents, + Node::Provider: Debug, + Node::Executor: Debug, +{ + fn from(ctx: ExExContext) -> Self { + // convert `NodeConfig` with generic over chainspec into `NodeConfig` + let chain: Arc> = + Arc::new(Box::new(ctx.config.chain) as Box); + let config = NodeConfig { + chain, + datadir: ctx.config.datadir, + config: ctx.config.config, + metrics: ctx.config.metrics, + instance: ctx.config.instance, + network: ctx.config.network, + rpc: ctx.config.rpc, + txpool: ctx.config.txpool, + builder: ctx.config.builder, + debug: ctx.config.debug, + db: ctx.config.db, + dev: ctx.config.dev, + pruning: ctx.config.pruning, + }; + let notifications = Box::new(ctx.notifications) as Box; + + Self { + head: ctx.head, + config, + reth_config: ctx.reth_config, + events: ctx.events, + notifications, + } + } +} diff --git a/crates/exex/exex/src/lib.rs b/crates/exex/exex/src/lib.rs index edc9e40d449d..ce6641ff6734 100644 --- a/crates/exex/exex/src/lib.rs +++ b/crates/exex/exex/src/lib.rs @@ -40,6 +40,9 @@ pub use backfill::*; mod context; pub use context::*; +mod dyn_context; +pub use dyn_context::*; + mod event; pub use event::*; diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index 90a0ee230a4d..ea112219a3f1 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -24,7 +24,9 @@ pub struct ExExNotifications { /// A trait, that represents a stream of [`ExExNotification`]s. The stream will emit notifications /// for all blocks. If the stream is configured with a head via [`ExExNotifications::set_with_head`] /// or [`ExExNotifications::with_head`], it will run backfill jobs to catch up to the node head. -pub trait ExExNotificationsStream: Stream> + Unpin { +pub trait ExExNotificationsStream: + Debug + Stream> + Unpin +{ /// Sets [`ExExNotificationsStream`] to a stream of [`ExExNotification`]s without a head. /// /// It's a no-op if the stream has already been configured without a head. @@ -90,8 +92,8 @@ impl ExExNotifications { impl ExExNotificationsStream for ExExNotifications where - P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, - E: BlockExecutorProvider + Clone + Unpin + 'static, + P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Debug + Unpin + 'static, + E: BlockExecutorProvider + Clone + Debug + Unpin + 'static, { fn set_without_head(&mut self) { let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); diff --git a/crates/node/types/Cargo.toml b/crates/node/types/Cargo.toml index b28dcfba5911..5747abe9c34b 100644 --- a/crates/node/types/Cargo.toml +++ b/crates/node/types/Cargo.toml @@ -16,4 +16,4 @@ reth-chainspec.workspace = true reth-db-api.workspace = true reth-engine-primitives.workspace = true reth-primitives.workspace = true -reth-primitives-traits.workspace = true \ No newline at end of file +reth-primitives-traits.workspace = true diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 83c499de5257..03ce75aec040 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -23,7 +23,6 @@ use alloy_genesis::Genesis; use alloy_primitives::{B256, U256}; pub use base::BASE_MAINNET; pub use base_sepolia::BASE_SEPOLIA; -use core::fmt::Display; use derive_more::{Constructor, Deref, From, Into}; pub use dev::OP_DEV; #[cfg(not(feature = "std"))] @@ -31,8 +30,8 @@ pub(crate) use once_cell::sync::Lazy as LazyLock; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; use reth_chainspec::{ - BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, DepositContract, EthChainSpec, - EthereumHardforks, ForkFilter, ForkId, Hardforks, Head, + BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, DepositContract, + DisplayHardforks, EthChainSpec, EthereumHardforks, ForkFilter, ForkId, Hardforks, Head, }; use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition, Hardfork}; use reth_network_peers::NodeRecord; @@ -203,7 +202,7 @@ impl EthChainSpec for OpChainSpec { self.inner.prune_delete_limit() } - fn display_hardforks(&self) -> impl Display { + fn display_hardforks(&self) -> DisplayHardforks { self.inner.display_hardforks() } From ba78e439385469925c6fc9df36444f1ef5a6a1bc Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 24 Oct 2024 17:52:13 +0100 Subject: [PATCH 081/239] fix(exex): relax `ExExContext` trait bounds (#12055) --- crates/exex/exex/src/context.rs | 33 +++++++++++++++++++++++++-- crates/exex/exex/src/dyn_context.rs | 2 +- crates/exex/exex/src/notifications.rs | 8 +++---- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/crates/exex/exex/src/context.rs b/crates/exex/exex/src/context.rs index 70972f9be7f7..23d772b738a8 100644 --- a/crates/exex/exex/src/context.rs +++ b/crates/exex/exex/src/context.rs @@ -70,8 +70,6 @@ where impl ExExContext where Node: FullNodeComponents, - Node::Provider: Debug, - Node::Executor: Debug, { /// Returns the transaction pool of the node. pub fn pool(&self) -> &Node::Pool { @@ -123,3 +121,34 @@ where self.notifications.set_with_head(head); } } + +#[cfg(test)] +mod tests { + use reth_exex_types::ExExHead; + use reth_node_api::FullNodeComponents; + + use crate::ExExContext; + + /// + #[test] + const fn issue_12054() { + #[allow(dead_code)] + struct ExEx { + ctx: ExExContext, + } + + impl ExEx { + async fn _test_bounds(mut self) -> eyre::Result<()> { + self.ctx.pool(); + self.ctx.block_executor(); + self.ctx.provider(); + self.ctx.network(); + self.ctx.payload_builder(); + self.ctx.task_executor(); + self.ctx.set_notifications_without_head(); + self.ctx.set_notifications_with_head(ExExHead { block: Default::default() }); + Ok(()) + } + } + } +} diff --git a/crates/exex/exex/src/dyn_context.rs b/crates/exex/exex/src/dyn_context.rs index 19d5a0e5062e..226f3a3feb9a 100644 --- a/crates/exex/exex/src/dyn_context.rs +++ b/crates/exex/exex/src/dyn_context.rs @@ -43,7 +43,7 @@ impl Debug for ExExContextDyn { .field("config", &self.config) .field("reth_config", &self.reth_config) .field("events", &self.events) - .field("notifications", &self.notifications) + .field("notifications", &"...") .finish() } } diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index ea112219a3f1..90a0ee230a4d 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -24,9 +24,7 @@ pub struct ExExNotifications { /// A trait, that represents a stream of [`ExExNotification`]s. The stream will emit notifications /// for all blocks. If the stream is configured with a head via [`ExExNotifications::set_with_head`] /// or [`ExExNotifications::with_head`], it will run backfill jobs to catch up to the node head. -pub trait ExExNotificationsStream: - Debug + Stream> + Unpin -{ +pub trait ExExNotificationsStream: Stream> + Unpin { /// Sets [`ExExNotificationsStream`] to a stream of [`ExExNotification`]s without a head. /// /// It's a no-op if the stream has already been configured without a head. @@ -92,8 +90,8 @@ impl ExExNotifications { impl ExExNotificationsStream for ExExNotifications where - P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Debug + Unpin + 'static, - E: BlockExecutorProvider + Clone + Debug + Unpin + 'static, + P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, + E: BlockExecutorProvider + Clone + Unpin + 'static, { fn set_without_head(&mut self) { let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); From 777417ad8a1b68d3c1b27e0790cb5be3918ffad1 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 25 Oct 2024 03:34:12 +0900 Subject: [PATCH 082/239] feat: add `reth test-vectors compact --write|--read` (#11954) --- .github/workflows/compact.yml | 42 +++ Cargo.lock | 18 ++ crates/cli/commands/Cargo.toml | 32 ++- .../cli/commands/src/test_vectors/compact.rs | 257 ++++++++++++++++++ crates/cli/commands/src/test_vectors/mod.rs | 26 +- .../cli/commands/src/test_vectors/tables.rs | 6 +- crates/engine/tree/Cargo.toml | 3 +- crates/evm/Cargo.toml | 15 +- crates/net/eth-wire/Cargo.toml | 6 +- crates/optimism/bin/Cargo.toml | 4 + crates/optimism/cli/Cargo.toml | 13 + crates/optimism/cli/src/commands/mod.rs | 7 + .../optimism/cli/src/commands/test_vectors.rs | 72 +++++ crates/optimism/cli/src/lib.rs | 2 + crates/primitives-traits/Cargo.toml | 3 +- crates/primitives/Cargo.toml | 3 +- crates/primitives/src/receipt.rs | 12 + crates/primitives/src/transaction/mod.rs | 6 +- crates/prune/types/Cargo.toml | 11 + crates/prune/types/src/checkpoint.rs | 2 +- crates/prune/types/src/mode.rs | 16 +- crates/revm/Cargo.toml | 9 +- crates/stages/api/Cargo.toml | 3 +- crates/stages/stages/Cargo.toml | 3 +- crates/stages/types/Cargo.toml | 13 + crates/stages/types/src/checkpoints.rs | 18 +- crates/storage/codecs/Cargo.toml | 27 +- .../storage/codecs/src/alloy/access_list.rs | 2 + .../codecs/src/alloy/authorization_list.rs | 9 +- .../codecs/src/alloy/genesis_account.rs | 20 +- crates/storage/codecs/src/alloy/header.rs | 14 +- crates/storage/codecs/src/alloy/mod.rs | 36 ++- crates/storage/codecs/src/alloy/signature.rs | 3 +- .../codecs/src/alloy/transaction/eip1559.rs | 10 +- .../codecs/src/alloy/transaction/eip2930.rs | 8 +- .../codecs/src/alloy/transaction/eip4844.rs | 60 +++- .../codecs/src/alloy/transaction/eip7702.rs | 8 +- .../codecs/src/alloy/transaction/legacy.rs | 10 +- .../codecs/src/alloy/transaction/mod.rs | 82 +++++- .../codecs/src/alloy/transaction/optimism.rs | 8 +- crates/storage/codecs/src/alloy/withdrawal.rs | 8 +- crates/storage/codecs/src/lib.rs | 5 + crates/storage/codecs/src/test_utils.rs | 9 + crates/storage/db-api/Cargo.toml | 9 +- crates/storage/db-models/Cargo.toml | 3 +- crates/storage/db-models/src/accounts.rs | 2 +- crates/storage/db/Cargo.toml | 8 +- crates/storage/provider/Cargo.toml | 7 +- crates/trie/common/Cargo.toml | 3 +- crates/trie/common/src/nibbles.rs | 2 + crates/trie/trie/Cargo.toml | 3 +- 51 files changed, 857 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/compact.yml create mode 100644 crates/cli/commands/src/test_vectors/compact.rs create mode 100644 crates/optimism/cli/src/commands/test_vectors.rs diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml new file mode 100644 index 000000000000..c7435220c0fd --- /dev/null +++ b/.github/workflows/compact.yml @@ -0,0 +1,42 @@ +# Ensures that `Compact` codec changes are backwards compatible. +# +# 1) checkout `main` +# 2) randomly generate and serialize to disk many different type vectors with `Compact` (eg. Header, Transaction, etc) +# 3) checkout `pr` +# 4) deserialize previously generated test vectors + +on: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +name: compact-codec +jobs: + compact-codec: + runs-on: + group: Reth + strategy: + matrix: + bin: + - cargo run --bin reth --features "dev" + - cargo run --bin op-reth --features "optimism dev" --manifest-path crates/optimism/bin/Cargo.toml + steps: + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Checkout base + uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref || 'main' }} + # On `main` branch, generates test vectors and serializes them to disk using `Compact`. + - name: Generate compact vectors + run: ${{ matrix.bin }} -- test-vectors compact --write + - name: Checkout PR + uses: actions/checkout@v4 + with: + clean: false + # On incoming merge try to read and decode previously generated vectors with `Compact` + - name: Read vectors + run: ${{ matrix.bin }} -- test-vectors compact --read diff --git a/Cargo.lock b/Cargo.lock index c3003c991fbc..a73a4231533f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6615,6 +6615,7 @@ dependencies = [ "reth-cli", "reth-cli-runner", "reth-cli-util", + "reth-codecs", "reth-config", "reth-consensus", "reth-db", @@ -6638,10 +6639,13 @@ dependencies = [ "reth-primitives", "reth-provider", "reth-prune", + "reth-prune-types", "reth-stages", + "reth-stages-types", "reth-static-file", "reth-static-file-types", "reth-trie", + "reth-trie-common", "reth-trie-db", "secp256k1", "serde", @@ -6696,6 +6700,7 @@ dependencies = [ "serde", "serde_json", "test-fuzz", + "visibility", ] [[package]] @@ -8090,6 +8095,8 @@ dependencies = [ "clap", "eyre", "futures-util", + "op-alloy-consensus", + "proptest", "reth-chainspec", "reth-cli", "reth-cli-commands", @@ -11233,6 +11240,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.80", +] + [[package]] name = "wait-timeout" version = "0.2.0" diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index 6f4b1008f29f..ef66a99410f0 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -17,6 +17,7 @@ reth-cli.workspace = true reth-ethereum-cli.workspace = true reth-cli-runner.workspace = true reth-cli-util.workspace = true +reth-codecs = { workspace = true, optional = true } reth-config.workspace = true reth-consensus.workspace = true reth-db = { workspace = true, features = ["mdbx"] } @@ -38,11 +39,14 @@ reth-node-metrics.workspace = true reth-primitives.workspace = true reth-provider.workspace = true reth-prune.workspace = true +reth-prune-types = { workspace = true, optional = true } reth-stages.workspace = true +reth-stages-types = { workspace = true, optional = true } reth-static-file-types = { workspace = true, features = ["clap"] } reth-static-file.workspace = true reth-trie = { workspace = true, features = ["metrics"] } reth-trie-db = { workspace = true, features = ["metrics"] } +reth-trie-common = { workspace = true, optional = true } # ethereum alloy-eips.workspace = true @@ -89,14 +93,22 @@ reth-discv4.workspace = true [features] default = [] arbitrary = [ - "dep:proptest", - "dep:arbitrary", - "dep:proptest-arbitrary-interop", - "reth-primitives/arbitrary", - "reth-db-api/arbitrary", - "reth-eth-wire/arbitrary", - "reth-db/arbitrary", - "reth-chainspec/arbitrary", - "alloy-eips/arbitrary", - "alloy-primitives/arbitrary", + "dep:proptest", + "dep:arbitrary", + "dep:proptest-arbitrary-interop", + "reth-primitives/arbitrary", + "reth-db-api/arbitrary", + "reth-eth-wire/arbitrary", + "reth-db/arbitrary", + "reth-chainspec/arbitrary", + "alloy-eips/arbitrary", + "alloy-primitives/arbitrary", + "reth-codecs/test-utils", + "reth-prune-types/test-utils", + "reth-stages-types/test-utils", + "reth-trie-common/test-utils", + "reth-codecs?/arbitrary", + "reth-prune-types?/arbitrary", + "reth-stages-types?/arbitrary", + "reth-trie-common?/arbitrary" ] diff --git a/crates/cli/commands/src/test_vectors/compact.rs b/crates/cli/commands/src/test_vectors/compact.rs new file mode 100644 index 000000000000..cda7d5bd578c --- /dev/null +++ b/crates/cli/commands/src/test_vectors/compact.rs @@ -0,0 +1,257 @@ +use alloy_primitives::{hex, private::getrandom::getrandom, TxKind}; +use arbitrary::Arbitrary; +use eyre::{Context, Result}; +use proptest::{ + prelude::{ProptestConfig, RngCore}, + test_runner::{TestRng, TestRunner}, +}; +use reth_codecs::alloy::{ + authorization_list::Authorization, + genesis_account::GenesisAccount, + header::{Header, HeaderExt}, + transaction::{ + eip1559::TxEip1559, eip2930::TxEip2930, eip4844::TxEip4844, eip7702::TxEip7702, + legacy::TxLegacy, + }, + withdrawal::Withdrawal, +}; +use reth_db::{ + models::{AccountBeforeTx, StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals}, + ClientVersion, +}; +use reth_fs_util as fs; +use reth_primitives::{ + Account, Log, LogData, Receipt, ReceiptWithBloom, StorageEntry, Transaction, + TransactionSignedNoHash, TxType, Withdrawals, +}; +use reth_prune_types::{PruneCheckpoint, PruneMode}; +use reth_stages_types::{ + AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint, + HeadersCheckpoint, IndexHistoryCheckpoint, StageCheckpoint, StageUnitCheckpoint, + StorageHashingCheckpoint, +}; +use reth_trie::{hash_builder::HashBuilderValue, TrieMask}; +use reth_trie_common::{hash_builder::HashBuilderState, StoredNibbles, StoredNibblesSubKey}; +use std::{fs::File, io::BufReader}; + +pub const VECTORS_FOLDER: &str = "testdata/micro/compact"; +pub const VECTOR_SIZE: usize = 100; + +#[macro_export] +macro_rules! compact_types { + (regular: [$($regular_ty:ident),*], identifier: [$($id_ty:ident),*]) => { + pub const GENERATE_VECTORS: &[fn(&mut TestRunner) -> eyre::Result<()>] = &[ + $( + generate_vector::<$regular_ty> as fn(&mut TestRunner) -> eyre::Result<()>, + )* + $( + generate_vector::<$id_ty> as fn(&mut TestRunner) -> eyre::Result<()>, + )* + ]; + + pub const READ_VECTORS: &[fn() -> eyre::Result<()>] = &[ + $( + read_vector::<$regular_ty> as fn() -> eyre::Result<()>, + )* + $( + read_vector::<$id_ty> as fn() -> eyre::Result<()>, + )* + ]; + + pub static IDENTIFIER_TYPE: std::sync::LazyLock> = std::sync::LazyLock::new(|| { + let mut map = std::collections::HashSet::new(); + $( + map.insert(type_name::<$id_ty>()); + )* + map + }); + }; +} + +// The type that **actually** implements `Compact` should go here. If it's an alloy type, import the +// auxiliary type from reth_codecs::alloy instead. +compact_types!( + regular: [ + // reth-primitives + Account, + Receipt, + Withdrawals, + ReceiptWithBloom, + // reth_codecs::alloy + Authorization, + GenesisAccount, + Header, + HeaderExt, + Withdrawal, + TxEip2930, + TxEip1559, + TxEip4844, + TxEip7702, + TxLegacy, + HashBuilderValue, + LogData, + Log, + // BranchNodeCompact, // todo requires arbitrary + TrieMask, + // TxDeposit, TODO(joshie): optimism + // reth_prune_types + PruneCheckpoint, + PruneMode, + // reth_stages_types + AccountHashingCheckpoint, + StorageHashingCheckpoint, + ExecutionCheckpoint, + HeadersCheckpoint, + IndexHistoryCheckpoint, + EntitiesCheckpoint, + CheckpointBlockRange, + StageCheckpoint, + StageUnitCheckpoint, + // reth_db_api + StoredBlockOmmers, + StoredBlockBodyIndices, + StoredBlockWithdrawals, + // Manual implementations + TransactionSignedNoHash, + // Bytecode, // todo revm arbitrary + StorageEntry, + // MerkleCheckpoint, // todo storedsubnode -> branchnodecompact arbitrary + AccountBeforeTx, + ClientVersion, + StoredNibbles, + StoredNibblesSubKey, + // StorageTrieEntry, // todo branchnodecompact arbitrary + // StoredSubNode, // todo branchnodecompact arbitrary + HashBuilderState + ], + // These types require an extra identifier which is usually stored elsewhere (eg. parent type). + identifier: [ + // Signature todo we for v we only store parity(true || false), while v can take more values + Transaction, + TxType, + TxKind + ] +); + +/// Generates a vector of type `T` to a file. +pub fn generate_vectors() -> Result<()> { + generate_vectors_with(GENERATE_VECTORS) +} + +pub fn read_vectors() -> Result<()> { + read_vectors_with(READ_VECTORS) +} + +/// Generates a vector of type `T` to a file. +pub fn generate_vectors_with(gen: &[fn(&mut TestRunner) -> eyre::Result<()>]) -> Result<()> { + // Prepare random seed for test (same method as used by proptest) + let mut seed = [0u8; 32]; + getrandom(&mut seed)?; + println!("Seed for compact test vectors: {:?}", hex::encode_prefixed(seed)); + + // Start the runner with the seed + let config = ProptestConfig::default(); + let rng = TestRng::from_seed(config.rng_algorithm, &seed); + let mut runner = TestRunner::new_with_rng(config, rng); + + fs::create_dir_all(VECTORS_FOLDER)?; + + for generate_fn in gen { + generate_fn(&mut runner)?; + } + + Ok(()) +} + +/// Reads multiple vectors of different types ensuring their correctness by decoding and +/// re-encoding. +pub fn read_vectors_with(read: &[fn() -> eyre::Result<()>]) -> Result<()> { + fs::create_dir_all(VECTORS_FOLDER)?; + + for read_fn in read { + read_fn()?; + } + + Ok(()) +} + +/// Generates test vectors for a specific type `T`. +pub fn generate_vector(runner: &mut TestRunner) -> Result<()> +where + T: for<'a> Arbitrary<'a> + reth_codecs::Compact, +{ + let type_name = type_name::(); + print!("{}", &type_name); + + let mut bytes = std::iter::repeat(0u8).take(256).collect::>(); + let mut compact_buffer = vec![]; + + let mut values = Vec::with_capacity(VECTOR_SIZE); + for _ in 0..VECTOR_SIZE { + runner.rng().fill_bytes(&mut bytes); + compact_buffer.clear(); + + let obj = T::arbitrary(&mut arbitrary::Unstructured::new(&bytes))?; + let res = obj.to_compact(&mut compact_buffer); + + if IDENTIFIER_TYPE.contains(&type_name) { + compact_buffer.push(res as u8); + } + + values.push(hex::encode(&compact_buffer)); + } + + serde_json::to_writer( + std::io::BufWriter::new( + std::fs::File::create(format!("{VECTORS_FOLDER}/{}.json", &type_name)).unwrap(), + ), + &values, + )?; + + println!(" ✅"); + + Ok(()) +} + +/// Reads a vector of type `T` from a file and compares each item with its reconstructed version +/// using `T::from_compact`. +pub fn read_vector() -> Result<()> +where + T: reth_codecs::Compact, +{ + let type_name = type_name::(); + print!("{}", &type_name); + + // Read the file where the vectors are stored + let file_path = format!("{VECTORS_FOLDER}/{}.json", &type_name); + let file = File::open(&file_path).wrap_err_with(|| { + "Failed to open vector. Make sure to run `reth test-vectors compact --write` first." + })?; + let reader = BufReader::new(file); + + let stored_values: Vec = serde_json::from_reader(reader)?; + let mut buffer = vec![]; + + for hex_str in stored_values { + let mut compact_bytes = hex::decode(hex_str)?; + let mut identifier = None; + buffer.clear(); + + if IDENTIFIER_TYPE.contains(&type_name) { + identifier = compact_bytes.pop().map(|b| b as usize); + } + let len_or_identifier = identifier.unwrap_or(compact_bytes.len()); + + let (reconstructed, _) = T::from_compact(&compact_bytes, len_or_identifier); + reconstructed.to_compact(&mut buffer); + assert_eq!(buffer, compact_bytes); + } + + println!(" ✅"); + + Ok(()) +} + +pub fn type_name() -> String { + std::any::type_name::().replace("::", "__") +} diff --git a/crates/cli/commands/src/test_vectors/mod.rs b/crates/cli/commands/src/test_vectors/mod.rs index 999c0bc91320..001d0c2e862d 100644 --- a/crates/cli/commands/src/test_vectors/mod.rs +++ b/crates/cli/commands/src/test_vectors/mod.rs @@ -2,7 +2,8 @@ use clap::{Parser, Subcommand}; -mod tables; +pub mod compact; +pub mod tables; /// Generate test-vectors for different data types. #[derive(Debug, Parser)] @@ -19,6 +20,22 @@ pub enum Subcommands { /// List of table names. Case-sensitive. names: Vec, }, + /// Randomly generate test vectors for each `Compact` type using the `--write` flag. + /// + /// The generated vectors are serialized in both `json` and `Compact` formats and saved to a + /// file. + /// + /// Use the `--read` flag to read and validate the previously generated vectors from file. + #[group(multiple = false, required = true)] + Compact { + /// Write test vectors to a file. + #[arg(long)] + write: bool, + + /// Read test vectors from a file. + #[arg(long)] + read: bool, + }, } impl Command { @@ -28,6 +45,13 @@ impl Command { Subcommands::Tables { names } => { tables::generate_vectors(names)?; } + Subcommands::Compact { write, .. } => { + if write { + compact::generate_vectors()?; + } else { + compact::read_vectors()?; + } + } } Ok(()) } diff --git a/crates/cli/commands/src/test_vectors/tables.rs b/crates/cli/commands/src/test_vectors/tables.rs index 112685251d08..29ba50c8d83e 100644 --- a/crates/cli/commands/src/test_vectors/tables.rs +++ b/crates/cli/commands/src/test_vectors/tables.rs @@ -1,4 +1,4 @@ -use alloy_primitives::private::getrandom::getrandom; +use alloy_primitives::{hex, private::getrandom::getrandom}; use arbitrary::Arbitrary; use eyre::Result; use proptest::{ @@ -17,11 +17,11 @@ const VECTORS_FOLDER: &str = "testdata/micro/db"; const PER_TABLE: usize = 1000; /// Generates test vectors for specified `tables`. If list is empty, then generate for all tables. -pub(crate) fn generate_vectors(mut tables: Vec) -> Result<()> { +pub fn generate_vectors(mut tables: Vec) -> Result<()> { // Prepare random seed for test (same method as used by proptest) let mut seed = [0u8; 32]; getrandom(&mut seed)?; - println!("Seed for test vectors: {:?}", seed); + println!("Seed for table test vectors: {:?}", hex::encode_prefixed(seed)); // Start the runner with the seed let config = ProptestConfig::default(); diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index 6fe741db8833..dee0bcaf7ce1 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -95,5 +95,6 @@ test-utils = [ "reth-revm/test-utils", "reth-stages-api/test-utils", "reth-provider/test-utils", - "reth-trie/test-utils" + "reth-trie/test-utils", + "reth-prune-types?/test-utils" ] diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index d97a57864193..90fd532828e1 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -57,11 +57,12 @@ std = [ "revm/std", ] test-utils = [ - "dep:parking_lot", - "reth-chainspec/test-utils", - "reth-consensus/test-utils", - "reth-primitives/test-utils", - "reth-primitives-traits/test-utils", - "reth-revm/test-utils", - "revm/test-utils", + "dep:parking_lot", + "reth-chainspec/test-utils", + "reth-consensus/test-utils", + "reth-primitives/test-utils", + "reth-primitives-traits/test-utils", + "reth-revm/test-utils", + "revm/test-utils", + "reth-prune-types/test-utils" ] diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index b0e256fdf637..83a3e163ebc1 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -71,7 +71,8 @@ arbitrary = [ "dep:arbitrary", "reth-chainspec/arbitrary", "alloy-eips/arbitrary", - "alloy-primitives/arbitrary" + "alloy-primitives/arbitrary", + "reth-codecs/arbitrary" ] serde = [ "dep:serde", @@ -80,7 +81,8 @@ serde = [ "alloy-primitives/serde", "bytes/serde", "rand/serde", - "secp256k1/serde" + "secp256k1/serde", + "reth-codecs/serde" ] [[test]] diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml index f60ef36a4669..77166763100a 100644 --- a/crates/optimism/bin/Cargo.toml +++ b/crates/optimism/bin/Cargo.toml @@ -47,6 +47,10 @@ optimism = [ "reth-provider/optimism" ] +dev = [ + "reth-optimism-cli/dev" +] + min-error-logs = ["tracing/release_max_level_error"] min-warn-logs = ["tracing/release_max_level_warn"] min-info-logs = ["tracing/release_max_level_info"] diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index 7db41ccbe843..a2ba71214f5c 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -65,6 +65,13 @@ tokio-util = { workspace = true, features = ["codec"] } tracing.workspace = true eyre.workspace = true +# reth test-vectors +proptest = { workspace = true, optional = true } +op-alloy-consensus = { workspace = true, features = [ + "arbitrary", +], optional = true } + + [dev-dependencies] tempfile.workspace = true reth-stages = { workspace = true, features = ["test-utils"] } @@ -94,3 +101,9 @@ jemalloc = [ "reth-node-core/jemalloc", "reth-node-metrics/jemalloc" ] + +dev = [ + "dep:proptest", + "reth-cli-commands/arbitrary", + "op-alloy-consensus" +] diff --git a/crates/optimism/cli/src/commands/mod.rs b/crates/optimism/cli/src/commands/mod.rs index a7674ec2c9bf..d51f89932965 100644 --- a/crates/optimism/cli/src/commands/mod.rs +++ b/crates/optimism/cli/src/commands/mod.rs @@ -16,6 +16,9 @@ pub mod import; pub mod import_receipts; pub mod init_state; +#[cfg(feature = "dev")] +pub mod test_vectors; + /// Commands to be executed #[derive(Debug, Subcommand)] pub enum Commands @@ -55,4 +58,8 @@ pub enum Commands), + /// Generate Test Vectors + #[cfg(feature = "dev")] + #[command(name = "test-vectors")] + TestVectors(test_vectors::Command), } diff --git a/crates/optimism/cli/src/commands/test_vectors.rs b/crates/optimism/cli/src/commands/test_vectors.rs new file mode 100644 index 000000000000..093d63148ee3 --- /dev/null +++ b/crates/optimism/cli/src/commands/test_vectors.rs @@ -0,0 +1,72 @@ +//! Command for generating test vectors. + +use clap::{Parser, Subcommand}; +use op_alloy_consensus::TxDeposit; +use proptest::test_runner::TestRunner; +use reth_cli_commands::{ + compact_types, + test_vectors::{ + compact, + compact::{ + generate_vector, read_vector, GENERATE_VECTORS as ETH_GENERATE_VECTORS, + READ_VECTORS as ETH_READ_VECTORS, + }, + tables, + }, +}; + +/// Generate test-vectors for different data types. +#[derive(Debug, Parser)] +pub struct Command { + #[command(subcommand)] + command: Subcommands, +} + +#[derive(Subcommand, Debug)] +/// `reth test-vectors` subcommands +pub enum Subcommands { + /// Generates test vectors for specified tables. If no table is specified, generate for all. + Tables { + /// List of table names. Case-sensitive. + names: Vec, + }, + /// Generates test vectors for `Compact` types with `--write`. Reads and checks generated + /// vectors with `--read`. + #[group(multiple = false, required = true)] + Compact { + /// Write test vectors to a file. + #[arg(long)] + write: bool, + + /// Read test vectors from a file. + #[arg(long)] + read: bool, + }, +} + +impl Command { + /// Execute the command + pub async fn execute(self) -> eyre::Result<()> { + match self.command { + Subcommands::Tables { names } => { + tables::generate_vectors(names)?; + } + Subcommands::Compact { write, .. } => { + compact_types!( + regular: [ + TxDeposit + ], identifier: [] + ); + + if write { + compact::generate_vectors_with(ETH_GENERATE_VECTORS)?; + compact::generate_vectors_with(GENERATE_VECTORS)?; + } else { + compact::read_vectors_with(ETH_READ_VECTORS)?; + compact::read_vectors_with(READ_VECTORS)?; + } + } + } + Ok(()) + } +} diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index 235b44559696..43d12616484d 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -169,6 +169,8 @@ where runner.run_command_until_exit(|ctx| command.execute::(ctx)) } Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::()), + #[cfg(feature = "dev")] + Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()), } } diff --git a/crates/primitives-traits/Cargo.toml b/crates/primitives-traits/Cargo.toml index 9634da40f47c..4319232f8248 100644 --- a/crates/primitives-traits/Cargo.toml +++ b/crates/primitives-traits/Cargo.toml @@ -74,7 +74,8 @@ arbitrary = [ "dep:proptest", "dep:proptest-arbitrary-interop", "alloy-eips/arbitrary", - "revm-primitives/arbitrary" + "revm-primitives/arbitrary", + "reth-codecs/arbitrary" ] serde-bincode-compat = [ "serde_with", diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 107c218c7589..5e761f41fe22 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -62,7 +62,7 @@ arbitrary = { workspace = true, features = ["derive"], optional = true } [dev-dependencies] # eth reth-chainspec.workspace = true -reth-codecs.workspace = true +reth-codecs = { workspace = true, features = ["test-utils"] } reth-primitives-traits = { workspace = true, features = ["arbitrary"] } reth-testing-utils.workspace = true revm-primitives = { workspace = true, features = ["arbitrary"] } @@ -120,6 +120,7 @@ arbitrary = [ "alloy-serde?/arbitrary", "op-alloy-consensus?/arbitrary", "op-alloy-rpc-types?/arbitrary", + "reth-codecs?/arbitrary" ] secp256k1 = ["dep:secp256k1"] c-kzg = [ diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index bb6c0841b8c8..940b491e3351 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -503,6 +503,18 @@ mod tests { use super::*; use alloy_primitives::{address, b256, bytes, hex_literal::hex}; + #[test] + fn test_decode_receipt() { + #[cfg(not(feature = "optimism"))] + reth_codecs::test_utils::test_decode::(&hex!( + "c428b52ffd23fc42696156b10200f034792b6a94c3850215c2fef7aea361a0c31b79d9a32652eefc0d4e2e730036061cff7344b6fc6132b50cda0ed810a991ae58ef013150c12b2522533cb3b3a8b19b7786a8b5ff1d3cdc84225e22b02def168c8858df" + )); + #[cfg(feature = "optimism")] + reth_codecs::test_utils::test_decode::(&hex!( + "c30328b52ffd23fc426961a00105007eb0042307705a97e503562eacf2b95060cce9de6de68386b6c155b73a9650021a49e2f8baad17f30faff5899d785c4c0873e45bc268bcf07560106424570d11f9a59e8f3db1efa4ceec680123712275f10d92c3411e1caaa11c7c5d591bc11487168e09934a9986848136da1b583babf3a7188e3aed007a1520f1cf4c1ca7d3482c6c28d37c298613c70a76940008816c4c95644579fd08471dc34732fd0f24" + )); + } + // Test vector from: https://eips.ethereum.org/EIPS/eip-2481 #[test] fn encode_legacy_receipt() { diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 7798433d05dd..b09fff9e2b6a 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -649,10 +649,12 @@ impl reth_codecs::Compact for Transaction { let (tx, buf) = TxDeposit::from_compact(buf, buf.len()); (Self::Deposit(tx), buf) } - _ => unreachable!("Junk data in database: unknown Transaction variant"), + _ => unreachable!( + "Junk data in database: unknown Transaction variant: {identifier}" + ), } } - _ => unreachable!("Junk data in database: unknown Transaction variant"), + _ => unreachable!("Junk data in database: unknown Transaction variant: {identifier}"), } } } diff --git a/crates/prune/types/Cargo.toml b/crates/prune/types/Cargo.toml index 13def8eaa8b0..5446d6f76ff5 100644 --- a/crates/prune/types/Cargo.toml +++ b/crates/prune/types/Cargo.toml @@ -20,6 +20,7 @@ derive_more.workspace = true modular-bitfield.workspace = true serde.workspace = true thiserror.workspace = true +arbitrary = { workspace = true, features = ["derive"], optional = true } [dev-dependencies] arbitrary = { workspace = true, features = ["derive"] } @@ -29,3 +30,13 @@ proptest-arbitrary-interop.workspace = true serde_json.workspace = true test-fuzz.workspace = true toml.workspace = true + +[features] +test-utils = [ + "dep:arbitrary", + "reth-codecs/test-utils" +] +arbitrary = [ + "alloy-primitives/arbitrary", + "reth-codecs/arbitrary" +] diff --git a/crates/prune/types/src/checkpoint.rs b/crates/prune/types/src/checkpoint.rs index f654fba7df19..e0397c5afc86 100644 --- a/crates/prune/types/src/checkpoint.rs +++ b/crates/prune/types/src/checkpoint.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; /// Saves the pruning progress of a stage. #[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(Default, arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(Default, arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct PruneCheckpoint { /// Highest pruned block number. If it's [None], the pruning for block `0` is not finished yet. diff --git a/crates/prune/types/src/mode.rs b/crates/prune/types/src/mode.rs index 3465882993bd..9a8e55bb383a 100644 --- a/crates/prune/types/src/mode.rs +++ b/crates/prune/types/src/mode.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; /// Prune mode. #[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)] #[serde(rename_all = "lowercase")] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub enum PruneMode { /// Prune all blocks. @@ -17,6 +17,13 @@ pub enum PruneMode { Before(BlockNumber), } +#[cfg(any(test, feature = "test-utils"))] +impl Default for PruneMode { + fn default() -> Self { + Self::Full + } +} + impl PruneMode { /// Prune blocks up to the specified block number. The specified block number is also pruned. /// @@ -69,13 +76,6 @@ impl PruneMode { } } -#[cfg(test)] -impl Default for PruneMode { - fn default() -> Self { - Self::Full - } -} - #[cfg(test)] mod tests { use crate::{ diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index 3f2a39a0b443..3ee680101084 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -43,10 +43,11 @@ std = [ "alloy-consensus/std", ] test-utils = [ - "dep:reth-trie", - "reth-primitives/test-utils", - "reth-trie?/test-utils", - "revm/test-utils", + "dep:reth-trie", + "reth-primitives/test-utils", + "reth-trie?/test-utils", + "revm/test-utils", + "reth-prune-types/test-utils" ] serde = [ "revm/serde", diff --git a/crates/stages/api/Cargo.toml b/crates/stages/api/Cargo.toml index cba569a2a431..88a8e3b96d13 100644 --- a/crates/stages/api/Cargo.toml +++ b/crates/stages/api/Cargo.toml @@ -50,5 +50,6 @@ test-utils = [ "reth-consensus/test-utils", "reth-network-p2p/test-utils", "reth-primitives-traits/test-utils", - "reth-provider/test-utils" + "reth-provider/test-utils", + "reth-stages-types/test-utils" ] diff --git a/crates/stages/stages/Cargo.toml b/crates/stages/stages/Cargo.toml index 0b26cb6a1e79..65bb2637b620 100644 --- a/crates/stages/stages/Cargo.toml +++ b/crates/stages/stages/Cargo.toml @@ -110,7 +110,8 @@ test-utils = [ "reth-codecs/test-utils", "reth-db-api/test-utils", "reth-trie-db/test-utils", - "reth-trie/test-utils" + "reth-trie/test-utils", + "reth-prune-types/test-utils" ] [[bench]] diff --git a/crates/stages/types/Cargo.toml b/crates/stages/types/Cargo.toml index 54b14b335cb9..a466b21b6f91 100644 --- a/crates/stages/types/Cargo.toml +++ b/crates/stages/types/Cargo.toml @@ -19,6 +19,7 @@ alloy-primitives.workspace = true modular-bitfield.workspace = true bytes.workspace = true serde.workspace = true +arbitrary = { workspace = true, features = ["derive"], optional = true } [dev-dependencies] arbitrary = { workspace = true, features = ["derive"] } @@ -26,3 +27,15 @@ proptest.workspace = true proptest-arbitrary-interop.workspace = true test-fuzz.workspace = true rand.workspace = true + +[features] +test-utils = [ + "dep:arbitrary", + "reth-codecs/test-utils", + "reth-trie-common/test-utils" +] +arbitrary = [ + "alloy-primitives/arbitrary", + "reth-codecs/arbitrary", + "reth-trie-common/arbitrary" +] diff --git a/crates/stages/types/src/checkpoints.rs b/crates/stages/types/src/checkpoints.rs index 79e896bf4d9b..87225f1eec4e 100644 --- a/crates/stages/types/src/checkpoints.rs +++ b/crates/stages/types/src/checkpoints.rs @@ -76,7 +76,7 @@ impl Compact for MerkleCheckpoint { /// Saves the progress of AccountHashing stage. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct AccountHashingCheckpoint { /// The next account to start hashing from. @@ -89,7 +89,7 @@ pub struct AccountHashingCheckpoint { /// Saves the progress of StorageHashing stage. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct StorageHashingCheckpoint { /// The next account to start hashing from. @@ -104,7 +104,7 @@ pub struct StorageHashingCheckpoint { /// Saves the progress of Execution stage. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct ExecutionCheckpoint { /// Block range which this checkpoint is valid for. @@ -115,7 +115,7 @@ pub struct ExecutionCheckpoint { /// Saves the progress of Headers stage. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct HeadersCheckpoint { /// Block range which this checkpoint is valid for. @@ -126,7 +126,7 @@ pub struct HeadersCheckpoint { /// Saves the progress of Index History stages. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct IndexHistoryCheckpoint { /// Block range which this checkpoint is valid for. @@ -137,7 +137,7 @@ pub struct IndexHistoryCheckpoint { /// Saves the progress of abstract stage iterating over or downloading entities. #[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct EntitiesCheckpoint { /// Number of entities already processed. @@ -166,7 +166,7 @@ impl EntitiesCheckpoint { /// Saves the block range. Usually, it's used to check the validity of some stage checkpoint across /// multiple executions. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct CheckpointBlockRange { /// The first block of the range, inclusive. @@ -189,7 +189,7 @@ impl From<&RangeInclusive> for CheckpointBlockRange { /// Saves the progress of a stage. #[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct StageCheckpoint { /// The maximum block processed by the stage. @@ -256,7 +256,7 @@ impl StageCheckpoint { // is not a Copy type. /// Stage-specific checkpoint metrics. #[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub enum StageUnitCheckpoint { /// Saves the progress of AccountHashing stage. diff --git a/crates/storage/codecs/Cargo.toml b/crates/storage/codecs/Cargo.toml index 2525b4e8d7f8..20a0673dff62 100644 --- a/crates/storage/codecs/Cargo.toml +++ b/crates/storage/codecs/Cargo.toml @@ -27,6 +27,9 @@ op-alloy-consensus = { workspace = true, optional = true } # misc bytes.workspace = true modular-bitfield = { workspace = true, optional = true } +visibility = { version = "0.1.1", optional = true} +serde.workspace = true +arbitrary = { workspace = true, features = ["derive"], optional = true } [dev-dependencies] alloy-eips = { workspace = true, default-features = false, features = [ @@ -45,7 +48,6 @@ serde_json.workspace = true arbitrary = { workspace = true, features = ["derive"] } proptest.workspace = true proptest-arbitrary-interop.workspace = true -serde.workspace = true [features] default = ["std", "alloy"] @@ -66,4 +68,25 @@ alloy = [ "dep:alloy-trie", ] optimism = ["alloy", "dep:op-alloy-consensus"] -test-utils = [] +test-utils = [ + "std", + "alloy", + "arbitrary", + "dep:visibility", + "dep:arbitrary" +] +serde = [ + "alloy-consensus?/serde", + "alloy-eips?/serde", + "alloy-primitives/serde", + "alloy-trie?/serde", + "bytes/serde", + "op-alloy-consensus?/serde" +] +arbitrary = [ + "alloy-consensus?/arbitrary", + "alloy-eips?/arbitrary", + "alloy-primitives/arbitrary", + "alloy-trie?/arbitrary", + "op-alloy-consensus?/arbitrary" +] diff --git a/crates/storage/codecs/src/alloy/access_list.rs b/crates/storage/codecs/src/alloy/access_list.rs index 306b64d7e4bf..304b6bd388cf 100644 --- a/crates/storage/codecs/src/alloy/access_list.rs +++ b/crates/storage/codecs/src/alloy/access_list.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AccessList`] + use crate::Compact; use alloc::vec::Vec; use alloy_eips::eip2930::{AccessList, AccessListItem}; diff --git a/crates/storage/codecs/src/alloy/authorization_list.rs b/crates/storage/codecs/src/alloy/authorization_list.rs index 6dc36956d249..3fc9518a6376 100644 --- a/crates/storage/codecs/src/alloy/authorization_list.rs +++ b/crates/storage/codecs/src/alloy/authorization_list.rs @@ -1,16 +1,21 @@ -use core::ops::Deref; +//! Compact implementation for [`AlloyAuthorization`] use crate::Compact; use alloy_eips::eip7702::{Authorization as AlloyAuthorization, SignedAuthorization}; use alloy_primitives::{Address, U256}; use bytes::Buf; +use core::ops::Deref; use reth_codecs_derive::add_arbitrary_tests; /// Authorization acts as bridge which simplifies Compact implementation for AlloyAuthorization. /// /// Notice: Make sure this struct is 1:1 with `alloy_eips::eip7702::Authorization` #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[add_arbitrary_tests(compact)] pub(crate) struct Authorization { chain_id: u64, diff --git a/crates/storage/codecs/src/alloy/genesis_account.rs b/crates/storage/codecs/src/alloy/genesis_account.rs index 938ad1375b15..b29fe526df4f 100644 --- a/crates/storage/codecs/src/alloy/genesis_account.rs +++ b/crates/storage/codecs/src/alloy/genesis_account.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyGenesisAccount`] + use crate::Compact; use alloc::vec::Vec; use alloy_genesis::GenesisAccount as AlloyGenesisAccount; @@ -22,8 +24,14 @@ pub(crate) struct GenesisAccountRef<'a> { private_key: Option<&'a B256>, } +/// Acts as bridge which simplifies Compact implementation for +/// `AlloyGenesisAccount`. #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[add_arbitrary_tests(compact)] pub(crate) struct GenesisAccount { /// The nonce of the account at genesis. @@ -39,14 +47,20 @@ pub(crate) struct GenesisAccount { } #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] #[add_arbitrary_tests(compact)] pub(crate) struct StorageEntries { entries: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] #[add_arbitrary_tests(compact)] pub(crate) struct StorageEntry { key: B256, diff --git a/crates/storage/codecs/src/alloy/header.rs b/crates/storage/codecs/src/alloy/header.rs index 90e67b1e312c..78f2029c32ea 100644 --- a/crates/storage/codecs/src/alloy/header.rs +++ b/crates/storage/codecs/src/alloy/header.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyHeader`] + use crate::Compact; use alloy_consensus::Header as AlloyHeader; use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, B256, U256}; @@ -10,7 +12,11 @@ use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, B256, U256}; /// will automatically apply to this type. /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::Header`] -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(serde::Serialize, serde::Deserialize, arbitrary::Arbitrary) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] pub(crate) struct Header { parent_hash: B256, @@ -42,7 +48,11 @@ pub(crate) struct Header { /// used as a field of [`Header`] for backwards compatibility. /// /// More information: & [`reth_codecs_derive::Compact`]. -#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(serde::Serialize, serde::Deserialize, arbitrary::Arbitrary) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] pub(crate) struct HeaderExt { requests_hash: Option, diff --git a/crates/storage/codecs/src/alloy/mod.rs b/crates/storage/codecs/src/alloy/mod.rs index ed77876c5ce1..f1bf6a00e694 100644 --- a/crates/storage/codecs/src/alloy/mod.rs +++ b/crates/storage/codecs/src/alloy/mod.rs @@ -1,13 +1,29 @@ -mod access_list; -mod authorization_list; -mod genesis_account; -mod header; -mod log; -mod signature; -mod transaction; -mod trie; -mod txkind; -mod withdrawal; +//! Implements Compact for alloy types. + +/// Will make it a pub mod if test-utils is enabled +macro_rules! cond_mod { + ($($mod_name:ident),*) => { + $( + #[cfg(feature = "test-utils")] + pub mod $mod_name; + #[cfg(not(feature = "test-utils"))] + mod $mod_name; + )* + }; +} + +cond_mod!( + access_list, + authorization_list, + genesis_account, + header, + log, + signature, + transaction, + trie, + txkind, + withdrawal +); #[cfg(test)] mod tests { diff --git a/crates/storage/codecs/src/alloy/signature.rs b/crates/storage/codecs/src/alloy/signature.rs index 70290ea96c18..0cc4774d0f8b 100644 --- a/crates/storage/codecs/src/alloy/signature.rs +++ b/crates/storage/codecs/src/alloy/signature.rs @@ -1,6 +1,7 @@ -use alloy_primitives::{Parity, Signature, U256}; +//! Compact implementation for [`Signature`] use crate::Compact; +use alloy_primitives::{Parity, Signature, U256}; impl Compact for Signature { fn to_compact(&self, buf: &mut B) -> usize diff --git a/crates/storage/codecs/src/alloy/transaction/eip1559.rs b/crates/storage/codecs/src/alloy/transaction/eip1559.rs index 8e7594951faa..0e7f44cdec18 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip1559.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip1559.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyTxEip1559`] + use crate::Compact; use alloy_consensus::TxEip1559 as AlloyTxEip1559; use alloy_eips::eip2930::AccessList; @@ -11,8 +13,12 @@ use alloy_primitives::{Bytes, ChainId, TxKind, U256}; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip1559`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Compact, Default)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, crate::add_arbitrary_tests(compact))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(any(test, feature = "test-utils"), crate::add_arbitrary_tests(compact))] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] pub(crate) struct TxEip1559 { chain_id: ChainId, nonce: u64, diff --git a/crates/storage/codecs/src/alloy/transaction/eip2930.rs b/crates/storage/codecs/src/alloy/transaction/eip2930.rs index e0c78a3e4c0c..75cab9e8a09c 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip2930.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip2930.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyTxEip2930`] + use crate::Compact; use alloy_consensus::TxEip2930 as AlloyTxEip2930; use alloy_eips::eip2930::AccessList; @@ -13,7 +15,11 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip2930`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[add_arbitrary_tests(compact)] pub(crate) struct TxEip2930 { chain_id: ChainId, diff --git a/crates/storage/codecs/src/alloy/transaction/eip4844.rs b/crates/storage/codecs/src/alloy/transaction/eip4844.rs index 27c6b9240902..5ec36e06bf5c 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip4844.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip4844.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyTxEip4844`] + use crate::{Compact, CompactPlaceholder}; use alloc::vec::Vec; use alloy_consensus::TxEip4844 as AlloyTxEip4844; @@ -14,7 +16,8 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip4844`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr(any(test, feature = "test-utils"), derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[add_arbitrary_tests(compact)] pub(crate) struct TxEip4844 { chain_id: ChainId, @@ -25,6 +28,13 @@ pub(crate) struct TxEip4844 { /// TODO(debt): this should be removed if we break the DB. /// Makes sure that the Compact bitflag struct has one bit after the above field: /// + #[cfg_attr( + feature = "test-utils", + serde( + serialize_with = "serialize_placeholder", + deserialize_with = "deserialize_placeholder" + ) + )] placeholder: Option, to: Address, value: U256, @@ -75,6 +85,54 @@ impl Compact for AlloyTxEip4844 { } } +#[cfg(any(test, feature = "test-utils"))] +impl<'a> arbitrary::Arbitrary<'a> for TxEip4844 { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + Ok(Self { + chain_id: ChainId::arbitrary(u)?, + nonce: u64::arbitrary(u)?, + gas_limit: u64::arbitrary(u)?, + max_fee_per_gas: u128::arbitrary(u)?, + max_priority_fee_per_gas: u128::arbitrary(u)?, + // Should always be Some for TxEip4844 + placeholder: Some(()), + to: Address::arbitrary(u)?, + value: U256::arbitrary(u)?, + access_list: AccessList::arbitrary(u)?, + blob_versioned_hashes: Vec::::arbitrary(u)?, + max_fee_per_blob_gas: u128::arbitrary(u)?, + input: Bytes::arbitrary(u)?, + }) + } +} + +#[cfg(any(test, feature = "test-utils"))] +fn serialize_placeholder(value: &Option<()>, serializer: S) -> Result +where + S: serde::Serializer, +{ + // Required otherwise `serde_json` will serialize it as null and would be `None` when decoding + // it again. + match value { + Some(()) => serializer.serialize_str("placeholder"), // Custom serialization + None => serializer.serialize_none(), + } +} + +#[cfg(any(test, feature = "test-utils"))] +fn deserialize_placeholder<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + use serde::de::Deserialize; + let s: Option = Option::deserialize(deserializer)?; + match s.as_deref() { + Some("placeholder") => Ok(Some(())), + None => Ok(None), + _ => Err(serde::de::Error::custom("unexpected value")), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/storage/codecs/src/alloy/transaction/eip7702.rs b/crates/storage/codecs/src/alloy/transaction/eip7702.rs index e714be1c3f66..8acf59425f2d 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip7702.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip7702.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyTxEip7702`] + use crate::Compact; use alloc::vec::Vec; use alloy_consensus::TxEip7702 as AlloyTxEip7702; @@ -14,7 +16,11 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip7702`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[add_arbitrary_tests(compact)] pub(crate) struct TxEip7702 { chain_id: ChainId, diff --git a/crates/storage/codecs/src/alloy/transaction/legacy.rs b/crates/storage/codecs/src/alloy/transaction/legacy.rs index 27e799a790e7..c83626aa4cf9 100644 --- a/crates/storage/codecs/src/alloy/transaction/legacy.rs +++ b/crates/storage/codecs/src/alloy/transaction/legacy.rs @@ -1,11 +1,17 @@ +//! Compact implementation for [`AlloyTxLegacy`] + use crate::Compact; use alloy_consensus::TxLegacy as AlloyTxLegacy; use alloy_primitives::{Bytes, ChainId, TxKind, U256}; /// Legacy transaction. #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] -#[cfg_attr(test, crate::add_arbitrary_tests(compact))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize), + crate::add_arbitrary_tests(compact) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] pub(crate) struct TxLegacy { /// Added as EIP-155: Simple replay attack protection chain_id: Option, diff --git a/crates/storage/codecs/src/alloy/transaction/mod.rs b/crates/storage/codecs/src/alloy/transaction/mod.rs index 5b1d173a528e..dc27eacfacc2 100644 --- a/crates/storage/codecs/src/alloy/transaction/mod.rs +++ b/crates/storage/codecs/src/alloy/transaction/mod.rs @@ -1,10 +1,18 @@ -pub(crate) mod eip1559; -pub(crate) mod eip2930; -pub(crate) mod eip4844; -pub(crate) mod eip7702; -pub(crate) mod legacy; -#[cfg(feature = "optimism")] -pub(crate) mod optimism; +//! Compact implementation for transaction types + +cond_mod!( + eip1559, + eip2930, + eip4844, + eip7702, + legacy +); + + +#[cfg(all(feature = "test-utils", feature = "optimism"))] +pub mod optimism; +#[cfg(all(not(feature = "test-utils"), feature = "optimism"))] +mod optimism; #[cfg(test)] mod tests { @@ -15,9 +23,13 @@ mod tests { // this check is to ensure we do not inadvertently add too many fields to a struct which would // expand the flags field and break backwards compatibility - use crate::alloy::transaction::{ - eip1559::TxEip1559, eip2930::TxEip2930, eip4844::TxEip4844, eip7702::TxEip7702, - legacy::TxLegacy, + use alloy_primitives::hex; + use crate::{ + alloy::{header::Header, transaction::{ + eip1559::TxEip1559, eip2930::TxEip2930, eip4844::TxEip4844, eip7702::TxEip7702, + legacy::TxLegacy, + }}, + test_utils::test_decode, }; #[test] @@ -34,4 +46,54 @@ mod tests { fn test_ensure_backwards_compatibility_optimism() { assert_eq!(crate::alloy::transaction::optimism::TxDeposit::bitflag_encoded_bytes(), 2); } + + #[test] + fn test_decode_header() { + test_decode::

(&hex!( + "01000000fbbb564baeafd064b979c2ac032df5cd987098066a8c6969514dfb8ecfbf043e667fa19efcc00d1dd197c309a3cc42dec820cd627af8f7f38f3274f842406891b22624431d0ea858422db8415b1181f8d19befbd21287debaf98a94e84b3ec20be846f35abfbf743ee3eda4fdda6a6f9124d295da97e26eaa1cedd09936f0a3c560b6bc10316dba5e82abd21afcf519a985feb09a6ce7fba2e8163b10f06c99828b8049c29b993d88d1d112dca60a03ebd8ebc6d69a7e1f301ca6d67c21fe0949d67bca251edf36c96a2cf7c84d98fc60a53988ac95820f434eb35280d98c8ba4d7484e7ee8fefd63591ad4c937ccaaea23871d05c77bac754c5759b34cf9b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + )); + } + + #[test] + fn test_decode_eip1559() { + test_decode::(&hex!( + "88086110b81b05bc5bb59ec3e4cd44e895a9dcb2656d5003e2f64ecb2e15443898cc1cc19af19ca96fc2b4eafc4abc26e4bbd70a3ddb10b7530b65eea128f4095c97164f712c04239902c1b08acf3949d4687123cdd72d5c73df113d2dc6ed7e519f410ace5553ca805975240a208b57013532de78c5cb407423ea11921ab11b13e93ef35d4d01c9a23166c4d627987545fe4675528d0ab111b0a1dc83fba0a4e1cd5c826a94db3f" + )); + } + + #[test] + fn test_decode_eip2930() { + test_decode::(&hex!( + "7810833fce14e3e2921e94fd3727eb71e91551d2c1e029697a654bfab510f3963aa57074015e152065d1c807f8830079fb0aeadc251d248eaec7147e78580ed638c4e667827775e24270edd5aad475776533ece65373afa71722bfeba3c900" + )); + } + + #[test] + fn test_decode_eip4844() { + test_decode::(&hex!( + "88086110025c359180ea680b5007c856f9e1ad4d1be7a5019feb42133f4fc4bdf74da1b457ab787462385a28a1bf8edb401adabf3ff21ac18f695e30180348ea67246fc4dc25e88add12b7c317651a0ce08946d98dbbe5b38883aa758a0f247e23b0fe3ac1bcc43d7212c984d6ccc770d70135890c9a07d715cacb9032c90d539d0b3d209a8d600178bcfb416fd489e5d5dd56d9cfc6addae810ae70bdaee65672b871dc2b3f35ec00dbaa0d872f78cb58b3199984c608c8ba" + )); + } + + #[test] + fn test_decode_eip7702() { + test_decode::(&hex!( + "8808210881415c034feba383d7a6efd3f2601309b33a6d682ad47168cac0f7a5c5136a33370e5e7ca7f570d5530d7a0d18bf5eac33583fdc27b6580f61e8cbd34d6de596f925c1f353188feb2c1e9e20de82a80b57f0be425d8c5896280d4f5f66cdcfba256d0c9ac8abd833859a62ec019501b4585fa176f048de4f88b93bdefecfcaf4d8f0dd04767bc683a4569c893632e44ba9d53f90d758125c9b24c0192a649166520cd5eecbc110b53eda400cf184b8ef9932c81d0deb2ea27dfa863392a87bfd53af3ec67379f20992501e76e387cbe3933861beead1b49649383cf8b2a2d5c6d04b7edc376981ed9b12cf7199fe7fabf5198659e001bed40922969b82a6cd000000000000" + )); + } + + #[test] + fn test_decode_legacy() { + test_decode::(&hex!( + "112210080a8ba06a8d108540bb3140e9f71a0812c46226f9ea77ae880d98d19fe27e5911801175c3b32620b2e887af0296af343526e439b775ee3b1c06750058e9e5fc4cd5965c3010f86184" + )); + } + + #[cfg(feature = "optimism")] + #[test] + fn test_decode_deposit() { + test_decode::(&hex!( + "8108ac8f15983d59b6ae4911a00ff7bfcd2e53d2950926f8c82c12afad02861c46fcb293e776204052725e1c08ff2e9ff602ca916357601fa972a14094891fe3598b718758f22c46f163c18bcaa6296ce87e5267ef3fd932112842fbbf79011548cdf067d93ce6098dfc0aaf5a94531e439f30d6dfd0c6" + )); + } } diff --git a/crates/storage/codecs/src/alloy/transaction/optimism.rs b/crates/storage/codecs/src/alloy/transaction/optimism.rs index f4fdcf5ee44c..22f508fd4ceb 100644 --- a/crates/storage/codecs/src/alloy/transaction/optimism.rs +++ b/crates/storage/codecs/src/alloy/transaction/optimism.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyTxDeposit`] + use crate::Compact; use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; use op_alloy_consensus::TxDeposit as AlloyTxDeposit; @@ -12,7 +14,11 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with [`op_alloy_consensus::TxDeposit`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[add_arbitrary_tests(compact)] pub(crate) struct TxDeposit { source_hash: B256, diff --git a/crates/storage/codecs/src/alloy/withdrawal.rs b/crates/storage/codecs/src/alloy/withdrawal.rs index 16324c280cc2..0f3347cec1a5 100644 --- a/crates/storage/codecs/src/alloy/withdrawal.rs +++ b/crates/storage/codecs/src/alloy/withdrawal.rs @@ -1,3 +1,5 @@ +//! Compact implementation for [`AlloyWithdrawal`] + use crate::Compact; use alloy_eips::eip4895::Withdrawal as AlloyWithdrawal; use alloy_primitives::Address; @@ -7,7 +9,11 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with `alloy_eips::eip4895::Withdrawal` #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] -#[cfg_attr(test, derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize))] +#[cfg_attr( + any(test, feature = "test-utils"), + derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) +)] +#[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[add_arbitrary_tests(compact)] pub(crate) struct Withdrawal { /// Monotonically increasing identifier issued by consensus layer. diff --git a/crates/storage/codecs/src/lib.rs b/crates/storage/codecs/src/lib.rs index c432400a5766..598f2131bde6 100644 --- a/crates/storage/codecs/src/lib.rs +++ b/crates/storage/codecs/src/lib.rs @@ -18,6 +18,7 @@ #![cfg_attr(not(feature = "std"), no_std)] pub use reth_codecs_derive::*; +use serde as _; use alloy_primitives::{Address, Bloom, Bytes, FixedBytes, U256}; use bytes::{Buf, BufMut}; @@ -25,6 +26,10 @@ use bytes::{Buf, BufMut}; extern crate alloc; use alloc::vec::Vec; +#[cfg(feature = "test-utils")] +pub mod alloy; + +#[cfg(not(feature = "test-utils"))] #[cfg(any(test, feature = "alloy"))] mod alloy; diff --git a/crates/storage/codecs/src/test_utils.rs b/crates/storage/codecs/src/test_utils.rs index bb377c691678..b845645cb1ab 100644 --- a/crates/storage/codecs/src/test_utils.rs +++ b/crates/storage/codecs/src/test_utils.rs @@ -79,3 +79,12 @@ impl UnusedBits { matches!(self, Self::NotZero) } } + +/// Tests decoding and re-encoding to ensure correctness. +pub fn test_decode(buf: &[u8]) { + let (decoded, _) = T::from_compact(buf, buf.len()); + let mut encoded = Vec::with_capacity(buf.len()); + + decoded.to_compact(&mut encoded); + assert_eq!(buf, &encoded[..]); +} diff --git a/crates/storage/db-api/Cargo.toml b/crates/storage/db-api/Cargo.toml index 932a94b98ebc..f827e48c8c3c 100644 --- a/crates/storage/db-api/Cargo.toml +++ b/crates/storage/db-api/Cargo.toml @@ -62,7 +62,9 @@ test-utils = [ "reth-primitives-traits/test-utils", "reth-codecs/test-utils", "reth-db-models/test-utils", - "reth-trie-common/test-utils" + "reth-trie-common/test-utils", + "reth-prune-types/test-utils", + "reth-stages-types/test-utils" ] arbitrary = [ "reth-primitives/arbitrary", @@ -72,7 +74,10 @@ arbitrary = [ "reth-primitives-traits/arbitrary", "reth-trie-common/arbitrary", "alloy-primitives/arbitrary", - "parity-scale-codec/arbitrary" + "parity-scale-codec/arbitrary", + "reth-codecs/arbitrary", + "reth-prune-types/arbitrary", + "reth-stages-types/arbitrary" ] optimism = [ "reth-primitives/optimism", diff --git a/crates/storage/db-models/Cargo.toml b/crates/storage/db-models/Cargo.toml index 31741207cad2..d5f773347b09 100644 --- a/crates/storage/db-models/Cargo.toml +++ b/crates/storage/db-models/Cargo.toml @@ -48,5 +48,6 @@ arbitrary = [ "reth-primitives/arbitrary", "dep:arbitrary", "dep:proptest", - "alloy-primitives/arbitrary" + "alloy-primitives/arbitrary", + "reth-codecs/arbitrary" ] diff --git a/crates/storage/db-models/src/accounts.rs b/crates/storage/db-models/src/accounts.rs index b0099d22d5fb..acfd45fe34e8 100644 --- a/crates/storage/db-models/src/accounts.rs +++ b/crates/storage/db-models/src/accounts.rs @@ -8,7 +8,7 @@ use reth_primitives::Account; /// /// [`Address`] is the subkey. #[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)] -#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary, serde::Deserialize))] #[add_arbitrary_tests(compact)] pub struct AccountBeforeTx { /// Address for the account. Acts as `DupSort::SubKey`. diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index 2f437e631095..324411613fcc 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -97,7 +97,9 @@ test-utils = [ "reth-primitives-traits/test-utils", "reth-db-api/test-utils", "reth-nippy-jar/test-utils", - "reth-trie-common/test-utils" + "reth-trie-common/test-utils", + "reth-prune-types/test-utils", + "reth-stages-types/test-utils" ] bench = [] arbitrary = [ @@ -105,7 +107,9 @@ arbitrary = [ "reth-db-api/arbitrary", "reth-primitives-traits/arbitrary", "reth-trie-common/arbitrary", - "alloy-primitives/arbitrary" + "alloy-primitives/arbitrary", + "reth-prune-types/arbitrary", + "reth-stages-types/arbitrary" ] optimism = [ "reth-primitives/optimism", diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index b93c22cdf671..04a0bf42908e 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -108,7 +108,8 @@ serde = [ "notify/serde", "parking_lot/serde", "rand/serde", - "revm/serde" + "revm/serde", + "reth-codecs/serde" ] test-utils = [ "reth-db/test-utils", @@ -124,5 +125,7 @@ test-utils = [ "reth-codecs/test-utils", "reth-db-api/test-utils", "reth-trie-db/test-utils", - "revm/test-utils" + "revm/test-utils", + "reth-prune-types/test-utils", + "reth-stages-types/test-utils" ] diff --git a/crates/trie/common/Cargo.toml b/crates/trie/common/Cargo.toml index 2c6ccbfe689b..0616e2597109 100644 --- a/crates/trie/common/Cargo.toml +++ b/crates/trie/common/Cargo.toml @@ -55,5 +55,6 @@ arbitrary = [ "alloy-consensus/arbitrary", "alloy-primitives/arbitrary", "nybbles/arbitrary", - "revm-primitives/arbitrary" + "revm-primitives/arbitrary", + "reth-codecs/arbitrary" ] diff --git a/crates/trie/common/src/nibbles.rs b/crates/trie/common/src/nibbles.rs index 991fb68f3c09..cf94f135f54b 100644 --- a/crates/trie/common/src/nibbles.rs +++ b/crates/trie/common/src/nibbles.rs @@ -19,6 +19,7 @@ pub use nybbles::Nibbles; Deserialize, derive_more::Index, )] +#[cfg_attr(feature = "test-utils", derive(arbitrary::Arbitrary))] pub struct StoredNibbles(pub Nibbles); impl From for StoredNibbles { @@ -74,6 +75,7 @@ impl Compact for StoredNibbles { /// The representation of nibbles of the merkle trie stored in the database. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord, Hash, Deref)] +#[cfg_attr(feature = "test-utils", derive(arbitrary::Arbitrary))] pub struct StoredNibblesSubKey(pub Nibbles); impl From for StoredNibblesSubKey { diff --git a/crates/trie/trie/Cargo.toml b/crates/trie/trie/Cargo.toml index 112e661c0278..134a3055c2b7 100644 --- a/crates/trie/trie/Cargo.toml +++ b/crates/trie/trie/Cargo.toml @@ -79,7 +79,8 @@ test-utils = [ "triehash", "reth-trie-common/test-utils", "reth-primitives/test-utils", - "revm/test-utils" + "revm/test-utils", + "reth-stages-types/test-utils" ] [[bench]] From ea4fb26063221f2b173412fa64ee1660741181e1 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 25 Oct 2024 09:23:51 +0900 Subject: [PATCH 083/239] ci: enable `compact-codec` job (#12056) --- .github/workflows/compact.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index c7435220c0fd..63f6f282fa26 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -6,7 +6,11 @@ # 4) deserialize previously generated test vectors on: - workflow_dispatch: + + pull_request: + merge_group: + push: + branches: [main] env: CARGO_TERM_COLOR: always From 965dabacad948030059c937ebab0711c9fa56748 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Oct 2024 05:57:09 +0200 Subject: [PATCH 084/239] chore: add missing helpers to BestTransactions (#12044) --- crates/transaction-pool/src/traits.rs | 18 ++++++++++++++++++ crates/transaction-pool/tests/it/best.rs | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index fa5e22ec0aa3..e35912726133 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -747,6 +747,15 @@ pub trait BestTransactions: Iterator + Send { /// listen to pool updates. fn no_updates(&mut self); + /// Convenience function for [`Self::no_updates`] that returns the iterator again. + fn without_updates(mut self) -> Self + where + Self: Sized, + { + self.no_updates(); + self + } + /// Skip all blob transactions. /// /// There's only limited blob space available in a block, once exhausted, EIP-4844 transactions @@ -762,6 +771,15 @@ pub trait BestTransactions: Iterator + Send { /// If set to true, no blob transactions will be returned. fn set_skip_blobs(&mut self, skip_blobs: bool); + /// Convenience function for [`Self::skip_blobs`] that returns the iterator again. + fn without_blobs(mut self) -> Self + where + Self: Sized, + { + self.skip_blobs(); + self + } + /// Creates an iterator which uses a closure to determine whether a transaction should be /// returned by the iterator. /// diff --git a/crates/transaction-pool/tests/it/best.rs b/crates/transaction-pool/tests/it/best.rs index cd7a93eaedb9..20e833676436 100644 --- a/crates/transaction-pool/tests/it/best.rs +++ b/crates/transaction-pool/tests/it/best.rs @@ -5,6 +5,7 @@ use reth_transaction_pool::{noop::NoopTransactionPool, BestTransactions, Transac #[test] fn test_best_transactions() { let noop = NoopTransactionPool::default(); - let mut best = noop.best_transactions().filter_transactions(|_| true); + let mut best = + noop.best_transactions().filter_transactions(|_| true).without_blobs().without_updates(); assert!(best.next().is_none()); } From 674616cab976d3d297566373127c6b10e2fbd3ca Mon Sep 17 00:00:00 2001 From: Panagiotis Ganelis <50522617+PanGan21@users.noreply.github.com> Date: Fri, 25 Oct 2024 05:56:21 +0200 Subject: [PATCH 085/239] chore: change `net::discv5` log target to `discv5` (#12045) --- crates/net/discv5/src/lib.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/net/discv5/src/lib.rs b/crates/net/discv5/src/lib.rs index d4e8e928fda0..a8154b7bd19a 100644 --- a/crates/net/discv5/src/lib.rs +++ b/crates/net/discv5/src/lib.rs @@ -167,7 +167,7 @@ impl Discv5 { // let (enr, bc_enr, fork_key, rlpx_ip_mode) = build_local_enr(sk, &discv5_config); - trace!(target: "net::discv5", + trace!(target: "discv5", ?enr, "local ENR" ); @@ -271,7 +271,7 @@ impl Discv5 { // to them over RLPx, to be compatible with EL discv5 implementations that don't // enforce this security measure. - trace!(target: "net::discv5", + trace!(target: "discv5", ?enr, %socket, "discovered unverifiable enr, source socket doesn't match socket advertised in ENR" @@ -296,7 +296,7 @@ impl Discv5 { let node_record = match self.try_into_reachable(enr, socket) { Ok(enr_bc) => enr_bc, Err(err) => { - trace!(target: "net::discv5", + trace!(target: "discv5", %err, ?enr, "discovered peer is unreachable" @@ -308,7 +308,7 @@ impl Discv5 { } }; if let FilterOutcome::Ignore { reason } = self.filter_discovered_peer(enr) { - trace!(target: "net::discv5", + trace!(target: "discv5", ?enr, reason, "filtered out discovered peer" @@ -324,7 +324,7 @@ impl Discv5 { .then(|| self.get_fork_id(enr).ok()) .flatten(); - trace!(target: "net::discv5", + trace!(target: "discv5", ?fork_id, ?enr, "discovered peer" @@ -491,7 +491,7 @@ pub async fn bootstrap( bootstrap_nodes: HashSet, discv5: &Arc, ) -> Result<(), Error> { - trace!(target: "net::discv5", + trace!(target: "discv5", ?bootstrap_nodes, "adding bootstrap nodes .." ); @@ -508,7 +508,7 @@ pub async fn bootstrap( let discv5 = discv5.clone(); enr_requests.push(async move { if let Err(err) = discv5.request_enr(enode.to_string()).await { - debug!(target: "net::discv5", + debug!(target: "discv5", ?enode, %err, "failed adding boot node" @@ -545,7 +545,7 @@ pub fn spawn_populate_kbuckets_bg( for i in (0..bootstrap_lookup_countdown).rev() { let target = discv5::enr::NodeId::random(); - trace!(target: "net::discv5", + trace!(target: "discv5", %target, bootstrap_boost_runs_countdown=i, lookup_interval=format!("{:#?}", pulse_lookup_interval), @@ -563,7 +563,7 @@ pub fn spawn_populate_kbuckets_bg( // selection (ref kademlia) let target = get_lookup_target(kbucket_index, local_node_id); - trace!(target: "net::discv5", + trace!(target: "discv5", %target, lookup_interval=format!("{:#?}", lookup_interval), "starting periodic lookup query" @@ -628,11 +628,11 @@ pub async fn lookup( ); match discv5.find_node(target).await { - Err(err) => trace!(target: "net::discv5", + Err(err) => trace!(target: "discv5", %err, "lookup query failed" ), - Ok(peers) => trace!(target: "net::discv5", + Ok(peers) => trace!(target: "discv5", target=format!("{:#?}", target), peers_count=peers.len(), peers=format!("[{:#}]", peers.iter() @@ -645,7 +645,7 @@ pub async fn lookup( // `Discv5::connected_peers` can be subset of sessions, not all peers make it // into kbuckets, e.g. incoming sessions from peers with // unreachable enrs - debug!(target: "net::discv5", + debug!(target: "discv5", connected_peers=discv5.connected_peers(), "connected peers in routing table" ); From c3182f2a644c2f363f75bdeeee4c2bc69a7b9faf Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 25 Oct 2024 06:37:20 +0200 Subject: [PATCH 086/239] primitives-traits: small refac for `IntegerList` and more doc (#12049) --- crates/primitives-traits/src/integer_list.rs | 31 +++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/crates/primitives-traits/src/integer_list.rs b/crates/primitives-traits/src/integer_list.rs index 570c96c9fdaf..682fa0cf822f 100644 --- a/crates/primitives-traits/src/integer_list.rs +++ b/crates/primitives-traits/src/integer_list.rs @@ -9,8 +9,16 @@ use serde::{ Deserialize, Deserializer, Serialize, Serializer, }; -/// Uses Roaring Bitmaps to hold a list of integers. It provides really good compression with the -/// capability to access its elements without decoding it. +/// A data structure that uses Roaring Bitmaps to efficiently store a list of integers. +/// +/// This structure provides excellent compression while allowing direct access to individual +/// elements without the need for full decompression. +/// +/// Key features: +/// - Efficient compression: the underlying Roaring Bitmaps significantly reduce memory usage. +/// - Direct access: elements can be accessed or queried without needing to decode the entire list. +/// - [`RoaringTreemap`] backing: internally backed by [`RoaringTreemap`], which supports 64-bit +/// integers. #[derive(Clone, PartialEq, Default, Deref)] pub struct IntegerList(pub RoaringTreemap); @@ -22,12 +30,12 @@ impl fmt::Debug for IntegerList { } impl IntegerList { - /// Creates a new empty `IntegerList`. + /// Creates a new empty [`IntegerList`]. pub fn empty() -> Self { Self(RoaringTreemap::new()) } - /// Creates an `IntegerList` from a list of integers. + /// Creates an [`IntegerList`] from a list of integers. /// /// Returns an error if the list is not pre-sorted. pub fn new(list: impl IntoIterator) -> Result { @@ -36,7 +44,7 @@ impl IntegerList { .map_err(|_| IntegerListError::UnsortedInput) } - // Creates an IntegerList from a pre-sorted list of integers. + /// Creates an [`IntegerList`] from a pre-sorted list of integers. /// /// # Panics /// @@ -54,11 +62,7 @@ impl IntegerList { /// Pushes a new integer to the list. pub fn push(&mut self, value: u64) -> Result<(), IntegerListError> { - if self.0.push(value) { - Ok(()) - } else { - Err(IntegerListError::UnsortedInput) - } + self.0.push(value).then_some(()).ok_or(IntegerListError::UnsortedInput) } /// Clears the list. @@ -80,10 +84,9 @@ impl IntegerList { /// Deserializes a sequence of bytes into a proper [`IntegerList`]. pub fn from_bytes(data: &[u8]) -> Result { - Ok(Self( - RoaringTreemap::deserialize_from(data) - .map_err(|_| IntegerListError::FailedToDeserialize)?, - )) + RoaringTreemap::deserialize_from(data) + .map(Self) + .map_err(|_| IntegerListError::FailedToDeserialize) } } From d9889787a777738f0a62d6f933587b64ebf2b2d6 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 25 Oct 2024 09:01:44 +0400 Subject: [PATCH 087/239] feat: improve e2e tests API + feeHistory test (#12058) --- Cargo.lock | 24 ++++++ Cargo.toml | 1 + crates/e2e-test-utils/Cargo.toml | 1 + crates/e2e-test-utils/src/engine_api.rs | 3 +- crates/e2e-test-utils/src/lib.rs | 6 +- crates/e2e-test-utils/src/node.rs | 24 ++---- crates/e2e-test-utils/src/payload.rs | 23 +++-- crates/ethereum/node/Cargo.toml | 3 + crates/ethereum/node/tests/e2e/blobs.rs | 16 ++-- crates/ethereum/node/tests/e2e/dev.rs | 4 +- crates/ethereum/node/tests/e2e/eth.rs | 9 +- crates/ethereum/node/tests/e2e/main.rs | 1 + crates/ethereum/node/tests/e2e/p2p.rs | 7 +- crates/ethereum/node/tests/e2e/rpc.rs | 109 ++++++++++++++++++++++++ crates/optimism/node/tests/e2e/p2p.rs | 2 - crates/optimism/node/tests/e2e/utils.rs | 31 +++---- 16 files changed, 202 insertions(+), 62 deletions(-) create mode 100644 crates/ethereum/node/tests/e2e/rpc.rs diff --git a/Cargo.lock b/Cargo.lock index a73a4231533f..d3aa0b2953a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -128,6 +128,26 @@ dependencies = [ "serde_with", ] +[[package]] +name = "alloy-contract" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460ab80ce4bda1c80bcf96fe7460520476f2c7b734581c6567fac2708e2a60ef" +dependencies = [ + "alloy-dyn-abi", + "alloy-json-abi", + "alloy-network", + "alloy-network-primitives", + "alloy-primitives", + "alloy-provider", + "alloy-rpc-types-eth", + "alloy-sol-types", + "alloy-transport", + "futures", + "futures-util", + "thiserror", +] + [[package]] name = "alloy-dyn-abi" version = "0.8.8" @@ -7018,6 +7038,7 @@ dependencies = [ "alloy-rpc-types", "alloy-signer", "alloy-signer-local", + "derive_more 1.0.0", "eyre", "futures-util", "jsonrpsee", @@ -7971,10 +7992,13 @@ name = "reth-node-ethereum" version = "1.1.0" dependencies = [ "alloy-consensus", + "alloy-contract", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-provider", "alloy-signer", + "alloy-sol-types", "eyre", "futures", "rand 0.8.5", diff --git a/Cargo.toml b/Cargo.toml index e23efdeb3156..d01ee01ce5cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -433,6 +433,7 @@ alloy-sol-types = "0.8.0" alloy-trie = { version = "0.7", default-features = false } alloy-consensus = { version = "0.5.4", default-features = false } +alloy-contract = { version = "0.5.4", default-features = false } alloy-eips = { version = "0.5.4", default-features = false } alloy-genesis = { version = "0.5.4", default-features = false } alloy-json-rpc = { version = "0.5.4", default-features = false } diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index 04f031daa580..67bb74555366 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -45,3 +45,4 @@ alloy-rpc-types.workspace = true alloy-network.workspace = true alloy-consensus = { workspace = true, features = ["kzg"] } tracing.workspace = true +derive_more.workspace = true diff --git a/crates/e2e-test-utils/src/engine_api.rs b/crates/e2e-test-utils/src/engine_api.rs index 5027b2620a6d..729205211ff0 100644 --- a/crates/e2e-test-utils/src/engine_api.rs +++ b/crates/e2e-test-utils/src/engine_api.rs @@ -50,12 +50,13 @@ impl EngineApiTestContext, ) -> eyre::Result where E::ExecutionPayloadEnvelopeV3: From + PayloadEnvelopeExt, E::ExecutionPayloadEnvelopeV4: From + PayloadEnvelopeExt, { + let versioned_hashes = + payload.block().blob_versioned_hashes_iter().copied().collect::>(); // submit payload to engine api let submission = if self .chain_spec diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index f5ee1e5e6696..1e9717d082cd 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -49,6 +49,7 @@ pub async fn setup( num_nodes: usize, chain_spec: Arc, is_dev: bool, + attributes_generator: impl Fn(u64) -> <::Engine as PayloadTypes>::PayloadBuilderAttributes + Copy + 'static, ) -> eyre::Result<(Vec>, TaskManager, Wallet)> where N: Default + Node> + NodeTypesWithEngine, @@ -84,7 +85,7 @@ where .launch() .await?; - let mut node = NodeTestContext::new(node).await?; + let mut node = NodeTestContext::new(node, attributes_generator).await?; // Connect each node in a chain. if let Some(previous_node) = nodes.last_mut() { @@ -109,6 +110,7 @@ pub async fn setup_engine( num_nodes: usize, chain_spec: Arc, is_dev: bool, + attributes_generator: impl Fn(u64) -> <::Engine as PayloadTypes>::PayloadBuilderAttributes + Copy + 'static, ) -> eyre::Result<( Vec>>>, TaskManager, @@ -166,7 +168,7 @@ where }) .await?; - let mut node = NodeTestContext::new(node).await?; + let mut node = NodeTestContext::new(node, attributes_generator).await?; // Connect each node in a chain. if let Some(previous_node) = nodes.last_mut() { diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index f40072018044..07df36a33e1a 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -59,12 +59,15 @@ where AddOns: RethRpcAddOns, { /// Creates a new test node - pub async fn new(node: FullNode) -> eyre::Result { + pub async fn new( + node: FullNode, + attributes_generator: impl Fn(u64) -> Engine::PayloadBuilderAttributes + 'static, + ) -> eyre::Result { let builder = node.payload_builder.clone(); Ok(Self { inner: node.clone(), - payload: PayloadTestContext::new(builder).await?, + payload: PayloadTestContext::new(builder, attributes_generator).await?, network: NetworkTestContext::new(node.network.clone()), engine_api: EngineApiTestContext { chain_spec: node.chain_spec(), @@ -90,7 +93,6 @@ where &mut self, length: u64, tx_generator: impl Fn(u64) -> Pin>>, - attributes_generator: impl Fn(u64) -> Engine::PayloadBuilderAttributes + Copy, ) -> eyre::Result> where Engine::ExecutionPayloadEnvelopeV3: From + PayloadEnvelopeExt, @@ -101,7 +103,7 @@ where for i in 0..length { let raw_tx = tx_generator(i).await; let tx_hash = self.rpc.inject_tx(raw_tx).await?; - let (payload, eth_attr) = self.advance_block(vec![], attributes_generator).await?; + let (payload, eth_attr) = self.advance_block().await?; let block_hash = payload.block().hash(); let block_number = payload.block().number; self.assert_new_block(tx_hash, block_hash, block_number).await?; @@ -116,14 +118,13 @@ where /// It triggers the resolve payload via engine api and expects the built payload event. pub async fn new_payload( &mut self, - attributes_generator: impl Fn(u64) -> Engine::PayloadBuilderAttributes, ) -> eyre::Result<(Engine::BuiltPayload, Engine::PayloadBuilderAttributes)> where ::ExecutionPayloadEnvelopeV3: From + PayloadEnvelopeExt, { // trigger new payload building draining the pool - let eth_attr = self.payload.new_payload(attributes_generator).await.unwrap(); + let eth_attr = self.payload.new_payload().await.unwrap(); // first event is the payload attributes self.payload.expect_attr_event(eth_attr.clone()).await?; // wait for the payload builder to have finished building @@ -137,8 +138,6 @@ where /// Advances the node forward one block pub async fn advance_block( &mut self, - versioned_hashes: Vec, - attributes_generator: impl Fn(u64) -> Engine::PayloadBuilderAttributes, ) -> eyre::Result<(Engine::BuiltPayload, Engine::PayloadBuilderAttributes)> where ::ExecutionPayloadEnvelopeV3: @@ -146,16 +145,11 @@ where ::ExecutionPayloadEnvelopeV4: From + PayloadEnvelopeExt, { - let (payload, eth_attr) = self.new_payload(attributes_generator).await?; + let (payload, eth_attr) = self.new_payload().await?; let block_hash = self .engine_api - .submit_payload( - payload.clone(), - eth_attr.clone(), - PayloadStatusEnum::Valid, - versioned_hashes, - ) + .submit_payload(payload.clone(), eth_attr.clone(), PayloadStatusEnum::Valid) .await?; // trigger forkchoice update via engine api to commit the block to the blockchain diff --git a/crates/e2e-test-utils/src/payload.rs b/crates/e2e-test-utils/src/payload.rs index a5e4f56ac429..29aa11895b7d 100644 --- a/crates/e2e-test-utils/src/payload.rs +++ b/crates/e2e-test-utils/src/payload.rs @@ -5,29 +5,36 @@ use reth_payload_primitives::{Events, PayloadBuilder, PayloadTypes}; use tokio_stream::wrappers::BroadcastStream; /// Helper for payload operations -#[derive(Debug)] +#[derive(derive_more::Debug)] pub struct PayloadTestContext { pub payload_event_stream: BroadcastStream>, payload_builder: PayloadBuilderHandle, pub timestamp: u64, + #[debug(skip)] + attributes_generator: Box T::PayloadBuilderAttributes>, } impl PayloadTestContext { /// Creates a new payload helper - pub async fn new(payload_builder: PayloadBuilderHandle) -> eyre::Result { + pub async fn new( + payload_builder: PayloadBuilderHandle, + attributes_generator: impl Fn(u64) -> T::PayloadBuilderAttributes + 'static, + ) -> eyre::Result { let payload_events = payload_builder.subscribe().await?; let payload_event_stream = payload_events.into_stream(); // Cancun timestamp - Ok(Self { payload_event_stream, payload_builder, timestamp: 1710338135 }) + Ok(Self { + payload_event_stream, + payload_builder, + timestamp: 1710338135, + attributes_generator: Box::new(attributes_generator), + }) } /// Creates a new payload job from static attributes - pub async fn new_payload( - &mut self, - attributes_generator: impl Fn(u64) -> T::PayloadBuilderAttributes, - ) -> eyre::Result { + pub async fn new_payload(&mut self) -> eyre::Result { self.timestamp += 1; - let attributes = attributes_generator(self.timestamp); + let attributes = (self.attributes_generator)(self.timestamp); self.payload_builder.send_new_payload(attributes.clone()).await.unwrap()?; Ok(attributes) } diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index e7784637a062..98224b99e26b 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -55,6 +55,9 @@ alloy-consensus.workspace = true alloy-provider.workspace = true rand.workspace = true alloy-signer.workspace = true +alloy-eips.workspace = true +alloy-sol-types.workspace = true +alloy-contract.workspace = true [features] default = [] diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index b4d9a532aebe..976727bc8158 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -41,7 +41,7 @@ async fn can_handle_blobs() -> eyre::Result<()> { .launch() .await?; - let mut node = NodeTestContext::new(node).await?; + let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; let wallets = Wallet::new(2).gen(); let blob_wallet = wallets.first().unwrap(); @@ -51,7 +51,7 @@ async fn can_handle_blobs() -> eyre::Result<()> { let raw_tx = TransactionTestContext::transfer_tx_bytes(1, second_wallet.clone()).await; let tx_hash = node.rpc.inject_tx(raw_tx).await?; // build payload with normal tx - let (payload, attributes) = node.new_payload(eth_payload_attributes).await?; + let (payload, attributes) = node.new_payload().await?; // clean the pool node.inner.pool.remove_transactions(vec![tx_hash]); @@ -64,16 +64,14 @@ async fn can_handle_blobs() -> eyre::Result<()> { // fetch it from rpc let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; // validate sidecar - let versioned_hashes = TransactionTestContext::validate_sidecar(envelope); + TransactionTestContext::validate_sidecar(envelope); // build a payload - let (blob_payload, blob_attr) = node.new_payload(eth_payload_attributes).await?; + let (blob_payload, blob_attr) = node.new_payload().await?; // submit the blob payload - let blob_block_hash = node - .engine_api - .submit_payload(blob_payload, blob_attr, PayloadStatusEnum::Valid, versioned_hashes.clone()) - .await?; + let blob_block_hash = + node.engine_api.submit_payload(blob_payload, blob_attr, PayloadStatusEnum::Valid).await?; let (_, _) = tokio::join!( // send fcu with blob hash @@ -83,7 +81,7 @@ async fn can_handle_blobs() -> eyre::Result<()> { ); // submit normal payload - node.engine_api.submit_payload(payload, attributes, PayloadStatusEnum::Valid, vec![]).await?; + node.engine_api.submit_payload(payload, attributes, PayloadStatusEnum::Valid).await?; tokio::time::sleep(std::time::Duration::from_secs(3)).await; diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index cad2fb34e5de..ead438b5a670 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use crate::utils::eth_payload_attributes; use alloy_genesis::Genesis; use alloy_primitives::{b256, hex}; use futures::StreamExt; @@ -17,7 +18,8 @@ use reth_tasks::TaskManager; #[tokio::test] async fn can_run_dev_node() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let (mut nodes, _tasks, _) = setup::(1, custom_chain(), true).await?; + let (mut nodes, _tasks, _) = + setup::(1, custom_chain(), true, eth_payload_attributes).await?; assert_chain_advances(nodes.pop().unwrap().inner).await; Ok(()) diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index 14bfb92d4770..cb7517c0c932 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -26,6 +26,7 @@ async fn can_run_eth_node() -> eyre::Result<()> { .build(), ), false, + eth_payload_attributes, ) .await?; @@ -36,7 +37,7 @@ async fn can_run_eth_node() -> eyre::Result<()> { let tx_hash = node.rpc.inject_tx(raw_tx).await?; // make the node advance - let (payload, _) = node.advance_block(vec![], eth_payload_attributes).await?; + let (payload, _) = node.advance_block().await?; let block_hash = payload.block().hash(); let block_number = payload.block().number; @@ -74,7 +75,7 @@ async fn can_run_eth_node_with_auth_engine_api_over_ipc() -> eyre::Result<()> { .node(EthereumNode::default()) .launch() .await?; - let mut node = NodeTestContext::new(node).await?; + let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; // Configure wallet from test mnemonic and create dummy transfer tx let wallet = Wallet::default(); @@ -84,7 +85,7 @@ async fn can_run_eth_node_with_auth_engine_api_over_ipc() -> eyre::Result<()> { let tx_hash = node.rpc.inject_tx(raw_tx).await?; // make the node advance - let (payload, _) = node.advance_block(vec![], eth_payload_attributes).await?; + let (payload, _) = node.advance_block().await?; let block_hash = payload.block().hash(); let block_number = payload.block().number; @@ -120,7 +121,7 @@ async fn test_failed_run_eth_node_with_no_auth_engine_api_over_ipc_opts() -> eyr .launch() .await?; - let node = NodeTestContext::new(node).await?; + let node = NodeTestContext::new(node, eth_payload_attributes).await?; // Ensure that the engine api client is not available let client = node.inner.engine_ipc_client().await; diff --git a/crates/ethereum/node/tests/e2e/main.rs b/crates/ethereum/node/tests/e2e/main.rs index 5dff7be17e10..4ed8ac5fcb63 100644 --- a/crates/ethereum/node/tests/e2e/main.rs +++ b/crates/ethereum/node/tests/e2e/main.rs @@ -4,6 +4,7 @@ mod blobs; mod dev; mod eth; mod p2p; +mod rpc; mod utils; const fn main() {} diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index 0fae23a08571..180b88bbd5a2 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -30,6 +30,7 @@ async fn can_sync() -> eyre::Result<()> { .build(), ), false, + eth_payload_attributes, ) .await?; @@ -41,7 +42,7 @@ async fn can_sync() -> eyre::Result<()> { let tx_hash = first_node.rpc.inject_tx(raw_tx).await?; // make the node advance - let (payload, _) = first_node.advance_block(vec![], eth_payload_attributes).await?; + let (payload, _) = first_node.advance_block().await?; let block_hash = payload.block().hash(); let block_number = payload.block().number; @@ -76,7 +77,7 @@ async fn e2e_test_send_transactions() -> eyre::Result<()> { ); let (mut nodes, _tasks, wallet) = - setup_engine::(2, chain_spec.clone(), false).await?; + setup_engine::(2, chain_spec.clone(), false, eth_payload_attributes).await?; let mut node = nodes.pop().unwrap(); let signers = wallet.gen(); let provider = ProviderBuilder::new().with_recommended_fillers().on_http(node.rpc_url()); @@ -139,7 +140,7 @@ async fn e2e_test_send_transactions() -> eyre::Result<()> { pending.push(provider.send_tx_envelope(tx).await?); } - let (payload, _) = node.advance_block(vec![], eth_payload_attributes).await?; + let (payload, _) = node.advance_block().await?; assert!(payload.block().raw_transactions().len() == tx_count); for pending in pending { diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs new file mode 100644 index 000000000000..ddf3d5cba2a6 --- /dev/null +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -0,0 +1,109 @@ +use crate::utils::eth_payload_attributes; +use alloy_eips::calc_next_block_base_fee; +use alloy_primitives::U256; +use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder}; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_e2e_test_utils::setup_engine; +use reth_node_ethereum::EthereumNode; +use std::sync::Arc; + +alloy_sol_types::sol! { + #[sol(rpc, bytecode = "6080604052348015600f57600080fd5b5060405160db38038060db833981016040819052602a91607a565b60005b818110156074576040805143602082015290810182905260009060600160408051601f19818403018152919052805160209091012080555080606d816092565b915050602d565b505060b8565b600060208284031215608b57600080fd5b5051919050565b60006001820160b157634e487b7160e01b600052601160045260246000fd5b5060010190565b60168060c56000396000f3fe6080604052600080fdfea164736f6c6343000810000a")] + contract GasWaster { + constructor(uint256 iterations) { + for (uint256 i = 0; i < iterations; i++) { + bytes32 slot = keccak256(abi.encode(block.number, i)); + assembly { + sstore(slot, slot) + } + } + } + } +} + +#[tokio::test] +async fn test_fee_history() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let seed: [u8; 32] = rand::thread_rng().gen(); + let mut rng = StdRng::from_seed(seed); + println!("Seed: {:?}", seed); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + + let (mut nodes, _tasks, wallet) = + setup_engine::(1, chain_spec.clone(), false, eth_payload_attributes).await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .with_recommended_fillers() + .wallet(EthereumWallet::new(wallet.gen().swap_remove(0))) + .on_http(node.rpc_url()); + + let fee_history = provider.get_fee_history(10, 0_u64.into(), &[]).await?; + + let genesis_base_fee = chain_spec.initial_base_fee().unwrap() as u128; + let expected_first_base_fee = genesis_base_fee - + genesis_base_fee / chain_spec.base_fee_params_at_block(0).max_change_denominator; + assert_eq!(fee_history.base_fee_per_gas[0], genesis_base_fee); + assert_eq!(fee_history.base_fee_per_gas[1], expected_first_base_fee,); + + // Spend some gas + let builder = GasWaster::deploy_builder(&provider, U256::from(500)).send().await?; + node.advance_block().await?; + let receipt = builder.get_receipt().await?; + assert!(receipt.status()); + + let block = provider.get_block_by_number(1.into(), false).await?.unwrap(); + assert_eq!(block.header.gas_used as u128, receipt.gas_used,); + assert_eq!(block.header.base_fee_per_gas.unwrap(), expected_first_base_fee as u64); + + for _ in 0..100 { + let _ = + GasWaster::deploy_builder(&provider, U256::from(rng.gen_range(0..1000))).send().await?; + + node.advance_block().await?; + } + + let latest_block = provider.get_block_number().await?; + + for _ in 0..100 { + let latest_block = rng.gen_range(0..=latest_block); + let block_count = rng.gen_range(1..=(latest_block + 1)); + + let fee_history = provider.get_fee_history(block_count, latest_block.into(), &[]).await?; + + let mut prev_header = provider + .get_block_by_number((latest_block + 1 - block_count).into(), false) + .await? + .unwrap() + .header; + for block in (latest_block + 2 - block_count)..=latest_block { + let expected_base_fee = calc_next_block_base_fee( + prev_header.gas_used, + prev_header.gas_limit, + prev_header.base_fee_per_gas.unwrap(), + chain_spec.base_fee_params_at_block(block), + ); + + let header = provider.get_block_by_number(block.into(), false).await?.unwrap().header; + + assert_eq!(header.base_fee_per_gas.unwrap(), expected_base_fee as u64); + assert_eq!( + header.base_fee_per_gas.unwrap(), + fee_history.base_fee_per_gas[(block + block_count - 1 - latest_block) as usize] + as u64 + ); + + prev_header = header; + } + } + + Ok(()) +} diff --git a/crates/optimism/node/tests/e2e/p2p.rs b/crates/optimism/node/tests/e2e/p2p.rs index ebd35cc8a5c8..30affa9bafb9 100644 --- a/crates/optimism/node/tests/e2e/p2p.rs +++ b/crates/optimism/node/tests/e2e/p2p.rs @@ -51,7 +51,6 @@ async fn can_sync() -> eyre::Result<()> { side_payload_chain[0].0.clone(), side_payload_chain[0].1.clone(), PayloadStatusEnum::Valid, - Default::default(), ) .await; @@ -81,7 +80,6 @@ async fn can_sync() -> eyre::Result<()> { } .to_string(), }, - Default::default(), ) .await; diff --git a/crates/optimism/node/tests/e2e/utils.rs b/crates/optimism/node/tests/e2e/utils.rs index 8ea8df380b0b..48175e5b21ac 100644 --- a/crates/optimism/node/tests/e2e/utils.rs +++ b/crates/optimism/node/tests/e2e/utils.rs @@ -21,6 +21,7 @@ pub(crate) async fn setup(num_nodes: usize) -> eyre::Result<(Vec, TaskMa num_nodes, Arc::new(OpChainSpecBuilder::base_mainnet().genesis(genesis).ecotone_activated().build()), false, + optimism_payload_attributes, ) .await } @@ -31,23 +32,19 @@ pub(crate) async fn advance_chain( node: &mut OpNode, wallet: Arc>, ) -> eyre::Result> { - node.advance( - length as u64, - |_| { - let wallet = wallet.clone(); - Box::pin(async move { - let mut wallet = wallet.lock().await; - let tx_fut = TransactionTestContext::optimism_l1_block_info_tx( - wallet.chain_id, - wallet.inner.clone(), - wallet.inner_nonce, - ); - wallet.inner_nonce += 1; - tx_fut.await - }) - }, - optimism_payload_attributes, - ) + node.advance(length as u64, |_| { + let wallet = wallet.clone(); + Box::pin(async move { + let mut wallet = wallet.lock().await; + let tx_fut = TransactionTestContext::optimism_l1_block_info_tx( + wallet.chain_id, + wallet.inner.clone(), + wallet.inner_nonce, + ); + wallet.inner_nonce += 1; + tx_fut.await + }) + }) .await } From 5a5ec73c37d2f84e27db7bc86d5c7893f641ca87 Mon Sep 17 00:00:00 2001 From: tedison <76473430+edisontim@users.noreply.github.com> Date: Fri, 25 Oct 2024 01:02:43 -0400 Subject: [PATCH 088/239] Change return type of ReceiptBuilder (#11987) --- crates/rpc/rpc-eth-types/src/receipt.rs | 17 ++++------------- crates/rpc/rpc/src/eth/helpers/block.rs | 3 ++- crates/rpc/rpc/src/eth/helpers/receipt.rs | 3 ++- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/receipt.rs b/crates/rpc/rpc-eth-types/src/receipt.rs index c3232f2383b8..198ca79aa2ac 100644 --- a/crates/rpc/rpc-eth-types/src/receipt.rs +++ b/crates/rpc/rpc-eth-types/src/receipt.rs @@ -2,10 +2,8 @@ use alloy_consensus::Transaction; use alloy_primitives::{Address, TxKind}; -use alloy_rpc_types::{ - AnyReceiptEnvelope, AnyTransactionReceipt, Log, ReceiptWithBloom, TransactionReceipt, -}; -use alloy_serde::{OtherFields, WithOtherFields}; +use alloy_rpc_types::{AnyReceiptEnvelope, Log, ReceiptWithBloom, TransactionReceipt}; +use alloy_serde::OtherFields; use reth_primitives::{Receipt, TransactionMeta, TransactionSigned}; use revm_primitives::calc_blob_gasprice; @@ -111,15 +109,8 @@ impl ReceiptBuilder { Ok(Self { base, other: Default::default() }) } - /// Adds fields to response body. - pub fn add_other_fields(mut self, mut fields: OtherFields) -> Self { - self.other.append(&mut fields); - self - } - /// Builds a receipt response from the base response body, and any set additional fields. - pub fn build(self) -> AnyTransactionReceipt { - let Self { base, other } = self; - WithOtherFields { inner: base, other } + pub fn build(self) -> TransactionReceipt> { + self.base } } diff --git a/crates/rpc/rpc/src/eth/helpers/block.rs b/crates/rpc/rpc/src/eth/helpers/block.rs index b2ff30b88f24..b29a24c38c4c 100644 --- a/crates/rpc/rpc/src/eth/helpers/block.rs +++ b/crates/rpc/rpc/src/eth/helpers/block.rs @@ -1,6 +1,7 @@ //! Contains RPC handler implementations specific to blocks. use alloy_rpc_types::{AnyTransactionReceipt, BlockId}; +use alloy_serde::WithOtherFields; use reth_primitives::TransactionMeta; use reth_provider::{BlockReaderIdExt, HeaderProvider}; use reth_rpc_eth_api::{ @@ -55,9 +56,9 @@ where excess_blob_gas, timestamp, }; - ReceiptBuilder::new(&tx, meta, receipt, &receipts) .map(|builder| builder.build()) + .map(WithOtherFields::new) }) .collect::, Self::Error>>() .map(Some) diff --git a/crates/rpc/rpc/src/eth/helpers/receipt.rs b/crates/rpc/rpc/src/eth/helpers/receipt.rs index 2ac360944943..570ec4fa3c0f 100644 --- a/crates/rpc/rpc/src/eth/helpers/receipt.rs +++ b/crates/rpc/rpc/src/eth/helpers/receipt.rs @@ -1,5 +1,6 @@ //! Builds an RPC receipt response w.r.t. data layout of network. +use alloy_serde::WithOtherFields; use reth_primitives::{Receipt, TransactionMeta, TransactionSigned}; use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcReceipt}; use reth_rpc_eth_types::{EthApiError, EthStateCache, ReceiptBuilder}; @@ -30,6 +31,6 @@ where .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(hash.into()))?; - Ok(ReceiptBuilder::new(&tx, meta, &receipt, &all_receipts)?.build()) + Ok(WithOtherFields::new(ReceiptBuilder::new(&tx, meta, &receipt, &all_receipts)?.build())) } } From 269d705c706479cebe303bd527d84a900b068070 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Oct 2024 10:38:07 +0200 Subject: [PATCH 089/239] test: ensure we acquire file lock in tests (#12064) --- crates/storage/db/src/lockfile.rs | 51 ++++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/crates/storage/db/src/lockfile.rs b/crates/storage/db/src/lockfile.rs index 6dc063a167ac..a87ab7393f1f 100644 --- a/crates/storage/db/src/lockfile.rs +++ b/crates/storage/db/src/lockfile.rs @@ -30,31 +30,35 @@ impl StorageLock { /// Note: In-process exclusivity is not on scope. If called from the same process (or another /// with the same PID), it will succeed. pub fn try_acquire(path: &Path) -> Result { - let file_path = path.join(LOCKFILE_NAME); - #[cfg(feature = "disable-lock")] { + let file_path = path.join(LOCKFILE_NAME); // Too expensive for ef-tests to write/read lock to/from disk. Ok(Self(Arc::new(StorageLockInner { file_path }))) } #[cfg(not(feature = "disable-lock"))] - { - if let Some(process_lock) = ProcessUID::parse(&file_path)? { - if process_lock.pid != (process::id() as usize) && process_lock.is_active() { - error!( - target: "reth::db::lockfile", - path = ?file_path, - pid = process_lock.pid, - start_time = process_lock.start_time, - "Storage lock already taken." - ); - return Err(StorageLockError::Taken(process_lock.pid)) - } - } + Self::try_acquire_file_lock(path) + } - Ok(Self(Arc::new(StorageLockInner::new(file_path)?))) + /// Acquire a file write lock. + #[cfg(any(test, not(feature = "disable-lock")))] + fn try_acquire_file_lock(path: &Path) -> Result { + let file_path = path.join(LOCKFILE_NAME); + if let Some(process_lock) = ProcessUID::parse(&file_path)? { + if process_lock.pid != (process::id() as usize) && process_lock.is_active() { + error!( + target: "reth::db::lockfile", + path = ?file_path, + pid = process_lock.pid, + start_time = process_lock.start_time, + "Storage lock already taken." + ); + return Err(StorageLockError::Taken(process_lock.pid)) + } } + + Ok(Self(Arc::new(StorageLockInner::new(file_path)?))) } } @@ -164,10 +168,10 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap(); - let lock = StorageLock::try_acquire(temp_dir.path()).unwrap(); + let lock = StorageLock::try_acquire_file_lock(temp_dir.path()).unwrap(); // Same process can re-acquire the lock - assert_eq!(Ok(lock.clone()), StorageLock::try_acquire(temp_dir.path())); + assert_eq!(Ok(lock.clone()), StorageLock::try_acquire_file_lock(temp_dir.path())); // A lock of a non existent PID can be acquired. let lock_file = temp_dir.path().join(LOCKFILE_NAME); @@ -177,18 +181,21 @@ mod tests { fake_pid += 1; } ProcessUID { pid: fake_pid, start_time: u64::MAX }.write(&lock_file).unwrap(); - assert_eq!(Ok(lock.clone()), StorageLock::try_acquire(temp_dir.path())); + assert_eq!(Ok(lock.clone()), StorageLock::try_acquire_file_lock(temp_dir.path())); let mut pid_1 = ProcessUID::new(1).unwrap(); // If a parsed `ProcessUID` exists, the lock can NOT be acquired. pid_1.write(&lock_file).unwrap(); - assert_eq!(Err(StorageLockError::Taken(1)), StorageLock::try_acquire(temp_dir.path())); + assert_eq!( + Err(StorageLockError::Taken(1)), + StorageLock::try_acquire_file_lock(temp_dir.path()) + ); // A lock of a different but existing PID can be acquired ONLY IF the start_time differs. pid_1.start_time += 1; pid_1.write(&lock_file).unwrap(); - assert_eq!(Ok(lock), StorageLock::try_acquire(temp_dir.path())); + assert_eq!(Ok(lock), StorageLock::try_acquire_file_lock(temp_dir.path())); } #[test] @@ -198,7 +205,7 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap(); let lock_file = temp_dir.path().join(LOCKFILE_NAME); - let lock = StorageLock::try_acquire(temp_dir.path()).unwrap(); + let lock = StorageLock::try_acquire_file_lock(temp_dir.path()).unwrap(); assert!(lock_file.exists()); drop(lock); From 2ae7ee51e0beb1920d4ab29e3736756321ccdee0 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 25 Oct 2024 19:13:46 +0900 Subject: [PATCH 090/239] fix: increase `arbitrary::Unstructured` buffer size if `NotEnoughData` is thrown (#12069) --- crates/cli/commands/src/test_vectors/compact.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/cli/commands/src/test_vectors/compact.rs b/crates/cli/commands/src/test_vectors/compact.rs index cda7d5bd578c..162ee1ceaa49 100644 --- a/crates/cli/commands/src/test_vectors/compact.rs +++ b/crates/cli/commands/src/test_vectors/compact.rs @@ -191,7 +191,21 @@ where runner.rng().fill_bytes(&mut bytes); compact_buffer.clear(); - let obj = T::arbitrary(&mut arbitrary::Unstructured::new(&bytes))?; + // Sometimes type T, might require extra arbitrary data, so we retry it a few times. + let mut tries = 0; + let obj = loop { + match T::arbitrary(&mut arbitrary::Unstructured::new(&bytes)) { + Ok(obj) => break obj, + Err(err) => { + if tries < 5 && matches!(err, arbitrary::Error::NotEnoughData) { + tries += 1; + bytes.extend(std::iter::repeat(0u8).take(256)); + } else { + return Err(err)? + } + } + } + }; let res = obj.to_compact(&mut compact_buffer); if IDENTIFIER_TYPE.contains(&type_name) { From a87d654c55eacc8f4878affd671e08df7fa536a6 Mon Sep 17 00:00:00 2001 From: tedison <76473430+edisontim@users.noreply.github.com> Date: Fri, 25 Oct 2024 07:48:58 -0400 Subject: [PATCH 091/239] feat: introduce iterator for default_ethereum_payload function (#11978) --- crates/ethereum/payload/src/lib.rs | 46 ++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 951a909b91ca..bb611441f032 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -33,7 +33,8 @@ use reth_primitives::{ use reth_provider::{ChainSpecProvider, StateProviderFactory}; use reth_revm::database::StateProviderDatabase; use reth_transaction_pool::{ - noop::NoopTransactionPool, BestTransactionsAttributes, TransactionPool, + noop::NoopTransactionPool, BestTransactions, BestTransactionsAttributes, TransactionPool, + ValidPoolTransaction, }; use reth_trie::HashedPostState; use revm::{ @@ -45,6 +46,10 @@ use revm_primitives::calc_excess_blob_gas; use std::sync::Arc; use tracing::{debug, trace, warn}; +type BestTransactionsIter = Box< + dyn BestTransactions::Transaction>>>, +>; + /// Ethereum payload builder #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct EthereumPayloadBuilder { @@ -94,7 +99,11 @@ where args: BuildArguments, ) -> Result, PayloadBuilderError> { let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); - default_ethereum_payload(self.evm_config.clone(), args, cfg_env, block_env) + + let pool = args.pool.clone(); + default_ethereum_payload(self.evm_config.clone(), args, cfg_env, block_env, |attributes| { + pool.best_transactions_with_attributes(attributes) + }) } fn build_empty_payload( @@ -102,19 +111,25 @@ where client: &Client, config: PayloadConfig, ) -> Result { - let args = BuildArguments { + let args = BuildArguments::new( client, - config, // we use defaults here because for the empty payload we don't need to execute anything - pool: NoopTransactionPool::default(), - cached_reads: Default::default(), - cancel: Default::default(), - best_payload: None, - }; + NoopTransactionPool::default(), + Default::default(), + config, + Default::default(), + None, + ); + let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); - default_ethereum_payload(self.evm_config.clone(), args, cfg_env, block_env)? - .into_payload() - .ok_or_else(|| PayloadBuilderError::MissingPayload) + + let pool = args.pool.clone(); + + default_ethereum_payload(self.evm_config.clone(), args, cfg_env, block_env, |attributes| { + pool.best_transactions_with_attributes(attributes) + })? + .into_payload() + .ok_or_else(|| PayloadBuilderError::MissingPayload) } } @@ -124,16 +139,18 @@ where /// and configuration, this function creates a transaction payload. Returns /// a result indicating success with the payload or an error in case of failure. #[inline] -pub fn default_ethereum_payload( +pub fn default_ethereum_payload( evm_config: EvmConfig, args: BuildArguments, initialized_cfg: CfgEnvWithHandlerCfg, initialized_block_env: BlockEnv, + best_txs: F, ) -> Result, PayloadBuilderError> where EvmConfig: ConfigureEvm
, Client: StateProviderFactory + ChainSpecProvider, Pool: TransactionPool, + F: FnOnce(BestTransactionsAttributes) -> BestTransactionsIter, { let BuildArguments { client, pool, mut cached_reads, config, cancel, best_payload } = args; @@ -153,11 +170,10 @@ where let mut executed_txs = Vec::new(); let mut executed_senders = Vec::new(); - let mut best_txs = pool.best_transactions_with_attributes(BestTransactionsAttributes::new( + let mut best_txs = best_txs(BestTransactionsAttributes::new( base_fee, initialized_block_env.get_blob_gasprice().map(|gasprice| gasprice as u64), )); - let mut total_fees = U256::ZERO; let block_number = initialized_block_env.number.to::(); From 58441c158b27986b904efccf6cbe5953d090e083 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 25 Oct 2024 16:10:20 +0400 Subject: [PATCH 092/239] fix: fail on unwind during `reth import` (#12062) --- crates/cli/commands/src/import.rs | 1 + .../cli/src/commands/build_pipeline.rs | 1 + crates/stages/api/src/error.rs | 3 +++ crates/stages/api/src/pipeline/builder.rs | 19 +++++++++++++++++-- crates/stages/api/src/pipeline/mod.rs | 8 ++++++++ 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/crates/cli/commands/src/import.rs b/crates/cli/commands/src/import.rs index 6b750d32a3df..a7c81e53052b 100644 --- a/crates/cli/commands/src/import.rs +++ b/crates/cli/commands/src/import.rs @@ -207,6 +207,7 @@ where .with_tip_sender(tip_tx) // we want to sync all blocks the file client provides or 0 if empty .with_max_block(max_block) + .with_fail_on_unwind(true) .add_stages( DefaultStages::new( provider_factory.clone(), diff --git a/crates/optimism/cli/src/commands/build_pipeline.rs b/crates/optimism/cli/src/commands/build_pipeline.rs index f23cb9a7c167..a197f93a8b42 100644 --- a/crates/optimism/cli/src/commands/build_pipeline.rs +++ b/crates/optimism/cli/src/commands/build_pipeline.rs @@ -75,6 +75,7 @@ where .with_tip_sender(tip_tx) // we want to sync all blocks the file client provides or 0 if empty .with_max_block(max_block) + .with_fail_on_unwind(true) .add_stages( DefaultStages::new( provider_factory.clone(), diff --git a/crates/stages/api/src/error.rs b/crates/stages/api/src/error.rs index 68e1d00fdae0..8562b10b6a55 100644 --- a/crates/stages/api/src/error.rs +++ b/crates/stages/api/src/error.rs @@ -188,4 +188,7 @@ pub enum PipelineError { /// Internal error #[error(transparent)] Internal(#[from] RethError), + /// The pipeline encountered an unwind when `fail_on_unwind` was set to `true`. + #[error("unexpected unwind")] + UnexpectedUnwind, } diff --git a/crates/stages/api/src/pipeline/builder.rs b/crates/stages/api/src/pipeline/builder.rs index 79a4c477ee6c..45bdc2d89427 100644 --- a/crates/stages/api/src/pipeline/builder.rs +++ b/crates/stages/api/src/pipeline/builder.rs @@ -14,6 +14,7 @@ pub struct PipelineBuilder { /// A receiver for the current chain tip to sync to. tip_tx: Option>, metrics_tx: Option, + fail_on_unwind: bool, } impl PipelineBuilder { @@ -62,6 +63,12 @@ impl PipelineBuilder { self } + /// Set whether pipeline should fail on unwind. + pub const fn with_fail_on_unwind(mut self, yes: bool) -> Self { + self.fail_on_unwind = yes; + self + } + /// Builds the final [`Pipeline`] using the given database. pub fn build( self, @@ -72,7 +79,7 @@ impl PipelineBuilder { N: ProviderNodeTypes, ProviderFactory: DatabaseProviderFactory, { - let Self { stages, max_block, tip_tx, metrics_tx } = self; + let Self { stages, max_block, tip_tx, metrics_tx, fail_on_unwind } = self; Pipeline { provider_factory, stages, @@ -82,13 +89,20 @@ impl PipelineBuilder { event_sender: Default::default(), progress: Default::default(), metrics_tx, + fail_on_unwind, } } } impl Default for PipelineBuilder { fn default() -> Self { - Self { stages: Vec::new(), max_block: None, tip_tx: None, metrics_tx: None } + Self { + stages: Vec::new(), + max_block: None, + tip_tx: None, + metrics_tx: None, + fail_on_unwind: false, + } } } @@ -97,6 +111,7 @@ impl std::fmt::Debug for PipelineBuilder { f.debug_struct("PipelineBuilder") .field("stages", &self.stages.iter().map(|stage| stage.id()).collect::>()) .field("max_block", &self.max_block) + .field("fail_on_unwind", &self.fail_on_unwind) .finish() } } diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 14225a595285..399a3ffb4b79 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -78,6 +78,9 @@ pub struct Pipeline { /// A receiver for the current chain tip to sync to. tip_tx: Option>, metrics_tx: Option, + /// Whether an unwind should fail the syncing process. Should only be set when downloading + /// blocks from trusted sources and expecting them to be valid. + fail_on_unwind: bool, } impl Pipeline { @@ -164,6 +167,10 @@ impl Pipeline { loop { let next_action = self.run_loop().await?; + if next_action.is_unwind() && self.fail_on_unwind { + return Err(PipelineError::UnexpectedUnwind) + } + // Terminate the loop early if it's reached the maximum user // configured block. if next_action.should_continue() && @@ -586,6 +593,7 @@ impl std::fmt::Debug for Pipeline { .field("stages", &self.stages.iter().map(|stage| stage.id()).collect::>()) .field("max_block", &self.max_block) .field("event_sender", &self.event_sender) + .field("fail_on_unwind", &self.fail_on_unwind) .finish() } } From 26d1b1524bea366e886ecf27a0e76e53d07aa0a5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Oct 2024 14:17:07 +0200 Subject: [PATCH 093/239] fix: ignore discovered peers with tcp port 0 (#12065) --- crates/net/network/src/discovery.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/net/network/src/discovery.rs b/crates/net/network/src/discovery.rs index d366027d680b..5b2bb788f478 100644 --- a/crates/net/network/src/discovery.rs +++ b/crates/net/network/src/discovery.rs @@ -214,6 +214,10 @@ impl Discovery { fn on_node_record_update(&mut self, record: NodeRecord, fork_id: Option) { let peer_id = record.id; let tcp_addr = record.tcp_addr(); + if tcp_addr.port() == 0 { + // useless peer for p2p + return + } let udp_addr = record.udp_addr(); let addr = PeerAddr::new(tcp_addr, Some(udp_addr)); _ = From 07bda5d453749a513fdd5afc5ea48dd90a3ba5a0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Oct 2024 14:17:32 +0200 Subject: [PATCH 094/239] chore: EthBuiltPayload touchups (#12067) --- .../ethereum/engine-primitives/src/payload.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/ethereum/engine-primitives/src/payload.rs b/crates/ethereum/engine-primitives/src/payload.rs index 1ad9c5450ee7..50c4852545b4 100644 --- a/crates/ethereum/engine-primitives/src/payload.rs +++ b/crates/ethereum/engine-primitives/src/payload.rs @@ -38,7 +38,9 @@ pub struct EthBuiltPayload { // === impl BuiltPayload === impl EthBuiltPayload { - /// Initializes the payload with the given initial block. + /// Initializes the payload with the given initial block + /// + /// Caution: This does not set any [`BlobTransactionSidecar`]. pub const fn new( id: PayloadId, block: SealedBlock, @@ -69,9 +71,18 @@ impl EthBuiltPayload { } /// Adds sidecars to the payload. - pub fn extend_sidecars(&mut self, sidecars: Vec) { + pub fn extend_sidecars(&mut self, sidecars: impl IntoIterator) { self.sidecars.extend(sidecars) } + + /// Same as [`Self::extend_sidecars`] but returns the type again. + pub fn with_sidecars( + mut self, + sidecars: impl IntoIterator, + ) -> Self { + self.extend_sidecars(sidecars); + self + } } impl BuiltPayload for EthBuiltPayload { @@ -134,7 +145,7 @@ impl From for ExecutionPayloadEnvelopeV3 { // Spec: // should_override_builder: false, - blobs_bundle: sidecars.into_iter().map(Into::into).collect::>().into(), + blobs_bundle: sidecars.into(), } } } From 09506aa130b4c0dae3fb7029f27845b4f2c58321 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Oct 2024 14:25:52 +0200 Subject: [PATCH 095/239] chore: rm TransactionFilter (#12066) --- crates/transaction-pool/src/traits.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index e35912726133..c21a7a4ea75a 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -815,25 +815,6 @@ where } } -/// A subtrait on the [`BestTransactions`] trait that allows to filter transactions. -pub trait BestTransactionsFilter: BestTransactions { - /// Creates an iterator which uses a closure to determine if a transaction should be yielded. - /// - /// Given an element the closure must return true or false. The returned iterator will yield - /// only the elements for which the closure returns true. - /// - /// Descendant transactions will be skipped. - fn filter

(self, predicate: P) -> BestTransactionFilter - where - P: FnMut(&Self::Item) -> bool, - Self: Sized, - { - BestTransactionFilter::new(self, predicate) - } -} - -impl BestTransactionsFilter for T where T: BestTransactions {} - /// A no-op implementation that yields no transactions. impl BestTransactions for std::iter::Empty { fn mark_invalid(&mut self, _tx: &T) {} From e93e373853ae2735fc6ca86ef0273bcacfd15c31 Mon Sep 17 00:00:00 2001 From: Debjit Bhowal Date: Fri, 25 Oct 2024 19:51:17 +0530 Subject: [PATCH 096/239] making `command` public (#12074) --- bin/reth/src/cli/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 01662eb4dcb6..192ab670028a 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -39,7 +39,7 @@ pub struct Cli, + pub command: Commands, /// The chain this node is running. /// @@ -52,7 +52,7 @@ pub struct Cli, + pub chain: Arc, /// Add a new instance of a node. /// @@ -68,10 +68,11 @@ pub struct Cli Date: Fri, 25 Oct 2024 18:39:52 +0200 Subject: [PATCH 097/239] primitive-traits: add unit tests for `Account` (#12048) --- crates/primitives-traits/src/account.rs | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/primitives-traits/src/account.rs b/crates/primitives-traits/src/account.rs index 063504b2a0e6..ae58973edd71 100644 --- a/crates/primitives-traits/src/account.rs +++ b/crates/primitives-traits/src/account.rs @@ -256,4 +256,50 @@ mod tests { assert_eq!(decoded, bytecode); assert!(remainder.is_empty()); } + + #[test] + fn test_account_has_bytecode() { + // Account with no bytecode (None) + let acc_no_bytecode = Account { nonce: 1, balance: U256::from(1000), bytecode_hash: None }; + assert!(!acc_no_bytecode.has_bytecode(), "Account should not have bytecode"); + + // Account with bytecode hash set to KECCAK_EMPTY (should have bytecode) + let acc_empty_bytecode = + Account { nonce: 1, balance: U256::from(1000), bytecode_hash: Some(KECCAK_EMPTY) }; + assert!(acc_empty_bytecode.has_bytecode(), "Account should have bytecode"); + + // Account with a non-empty bytecode hash + let acc_with_bytecode = Account { + nonce: 1, + balance: U256::from(1000), + bytecode_hash: Some(B256::from_slice(&[0x11u8; 32])), + }; + assert!(acc_with_bytecode.has_bytecode(), "Account should have bytecode"); + } + + #[test] + fn test_account_get_bytecode_hash() { + // Account with no bytecode (should return KECCAK_EMPTY) + let acc_no_bytecode = Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None }; + assert_eq!(acc_no_bytecode.get_bytecode_hash(), KECCAK_EMPTY, "Should return KECCAK_EMPTY"); + + // Account with bytecode hash set to KECCAK_EMPTY + let acc_empty_bytecode = + Account { nonce: 1, balance: U256::from(1000), bytecode_hash: Some(KECCAK_EMPTY) }; + assert_eq!( + acc_empty_bytecode.get_bytecode_hash(), + KECCAK_EMPTY, + "Should return KECCAK_EMPTY" + ); + + // Account with a valid bytecode hash + let bytecode_hash = B256::from_slice(&[0x11u8; 32]); + let acc_with_bytecode = + Account { nonce: 1, balance: U256::from(1000), bytecode_hash: Some(bytecode_hash) }; + assert_eq!( + acc_with_bytecode.get_bytecode_hash(), + bytecode_hash, + "Should return the bytecode hash" + ); + } } From e676d71d0b5c4f372435277031e41afde349a073 Mon Sep 17 00:00:00 2001 From: tedison <76473430+edisontim@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:08:01 -0400 Subject: [PATCH 098/239] feat: Freeze payload if final (#12078) Co-authored-by: Matthias Seitz --- crates/optimism/payload/src/builder.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index c85abfad7c4e..85f687aa803b 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -522,6 +522,8 @@ where trie: Arc::new(trie_output), }; + let no_tx_pool = attributes.no_tx_pool; + let mut payload = OptimismBuiltPayload::new( attributes.payload_attributes.id, sealed_block, @@ -534,5 +536,12 @@ where // extend the payload with the blob sidecars from the executed txs payload.extend_sidecars(blob_sidecars); - Ok(BuildOutcome::Better { payload, cached_reads }) + if no_tx_pool { + // if `no_tx_pool` is set only transactions from the payload attributes will be included in + // the payload. In other words, the payload is deterministic and we can freeze it once we've + // successfully built it. + Ok(BuildOutcome::Freeze(payload)) + } else { + Ok(BuildOutcome::Better { payload, cached_reads }) + } } From d91cacd14ac56495100a7785352e1a1ed8c8a42d Mon Sep 17 00:00:00 2001 From: Jeff Date: Fri, 25 Oct 2024 13:11:24 -0400 Subject: [PATCH 099/239] feat(rpc): rpc rate limiter impl (#11952) Co-authored-by: Matthias Seitz --- Cargo.lock | 1 + crates/rpc/rpc-builder/Cargo.toml | 2 + crates/rpc/rpc-builder/src/lib.rs | 3 + crates/rpc/rpc-builder/src/rate_limiter.rs | 116 +++++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 crates/rpc/rpc-builder/src/rate_limiter.rs diff --git a/Cargo.lock b/Cargo.lock index d3aa0b2953a0..a99803e1bc4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8744,6 +8744,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "tokio-util", "tower 0.4.13", "tower-http", "tracing", diff --git a/crates/rpc/rpc-builder/Cargo.toml b/crates/rpc/rpc-builder/Cargo.toml index cc72c2ebf92e..b9b511a078bc 100644 --- a/crates/rpc/rpc-builder/Cargo.toml +++ b/crates/rpc/rpc-builder/Cargo.toml @@ -50,6 +50,8 @@ metrics.workspace = true serde = { workspace = true, features = ["derive"] } thiserror.workspace = true tracing.workspace = true +tokio-util = { workspace = true } +tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } [dev-dependencies] reth-chainspec.workspace = true diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 72b53efe674c..ceafe206531a 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -226,6 +226,9 @@ pub use eth::EthHandlers; mod metrics; pub use metrics::{MeteredRequestFuture, RpcRequestMetricsService}; +// Rpc rate limiter +pub mod rate_limiter; + /// Convenience function for starting a server in one step. #[allow(clippy::too_many_arguments)] pub async fn launch( diff --git a/crates/rpc/rpc-builder/src/rate_limiter.rs b/crates/rpc/rpc-builder/src/rate_limiter.rs new file mode 100644 index 000000000000..85df0eee61c6 --- /dev/null +++ b/crates/rpc/rpc-builder/src/rate_limiter.rs @@ -0,0 +1,116 @@ +//! [`jsonrpsee`] helper layer for rate limiting certain methods. + +use jsonrpsee::{server::middleware::rpc::RpcServiceT, types::Request, MethodResponse}; +use std::{ + future::Future, + pin::Pin, + sync::Arc, + task::{ready, Context, Poll}, +}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio_util::sync::PollSemaphore; +use tower::Layer; + +/// Rate limiter for the RPC server. +/// +/// Rate limits expensive calls such as debug_ and trace_. +#[derive(Debug, Clone)] +pub struct RpcRequestRateLimiter { + inner: Arc, +} + +impl RpcRequestRateLimiter { + /// Create a new rate limit layer with the given number of permits. + pub fn new(rate_limit: usize) -> Self { + Self { + inner: Arc::new(RpcRequestRateLimiterInner { + call_guard: PollSemaphore::new(Arc::new(Semaphore::new(rate_limit))), + }), + } + } +} + +impl Layer for RpcRequestRateLimiter { + type Service = RpcRequestRateLimitingService; + + fn layer(&self, inner: S) -> Self::Service { + RpcRequestRateLimitingService::new(inner, self.clone()) + } +} + +/// Rate Limiter for the RPC server +#[derive(Debug, Clone)] +struct RpcRequestRateLimiterInner { + /// Semaphore to rate limit calls + call_guard: PollSemaphore, +} + +/// A [`RpcServiceT`] middleware that rate limits RPC calls to the server. +#[derive(Debug, Clone)] +pub struct RpcRequestRateLimitingService { + /// The rate limiter for RPC requests + rate_limiter: RpcRequestRateLimiter, + /// The inner service being wrapped + inner: S, +} + +impl RpcRequestRateLimitingService { + /// Create a new rate limited service. + pub const fn new(service: S, rate_limiter: RpcRequestRateLimiter) -> Self { + Self { inner: service, rate_limiter } + } +} + +impl<'a, S> RpcServiceT<'a> for RpcRequestRateLimitingService +where + S: RpcServiceT<'a> + Send + Sync + Clone + 'static, +{ + type Future = RateLimitingRequestFuture; + + fn call(&self, req: Request<'a>) -> Self::Future { + let method_name = req.method_name(); + if method_name.starts_with("trace_") || method_name.starts_with("debug_") { + RateLimitingRequestFuture { + fut: self.inner.call(req), + guard: Some(self.rate_limiter.inner.call_guard.clone()), + permit: None, + } + } else { + // if we don't need to rate limit, then there + // is no need to get a semaphore permit + RateLimitingRequestFuture { fut: self.inner.call(req), guard: None, permit: None } + } + } +} + +/// Response future. +#[pin_project::pin_project] +pub struct RateLimitingRequestFuture { + #[pin] + fut: F, + guard: Option, + permit: Option, +} + +impl std::fmt::Debug for RateLimitingRequestFuture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("RateLimitingRequestFuture") + } +} + +impl> Future for RateLimitingRequestFuture { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + if let Some(guard) = this.guard.as_mut() { + *this.permit = ready!(guard.poll_acquire(cx)); + *this.guard = None; + } + let res = this.fut.poll(cx); + if res.is_ready() { + *this.permit = None; + } + res + } +} From 16b64d8284a87888bae296992a7e4789594e8d16 Mon Sep 17 00:00:00 2001 From: Kien Trinh <51135161+kien6034@users.noreply.github.com> Date: Sat, 26 Oct 2024 03:22:02 +0700 Subject: [PATCH 100/239] feat(make): add docs lint (#12082) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 5ad7abac6755..ac4ad1038582 100644 --- a/Makefile +++ b/Makefile @@ -497,6 +497,7 @@ test: pr: make lint && \ make update-book-cli && \ + cargo docs --document-private-items && \ make test check-features: From a349919b5c1b3f4b28959acebfa55f6786845fe9 Mon Sep 17 00:00:00 2001 From: AJStonewee Date: Fri, 25 Oct 2024 18:44:00 -0400 Subject: [PATCH 101/239] docs: remove deleted op-sync workflow from docs (#12086) Co-authored-by: Oliver --- docs/repo/ci.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/repo/ci.md b/docs/repo/ci.md index 5ed2cec0091e..863a18f9c383 100644 --- a/docs/repo/ci.md +++ b/docs/repo/ci.md @@ -7,8 +7,7 @@ The CI runs a couple of workflows: - **[unit]**: Runs unit tests (tests in `src/`) and doc tests - **[integration]**: Runs integration tests (tests in `tests/` and sync tests) - **[bench]**: Runs benchmarks -- **[eth-sync]**: Runs Ethereum mainnet sync tests -- **[op-sync]**: Runs base mainnet sync tests for Optimism +- **[sync]**: Runs sync tests - **[stage]**: Runs all `stage run` commands ### Docs @@ -38,8 +37,7 @@ The CI runs a couple of workflows: [unit]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/unit.yml [integration]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/integration.yml [bench]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/bench.yml -[eth-sync]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/eth-sync.yml -[op-sync]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/op-sync.yml +[sync]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/sync.yml [stage]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/stage.yml [book]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/book.yml [deny]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/deny.yml From fa59bd512e877e4677890b5c154b80816c07480e Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sat, 26 Oct 2024 03:18:34 +0400 Subject: [PATCH 102/239] fix: correctly detect first sync on headers stage (#12085) --- crates/stages/stages/src/stages/headers.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index 199e015c2dce..49e687a96a1d 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -2,7 +2,7 @@ use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256}; use futures_util::StreamExt; use reth_config::config::EtlConfig; use reth_consensus::Consensus; -use reth_db::{tables, RawKey, RawTable, RawValue}; +use reth_db::{tables, transaction::DbTx, RawKey, RawTable, RawValue}; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW}, transaction::DbTxMut, @@ -155,11 +155,13 @@ where // If we only have the genesis block hash, then we are at first sync, and we can remove it, // add it to the collector and use tx.append on all hashes. - if let Some((hash, block_number)) = cursor_header_numbers.last()? { - if block_number.value()? == 0 { - self.hash_collector.insert(hash.key()?, 0)?; - cursor_header_numbers.delete_current()?; - first_sync = true; + if provider.tx_ref().entries::>()? == 1 { + if let Some((hash, block_number)) = cursor_header_numbers.last()? { + if block_number.value()? == 0 { + self.hash_collector.insert(hash.key()?, 0)?; + cursor_header_numbers.delete_current()?; + first_sync = true; + } } } From e0ad59834de2e77c0283f1a5c1fc2fd7718a9875 Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Sat, 26 Oct 2024 08:10:15 +0200 Subject: [PATCH 103/239] dev: add `requests` to `EthBuiltPayload` (#12072) --- Cargo.lock | 3 ++- crates/e2e-test-utils/src/engine_api.rs | 9 +------ .../ethereum/engine-primitives/src/payload.rs | 26 +++++++++++-------- crates/ethereum/payload/src/lib.rs | 5 ++-- crates/optimism/payload/src/payload.rs | 10 ++++++- crates/payload/builder/src/lib.rs | 2 +- crates/payload/builder/src/test_utils.rs | 1 + crates/payload/primitives/Cargo.toml | 1 + crates/payload/primitives/src/traits.rs | 4 +++ 9 files changed, 37 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a99803e1bc4d..bf73d7eef39f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8373,6 +8373,7 @@ dependencies = [ name = "reth-payload-primitives" version = "1.1.0" dependencies = [ + "alloy-eips", "alloy-primitives", "alloy-rpc-types", "async-trait", @@ -11956,4 +11957,4 @@ checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", -] \ No newline at end of file +] diff --git a/crates/e2e-test-utils/src/engine_api.rs b/crates/e2e-test-utils/src/engine_api.rs index 729205211ff0..cfa245e1de01 100644 --- a/crates/e2e-test-utils/src/engine_api.rs +++ b/crates/e2e-test-utils/src/engine_api.rs @@ -62,14 +62,7 @@ impl EngineApiTestContext::ExecutionPayloadEnvelopeV4 = payload.into(); EngineApiClient::::new_payload_v4( &self.engine_api_client, diff --git a/crates/ethereum/engine-primitives/src/payload.rs b/crates/ethereum/engine-primitives/src/payload.rs index 50c4852545b4..420352cf2b98 100644 --- a/crates/ethereum/engine-primitives/src/payload.rs +++ b/crates/ethereum/engine-primitives/src/payload.rs @@ -33,6 +33,8 @@ pub struct EthBuiltPayload { /// The blobs, proofs, and commitments in the block. If the block is pre-cancun, this will be /// empty. pub(crate) sidecars: Vec, + /// The requests of the payload + pub(crate) requests: Option, } // === impl BuiltPayload === @@ -46,8 +48,9 @@ impl EthBuiltPayload { block: SealedBlock, fees: U256, executed_block: Option, + requests: Option, ) -> Self { - Self { id, block, executed_block, fees, sidecars: Vec::new() } + Self { id, block, executed_block, fees, sidecars: Vec::new(), requests } } /// Returns the identifier of the payload. @@ -97,6 +100,10 @@ impl BuiltPayload for EthBuiltPayload { fn executed_block(&self) -> Option { self.executed_block.clone() } + + fn requests(&self) -> Option { + self.requests.clone() + } } impl BuiltPayload for &EthBuiltPayload { @@ -111,6 +118,10 @@ impl BuiltPayload for &EthBuiltPayload { fn executed_block(&self) -> Option { self.executed_block.clone() } + + fn requests(&self) -> Option { + self.requests.clone() + } } // V1 engine_getPayloadV1 response @@ -152,15 +163,8 @@ impl From for ExecutionPayloadEnvelopeV3 { impl From for ExecutionPayloadEnvelopeV4 { fn from(value: EthBuiltPayload) -> Self { - let EthBuiltPayload { block, fees, sidecars, executed_block, .. } = value; - - // if we have an executed block, we pop off the first set of requests from the execution - // outcome. the assumption here is that there will always only be one block in the execution - // outcome. - let execution_requests = executed_block - .and_then(|block| block.execution_outcome().requests.first().cloned()) - .map(Requests::take) - .unwrap_or_default(); + let EthBuiltPayload { block, fees, sidecars, requests, .. } = value; + Self { execution_payload: block_to_payload_v3(block), block_value: fees, @@ -174,7 +178,7 @@ impl From for ExecutionPayloadEnvelopeV4 { // should_override_builder: false, blobs_bundle: sidecars.into_iter().map(Into::into).collect::>().into(), - execution_requests, + execution_requests: requests.unwrap_or_default().take(), } } } diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index bb611441f032..f14c145889c9 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -360,7 +360,7 @@ where db.take_bundle(), vec![receipts].into(), block_number, - vec![requests.unwrap_or_default()], + vec![requests.clone().unwrap_or_default()], ); let receipts_root = execution_outcome.receipts_root_slow(block_number).expect("Number is in range"); @@ -449,7 +449,8 @@ where trie: Arc::new(trie_output), }; - let mut payload = EthBuiltPayload::new(attributes.id, sealed_block, total_fees, Some(executed)); + let mut payload = + EthBuiltPayload::new(attributes.id, sealed_block, total_fees, Some(executed), requests); // extend the payload with the blob sidecars from the executed txs payload.extend_sidecars(blob_sidecars); diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index d5d1620e54b9..98b0e41b0f57 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -2,7 +2,7 @@ //! Optimism builder support -use alloy_eips::eip2718::Decodable2718; +use alloy_eips::{eip2718::Decodable2718, eip7685::Requests}; use alloy_primitives::{Address, B256, U256}; use alloy_rlp::Encodable; use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV2, ExecutionPayloadV1, PayloadId}; @@ -178,6 +178,10 @@ impl BuiltPayload for OptimismBuiltPayload { fn executed_block(&self) -> Option { self.executed_block.clone() } + + fn requests(&self) -> Option { + None + } } impl BuiltPayload for &OptimismBuiltPayload { @@ -192,6 +196,10 @@ impl BuiltPayload for &OptimismBuiltPayload { fn executed_block(&self) -> Option { self.executed_block.clone() } + + fn requests(&self) -> Option { + None + } } // V1 engine_getPayloadV1 response diff --git a/crates/payload/builder/src/lib.rs b/crates/payload/builder/src/lib.rs index 0df15f5b0de0..7af61ac4c682 100644 --- a/crates/payload/builder/src/lib.rs +++ b/crates/payload/builder/src/lib.rs @@ -65,7 +65,7 @@ //! }, //! ..Default::default() //! }; -//! let payload = EthBuiltPayload::new(self.attributes.id, payload.seal_slow(), U256::ZERO, None); +//! let payload = EthBuiltPayload::new(self.attributes.id, payload.seal_slow(), U256::ZERO, None, None); //! Ok(payload) //! } //! diff --git a/crates/payload/builder/src/test_utils.rs b/crates/payload/builder/src/test_utils.rs index 6990dc9b1744..676e60d912f0 100644 --- a/crates/payload/builder/src/test_utils.rs +++ b/crates/payload/builder/src/test_utils.rs @@ -89,6 +89,7 @@ impl PayloadJob for TestPayloadJob { Block::default().seal_slow(), U256::ZERO, Some(ExecutedBlock::default()), + Some(Default::default()), )) } diff --git a/crates/payload/primitives/Cargo.toml b/crates/payload/primitives/Cargo.toml index 27418ccd899f..ad8ce63a7e9b 100644 --- a/crates/payload/primitives/Cargo.toml +++ b/crates/payload/primitives/Cargo.toml @@ -20,6 +20,7 @@ reth-transaction-pool.workspace = true reth-chain-state.workspace = true # alloy +alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rpc-types = { workspace = true, features = ["engine"] } op-alloy-rpc-types-engine.workspace = true diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index ce98fcad32ea..df76149028aa 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -1,4 +1,5 @@ use crate::{PayloadEvents, PayloadKind, PayloadTypes}; +use alloy_eips::eip7685::Requests; use alloy_primitives::{Address, B256, U256}; use alloy_rpc_types::{ engine::{PayloadAttributes as EthPayloadAttributes, PayloadId}, @@ -65,6 +66,9 @@ pub trait BuiltPayload: Send + Sync + std::fmt::Debug { fn executed_block(&self) -> Option { None } + + /// Returns the EIP-7865 requests for the payload if any. + fn requests(&self) -> Option; } /// This can be implemented by types that describe a currently running payload job. From cecdf611e948257a79e7c262501bdadbd174e44d Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Sat, 26 Oct 2024 08:11:27 +0200 Subject: [PATCH 104/239] feat: `map_chainspec` for `NodeConfig` (#12068) --- crates/exex/exex/src/dyn_context.rs | 22 +++------------------- crates/node/core/src/node_config.rs | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/crates/exex/exex/src/dyn_context.rs b/crates/exex/exex/src/dyn_context.rs index 226f3a3feb9a..b48a6ebc951f 100644 --- a/crates/exex/exex/src/dyn_context.rs +++ b/crates/exex/exex/src/dyn_context.rs @@ -1,7 +1,7 @@ //! Mirrored version of [`ExExContext`](`crate::ExExContext`) //! without generic abstraction over [Node](`reth_node_api::FullNodeComponents`) -use std::{fmt::Debug, sync::Arc}; +use std::fmt::Debug; use reth_chainspec::{EthChainSpec, Head}; use reth_node_api::FullNodeComponents; @@ -55,24 +55,8 @@ where Node::Executor: Debug, { fn from(ctx: ExExContext) -> Self { - // convert `NodeConfig` with generic over chainspec into `NodeConfig` - let chain: Arc> = - Arc::new(Box::new(ctx.config.chain) as Box); - let config = NodeConfig { - chain, - datadir: ctx.config.datadir, - config: ctx.config.config, - metrics: ctx.config.metrics, - instance: ctx.config.instance, - network: ctx.config.network, - rpc: ctx.config.rpc, - txpool: ctx.config.txpool, - builder: ctx.config.builder, - debug: ctx.config.debug, - db: ctx.config.db, - dev: ctx.config.dev, - pruning: ctx.config.pruning, - }; + let config = + ctx.config.map_chainspec(|chainspec| Box::new(chainspec) as Box); let notifications = Box::new(ctx.notifications) as Box; Self { diff --git a/crates/node/core/src/node_config.rs b/crates/node/core/src/node_config.rs index a8799d80df1c..80fb5152e7bb 100644 --- a/crates/node/core/src/node_config.rs +++ b/crates/node/core/src/node_config.rs @@ -422,6 +422,29 @@ impl NodeConfig { Err(e) => Err(eyre!("Failed to load configuration: {e}")), } } + + /// Modifies the [`ChainSpec`] generic of the config using the provided closure. + pub fn map_chainspec(self, f: F) -> NodeConfig + where + F: FnOnce(Arc) -> C, + { + let chain = Arc::new(f(self.chain)); + NodeConfig { + chain, + datadir: self.datadir, + config: self.config, + metrics: self.metrics, + instance: self.instance, + network: self.network, + rpc: self.rpc, + txpool: self.txpool, + builder: self.builder, + debug: self.debug, + db: self.db, + dev: self.dev, + pruning: self.pruning, + } + } } impl Default for NodeConfig { From ac329bfce1d6ebf90e06fc977ff43e40f20cf084 Mon Sep 17 00:00:00 2001 From: Darshan Kathiriya <8559992+lakshya-sky@users.noreply.github.com> Date: Sat, 26 Oct 2024 03:44:47 -0400 Subject: [PATCH 105/239] perf: improve debug_traceBlock performance (#11979) Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/debug.rs | 78 +++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 2d9d6f7822e1..5a20bee975ff 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -112,6 +112,7 @@ where let mut results = Vec::with_capacity(transactions.len()); let mut db = CacheDB::new(StateProviderDatabase::new(state)); let mut transactions = transactions.into_iter().enumerate().peekable(); + let mut inspector = None; while let Some((index, tx)) = transactions.next() { let tx_hash = tx.hash; @@ -124,7 +125,7 @@ where handler_cfg: cfg.handler_cfg, }; let (result, state_changes) = this.trace_transaction( - opts.clone(), + &opts, env, &mut db, Some(TransactionContext { @@ -132,8 +133,11 @@ where tx_hash: Some(tx_hash), tx_index: Some(index), }), + &mut inspector, )?; + inspector = inspector.map(|insp| insp.fused()); + results.push(TraceResult::Success { result, tx_hash: Some(tx_hash) }); if transactions.peek().is_some() { // need to apply the state changes of this transaction before executing the @@ -295,7 +299,7 @@ where }; this.trace_transaction( - opts, + &opts, env, &mut db, Some(TransactionContext { @@ -303,6 +307,7 @@ where tx_index: Some(index), tx_hash: Some(tx.hash), }), + &mut None, ) .map(|(trace, _)| trace) }) @@ -573,6 +578,7 @@ where let Bundle { transactions, block_override } = bundle; let block_overrides = block_override.map(Box::new); + let mut inspector = None; let mut transactions = transactions.into_iter().peekable(); while let Some(tx) = transactions.next() { @@ -588,8 +594,15 @@ where overrides, )?; - let (trace, state) = - this.trace_transaction(tracing_options.clone(), env, &mut db, None)?; + let (trace, state) = this.trace_transaction( + &tracing_options, + env, + &mut db, + None, + &mut inspector, + )?; + + inspector = inspector.map(|insp| insp.fused()); // If there is more transactions, commit the database // If there is no transactions, but more bundles, commit to the database too @@ -692,6 +705,13 @@ where /// Executes the configured transaction with the environment on the given database. /// + /// It optionally takes fused inspector ([`TracingInspector::fused`]) to avoid re-creating the + /// inspector for each transaction. This is useful when tracing multiple transactions in a + /// block. This is only useful for block tracing which uses the same tracer for all transactions + /// in the block. + /// + /// Caution: If the inspector is provided then `opts.tracer_config` is ignored. + /// /// Returns the trace frame and the state that got updated after executing the transaction. /// /// Note: this does not apply any state overrides if they're configured in the `opts`. @@ -699,10 +719,11 @@ where /// Caution: this is blocking and should be performed on a blocking task. fn trace_transaction( &self, - opts: GethDebugTracingOptions, + opts: &GethDebugTracingOptions, env: EnvWithHandlerCfg, db: &mut StateCacheDb<'_>, transaction_context: Option, + fused_inspector: &mut Option, ) -> Result<(GethTrace, revm_primitives::EvmState), Eth::Error> { let GethDebugTracingOptions { config, tracer, tracer_config, .. } = opts; @@ -716,35 +737,42 @@ where } GethDebugBuiltInTracerType::CallTracer => { let call_config = tracer_config + .clone() .into_call_config() .map_err(|_| EthApiError::InvalidTracerConfig)?; - let mut inspector = TracingInspector::new( - TracingInspectorConfig::from_geth_call_config(&call_config), - ); + let mut inspector = fused_inspector.get_or_insert_with(|| { + TracingInspector::new(TracingInspectorConfig::from_geth_call_config( + &call_config, + )) + }); let (res, env) = self.eth_api().inspect(db, env, &mut inspector)?; + inspector.set_transaction_gas_limit(env.tx.gas_limit); + let frame = inspector - .with_transaction_gas_limit(env.tx.gas_limit) - .into_geth_builder() + .geth_builder() .geth_call_traces(call_config, res.result.gas_used()); return Ok((frame.into(), res.state)) } GethDebugBuiltInTracerType::PreStateTracer => { let prestate_config = tracer_config + .clone() .into_pre_state_config() .map_err(|_| EthApiError::InvalidTracerConfig)?; - let mut inspector = TracingInspector::new( - TracingInspectorConfig::from_geth_prestate_config(&prestate_config), - ); + let mut inspector = fused_inspector.get_or_insert_with(|| { + TracingInspector::new( + TracingInspectorConfig::from_geth_prestate_config(&prestate_config), + ) + }); let (res, env) = self.eth_api().inspect(&mut *db, env, &mut inspector)?; + inspector.set_transaction_gas_limit(env.tx.gas_limit); let frame = inspector - .with_transaction_gas_limit(env.tx.gas_limit) - .into_geth_builder() + .geth_builder() .geth_prestate_traces(&res, &prestate_config, db) .map_err(Eth::Error::from_eth_err)?; @@ -755,6 +783,7 @@ where } GethDebugBuiltInTracerType::MuxTracer => { let mux_config = tracer_config + .clone() .into_mux_config() .map_err(|_| EthApiError::InvalidTracerConfig)?; @@ -769,6 +798,7 @@ where } GethDebugBuiltInTracerType::FlatCallTracer => { let flat_call_config = tracer_config + .clone() .into_flat_call_config() .map_err(|_| EthApiError::InvalidTracerConfig)?; @@ -799,10 +829,10 @@ where } #[cfg(feature = "js-tracer")] GethDebugTracerType::JsTracer(code) => { - let config = tracer_config.into_json(); + let config = tracer_config.clone().into_json(); let mut inspector = revm_inspectors::tracing::js::JsInspector::with_transaction_context( - code, + code.clone(), config, transaction_context.unwrap_or_default(), ) @@ -818,17 +848,15 @@ where } // default structlog tracer - let inspector_config = TracingInspectorConfig::from_geth_config(&config); - - let mut inspector = TracingInspector::new(inspector_config); - + let mut inspector = fused_inspector.get_or_insert_with(|| { + let inspector_config = TracingInspectorConfig::from_geth_config(config); + TracingInspector::new(inspector_config) + }); let (res, env) = self.eth_api().inspect(db, env, &mut inspector)?; let gas_used = res.result.gas_used(); let return_value = res.result.into_output().unwrap_or_default(); - let frame = inspector - .with_transaction_gas_limit(env.tx.gas_limit) - .into_geth_builder() - .geth_traces(gas_used, return_value, config); + inspector.set_transaction_gas_limit(env.tx.gas_limit); + let frame = inspector.geth_builder().geth_traces(gas_used, return_value, *config); Ok((frame.into(), res.state)) } From 44e4c47803f26585872ddf0f25848092a4f29c73 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sat, 26 Oct 2024 18:55:26 +0800 Subject: [PATCH 106/239] chore(sdk): add helper trait to node API to simplify type definition (#10616) --- Cargo.lock | 5 +- crates/consensus/debug-client/Cargo.toml | 2 +- crates/e2e-test-utils/src/rpc.rs | 7 ++- crates/ethereum/node/tests/e2e/dev.rs | 2 +- crates/node/api/Cargo.toml | 1 - crates/node/api/src/lib.rs | 2 - crates/node/builder/src/launch/mod.rs | 17 ++++--- crates/node/builder/src/node.rs | 2 +- crates/node/builder/src/rpc.rs | 6 ++- crates/node/core/Cargo.toml | 4 +- crates/node/core/src/lib.rs | 9 ---- crates/optimism/rpc/src/eth/mod.rs | 40 ++++++++-------- crates/rpc/rpc-eth-api/Cargo.toml | 1 + crates/rpc/rpc-eth-api/src/lib.rs | 2 + crates/rpc/rpc-eth-api/src/node.rs | 58 ++++++++++++++++++++++++ crates/rpc/rpc/Cargo.toml | 1 - crates/rpc/rpc/src/eth/filter.rs | 5 +- crates/rpc/rpc/src/eth/mod.rs | 2 +- 18 files changed, 105 insertions(+), 61 deletions(-) create mode 100644 crates/rpc/rpc-eth-api/src/node.rs diff --git a/Cargo.lock b/Cargo.lock index bf73d7eef39f..f4dc7dae4e98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7866,7 +7866,6 @@ dependencies = [ "reth-payload-primitives", "reth-primitives", "reth-provider", - "reth-rpc-eth-api", "reth-tasks", "reth-transaction-pool", ] @@ -7966,8 +7965,6 @@ dependencies = [ "reth-network-peers", "reth-primitives", "reth-prune-types", - "reth-rpc-api", - "reth-rpc-eth-api", "reth-rpc-eth-types", "reth-rpc-server-types", "reth-rpc-types-compat", @@ -8629,7 +8626,6 @@ dependencies = [ "reth-network-api", "reth-network-peers", "reth-network-types", - "reth-node-api", "reth-primitives", "reth-provider", "reth-revm", @@ -8812,6 +8808,7 @@ dependencies = [ "reth-evm", "reth-execution-types", "reth-network-api", + "reth-node-api", "reth-primitives", "reth-provider", "reth-revm", diff --git a/crates/consensus/debug-client/Cargo.toml b/crates/consensus/debug-client/Cargo.toml index c37beef10742..e73125a80bd7 100644 --- a/crates/consensus/debug-client/Cargo.toml +++ b/crates/consensus/debug-client/Cargo.toml @@ -13,7 +13,7 @@ workspace = true [dependencies] # reth reth-node-api.workspace = true -reth-rpc-api.workspace = true +reth-rpc-api = { workspace = true, features = ["client"] } reth-rpc-builder.workspace = true reth-tracing.workspace = true diff --git a/crates/e2e-test-utils/src/rpc.rs b/crates/e2e-test-utils/src/rpc.rs index b8cbe4d77add..7b7dabdf2404 100644 --- a/crates/e2e-test-utils/src/rpc.rs +++ b/crates/e2e-test-utils/src/rpc.rs @@ -4,12 +4,15 @@ use alloy_primitives::{Bytes, B256}; use reth::{ builder::{rpc::RpcRegistry, FullNodeComponents}, rpc::api::{ - eth::helpers::{EthApiSpec, EthTransactions, TraceExt}, + eth::{ + helpers::{EthApiSpec, EthTransactions, TraceExt}, + EthApiTypes, + }, DebugApiServer, }, }; use reth_chainspec::EthereumHardforks; -use reth_node_builder::{EthApiTypes, NodeTypes}; +use reth_node_builder::NodeTypes; #[allow(missing_debug_implementations)] pub struct RpcTestContext { diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index ead438b5a670..f0fcaf64524e 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -4,7 +4,7 @@ use crate::utils::eth_payload_attributes; use alloy_genesis::Genesis; use alloy_primitives::{b256, hex}; use futures::StreamExt; -use reth::{args::DevArgs, core::rpc::eth::helpers::EthTransactions}; +use reth::{args::DevArgs, rpc::api::eth::helpers::EthTransactions}; use reth_chainspec::ChainSpec; use reth_e2e_test_utils::setup; use reth_node_api::FullNodeComponents; diff --git a/crates/node/api/Cargo.toml b/crates/node/api/Cargo.toml index c2c3eb46326b..6b263d6c532b 100644 --- a/crates/node/api/Cargo.toml +++ b/crates/node/api/Cargo.toml @@ -21,7 +21,6 @@ reth-transaction-pool.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true reth-tasks.workspace = true -reth-rpc-eth-api.workspace = true reth-network-api.workspace = true reth-node-types.workspace = true reth-primitives.workspace = true diff --git a/crates/node/api/src/lib.rs b/crates/node/api/src/lib.rs index 7692ed6f2cae..099cf82b5fe0 100644 --- a/crates/node/api/src/lib.rs +++ b/crates/node/api/src/lib.rs @@ -25,5 +25,3 @@ pub use node::*; // re-export for convenience pub use reth_node_types::*; pub use reth_provider::FullProvider; - -pub use reth_rpc_eth_api::EthApiTypes; diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index 36aa55541e00..50438e79d2b6 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -23,15 +23,14 @@ use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, use reth_engine_util::EngineMessageStreamExt; use reth_exex::ExExManagerHandle; use reth_network::{BlockDownloaderProvider, NetworkEventListenerProvider}; -use reth_node_api::{ - AddOnsContext, FullNodeComponents, FullNodeTypes, NodeTypesWithDB, NodeTypesWithEngine, -}; +use reth_node_api::{AddOnsContext, FullNodeTypes, NodeTypesWithDB, NodeTypesWithEngine}; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, exit::NodeExitFuture, }; use reth_node_events::{cl::ConsensusLayerHealthEvents, node}; use reth_provider::providers::BlockchainProvider; +use reth_rpc::eth::RpcNodeCore; use reth_tasks::TaskExecutor; use reth_tracing::tracing::{debug, info}; use reth_transaction_pool::TransactionPool; @@ -47,14 +46,14 @@ use crate::{ AddOns, NodeBuilderWithComponents, NodeHandle, }; -/// Alias for [`reth_rpc_eth_types::EthApiBuilderCtx`], adapter for [`FullNodeComponents`]. +/// Alias for [`reth_rpc_eth_types::EthApiBuilderCtx`], adapter for [`RpcNodeCore`]. pub type EthApiBuilderCtx = reth_rpc_eth_types::EthApiBuilderCtx< - ::Provider, - ::Pool, - ::Evm, - ::Network, + ::Provider, + ::Pool, + ::Evm, + ::Network, TaskExecutor, - ::Provider, + ::Provider, >; /// A general purpose trait that launches a new node of any kind. diff --git a/crates/node/builder/src/node.rs b/crates/node/builder/src/node.rs index 3e3d5b696c39..3b2f467d61c0 100644 --- a/crates/node/builder/src/node.rs +++ b/crates/node/builder/src/node.rs @@ -11,10 +11,10 @@ use reth_node_api::{EngineTypes, FullNodeComponents}; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, node_config::NodeConfig, - rpc::api::EngineApiClient, }; use reth_payload_builder::PayloadBuilderHandle; use reth_provider::ChainSpecProvider; +use reth_rpc_api::EngineApiClient; use reth_rpc_builder::{auth::AuthServerHandle, RpcServerHandle}; use reth_tasks::TaskExecutor; diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 18293118dc66..4c1ea32d045d 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -14,12 +14,14 @@ use reth_node_api::{ }; use reth_node_core::{ node_config::NodeConfig, - rpc::eth::{EthApiTypes, FullEthApiServer}, version::{CARGO_PKG_VERSION, CLIENT_CODE, NAME_CLIENT, VERGEN_GIT_SHA}, }; use reth_payload_builder::PayloadBuilderHandle; use reth_provider::providers::ProviderNodeTypes; -use reth_rpc::EthApi; +use reth_rpc::{ + eth::{EthApiTypes, FullEthApiServer}, + EthApi, +}; use reth_rpc_api::eth::helpers::AddDevSigners; use reth_rpc_builder::{ auth::{AuthRpcModule, AuthServerHandle}, diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 0c9672d17779..1c6c9d98c80b 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -23,8 +23,6 @@ reth-network-p2p.workspace = true reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true reth-rpc-types-compat.workspace = true -reth-rpc-api = { workspace = true, features = ["client"] } -reth-rpc-eth-api = { workspace = true, features = ["client"] } reth-transaction-pool.workspace = true reth-tracing.workspace = true reth-config.workspace = true @@ -38,7 +36,7 @@ reth-stages-types.workspace = true # ethereum alloy-primitives.workspace = true -alloy-rpc-types-engine = { workspace = true, features = ["jwt"] } +alloy-rpc-types-engine = { workspace = true, features = ["std", "jwt"] } alloy-consensus.workspace = true alloy-eips.workspace = true diff --git a/crates/node/core/src/lib.rs b/crates/node/core/src/lib.rs index 6af822e22eeb..a69a255a3c67 100644 --- a/crates/node/core/src/lib.rs +++ b/crates/node/core/src/lib.rs @@ -22,15 +22,6 @@ pub mod primitives { /// Re-export of `reth_rpc_*` crates. pub mod rpc { - /// Re-exported from `reth_rpc_api`. - pub mod api { - pub use reth_rpc_api::*; - } - /// Re-exported from `reth_rpc::eth`. - pub mod eth { - pub use reth_rpc_eth_api::*; - } - /// Re-exported from `reth_rpc::rpc`. pub mod result { pub use reth_rpc_server_types::result::*; diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index 04774a4651c0..a1a9f6e8f04f 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -17,12 +17,12 @@ use op_alloy_network::Optimism; use reth_chainspec::EthereumHardforks; use reth_evm::ConfigureEvm; use reth_network_api::NetworkInfo; -use reth_node_api::{FullNodeComponents, FullNodeTypes, NodeTypes}; +use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_node_builder::EthApiBuilderCtx; use reth_primitives::Header; use reth_provider::{ - BlockIdReader, BlockNumReader, BlockReaderIdExt, ChainSpecProvider, HeaderProvider, - StageCheckpointReader, StateProviderFactory, + BlockIdReader, BlockNumReader, BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, + HeaderProvider, StageCheckpointReader, StateProviderFactory, }; use reth_rpc::eth::{core::EthApiInner, DevSigner}; use reth_rpc_eth_api::{ @@ -30,7 +30,7 @@ use reth_rpc_eth_api::{ AddDevSigners, EthApiSpec, EthFees, EthSigner, EthState, LoadBlock, LoadFee, LoadState, SpawnBlocking, Trace, }, - EthApiTypes, + EthApiTypes, RpcNodeCore, }; use reth_rpc_eth_types::{EthStateCache, FeeHistoryCache, GasPriceOracle}; use reth_tasks::{ @@ -43,10 +43,10 @@ use crate::{OpEthApiError, SequencerClient}; /// Adapter for [`EthApiInner`], which holds all the data required to serve core `eth_` API. pub type EthApiNodeBackend = EthApiInner< - ::Provider, - ::Pool, - ::Network, - ::Evm, + ::Provider, + ::Pool, + ::Network, + ::Evm, >; /// OP-Reth `Eth` API implementation. @@ -59,8 +59,8 @@ pub type EthApiNodeBackend = EthApiInner< /// /// This type implements the [`FullEthApi`](reth_rpc_eth_api::helpers::FullEthApi) by implemented /// all the `Eth` helper traits and prerequisite traits. -#[derive(Deref)] -pub struct OpEthApi { +#[derive(Deref, Clone)] +pub struct OpEthApi { /// Gateway to node's core components. #[deref] inner: Arc>, @@ -69,7 +69,12 @@ pub struct OpEthApi { sequencer_client: Option, } -impl OpEthApi { +impl OpEthApi +where + N: RpcNodeCore< + Provider: BlockReaderIdExt + ChainSpecProvider + CanonStateSubscriptions + Clone + 'static, + >, +{ /// Creates a new instance for given context. pub fn new(ctx: &EthApiBuilderCtx, sequencer_http: Option) -> Self { let blocking_task_pool = @@ -98,7 +103,7 @@ impl OpEthApi { impl EthApiTypes for OpEthApi where Self: Send + Sync, - N: FullNodeComponents, + N: RpcNodeCore, { type Error = OpEthApiError; type NetworkTypes = Optimism; @@ -248,17 +253,8 @@ where } } -impl fmt::Debug for OpEthApi { +impl fmt::Debug for OpEthApi { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OpEthApi").finish_non_exhaustive() } } - -impl Clone for OpEthApi -where - N: FullNodeComponents, -{ - fn clone(&self) -> Self { - Self { inner: self.inner.clone(), sequencer_client: self.sequencer_client.clone() } - } -} diff --git a/crates/rpc/rpc-eth-api/Cargo.toml b/crates/rpc/rpc-eth-api/Cargo.toml index 9d0f6cfd83d6..edfd57b201d9 100644 --- a/crates/rpc/rpc-eth-api/Cargo.toml +++ b/crates/rpc/rpc-eth-api/Cargo.toml @@ -30,6 +30,7 @@ reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true reth-network-api.workspace = true reth-trie.workspace = true +reth-node-api.workspace = true # ethereum alloy-eips.workspace = true diff --git a/crates/rpc/rpc-eth-api/src/lib.rs b/crates/rpc/rpc-eth-api/src/lib.rs index 849c8e2e4c8f..bc46d526c6f0 100644 --- a/crates/rpc/rpc-eth-api/src/lib.rs +++ b/crates/rpc/rpc-eth-api/src/lib.rs @@ -16,6 +16,7 @@ pub mod bundle; pub mod core; pub mod filter; pub mod helpers; +pub mod node; pub mod pubsub; pub mod types; @@ -25,6 +26,7 @@ pub use bundle::{EthBundleApiServer, EthCallBundleApiServer}; pub use core::{EthApiServer, FullEthApiServer}; pub use filter::EthFilterApiServer; pub use helpers::error::{AsEthApiError, FromEthApiError, FromEvmError, IntoEthApiError}; +pub use node::RpcNodeCore; pub use pubsub::EthPubSubApiServer; pub use types::{EthApiTypes, FullEthApiTypes, RpcBlock, RpcReceipt, RpcTransaction}; diff --git a/crates/rpc/rpc-eth-api/src/node.rs b/crates/rpc/rpc-eth-api/src/node.rs new file mode 100644 index 000000000000..8488677e32fb --- /dev/null +++ b/crates/rpc/rpc-eth-api/src/node.rs @@ -0,0 +1,58 @@ +//! Helper trait for interfacing with [`FullNodeComponents`]. + +use reth_node_api::FullNodeComponents; + +/// Helper trait to relax trait bounds on [`FullNodeComponents`]. +/// +/// Helpful when defining types that would otherwise have a generic `N: FullNodeComponents`. Using +/// `N: RpcNodeCore` instead, allows access to all the associated types on [`FullNodeComponents`] +/// that are used in RPC, but with more flexibility since they have no trait bounds (asides auto +/// traits). +pub trait RpcNodeCore: Clone { + /// The provider type used to interact with the node. + type Provider: Send + Sync + Clone + Unpin; + /// The transaction pool of the node. + type Pool: Send + Sync + Clone + Unpin; + /// The node's EVM configuration, defining settings for the Ethereum Virtual Machine. + type Evm: Send + Sync + Clone + Unpin; + /// Network API. + type Network: Send + Sync + Clone; + + /// Returns the transaction pool of the node. + fn pool(&self) -> &Self::Pool; + + /// Returns the node's evm config. + fn evm_config(&self) -> &Self::Evm; + + /// Returns the handle to the network + fn network(&self) -> &Self::Network; + + /// Returns the provider of the node. + fn provider(&self) -> &Self::Provider; +} + +impl RpcNodeCore for T +where + T: FullNodeComponents, +{ + type Provider = T::Provider; + type Pool = T::Pool; + type Network = ::Network; + type Evm = ::Evm; + + fn pool(&self) -> &Self::Pool { + FullNodeComponents::pool(self) + } + + fn evm_config(&self) -> &Self::Evm { + FullNodeComponents::evm_config(self) + } + + fn network(&self) -> &Self::Network { + FullNodeComponents::network(self) + } + + fn provider(&self) -> &Self::Provider { + FullNodeComponents::provider(self) + } +} diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index fe150e36eed5..dab86ac25872 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -31,7 +31,6 @@ reth-network-peers = { workspace = true, features = ["secp256k1"] } reth-evm.workspace = true reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true -reth-node-api.workspace = true reth-network-types.workspace = true reth-trie.workspace = true diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 24058da1734c..5ef224609c5b 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -17,10 +17,11 @@ use alloy_rpc_types::{ use async_trait::async_trait; use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_chainspec::ChainInfo; -use reth_node_api::EthApiTypes; use reth_primitives::{Receipt, SealedBlockWithSenders, TransactionSignedEcRecovered}; use reth_provider::{BlockIdReader, BlockReader, EvmEnvProvider, ProviderError}; -use reth_rpc_eth_api::{EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat}; +use reth_rpc_eth_api::{ + EthApiTypes, EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat, +}; use reth_rpc_eth_types::{ logs_utils::{self, append_matching_block_logs, ProviderOrBlock}, EthApiError, EthFilterConfig, EthStateCache, EthSubscriptionIdProvider, diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 99919110da7b..4d1833add3e7 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -15,4 +15,4 @@ pub use pubsub::EthPubSub; pub use helpers::{signer::DevSigner, types::EthTxBuilder}; -pub use reth_rpc_eth_api::EthApiServer; +pub use reth_rpc_eth_api::{EthApiServer, EthApiTypes, FullEthApiServer, RpcNodeCore}; From a06c3af8320caaa95adeffbca38b9ed945019557 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sat, 26 Oct 2024 21:03:22 +0800 Subject: [PATCH 107/239] chore(rpc): Remove provider and network trait methods from `EthApiSpec` (#12050) --- crates/optimism/rpc/src/eth/mod.rs | 41 ++++++++++++++++------ crates/rpc/rpc-builder/src/lib.rs | 5 +-- crates/rpc/rpc-eth-api/src/helpers/spec.rs | 20 +++++------ crates/rpc/rpc/src/eth/core.rs | 31 +++++++++++++++- crates/rpc/rpc/src/eth/helpers/spec.rs | 14 ++------ 5 files changed, 75 insertions(+), 36 deletions(-) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index a1a9f6e8f04f..ccff477892f2 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -114,24 +114,43 @@ where } } -impl EthApiSpec for OpEthApi +impl RpcNodeCore for OpEthApi where - Self: Send + Sync, - N: FullNodeComponents>, + Self: Clone, + N: RpcNodeCore, { - #[inline] - fn provider( - &self, - ) -> impl ChainSpecProvider + BlockNumReader + StageCheckpointReader - { - self.inner.provider() + type Provider = N::Provider; + type Pool = N::Pool; + type Network = ::Network; + type Evm = ::Evm; + + fn pool(&self) -> &Self::Pool { + self.inner.pool() } - #[inline] - fn network(&self) -> impl NetworkInfo { + fn evm_config(&self) -> &Self::Evm { + self.inner.evm_config() + } + + fn network(&self) -> &Self::Network { self.inner.network() } + fn provider(&self) -> &Self::Provider { + self.inner.provider() + } +} + +impl EthApiSpec for OpEthApi +where + Self: Send + Sync, + N: RpcNodeCore< + Provider: ChainSpecProvider + + BlockNumReader + + StageCheckpointReader, + Network: NetworkInfo, + >, +{ #[inline] fn starting_block(&self) -> U256 { self.inner.starting_block() diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index ceafe206531a..fc98cd6ff4ef 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -36,7 +36,7 @@ //! block_executor: BlockExecutor, //! ) where //! Provider: FullRpcProvider + AccountReader + ChangeSetReader, -//! Pool: TransactionPool + 'static, +//! Pool: TransactionPool + Unpin + 'static, //! Network: NetworkInfo + Peers + Clone + 'static, //! Events: CanonStateSubscriptions + Clone + 'static, //! EvmConfig: ConfigureEvm

, @@ -85,6 +85,7 @@ //! use reth_tasks::TokioTaskExecutor; //! use reth_transaction_pool::TransactionPool; //! use tokio::try_join; +//! //! pub async fn launch< //! Provider, //! Pool, @@ -104,7 +105,7 @@ //! block_executor: BlockExecutor, //! ) where //! Provider: FullRpcProvider + AccountReader + ChangeSetReader, -//! Pool: TransactionPool + 'static, +//! Pool: TransactionPool + Unpin + 'static, //! Network: NetworkInfo + Peers + Clone + 'static, //! Events: CanonStateSubscriptions + Clone + 'static, //! EngineApi: EngineApiServer, diff --git a/crates/rpc/rpc-eth-api/src/helpers/spec.rs b/crates/rpc/rpc-eth-api/src/helpers/spec.rs index 5976cf29c07d..5aa0509e8bbe 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/spec.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/spec.rs @@ -8,21 +8,21 @@ use reth_errors::{RethError, RethResult}; use reth_network_api::NetworkInfo; use reth_provider::{BlockNumReader, ChainSpecProvider, StageCheckpointReader}; -use super::EthSigner; +use crate::{helpers::EthSigner, RpcNodeCore}; /// `Eth` API trait. /// /// Defines core functionality of the `eth` API implementation. #[auto_impl::auto_impl(&, Arc)] -pub trait EthApiSpec: Send + Sync { - /// Returns a handle for reading data from disk. - fn provider( - &self, - ) -> impl ChainSpecProvider + BlockNumReader + StageCheckpointReader; - - /// Returns a handle for reading network data summary. - fn network(&self) -> impl NetworkInfo; - +pub trait EthApiSpec: + RpcNodeCore< + Provider: ChainSpecProvider + + BlockNumReader + + StageCheckpointReader, + Network: NetworkInfo, + > + Send + + Sync +{ /// Returns the block node is started on. fn starting_block(&self) -> U256; diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index 21787873e966..3fca76e8b0cc 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -10,7 +10,7 @@ use reth_primitives::BlockNumberOrTag; use reth_provider::{BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider}; use reth_rpc_eth_api::{ helpers::{EthSigner, SpawnBlocking}, - EthApiTypes, + EthApiTypes, RpcNodeCore, }; use reth_rpc_eth_types::{ EthApiBuilderCtx, EthApiError, EthStateCache, FeeHistoryCache, GasCap, GasPriceOracle, @@ -140,6 +140,35 @@ where } } +impl RpcNodeCore for EthApi +where + Provider: Send + Sync + Clone + Unpin, + Pool: Send + Sync + Clone + Unpin, + Network: Send + Sync + Clone, + EvmConfig: Send + Sync + Clone + Unpin, +{ + type Provider = Provider; + type Pool = Pool; + type Network = Network; + type Evm = EvmConfig; + + fn pool(&self) -> &Self::Pool { + self.inner.pool() + } + + fn evm_config(&self) -> &Self::Evm { + self.inner.evm_config() + } + + fn network(&self) -> &Self::Network { + self.inner.network() + } + + fn provider(&self) -> &Self::Provider { + self.inner.provider() + } +} + impl std::fmt::Debug for EthApi { diff --git a/crates/rpc/rpc/src/eth/helpers/spec.rs b/crates/rpc/rpc/src/eth/helpers/spec.rs index 92445bf5ed1b..c5c8d54c64bc 100644 --- a/crates/rpc/rpc/src/eth/helpers/spec.rs +++ b/crates/rpc/rpc/src/eth/helpers/spec.rs @@ -2,13 +2,14 @@ use alloy_primitives::U256; use reth_chainspec::EthereumHardforks; use reth_network_api::NetworkInfo; use reth_provider::{BlockNumReader, ChainSpecProvider, StageCheckpointReader}; -use reth_rpc_eth_api::helpers::EthApiSpec; +use reth_rpc_eth_api::{helpers::EthApiSpec, RpcNodeCore}; use reth_transaction_pool::TransactionPool; use crate::EthApi; impl EthApiSpec for EthApi where + Self: RpcNodeCore, Pool: TransactionPool + 'static, Provider: ChainSpecProvider + BlockNumReader @@ -17,17 +18,6 @@ where Network: NetworkInfo + 'static, EvmConfig: Send + Sync, { - fn provider( - &self, - ) -> impl ChainSpecProvider + BlockNumReader + StageCheckpointReader - { - self.inner.provider() - } - - fn network(&self) -> impl NetworkInfo { - self.inner.network() - } - fn starting_block(&self) -> U256 { self.inner.starting_block() } From 019f347385741d75638effee1fb0b640ae4a363b Mon Sep 17 00:00:00 2001 From: Yu Zeng Date: Sat, 26 Oct 2024 23:04:17 +0800 Subject: [PATCH 108/239] chore: move optimism execution types test to optimism crate (#12026) --- .../execution-types/src/execution_outcome.rs | 56 +-- crates/optimism/evm/src/lib.rs | 414 +++++++++++++++++- 2 files changed, 423 insertions(+), 47 deletions(-) diff --git a/crates/evm/execution-types/src/execution_outcome.rs b/crates/evm/execution-types/src/execution_outcome.rs index 0fde01547f7e..026e6b37c42c 100644 --- a/crates/evm/execution-types/src/execution_outcome.rs +++ b/crates/evm/execution-types/src/execution_outcome.rs @@ -168,7 +168,7 @@ impl ExecutionOutcome { } /// Transform block number to the index of block. - fn block_number_to_index(&self, block_number: BlockNumber) -> Option { + pub fn block_number_to_index(&self, block_number: BlockNumber) -> Option { if self.first_block > block_number { return None } @@ -366,12 +366,15 @@ impl From<(BlockExecutionOutput, BlockNumber)> for ExecutionOutcome { #[cfg(test)] mod tests { use super::*; - use alloy_eips::eip7685::Requests; - use alloy_primitives::{bytes, Address, LogData, B256}; - use reth_primitives::{Receipts, TxType}; - use std::collections::HashMap; + #[cfg(not(feature = "optimism"))] + use alloy_primitives::bytes; + use alloy_primitives::{Address, B256}; + use reth_primitives::Receipts; + #[cfg(not(feature = "optimism"))] + use reth_primitives::{LogData, TxType}; #[test] + #[cfg(not(feature = "optimism"))] fn test_initialisation() { // Create a new BundleState object with initial data let bundle = BundleState::new( @@ -387,10 +390,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), })]], }; @@ -444,6 +443,7 @@ mod tests { } #[test] + #[cfg(not(feature = "optimism"))] fn test_block_number_to_index() { // Create a Receipts object with a vector of receipt vectors let receipts = Receipts { @@ -452,10 +452,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), })]], }; @@ -482,6 +478,7 @@ mod tests { } #[test] + #[cfg(not(feature = "optimism"))] fn test_get_logs() { // Create a Receipts object with a vector of receipt vectors let receipts = Receipts { @@ -490,10 +487,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![Log::::default()], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), })]], }; @@ -517,6 +510,7 @@ mod tests { } #[test] + #[cfg(not(feature = "optimism"))] fn test_receipts_by_block() { // Create a Receipts object with a vector of receipt vectors let receipts = Receipts { @@ -525,10 +519,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![Log::::default()], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), })]], }; @@ -555,15 +545,12 @@ mod tests { cumulative_gas_used: 46913, logs: vec![Log::::default()], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), })] ); } #[test] + #[cfg(not(feature = "optimism"))] fn test_receipts_len() { // Create a Receipts object with a vector of receipt vectors let receipts = Receipts { @@ -572,10 +559,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![Log::::default()], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), })]], }; @@ -616,6 +599,7 @@ mod tests { } #[test] + #[cfg(not(feature = "optimism"))] fn test_revert_to() { // Create a random receipt object let receipt = Receipt { @@ -623,10 +607,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), }; // Create a Receipts object with a vector of receipt vectors @@ -668,6 +648,7 @@ mod tests { } #[test] + #[cfg(not(feature = "optimism"))] fn test_extend_execution_outcome() { // Create a Receipt object with specific attributes. let receipt = Receipt { @@ -675,10 +656,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), }; // Create a Receipts object containing the receipt. @@ -715,6 +692,7 @@ mod tests { } #[test] + #[cfg(not(feature = "optimism"))] fn test_split_at_execution_outcome() { // Create a random receipt object let receipt = Receipt { @@ -722,10 +700,6 @@ mod tests { cumulative_gas_used: 46913, logs: vec![], success: true, - #[cfg(feature = "optimism")] - deposit_nonce: Some(18), - #[cfg(feature = "optimism")] - deposit_receipt_version: Some(34), }; // Create a Receipts object with a vector of receipt vectors diff --git a/crates/optimism/evm/src/lib.rs b/crates/optimism/evm/src/lib.rs index 60aa9f7db083..bc46f3ea9c20 100644 --- a/crates/optimism/evm/src/lib.rs +++ b/crates/optimism/evm/src/lib.rs @@ -206,23 +206,30 @@ impl ConfigureEvm for OptimismEvmConfig { mod tests { use super::*; use alloy_consensus::constants::KECCAK_EMPTY; + use alloy_eips::eip7685::Requests; use alloy_genesis::Genesis; - use alloy_primitives::{B256, U256}; + use alloy_primitives::{bytes, Address, LogData, B256, U256}; use reth_chainspec::ChainSpec; use reth_evm::execute::ProviderError; - use reth_execution_types::{Chain, ExecutionOutcome}; + use reth_execution_types::{ + AccountRevertInit, BundleStateInit, Chain, ExecutionOutcome, RevertsInit, + }; use reth_optimism_chainspec::BASE_MAINNET; use reth_primitives::{ - revm_primitives::{BlockEnv, CfgEnv, SpecId}, - Header, Receipt, Receipts, SealedBlockWithSenders, TxType, + revm_primitives::{AccountInfo, BlockEnv, CfgEnv, SpecId}, + Account, Header, Log, Receipt, Receipts, SealedBlockWithSenders, TxType, }; + use reth_revm::{ - db::{CacheDB, EmptyDBTyped}, + db::{BundleState, CacheDB, EmptyDBTyped}, inspectors::NoOpInspector, JournaledState, }; use revm_primitives::{CfgEnvWithHandlerCfg, EnvWithHandlerCfg, HandlerCfg}; - use std::{collections::HashSet, sync::Arc}; + use std::{ + collections::{HashMap, HashSet}, + sync::Arc, + }; fn test_evm_config() -> OptimismEvmConfig { OptimismEvmConfig::new(BASE_MAINNET.clone()) @@ -620,4 +627,399 @@ mod tests { // Assert that the execution outcome at the tip block contains the whole execution outcome assert_eq!(chain.execution_outcome_at_block(11), Some(execution_outcome)); } + + #[test] + fn test_initialisation() { + // Create a new BundleState object with initial data + let bundle = BundleState::new( + vec![(Address::new([2; 20]), None, Some(AccountInfo::default()), HashMap::default())], + vec![vec![(Address::new([2; 20]), None, vec![])]], + vec![], + ); + + // Create a Receipts object with a vector of receipt vectors + let receipts = Receipts { + receipt_vec: vec![vec![Some(Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + })]], + }; + + // Create a Requests object with a vector of requests + let requests = vec![Requests::new(vec![bytes!("dead"), bytes!("beef"), bytes!("beebee")])]; + + // Define the first block number + let first_block = 123; + + // Create a ExecutionOutcome object with the created bundle, receipts, requests, and + // first_block + let exec_res = ExecutionOutcome { + bundle: bundle.clone(), + receipts: receipts.clone(), + requests: requests.clone(), + first_block, + }; + + // Assert that creating a new ExecutionOutcome using the constructor matches exec_res + assert_eq!( + ExecutionOutcome::new(bundle, receipts.clone(), first_block, requests.clone()), + exec_res + ); + + // Create a BundleStateInit object and insert initial data + let mut state_init: BundleStateInit = HashMap::default(); + state_init + .insert(Address::new([2; 20]), (None, Some(Account::default()), HashMap::default())); + + // Create a HashMap for account reverts and insert initial data + let mut revert_inner: HashMap = HashMap::default(); + revert_inner.insert(Address::new([2; 20]), (None, vec![])); + + // Create a RevertsInit object and insert the revert_inner data + let mut revert_init: RevertsInit = HashMap::default(); + revert_init.insert(123, revert_inner); + + // Assert that creating a new ExecutionOutcome using the new_init method matches + // exec_res + assert_eq!( + ExecutionOutcome::new_init( + state_init, + revert_init, + vec![], + receipts, + first_block, + requests, + ), + exec_res + ); + } + + #[test] + fn test_block_number_to_index() { + // Create a Receipts object with a vector of receipt vectors + let receipts = Receipts { + receipt_vec: vec![vec![Some(Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + })]], + }; + + // Define the first block number + let first_block = 123; + + // Create a ExecutionOutcome object with the created bundle, receipts, requests, and + // first_block + let exec_res = ExecutionOutcome { + bundle: Default::default(), + receipts, + requests: vec![], + first_block, + }; + + // Test before the first block + assert_eq!(exec_res.block_number_to_index(12), None); + + // Test after after the first block but index larger than receipts length + assert_eq!(exec_res.block_number_to_index(133), None); + + // Test after the first block + assert_eq!(exec_res.block_number_to_index(123), Some(0)); + } + + #[test] + fn test_get_logs() { + // Create a Receipts object with a vector of receipt vectors + let receipts = Receipts { + receipt_vec: vec![vec![Some(Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![Log::::default()], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + })]], + }; + + // Define the first block number + let first_block = 123; + + // Create a ExecutionOutcome object with the created bundle, receipts, requests, and + // first_block + let exec_res = ExecutionOutcome { + bundle: Default::default(), + receipts, + requests: vec![], + first_block, + }; + + // Get logs for block number 123 + let logs: Vec<&Log> = exec_res.logs(123).unwrap().collect(); + + // Assert that the logs match the expected logs + assert_eq!(logs, vec![&Log::::default()]); + } + + #[test] + fn test_receipts_by_block() { + // Create a Receipts object with a vector of receipt vectors + let receipts = Receipts { + receipt_vec: vec![vec![Some(Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![Log::::default()], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + })]], + }; + + // Define the first block number + let first_block = 123; + + // Create a ExecutionOutcome object with the created bundle, receipts, requests, and + // first_block + let exec_res = ExecutionOutcome { + bundle: Default::default(), // Default value for bundle + receipts, // Include the created receipts + requests: vec![], // Empty vector for requests + first_block, // Set the first block number + }; + + // Get receipts for block number 123 and convert the result into a vector + let receipts_by_block: Vec<_> = exec_res.receipts_by_block(123).iter().collect(); + + // Assert that the receipts for block number 123 match the expected receipts + assert_eq!( + receipts_by_block, + vec![&Some(Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![Log::::default()], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + })] + ); + } + + #[test] + fn test_receipts_len() { + // Create a Receipts object with a vector of receipt vectors + let receipts = Receipts { + receipt_vec: vec![vec![Some(Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![Log::::default()], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + })]], + }; + + // Create an empty Receipts object + let receipts_empty = Receipts { receipt_vec: vec![] }; + + // Define the first block number + let first_block = 123; + + // Create a ExecutionOutcome object with the created bundle, receipts, requests, and + // first_block + let exec_res = ExecutionOutcome { + bundle: Default::default(), // Default value for bundle + receipts, // Include the created receipts + requests: vec![], // Empty vector for requests + first_block, // Set the first block number + }; + + // Assert that the length of receipts in exec_res is 1 + assert_eq!(exec_res.len(), 1); + + // Assert that exec_res is not empty + assert!(!exec_res.is_empty()); + + // Create a ExecutionOutcome object with an empty Receipts object + let exec_res_empty_receipts = ExecutionOutcome { + bundle: Default::default(), // Default value for bundle + receipts: receipts_empty, // Include the empty receipts + requests: vec![], // Empty vector for requests + first_block, // Set the first block number + }; + + // Assert that the length of receipts in exec_res_empty_receipts is 0 + assert_eq!(exec_res_empty_receipts.len(), 0); + + // Assert that exec_res_empty_receipts is empty + assert!(exec_res_empty_receipts.is_empty()); + } + + #[test] + fn test_revert_to() { + // Create a random receipt object + let receipt = Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + }; + + // Create a Receipts object with a vector of receipt vectors + let receipts = Receipts { + receipt_vec: vec![vec![Some(receipt.clone())], vec![Some(receipt.clone())]], + }; + + // Define the first block number + let first_block = 123; + + // Create a request. + let request = bytes!("deadbeef"); + + // Create a vector of Requests containing the request. + let requests = + vec![Requests::new(vec![request.clone()]), Requests::new(vec![request.clone()])]; + + // Create a ExecutionOutcome object with the created bundle, receipts, requests, and + // first_block + let mut exec_res = + ExecutionOutcome { bundle: Default::default(), receipts, requests, first_block }; + + // Assert that the revert_to method returns true when reverting to the initial block number. + assert!(exec_res.revert_to(123)); + + // Assert that the receipts are properly cut after reverting to the initial block number. + assert_eq!(exec_res.receipts, Receipts { receipt_vec: vec![vec![Some(receipt)]] }); + + // Assert that the requests are properly cut after reverting to the initial block number. + assert_eq!(exec_res.requests, vec![Requests::new(vec![request])]); + + // Assert that the revert_to method returns false when attempting to revert to a block + // number greater than the initial block number. + assert!(!exec_res.revert_to(133)); + + // Assert that the revert_to method returns false when attempting to revert to a block + // number less than the initial block number. + assert!(!exec_res.revert_to(10)); + } + + #[test] + fn test_extend_execution_outcome() { + // Create a Receipt object with specific attributes. + let receipt = Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + }; + + // Create a Receipts object containing the receipt. + let receipts = Receipts { receipt_vec: vec![vec![Some(receipt.clone())]] }; + + // Create a request. + let request = bytes!("deadbeef"); + + // Create a vector of Requests containing the request. + let requests = vec![Requests::new(vec![request.clone()])]; + + // Define the initial block number. + let first_block = 123; + + // Create an ExecutionOutcome object. + let mut exec_res = + ExecutionOutcome { bundle: Default::default(), receipts, requests, first_block }; + + // Extend the ExecutionOutcome object by itself. + exec_res.extend(exec_res.clone()); + + // Assert the extended ExecutionOutcome matches the expected outcome. + assert_eq!( + exec_res, + ExecutionOutcome { + bundle: Default::default(), + receipts: Receipts { + receipt_vec: vec![vec![Some(receipt.clone())], vec![Some(receipt)]] + }, + requests: vec![Requests::new(vec![request.clone()]), Requests::new(vec![request])], + first_block: 123, + } + ); + } + + #[test] + fn test_split_at_execution_outcome() { + // Create a random receipt object + let receipt = Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 46913, + logs: vec![], + success: true, + deposit_nonce: Some(18), + deposit_receipt_version: Some(34), + }; + + // Create a Receipts object with a vector of receipt vectors + let receipts = Receipts { + receipt_vec: vec![ + vec![Some(receipt.clone())], + vec![Some(receipt.clone())], + vec![Some(receipt.clone())], + ], + }; + + // Define the first block number + let first_block = 123; + + // Create a request. + let request = bytes!("deadbeef"); + + // Create a vector of Requests containing the request. + let requests = vec![ + Requests::new(vec![request.clone()]), + Requests::new(vec![request.clone()]), + Requests::new(vec![request.clone()]), + ]; + + // Create a ExecutionOutcome object with the created bundle, receipts, requests, and + // first_block + let exec_res = + ExecutionOutcome { bundle: Default::default(), receipts, requests, first_block }; + + // Split the ExecutionOutcome at block number 124 + let result = exec_res.clone().split_at(124); + + // Define the expected lower ExecutionOutcome after splitting + let lower_execution_outcome = ExecutionOutcome { + bundle: Default::default(), + receipts: Receipts { receipt_vec: vec![vec![Some(receipt.clone())]] }, + requests: vec![Requests::new(vec![request.clone()])], + first_block, + }; + + // Define the expected higher ExecutionOutcome after splitting + let higher_execution_outcome = ExecutionOutcome { + bundle: Default::default(), + receipts: Receipts { + receipt_vec: vec![vec![Some(receipt.clone())], vec![Some(receipt)]], + }, + requests: vec![Requests::new(vec![request.clone()]), Requests::new(vec![request])], + first_block: 124, + }; + + // Assert that the split result matches the expected lower and higher outcomes + assert_eq!(result.0, Some(lower_execution_outcome)); + assert_eq!(result.1, higher_execution_outcome); + + // Assert that splitting at the first block number returns None for the lower outcome + assert_eq!(exec_res.clone().split_at(123), (None, exec_res)); + } } From d5f5c0f11226879b3c7e46cf7226506871d689da Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sat, 26 Oct 2024 23:40:46 +0800 Subject: [PATCH 109/239] chore(rpc): set `RpcNodeCore` as supertrait for `LoadState` (#12094) --- crates/optimism/rpc/src/eth/mod.rs | 19 +++-------- crates/rpc/rpc-eth-api/src/helpers/call.rs | 8 ++--- crates/rpc/rpc-eth-api/src/helpers/spec.rs | 11 +++--- crates/rpc/rpc-eth-api/src/helpers/state.rs | 34 ++++++++----------- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 8 ++--- .../rpc-eth-api/src/helpers/transaction.rs | 6 ++-- crates/rpc/rpc-eth-api/src/node.rs | 2 +- crates/rpc/rpc/src/debug.rs | 6 ++-- crates/rpc/rpc/src/eth/helpers/state.rs | 24 +++++-------- 9 files changed, 48 insertions(+), 70 deletions(-) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index ccff477892f2..bc1692dff4ea 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -143,7 +143,6 @@ where impl EthApiSpec for OpEthApi where - Self: Send + Sync, N: RpcNodeCore< Provider: ChainSpecProvider + BlockNumReader @@ -213,25 +212,15 @@ where impl LoadState for OpEthApi where - Self: Send + Sync + Clone, - N: FullNodeComponents>, + N: RpcNodeCore< + Provider: StateProviderFactory + ChainSpecProvider, + Pool: TransactionPool, + >, { - #[inline] - fn provider( - &self, - ) -> impl StateProviderFactory + ChainSpecProvider { - self.inner.provider() - } - #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() } - - #[inline] - fn pool(&self) -> impl TransactionPool { - self.inner.pool() - } } impl EthState for OpEthApi diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 1510233c5059..89ae1c8ac96e 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -486,7 +486,7 @@ pub trait Call: LoadState + SpawnBlocking { DB: Database, EthApiError: From, { - let mut evm = self.evm_config().evm_with_env(db, env); + let mut evm = Call::evm_config(self).evm_with_env(db, env); let res = evm.transact().map_err(Self::Error::from_evm_err)?; let (_, env) = evm.into_db_and_env_with_handler_cfg(); Ok((res, env)) @@ -504,7 +504,7 @@ pub trait Call: LoadState + SpawnBlocking { DB: Database, EthApiError: From, { - let mut evm = self.evm_config().evm_with_env_and_inspector(db, env, inspector); + let mut evm = Call::evm_config(self).evm_with_env_and_inspector(db, env, inspector); let res = evm.transact().map_err(Self::Error::from_evm_err)?; let (_, env) = evm.into_db_and_env_with_handler_cfg(); Ok((res, env)) @@ -669,7 +669,7 @@ pub trait Call: LoadState + SpawnBlocking { { let env = EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()); - let mut evm = self.evm_config().evm_with_env(db, env); + let mut evm = Call::evm_config(self).evm_with_env(db, env); let mut index = 0; for (sender, tx) in transactions { if tx.hash() == target_tx_hash { @@ -677,7 +677,7 @@ pub trait Call: LoadState + SpawnBlocking { break } - self.evm_config().fill_tx_env(evm.tx_mut(), tx, *sender); + Call::evm_config(self).fill_tx_env(evm.tx_mut(), tx, *sender); evm.transact_commit().map_err(Self::Error::from_evm_err)?; index += 1; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/spec.rs b/crates/rpc/rpc-eth-api/src/helpers/spec.rs index 5aa0509e8bbe..a6213017af8a 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/spec.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/spec.rs @@ -16,12 +16,11 @@ use crate::{helpers::EthSigner, RpcNodeCore}; #[auto_impl::auto_impl(&, Arc)] pub trait EthApiSpec: RpcNodeCore< - Provider: ChainSpecProvider - + BlockNumReader - + StageCheckpointReader, - Network: NetworkInfo, - > + Send - + Sync + Provider: ChainSpecProvider + + BlockNumReader + + StageCheckpointReader, + Network: NetworkInfo, +> { /// Returns the block node is started on. fn starting_block(&self) -> U256; diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 080d90dc3b00..2a15b194f13d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -18,7 +18,7 @@ use reth_rpc_types_compat::proof::from_primitive_account_proof; use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg, SpecId}; -use crate::{EthApiTypes, FromEthApiError}; +use crate::{EthApiTypes, FromEthApiError, RpcNodeCore}; use super::{EthApiSpec, LoadPendingBlock, SpawnBlocking}; @@ -105,7 +105,8 @@ pub trait EthState: LoadState + SpawnBlocking { let block_id = block_id.unwrap_or_default(); // Check whether the distance to the block exceeds the maximum configured window. - let block_number = LoadState::provider(self) + let block_number = self + .provider() .block_number_for_id(block_id) .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(block_id))?; @@ -138,9 +139,9 @@ pub trait EthState: LoadState + SpawnBlocking { let Some(account) = account else { return Ok(None) }; // Check whether the distance to the block exceeds the maximum configured proof window. - let chain_info = - LoadState::provider(&this).chain_info().map_err(Self::Error::from_eth_err)?; - let block_number = LoadState::provider(&this) + let chain_info = this.provider().chain_info().map_err(Self::Error::from_eth_err)?; + let block_number = this + .provider() .block_number_for_id(block_id) .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(block_id))?; @@ -167,24 +168,19 @@ pub trait EthState: LoadState + SpawnBlocking { /// Loads state from database. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` state RPC methods. -pub trait LoadState: EthApiTypes { - /// Returns a handle for reading state from database. - /// - /// Data access in default trait method implementations. - fn provider( - &self, - ) -> impl StateProviderFactory + ChainSpecProvider; - +pub trait LoadState: + EthApiTypes + + RpcNodeCore< + Provider: StateProviderFactory + + ChainSpecProvider, + Pool: TransactionPool, + > +{ /// Returns a handle for reading data from memory. /// /// Data access in default (L1) trait method implementations. fn cache(&self) -> &EthStateCache; - /// Returns a handle for reading data from transaction pool. - /// - /// Data access in default trait method implementations. - fn pool(&self) -> impl TransactionPool; - /// Returns the state at the given block number fn state_at_hash(&self, block_hash: B256) -> Result { self.provider().history_by_block_hash(block_hash).map_err(Self::Error::from_eth_err) @@ -266,7 +262,7 @@ pub trait LoadState: EthApiTypes { let (cfg, mut block_env, _) = self.evm_env_at(header.parent_hash.into()).await?; let after_merge = cfg.handler_cfg.spec_id >= SpecId::MERGE; - self.evm_config().fill_block_env(&mut block_env, header, after_merge); + LoadPendingBlock::evm_config(self).fill_block_env(&mut block_env, header, after_merge); Ok((cfg, block_env)) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 64056148cd38..4f11734849a3 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -2,7 +2,7 @@ use std::sync::Arc; -use crate::FromEvmError; +use crate::{FromEvmError, RpcNodeCore}; use alloy_primitives::B256; use alloy_rpc_types::{BlockId, TransactionInfo}; use futures::Future; @@ -60,7 +60,7 @@ pub trait Trace: LoadState { I: GetInspector, { - let mut evm = self.evm_config().evm_with_env_and_inspector(db, env, inspector); + let mut evm = Trace::evm_config(self).evm_with_env_and_inspector(db, env, inspector); let res = evm.transact().map_err(Self::Error::from_evm_err)?; let (db, env) = evm.into_db_and_env_with_handler_cfg(); Ok((res, env, db)) @@ -202,7 +202,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( Trace::evm_config(&this).clone(), - LoadState::provider(&this).chain_spec(), + RpcNodeCore::provider(&this).chain_spec(), ); system_caller .pre_block_beacon_root_contract_call( @@ -345,7 +345,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( Trace::evm_config(&this).clone(), - LoadState::provider(&this).chain_spec(), + RpcNodeCore::provider(&this).chain_spec(), ); system_caller .pre_block_beacon_root_contract_call( diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 0d16a5c9145b..a91e4e6faef7 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -21,7 +21,9 @@ use reth_rpc_types_compat::transaction::{from_recovered, from_recovered_with_blo use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; use std::sync::Arc; -use crate::{FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcReceipt, RpcTransaction}; +use crate::{ + FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcNodeCore, RpcReceipt, RpcTransaction, +}; use super::{ Call, EthApiSpec, EthSigner, LoadBlock, LoadPendingBlock, LoadReceipt, LoadState, SpawnBlocking, @@ -235,7 +237,7 @@ pub trait EthTransactions: LoadTransaction { // Check the pool first if include_pending { if let Some(tx) = - LoadState::pool(self).get_transaction_by_sender_and_nonce(sender, nonce) + RpcNodeCore::pool(self).get_transaction_by_sender_and_nonce(sender, nonce) { let transaction = tx.transaction.clone().into_consensus(); return Ok(Some(from_recovered(transaction.into(), self.tx_resp_builder()))); diff --git a/crates/rpc/rpc-eth-api/src/node.rs b/crates/rpc/rpc-eth-api/src/node.rs index 8488677e32fb..950271dfcb17 100644 --- a/crates/rpc/rpc-eth-api/src/node.rs +++ b/crates/rpc/rpc-eth-api/src/node.rs @@ -8,7 +8,7 @@ use reth_node_api::FullNodeComponents; /// `N: RpcNodeCore` instead, allows access to all the associated types on [`FullNodeComponents`] /// that are used in RPC, but with more flexibility since they have no trait bounds (asides auto /// traits). -pub trait RpcNodeCore: Clone { +pub trait RpcNodeCore: Clone + Send + Sync { /// The provider type used to interact with the node. type Provider: Send + Sync + Clone + Unpin; /// The transaction pool of the node. diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 5a20bee975ff..eeab734a6437 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -27,8 +27,8 @@ use reth_provider::{ use reth_revm::database::StateProviderDatabase; use reth_rpc_api::DebugApiServer; use reth_rpc_eth_api::{ - helpers::{Call, EthApiSpec, EthTransactions, LoadState, TraceExt}, - EthApiTypes, FromEthApiError, + helpers::{Call, EthApiSpec, EthTransactions, TraceExt}, + EthApiTypes, FromEthApiError, RpcNodeCore, }; use reth_rpc_eth_types::{EthApiError, StateCacheDb}; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; @@ -264,7 +264,7 @@ where // apply relevant system calls let mut system_caller = SystemCaller::new( Call::evm_config(this.eth_api()).clone(), - LoadState::provider(this.eth_api()).chain_spec(), + RpcNodeCore::provider(this.eth_api()).chain_spec(), ); system_caller diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 429a10333d1b..8c958ea2ae2d 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -4,7 +4,10 @@ use reth_chainspec::EthereumHardforks; use reth_provider::{ChainSpecProvider, StateProviderFactory}; use reth_transaction_pool::TransactionPool; -use reth_rpc_eth_api::helpers::{EthState, LoadState, SpawnBlocking}; +use reth_rpc_eth_api::{ + helpers::{EthState, LoadState, SpawnBlocking}, + RpcNodeCore, +}; use reth_rpc_eth_types::EthStateCache; use crate::EthApi; @@ -20,26 +23,15 @@ where impl LoadState for EthApi where - Self: Send + Sync, - Provider: StateProviderFactory + ChainSpecProvider, - Pool: TransactionPool, + Self: RpcNodeCore< + Provider: StateProviderFactory + ChainSpecProvider, + Pool: TransactionPool, + >, { - #[inline] - fn provider( - &self, - ) -> impl StateProviderFactory + ChainSpecProvider { - self.inner.provider() - } - #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() } - - #[inline] - fn pool(&self) -> impl TransactionPool { - self.inner.pool() - } } #[cfg(test)] From 09ebecffc7ffafbe965f6f652032b95e6acac473 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sat, 26 Oct 2024 18:11:57 +0200 Subject: [PATCH 110/239] prune: add unit tests for `PruneInput` `get_next_tx_num_range` (#12081) --- crates/prune/prune/src/segments/mod.rs | 209 ++++++++++++++++++++++++- crates/prune/prune/src/segments/set.rs | 4 +- 2 files changed, 208 insertions(+), 5 deletions(-) diff --git a/crates/prune/prune/src/segments/mod.rs b/crates/prune/prune/src/segments/mod.rs index d1b7819ac763..b3b40aab5b3b 100644 --- a/crates/prune/prune/src/segments/mod.rs +++ b/crates/prune/prune/src/segments/mod.rs @@ -23,9 +23,9 @@ pub use user::{ /// A segment represents a pruning of some portion of the data. /// -/// Segments are called from [Pruner](crate::Pruner) with the following lifecycle: +/// Segments are called from [`Pruner`](crate::Pruner) with the following lifecycle: /// 1. Call [`Segment::prune`] with `delete_limit` of [`PruneInput`]. -/// 2. If [`Segment::prune`] returned a [Some] in `checkpoint` of [`SegmentOutput`], call +/// 2. If [`Segment::prune`] returned a [`Some`] in `checkpoint` of [`SegmentOutput`], call /// [`Segment::save_checkpoint`]. /// 3. Subtract `pruned` of [`SegmentOutput`] from `delete_limit` of next [`PruneInput`]. pub trait Segment: Debug + Send + Sync { @@ -88,7 +88,7 @@ impl PruneInput { }, }) // No checkpoint exists, prune from genesis - .unwrap_or(0); + .unwrap_or_default(); let to_tx_number = match provider.block_body_indices(self.to_block)? { Some(body) => { @@ -143,3 +143,206 @@ impl PruneInput { .unwrap_or(0) } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use reth_provider::{ + providers::BlockchainProvider2, + test_utils::{create_test_provider_factory, MockEthProvider}, + }; + use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; + + #[test] + fn test_prune_input_get_next_tx_num_range_no_to_block() { + let input = PruneInput { + previous_checkpoint: None, + to_block: 10, + limiter: PruneLimiter::default(), + }; + + // Default provider with no block corresponding to block 10 + let provider = MockEthProvider::default(); + + // No block body for block 10, expected None + let range = input.get_next_tx_num_range(&provider).expect("Expected range"); + assert!(range.is_none()); + } + + #[test] + fn test_prune_input_get_next_tx_num_range_no_tx() { + let input = PruneInput { + previous_checkpoint: None, + to_block: 10, + limiter: PruneLimiter::default(), + }; + + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + + // Generate 10 random blocks with no transactions + let blocks = random_block_range( + &mut rng, + 0..=10, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, + ); + + // Insert the blocks into the database + let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); + for block in &blocks { + provider_rw + .insert_historical_block( + block.clone().seal_with_senders().expect("failed to seal block with senders"), + ) + .expect("failed to insert block"); + } + provider_rw.commit().expect("failed to commit"); + + // Create a new provider + let provider = BlockchainProvider2::new(factory).unwrap(); + + // Since there are no transactions, expected None + let range = input.get_next_tx_num_range(&provider).expect("Expected range"); + assert!(range.is_none()); + } + + #[test] + fn test_prune_input_get_next_tx_num_range_valid() { + // Create a new prune input + let input = PruneInput { + previous_checkpoint: None, + to_block: 10, + limiter: PruneLimiter::default(), + }; + + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + + // Generate 10 random blocks with some transactions + let blocks = random_block_range( + &mut rng, + 0..=10, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..5, ..Default::default() }, + ); + + // Insert the blocks into the database + let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); + for block in &blocks { + provider_rw + .insert_historical_block( + block.clone().seal_with_senders().expect("failed to seal block with senders"), + ) + .expect("failed to insert block"); + } + provider_rw.commit().expect("failed to commit"); + + // Create a new provider + let provider = BlockchainProvider2::new(factory).unwrap(); + + // Get the next tx number range + let range = input.get_next_tx_num_range(&provider).expect("Expected range").unwrap(); + + // Calculate the total number of transactions + let num_txs = + blocks.iter().map(|block| block.body.transactions().count() as u64).sum::(); + + assert_eq!(range, 0..=num_txs - 1); + } + + #[test] + fn test_prune_input_get_next_tx_checkpoint_without_tx_number() { + // Create a prune input with a previous checkpoint without a tx number (unexpected) + let input = PruneInput { + previous_checkpoint: Some(PruneCheckpoint { + block_number: Some(5), + tx_number: None, + prune_mode: PruneMode::Full, + }), + to_block: 10, + limiter: PruneLimiter::default(), + }; + + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + + // Generate 10 random blocks + let blocks = random_block_range( + &mut rng, + 0..=10, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..5, ..Default::default() }, + ); + + // Insert the blocks into the database + let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); + for block in &blocks { + provider_rw + .insert_historical_block( + block.clone().seal_with_senders().expect("failed to seal block with senders"), + ) + .expect("failed to insert block"); + } + provider_rw.commit().expect("failed to commit"); + + // Create a new provider + let provider = BlockchainProvider2::new(factory).unwrap(); + + // Fetch the range and check if it is correct + let range = input.get_next_tx_num_range(&provider).expect("Expected range").unwrap(); + + // Calculate the total number of transactions + let num_txs = + blocks.iter().map(|block| block.body.transactions().count() as u64).sum::(); + + assert_eq!(range, 0..=num_txs - 1,); + } + + #[test] + fn test_prune_input_get_next_tx_empty_range() { + // Create a new provider via factory + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + + // Generate 10 random blocks + let blocks = random_block_range( + &mut rng, + 0..=10, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..5, ..Default::default() }, + ); + + // Insert the blocks into the database + let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); + for block in &blocks { + provider_rw + .insert_historical_block( + block.clone().seal_with_senders().expect("failed to seal block with senders"), + ) + .expect("failed to insert block"); + } + provider_rw.commit().expect("failed to commit"); + + // Create a new provider + let provider = BlockchainProvider2::new(factory).unwrap(); + + // Get the last tx number + // Calculate the total number of transactions + let num_txs = + blocks.iter().map(|block| block.body.transactions().count() as u64).sum::(); + let max_range = num_txs - 1; + + // Create a prune input with a previous checkpoint that is the last tx number + let input = PruneInput { + previous_checkpoint: Some(PruneCheckpoint { + block_number: Some(5), + tx_number: Some(max_range), + prune_mode: PruneMode::Full, + }), + to_block: 10, + limiter: PruneLimiter::default(), + }; + + // We expect an empty range since the previous checkpoint is the last tx number + let range = input.get_next_tx_num_range(&provider).expect("Expected range"); + assert!(range.is_none()); + } +} diff --git a/crates/prune/prune/src/segments/set.rs b/crates/prune/prune/src/segments/set.rs index 710b2b721cd6..23d03345b096 100644 --- a/crates/prune/prune/src/segments/set.rs +++ b/crates/prune/prune/src/segments/set.rs @@ -11,7 +11,7 @@ use reth_prune_types::PruneModes; use super::{StaticFileHeaders, StaticFileReceipts, StaticFileTransactions}; -/// Collection of [Segment]. Thread-safe, allocated on the heap. +/// Collection of [`Segment`]. Thread-safe, allocated on the heap. #[derive(Debug)] pub struct SegmentSet { inner: Vec>>, @@ -23,7 +23,7 @@ impl SegmentSet { Self::default() } - /// Adds new [Segment] to collection. + /// Adds new [`Segment`] to collection. pub fn segment + 'static>(mut self, segment: S) -> Self { self.inner.push(Box::new(segment)); self From f616de6d94670687aedfd46e43fb991041cd5936 Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Sat, 26 Oct 2024 09:15:08 -0700 Subject: [PATCH 111/239] feat(rpc): Start to implement flashbots_validateBuilderSubmissionV3 (#12061) --- Cargo.lock | 1 + book/cli/reth/node.md | 4 +- crates/rpc/rpc-api/src/validation.rs | 9 ++- crates/rpc/rpc-builder/src/lib.rs | 10 ++- crates/rpc/rpc-server-types/src/module.rs | 3 + crates/rpc/rpc/Cargo.toml | 1 + crates/rpc/rpc/src/lib.rs | 3 + crates/rpc/rpc/src/validation.rs | 94 +++++++++++++++++++++++ 8 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 crates/rpc/rpc/src/validation.rs diff --git a/Cargo.lock b/Cargo.lock index f4dc7dae4e98..b1ca1c9259c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8598,6 +8598,7 @@ dependencies = [ "alloy-rlp", "alloy-rpc-types", "alloy-rpc-types-admin", + "alloy-rpc-types-beacon", "alloy-rpc-types-debug", "alloy-rpc-types-eth", "alloy-rpc-types-mev", diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 4cd55db1fe08..a3ff8f6a57b1 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -245,7 +245,7 @@ RPC: --http.api Rpc Modules to be configured for the HTTP server - [possible values: admin, debug, eth, net, trace, txpool, web3, rpc, reth, ots] + [possible values: admin, debug, eth, net, trace, txpool, web3, rpc, reth, ots, flashbots] --http.corsdomain Http Corsdomain to allow request from @@ -269,7 +269,7 @@ RPC: --ws.api Rpc Modules to be configured for the WS server - [possible values: admin, debug, eth, net, trace, txpool, web3, rpc, reth, ots] + [possible values: admin, debug, eth, net, trace, txpool, web3, rpc, reth, ots, flashbots] --ipcdisable Disable the IPC-RPC server diff --git a/crates/rpc/rpc-api/src/validation.rs b/crates/rpc/rpc-api/src/validation.rs index bbfa673d2598..e1819dde440c 100644 --- a/crates/rpc/rpc-api/src/validation.rs +++ b/crates/rpc/rpc-api/src/validation.rs @@ -1,7 +1,7 @@ //! API for block submission validation. use alloy_rpc_types_beacon::relay::{ - BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, + BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, BuilderBlockValidationRequestV3, }; use jsonrpsee::proc_macros::rpc; @@ -22,4 +22,11 @@ pub trait BlockSubmissionValidationApi { &self, request: BuilderBlockValidationRequestV2, ) -> jsonrpsee::core::RpcResult<()>; + + /// A Request to validate a block submission. + #[method(name = "validateBuilderSubmissionV3")] + async fn validate_builder_submission_v3( + &self, + request: BuilderBlockValidationRequestV3, + ) -> jsonrpsee::core::RpcResult<()>; } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index fc98cd6ff4ef..787dce08b8d0 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -180,7 +180,7 @@ use reth_provider::{ }; use reth_rpc::{ AdminApi, DebugApi, EngineEthApi, EthBundle, NetApi, OtterscanApi, RPCApi, RethApi, TraceApi, - TxPoolApi, Web3Api, + TxPoolApi, ValidationApi, Web3Api, }; use reth_rpc_api::servers::*; use reth_rpc_eth_api::{ @@ -1067,6 +1067,11 @@ where pub fn reth_api(&self) -> RethApi { RethApi::new(self.provider.clone(), Box::new(self.executor.clone())) } + + /// Instantiates `ValidationApi` + pub fn validation_api(&self) -> ValidationApi { + ValidationApi::new(self.provider.clone()) + } } impl @@ -1223,6 +1228,9 @@ where .into_rpc() .into() } + RethRpcModule::Flashbots => { + ValidationApi::new(self.provider.clone()).into_rpc().into() + } }) .clone() }) diff --git a/crates/rpc/rpc-server-types/src/module.rs b/crates/rpc/rpc-server-types/src/module.rs index 56417dda701c..9f96ff0cef35 100644 --- a/crates/rpc/rpc-server-types/src/module.rs +++ b/crates/rpc/rpc-server-types/src/module.rs @@ -258,6 +258,8 @@ pub enum RethRpcModule { Reth, /// `ots_` module Ots, + /// `flashbots_` module + Flashbots, } // === impl RethRpcModule === @@ -306,6 +308,7 @@ impl FromStr for RethRpcModule { "rpc" => Self::Rpc, "reth" => Self::Reth, "ots" => Self::Ots, + "flashbots" => Self::Flashbots, _ => return Err(ParseError::VariantNotFound), }) } diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index dab86ac25872..00799d761d39 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -45,6 +45,7 @@ alloy-network.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true alloy-rpc-types.workspace = true +alloy-rpc-types-beacon.workspace = true alloy-rpc-types-eth = { workspace = true, features = ["jsonrpsee-types"] } alloy-rpc-types-debug.workspace = true alloy-rpc-types-trace.workspace = true diff --git a/crates/rpc/rpc/src/lib.rs b/crates/rpc/rpc/src/lib.rs index eec14981bf57..027edea3cc1e 100644 --- a/crates/rpc/rpc/src/lib.rs +++ b/crates/rpc/rpc/src/lib.rs @@ -42,7 +42,9 @@ mod reth; mod rpc; mod trace; mod txpool; +mod validation; mod web3; + pub use admin::AdminApi; pub use debug::DebugApi; pub use engine::{EngineApi, EngineEthApi}; @@ -53,4 +55,5 @@ pub use reth::RethApi; pub use rpc::RPCApi; pub use trace::TraceApi; pub use txpool::TxPoolApi; +pub use validation::ValidationApi; pub use web3::Web3Api; diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs new file mode 100644 index 000000000000..c6419dc12c01 --- /dev/null +++ b/crates/rpc/rpc/src/validation.rs @@ -0,0 +1,94 @@ +use alloy_rpc_types_beacon::relay::{ + BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, BuilderBlockValidationRequestV3, +}; +use async_trait::async_trait; +use jsonrpsee::core::RpcResult; +use reth_chainspec::ChainSpecProvider; +use reth_provider::{ + AccountReader, BlockReaderIdExt, HeaderProvider, StateProviderFactory, WithdrawalsProvider, +}; +use reth_rpc_api::BlockSubmissionValidationApiServer; +use reth_rpc_server_types::result::internal_rpc_err; +use std::sync::Arc; +use tracing::warn; + +/// The type that implements the `validation` rpc namespace trait +pub struct ValidationApi { + inner: Arc>, +} + +impl ValidationApi +where + Provider: BlockReaderIdExt + + ChainSpecProvider + + StateProviderFactory + + HeaderProvider + + AccountReader + + WithdrawalsProvider + + Clone + + 'static, +{ + /// The provider that can interact with the chain. + pub fn provider(&self) -> Provider { + self.inner.provider.clone() + } + + /// Create a new instance of the [`ValidationApi`] + pub fn new(provider: Provider) -> Self { + let inner = Arc::new(ValidationApiInner { provider }); + Self { inner } + } +} + +#[async_trait] +impl BlockSubmissionValidationApiServer for ValidationApi +where + Provider: BlockReaderIdExt + + ChainSpecProvider + + StateProviderFactory + + HeaderProvider + + AccountReader + + WithdrawalsProvider + + Clone + + 'static, +{ + async fn validate_builder_submission_v1( + &self, + _request: BuilderBlockValidationRequest, + ) -> RpcResult<()> { + Err(internal_rpc_err("unimplemented")) + } + + async fn validate_builder_submission_v2( + &self, + _request: BuilderBlockValidationRequestV2, + ) -> RpcResult<()> { + Err(internal_rpc_err("unimplemented")) + } + + /// Validates a block submitted to the relay + async fn validate_builder_submission_v3( + &self, + request: BuilderBlockValidationRequestV3, + ) -> RpcResult<()> { + warn!("flashbots_validateBuilderSubmissionV3: blindly accepting request without validation {:?}", request); + Ok(()) + } +} + +impl std::fmt::Debug for ValidationApi { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ValidationApi").finish_non_exhaustive() + } +} + +impl Clone for ValidationApi { + fn clone(&self) -> Self { + Self { inner: Arc::clone(&self.inner) } + } +} + +struct ValidationApiInner { + /// The provider that can interact with the chain. + provider: Provider, +} From b2574080609f8fd2584c0c95aa2555b45ff91288 Mon Sep 17 00:00:00 2001 From: lazymio Date: Sun, 27 Oct 2024 00:17:21 +0800 Subject: [PATCH 112/239] Fix readonly check in libmdbx-rs (#12096) --- crates/cli/commands/src/common.rs | 2 +- crates/storage/libmdbx-rs/src/environment.rs | 32 ++++++++++++++----- crates/storage/libmdbx-rs/src/transaction.rs | 5 ++- .../storage/libmdbx-rs/tests/environment.rs | 12 +++++++ 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 956a63a5aa0e..49fee347ed45 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -126,7 +126,7 @@ impl> Environmen .static_file_provider() .check_consistency(&factory.provider()?, has_receipt_pruning)? { - if factory.db_ref().is_read_only() { + if factory.db_ref().is_read_only()? { warn!(target: "reth::cli", ?unwind_target, "Inconsistent storage. Restart node to heal."); return Ok(factory) } diff --git a/crates/storage/libmdbx-rs/src/environment.rs b/crates/storage/libmdbx-rs/src/environment.rs index edf9321ace40..480f5aaab65c 100644 --- a/crates/storage/libmdbx-rs/src/environment.rs +++ b/crates/storage/libmdbx-rs/src/environment.rs @@ -4,7 +4,7 @@ use crate::{ flags::EnvironmentFlags, transaction::{RO, RW}, txn_manager::{TxnManager, TxnManagerMessage, TxnPtr}, - Transaction, TransactionKind, + Mode, SyncMode, Transaction, TransactionKind, }; use byteorder::{ByteOrder, NativeEndian}; use mem::size_of; @@ -72,14 +72,14 @@ impl Environment { /// Returns true if the environment was opened in [`crate::Mode::ReadWrite`] mode. #[inline] - pub fn is_read_write(&self) -> bool { - self.inner.env_kind.is_write_map() + pub fn is_read_write(&self) -> Result { + Ok(!self.is_read_only()?) } /// Returns true if the environment was opened in [`crate::Mode::ReadOnly`] mode. #[inline] - pub fn is_read_only(&self) -> bool { - !self.inner.env_kind.is_write_map() + pub fn is_read_only(&self) -> Result { + Ok(matches!(self.info()?.mode(), Mode::ReadOnly)) } /// Returns the transaction manager. @@ -425,6 +425,23 @@ impl Info { fsync: self.0.mi_pgop_stat.fsync, } } + + /// Return the mode of the database + #[inline] + pub const fn mode(&self) -> Mode { + let mode = self.0.mi_mode; + if (mode & ffi::MDBX_RDONLY) != 0 { + Mode::ReadOnly + } else if (mode & ffi::MDBX_UTTERLY_NOSYNC) != 0 { + Mode::ReadWrite { sync_mode: SyncMode::UtterlyNoSync } + } else if (mode & ffi::MDBX_NOMETASYNC) != 0 { + Mode::ReadWrite { sync_mode: SyncMode::NoMetaSync } + } else if (mode & ffi::MDBX_SAFE_NOSYNC) != 0 { + Mode::ReadWrite { sync_mode: SyncMode::SafeNoSync } + } else { + Mode::ReadWrite { sync_mode: SyncMode::Durable } + } + } } impl fmt::Debug for Environment { @@ -781,15 +798,14 @@ impl EnvironmentBuilder { } /// Sets the interprocess/shared threshold to force flush the data buffers to disk, if - /// [`SyncMode::SafeNoSync`](crate::flags::SyncMode::SafeNoSync) is used. + /// [`SyncMode::SafeNoSync`] is used. pub fn set_sync_bytes(&mut self, v: usize) -> &mut Self { self.sync_bytes = Some(v as u64); self } /// Sets the interprocess/shared relative period since the last unsteady commit to force flush - /// the data buffers to disk, if [`SyncMode::SafeNoSync`](crate::flags::SyncMode::SafeNoSync) is - /// used. + /// the data buffers to disk, if [`SyncMode::SafeNoSync`] is used. pub fn set_sync_period(&mut self, v: Duration) -> &mut Self { // For this option, mdbx uses units of 1/65536 of a second. let as_mdbx_units = (v.as_secs_f64() * 65536f64) as u64; diff --git a/crates/storage/libmdbx-rs/src/transaction.rs b/crates/storage/libmdbx-rs/src/transaction.rs index 88236ebe9914..84b2dabc90a8 100644 --- a/crates/storage/libmdbx-rs/src/transaction.rs +++ b/crates/storage/libmdbx-rs/src/transaction.rs @@ -6,7 +6,7 @@ use crate::{ txn_manager::{TxnManagerMessage, TxnPtr}, Cursor, Error, Stat, TableObject, }; -use ffi::{mdbx_txn_renew, MDBX_txn_flags_t, MDBX_TXN_RDONLY, MDBX_TXN_READWRITE}; +use ffi::{MDBX_txn_flags_t, MDBX_TXN_RDONLY, MDBX_TXN_READWRITE}; use indexmap::IndexSet; use parking_lot::{Mutex, MutexGuard}; use std::{ @@ -18,6 +18,9 @@ use std::{ time::Duration, }; +#[cfg(feature = "read-tx-timeouts")] +use ffi::mdbx_txn_renew; + mod private { use super::*; diff --git a/crates/storage/libmdbx-rs/tests/environment.rs b/crates/storage/libmdbx-rs/tests/environment.rs index 99453ef113af..007418f76bb9 100644 --- a/crates/storage/libmdbx-rs/tests/environment.rs +++ b/crates/storage/libmdbx-rs/tests/environment.rs @@ -128,6 +128,18 @@ fn test_info() { // assert_eq!(info.last_pgno(), 1); // assert_eq!(info.last_txnid(), 0); assert_eq!(info.num_readers(), 0); + assert!(matches!(info.mode(), Mode::ReadWrite { sync_mode: SyncMode::Durable })); + assert!(env.is_read_write().unwrap()); + + drop(env); + let env = Environment::builder() + .set_geometry(Geometry { size: Some(map_size..), ..Default::default() }) + .set_flags(EnvironmentFlags { mode: Mode::ReadOnly, ..Default::default() }) + .open(dir.path()) + .unwrap(); + let info = env.info().unwrap(); + assert!(matches!(info.mode(), Mode::ReadOnly)); + assert!(env.is_read_only().unwrap()); } #[test] From 1bdf429af5092a0b737e900977c1418bc070ec59 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sun, 27 Oct 2024 00:20:08 +0800 Subject: [PATCH 113/239] Remove trait method `Call::evm_config` (#12095) --- crates/optimism/rpc/src/eth/call.rs | 11 +++-------- crates/rpc/rpc-eth-api/src/helpers/call.rs | 20 ++++++++------------ crates/rpc/rpc-eth-api/src/helpers/trace.rs | 2 +- crates/rpc/rpc/src/debug.rs | 11 ++++++----- crates/rpc/rpc/src/eth/bundle.rs | 6 +++--- crates/rpc/rpc/src/eth/helpers/call.rs | 7 +------ crates/rpc/rpc/src/trace.rs | 7 ++----- 7 files changed, 24 insertions(+), 40 deletions(-) diff --git a/crates/optimism/rpc/src/eth/call.rs b/crates/optimism/rpc/src/eth/call.rs index f1c10e6f1726..0402165f7079 100644 --- a/crates/optimism/rpc/src/eth/call.rs +++ b/crates/optimism/rpc/src/eth/call.rs @@ -9,7 +9,7 @@ use reth_primitives::{ }; use reth_rpc_eth_api::{ helpers::{Call, EthCall, LoadState, SpawnBlocking}, - FromEthApiError, IntoEthApiError, + FromEthApiError, IntoEthApiError, RpcNodeCore, }; use reth_rpc_eth_types::{revm_utils::CallFees, RpcInvalidTransactionError}; @@ -24,9 +24,9 @@ where impl Call for OpEthApi where - Self: LoadState + SpawnBlocking, + N: RpcNodeCore, + Self: LoadState> + SpawnBlocking, Self::Error: From, - N: FullNodeComponents, { #[inline] fn call_gas_limit(&self) -> u64 { @@ -38,11 +38,6 @@ where self.inner.max_simulate_blocks() } - #[inline] - fn evm_config(&self) -> &impl ConfigureEvm
{ - self.inner.evm_config() - } - fn create_txn_env( &self, block_env: &BlockEnv, diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 89ae1c8ac96e..f2662a86a74f 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -3,6 +3,7 @@ use crate::{ AsEthApiError, FromEthApiError, FromEvmError, FullEthApiTypes, IntoEthApiError, RpcBlock, + RpcNodeCore, }; use alloy_eips::{eip1559::calc_next_block_base_fee, eip2930::AccessListResult}; use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; @@ -300,7 +301,7 @@ pub trait EthCall: Call + LoadPendingBlock { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg.clone(), block_env.clone(), - Call::evm_config(&this).tx_env(tx, *signer), + RpcNodeCore::evm_config(&this).tx_env(tx, *signer), ); let (res, _) = this.transact(&mut db, env)?; db.commit(res.state); @@ -452,7 +453,7 @@ pub trait EthCall: Call + LoadPendingBlock { } /// Executes code on state. -pub trait Call: LoadState + SpawnBlocking { +pub trait Call: LoadState> + SpawnBlocking { /// Returns default gas limit to use for `eth_call` and tracing RPC methods. /// /// Data access in default trait method implementations. @@ -461,11 +462,6 @@ pub trait Call: LoadState + SpawnBlocking { /// Returns the maximum number of blocks accepted for `eth_simulateV1`. fn max_simulate_blocks(&self) -> u64; - /// Returns a handle for reading evm config. - /// - /// Data access in default (L1) trait method implementations. - fn evm_config(&self) -> &impl ConfigureEvm
; - /// Executes the closure with the state that corresponds to the given [`BlockId`]. fn with_state_at_block(&self, at: BlockId, f: F) -> Result where @@ -486,7 +482,7 @@ pub trait Call: LoadState + SpawnBlocking { DB: Database, EthApiError: From, { - let mut evm = Call::evm_config(self).evm_with_env(db, env); + let mut evm = self.evm_config().evm_with_env(db, env); let res = evm.transact().map_err(Self::Error::from_evm_err)?; let (_, env) = evm.into_db_and_env_with_handler_cfg(); Ok((res, env)) @@ -504,7 +500,7 @@ pub trait Call: LoadState + SpawnBlocking { DB: Database, EthApiError: From, { - let mut evm = Call::evm_config(self).evm_with_env_and_inspector(db, env, inspector); + let mut evm = self.evm_config().evm_with_env_and_inspector(db, env, inspector); let res = evm.transact().map_err(Self::Error::from_evm_err)?; let (_, env) = evm.into_db_and_env_with_handler_cfg(); Ok((res, env)) @@ -636,7 +632,7 @@ pub trait Call: LoadState + SpawnBlocking { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block_env, - Call::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), + RpcNodeCore::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), ); let (res, _) = this.transact(&mut db, env)?; @@ -669,7 +665,7 @@ pub trait Call: LoadState + SpawnBlocking { { let env = EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()); - let mut evm = Call::evm_config(self).evm_with_env(db, env); + let mut evm = self.evm_config().evm_with_env(db, env); let mut index = 0; for (sender, tx) in transactions { if tx.hash() == target_tx_hash { @@ -677,7 +673,7 @@ pub trait Call: LoadState + SpawnBlocking { break } - Call::evm_config(self).fill_tx_env(evm.tx_mut(), tx, *sender); + self.evm_config().fill_tx_env(evm.tx_mut(), tx, *sender); evm.transact_commit().map_err(Self::Error::from_evm_err)?; index += 1; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 4f11734849a3..6c7dd0f6f8d9 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -229,7 +229,7 @@ pub trait Trace: LoadState { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block_env, - Call::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), + RpcNodeCore::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), ); let (res, _) = this.inspect(StateCacheDbRefMutWrapper(&mut db), env, &mut inspector)?; diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index eeab734a6437..dd1cd9739ed1 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -27,7 +27,7 @@ use reth_provider::{ use reth_revm::database::StateProviderDatabase; use reth_rpc_api::DebugApiServer; use reth_rpc_eth_api::{ - helpers::{Call, EthApiSpec, EthTransactions, TraceExt}, + helpers::{EthApiSpec, EthTransactions, TraceExt}, EthApiTypes, FromEthApiError, RpcNodeCore, }; use reth_rpc_eth_types::{EthApiError, StateCacheDb}; @@ -120,7 +120,8 @@ where env: Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Call::evm_config(this.eth_api()).tx_env(tx.as_signed(), tx.signer()), + RpcNodeCore::evm_config(this.eth_api()) + .tx_env(tx.as_signed(), tx.signer()), ), handler_cfg: cfg.handler_cfg, }; @@ -263,7 +264,7 @@ where // apply relevant system calls let mut system_caller = SystemCaller::new( - Call::evm_config(this.eth_api()).clone(), + RpcNodeCore::evm_config(this.eth_api()).clone(), RpcNodeCore::provider(this.eth_api()).chain_spec(), ); @@ -293,7 +294,7 @@ where env: Env::boxed( cfg.cfg_env.clone(), block_env, - Call::evm_config(this.eth_api()).tx_env(tx.as_signed(), tx.signer()), + RpcNodeCore::evm_config(this.eth_api()).tx_env(tx.as_signed(), tx.signer()), ), handler_cfg: cfg.handler_cfg, }; @@ -562,7 +563,7 @@ where env: Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Call::evm_config(this.eth_api()).tx_env(tx, *signer), + RpcNodeCore::evm_config(this.eth_api()).tx_env(tx, *signer), ), handler_cfg: cfg.handler_cfg, }; diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index e97497786ede..9ce9ee1ebcbf 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -12,7 +12,7 @@ use reth_primitives::{ PooledTransactionsElement, }; use reth_revm::database::StateProviderDatabase; -use reth_rpc_eth_api::{FromEthApiError, FromEvmError}; +use reth_rpc_eth_api::{FromEthApiError, FromEvmError, RpcNodeCore}; use reth_tasks::pool::BlockingTaskGuard; use revm::{ db::CacheDB, @@ -166,7 +166,7 @@ where let mut total_gas_fess = U256::ZERO; let mut hasher = Keccak256::new(); - let mut evm = Call::evm_config(ð_api).evm_with_env(db, env); + let mut evm = RpcNodeCore::evm_config(ð_api).evm_with_env(db, env); let mut results = Vec::with_capacity(transactions.len()); let mut transactions = transactions.into_iter().peekable(); @@ -187,7 +187,7 @@ where .effective_tip_per_gas(basefee) .ok_or_else(|| RpcInvalidTransactionError::FeeCapTooLow) .map_err(Eth::Error::from_eth_err)?; - Call::evm_config(ð_api).fill_tx_env(evm.tx_mut(), &tx, signer); + RpcNodeCore::evm_config(ð_api).fill_tx_env(evm.tx_mut(), &tx, signer); let ResultAndState { result, state } = evm.transact().map_err(Eth::Error::from_evm_err)?; diff --git a/crates/rpc/rpc/src/eth/helpers/call.rs b/crates/rpc/rpc/src/eth/helpers/call.rs index 396bf9bd08e4..d1d33190a7c7 100644 --- a/crates/rpc/rpc/src/eth/helpers/call.rs +++ b/crates/rpc/rpc/src/eth/helpers/call.rs @@ -13,7 +13,7 @@ impl EthCall for EthApi Call for EthApi where - Self: LoadState + SpawnBlocking, + Self: LoadState> + SpawnBlocking, EvmConfig: ConfigureEvm
, { #[inline] @@ -25,9 +25,4 @@ where fn max_simulate_blocks(&self) -> u64 { self.inner.max_simulate_blocks() } - - #[inline] - fn evm_config(&self) -> &impl ConfigureEvm
{ - self.inner.evm_config() - } } diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index b9b15b5366df..2883818afd9f 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -21,10 +21,7 @@ use reth_primitives::{BlockId, Header}; use reth_provider::{BlockReader, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_revm::database::StateProviderDatabase; use reth_rpc_api::TraceApiServer; -use reth_rpc_eth_api::{ - helpers::{Call, TraceExt}, - FromEthApiError, -}; +use reth_rpc_eth_api::{helpers::TraceExt, FromEthApiError, RpcNodeCore}; use reth_rpc_eth_types::{error::EthApiError, utils::recover_raw_transaction}; use reth_tasks::pool::BlockingTaskGuard; use revm::{ @@ -124,7 +121,7 @@ where let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block, - Call::evm_config(self.eth_api()).tx_env(tx.as_signed(), tx.signer()), + RpcNodeCore::evm_config(self.eth_api()).tx_env(tx.as_signed(), tx.signer()), ); let config = TracingInspectorConfig::from_parity_config(&trace_types); From ab07fcfb113cb0b579e1f9f55a5dd9511b576687 Mon Sep 17 00:00:00 2001 From: Hai | RISE <150876604+hai-rise@users.noreply.github.com> Date: Sun, 27 Oct 2024 00:02:14 +0700 Subject: [PATCH 114/239] chore(op): simplify blob fields in newly built block header (#12035) --- Cargo.lock | 1 - crates/optimism/payload/Cargo.toml | 2 -- crates/optimism/payload/src/builder.rs | 36 ++++++++------------------ 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1ca1c9259c4..b483fd6641db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8279,7 +8279,6 @@ dependencies = [ "reth-transaction-pool", "reth-trie", "revm", - "revm-primitives", "sha2 0.10.8", "thiserror", "tracing", diff --git a/crates/optimism/payload/Cargo.toml b/crates/optimism/payload/Cargo.toml index de61def83506..ba0b105e8329 100644 --- a/crates/optimism/payload/Cargo.toml +++ b/crates/optimism/payload/Cargo.toml @@ -39,7 +39,6 @@ alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true op-alloy-rpc-types-engine.workspace = true -revm-primitives.workspace = true alloy-rpc-types-engine.workspace = true alloy-consensus.workspace = true @@ -56,5 +55,4 @@ optimism = [ "revm/optimism", "reth-execution-types/optimism", "reth-optimism-consensus/optimism", - "revm-primitives/optimism" ] diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 85f687aa803b..5523770e09c3 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -7,12 +7,12 @@ use alloy_eips::merge::BEACON_NONCE; use alloy_primitives::U256; use reth_basic_payload_builder::*; use reth_chain_state::ExecutedBlock; -use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; +use reth_chainspec::ChainSpecProvider; use reth_evm::{system_calls::SystemCaller, ConfigureEvm, ConfigureEvmEnv, NextBlockEnvAttributes}; use reth_execution_types::ExecutionOutcome; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_consensus::calculate_receipt_root_no_memo_optimism; -use reth_optimism_forks::OptimismHardfork; +use reth_optimism_forks::{OptimismHardfork, OptimismHardforks}; use reth_payload_primitives::{PayloadBuilderAttributes, PayloadBuilderError}; use reth_primitives::{ proofs, @@ -30,7 +30,6 @@ use revm::{ primitives::{EVMError, EnvWithHandlerCfg, InvalidTransaction, ResultAndState}, DatabaseCommit, }; -use revm_primitives::calc_excess_blob_gas; use tracing::{debug, trace, warn}; use crate::{ @@ -460,26 +459,16 @@ where // create the block header let transactions_root = proofs::calculate_transaction_root(&executed_txs); - // initialize empty blob sidecars. There are no blob transactions on L2. - let blob_sidecars = Vec::new(); - let mut excess_blob_gas = None; - let mut blob_gas_used = None; - - // only determine cancun fields when active - if chain_spec.is_cancun_active_at_timestamp(attributes.payload_attributes.timestamp) { - excess_blob_gas = if chain_spec.is_cancun_active_at_timestamp(parent_block.timestamp) { - let parent_excess_blob_gas = parent_block.excess_blob_gas.unwrap_or_default(); - let parent_blob_gas_used = parent_block.blob_gas_used.unwrap_or_default(); - Some(calc_excess_blob_gas(parent_excess_blob_gas, parent_blob_gas_used)) + // OP doesn't support blobs/EIP-4844. + // https://specs.optimism.io/protocol/exec-engine.html#ecotone-disable-blob-transactions + // Need [Some] or [None] based on hardfork to match block hash. + let (excess_blob_gas, blob_gas_used) = + if chain_spec.is_ecotone_active_at_timestamp(attributes.payload_attributes.timestamp) { + (Some(0), Some(0)) } else { - // for the first post-fork block, both parent.blob_gas_used and - // parent.excess_blob_gas are evaluated as 0 - Some(calc_excess_blob_gas(0, 0)) + (None, None) }; - blob_gas_used = Some(0); - } - let header = Header { parent_hash: parent_block.hash(), ommers_hash: EMPTY_OMMER_ROOT_HASH, @@ -500,7 +489,7 @@ where extra_data, parent_beacon_block_root: attributes.payload_attributes.parent_beacon_block_root, blob_gas_used, - excess_blob_gas: excess_blob_gas.map(Into::into), + excess_blob_gas, requests_hash: None, }; @@ -524,7 +513,7 @@ where let no_tx_pool = attributes.no_tx_pool; - let mut payload = OptimismBuiltPayload::new( + let payload = OptimismBuiltPayload::new( attributes.payload_attributes.id, sealed_block, total_fees, @@ -533,9 +522,6 @@ where Some(executed), ); - // extend the payload with the blob sidecars from the executed txs - payload.extend_sidecars(blob_sidecars); - if no_tx_pool { // if `no_tx_pool` is set only transactions from the payload attributes will be included in // the payload. In other words, the payload is deterministic and we can freeze it once we've From 923f4ffa92c6fe387050f099773b61dd6983bff2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 26 Oct 2024 20:22:46 +0200 Subject: [PATCH 115/239] chore: only check for better payload if tx_pool (#12097) --- crates/optimism/payload/src/builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 5523770e09c3..36b7d17a07d8 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -413,8 +413,8 @@ where } } - // check if we have a better block - if !is_better_payload(best_payload.as_ref(), total_fees) { + // check if we have a better block, but only if we included transactions from the pool + if !attributes.no_tx_pool && !is_better_payload(best_payload.as_ref(), total_fees) { // can skip building the block return Ok(BuildOutcome::Aborted { fees: total_fees, cached_reads }) } From a98dc3973ff733feb11a7c8b50c60459dfaa9351 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sun, 27 Oct 2024 02:56:29 +0800 Subject: [PATCH 116/239] chore(rpc): simplify trait bounds on `EthApiSpec` impl (#12101) --- crates/rpc/rpc/src/eth/helpers/spec.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/rpc/rpc/src/eth/helpers/spec.rs b/crates/rpc/rpc/src/eth/helpers/spec.rs index c5c8d54c64bc..a44692e18a35 100644 --- a/crates/rpc/rpc/src/eth/helpers/spec.rs +++ b/crates/rpc/rpc/src/eth/helpers/spec.rs @@ -3,20 +3,17 @@ use reth_chainspec::EthereumHardforks; use reth_network_api::NetworkInfo; use reth_provider::{BlockNumReader, ChainSpecProvider, StageCheckpointReader}; use reth_rpc_eth_api::{helpers::EthApiSpec, RpcNodeCore}; -use reth_transaction_pool::TransactionPool; use crate::EthApi; impl EthApiSpec for EthApi where - Self: RpcNodeCore, - Pool: TransactionPool + 'static, - Provider: ChainSpecProvider - + BlockNumReader - + StageCheckpointReader - + 'static, - Network: NetworkInfo + 'static, - EvmConfig: Send + Sync, + Self: RpcNodeCore< + Provider: ChainSpecProvider + + BlockNumReader + + StageCheckpointReader, + Network: NetworkInfo, + >, { fn starting_block(&self) -> U256 { self.inner.starting_block() From 988c5ee4c5d4b9e1c71d81aaf54ff7e6c1855895 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sun, 27 Oct 2024 07:11:53 +0800 Subject: [PATCH 117/239] chore(rpc): Add super trait `RpcNodeCore` to `LoadPendingBlock` (#12098) --- crates/optimism/rpc/src/eth/call.rs | 8 ++-- crates/optimism/rpc/src/eth/pending_block.rs | 34 +++++---------- crates/rpc/rpc-eth-api/src/helpers/block.rs | 6 +-- crates/rpc/rpc-eth-api/src/helpers/call.rs | 4 +- .../rpc-eth-api/src/helpers/pending_block.rs | 34 ++++++--------- crates/rpc/rpc-eth-api/src/helpers/state.rs | 4 +- crates/rpc/rpc/src/eth/bundle.rs | 4 +- .../rpc/rpc/src/eth/helpers/pending_block.rs | 43 ++++++------------- 8 files changed, 48 insertions(+), 89 deletions(-) diff --git a/crates/optimism/rpc/src/eth/call.rs b/crates/optimism/rpc/src/eth/call.rs index 0402165f7079..c8f8200bc69c 100644 --- a/crates/optimism/rpc/src/eth/call.rs +++ b/crates/optimism/rpc/src/eth/call.rs @@ -1,14 +1,12 @@ use alloy_primitives::{Bytes, TxKind, U256}; use alloy_rpc_types_eth::transaction::TransactionRequest; -use reth_chainspec::EthereumHardforks; use reth_evm::ConfigureEvm; -use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_primitives::{ revm_primitives::{BlockEnv, OptimismFields, TxEnv}, Header, }; use reth_rpc_eth_api::{ - helpers::{Call, EthCall, LoadState, SpawnBlocking}, + helpers::{Call, EthCall, LoadPendingBlock, LoadState, SpawnBlocking}, FromEthApiError, IntoEthApiError, RpcNodeCore, }; use reth_rpc_eth_types::{revm_utils::CallFees, RpcInvalidTransactionError}; @@ -17,8 +15,8 @@ use crate::{OpEthApi, OpEthApiError}; impl EthCall for OpEthApi where - Self: Call, - N: FullNodeComponents>, + Self: Call + LoadPendingBlock, + N: RpcNodeCore, { } diff --git a/crates/optimism/rpc/src/eth/pending_block.rs b/crates/optimism/rpc/src/eth/pending_block.rs index 5b716f39320a..3b3b7845cc19 100644 --- a/crates/optimism/rpc/src/eth/pending_block.rs +++ b/crates/optimism/rpc/src/eth/pending_block.rs @@ -1,9 +1,8 @@ //! Loads OP pending block for a RPC response. use alloy_primitives::{BlockNumber, B256}; -use reth_chainspec::EthereumHardforks; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_evm::ConfigureEvm; -use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_optimism_consensus::calculate_receipt_root_no_memo_optimism; use reth_primitives::{ revm_primitives::BlockEnv, BlockNumberOrTag, Header, Receipt, SealedBlockWithSenders, @@ -14,7 +13,7 @@ use reth_provider::{ }; use reth_rpc_eth_api::{ helpers::{LoadPendingBlock, SpawnBlocking}, - FromEthApiError, + FromEthApiError, RpcNodeCore, }; use reth_rpc_eth_types::{EthApiError, PendingBlock}; use reth_transaction_pool::TransactionPool; @@ -24,33 +23,20 @@ use crate::OpEthApi; impl LoadPendingBlock for OpEthApi where Self: SpawnBlocking, - N: FullNodeComponents>, + N: RpcNodeCore< + Provider: BlockReaderIdExt + + EvmEnvProvider + + ChainSpecProvider + + StateProviderFactory, + Pool: TransactionPool, + Evm: ConfigureEvm
, + >, { - #[inline] - fn provider( - &self, - ) -> impl BlockReaderIdExt - + EvmEnvProvider - + ChainSpecProvider - + StateProviderFactory { - self.inner.provider() - } - - #[inline] - fn pool(&self) -> impl TransactionPool { - self.inner.pool() - } - #[inline] fn pending_block(&self) -> &tokio::sync::Mutex> { self.inner.pending_block() } - #[inline] - fn evm_config(&self) -> &impl ConfigureEvm
{ - self.inner.evm_config() - } - /// Returns the locally built pending block async fn local_pending_block( &self, diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index da5f275ef0cf..7599cbb599d1 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -9,7 +9,7 @@ use reth_provider::{BlockIdReader, BlockReader, BlockReaderIdExt, HeaderProvider use reth_rpc_eth_types::EthStateCache; use reth_rpc_types_compat::block::{from_block, uncle_block_from_header}; -use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; +use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcNodeCore, RpcReceipt}; use super::{LoadPendingBlock, LoadReceipt, SpawnBlocking}; @@ -220,7 +220,7 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching - if let Some(pending_block) = LoadPendingBlock::provider(self) + if let Some(pending_block) = RpcNodeCore::provider(self) .pending_block_with_senders() .map_err(Self::Error::from_eth_err)? { @@ -234,7 +234,7 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { }; } - let block_hash = match LoadPendingBlock::provider(self) + let block_hash = match RpcNodeCore::provider(self) .block_hash_for_id(block_id) .map_err(Self::Error::from_eth_err)? { diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index f2662a86a74f..c2b6524b3efe 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -97,7 +97,7 @@ pub trait EthCall: Call + LoadPendingBlock { let base_block = self.block_with_senders(block).await?.ok_or(EthApiError::HeaderNotFound(block))?; let mut parent_hash = base_block.header.hash(); - let total_difficulty = LoadPendingBlock::provider(self) + let total_difficulty = RpcNodeCore::provider(self) .header_td_by_number(block_env.number.to()) .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(block))?; @@ -119,7 +119,7 @@ pub trait EthCall: Call + LoadPendingBlock { block_env.timestamp += U256::from(1); if validation { - let chain_spec = LoadPendingBlock::provider(&this).chain_spec(); + let chain_spec = RpcNodeCore::provider(&this).chain_spec(); let base_fee_params = chain_spec.base_fee_params_at_timestamp(block_env.timestamp.to()); let base_fee = if let Some(latest) = blocks.last() { diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 872f17ee9107..2c04f3beb3b3 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -3,7 +3,7 @@ use std::time::{Duration, Instant}; -use crate::{EthApiTypes, FromEthApiError, FromEvmError}; +use crate::{EthApiTypes, FromEthApiError, FromEvmError, RpcNodeCore}; use alloy_consensus::EMPTY_OMMER_ROOT_HASH; use alloy_eips::{eip7685::EMPTY_REQUESTS_HASH, merge::BEACON_NONCE}; @@ -43,32 +43,22 @@ use super::SpawnBlocking; /// Loads a pending block from database. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` blocks RPC methods. -pub trait LoadPendingBlock: EthApiTypes { - /// Returns a handle for reading data from disk. - /// - /// Data access in default (L1) trait method implementations. - fn provider( - &self, - ) -> impl BlockReaderIdExt - + EvmEnvProvider - + ChainSpecProvider - + StateProviderFactory; - - /// Returns a handle for reading data from transaction pool. - /// - /// Data access in default (L1) trait method implementations. - fn pool(&self) -> impl TransactionPool; - +pub trait LoadPendingBlock: + EthApiTypes + + RpcNodeCore< + Provider: BlockReaderIdExt + + EvmEnvProvider + + ChainSpecProvider + + StateProviderFactory, + Pool: TransactionPool, + Evm: ConfigureEvm
, + > +{ /// Returns a handle to the pending block. /// /// Data access in default (L1) trait method implementations. fn pending_block(&self) -> &Mutex>; - /// Returns a handle for reading evm config. - /// - /// Data access in default (L1) trait method implementations. - fn evm_config(&self) -> &impl ConfigureEvm
; - /// Configures the [`CfgEnvWithHandlerCfg`] and [`BlockEnv`] for the pending block /// /// If no pending block is available, this will derive it from the `latest` block diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 2a15b194f13d..87e66cb74817 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -233,7 +233,7 @@ pub trait LoadState: Ok((cfg, block_env, origin.state_block_id())) } else { // Use cached values if there is no pending block - let block_hash = LoadPendingBlock::provider(self) + let block_hash = RpcNodeCore::provider(self) .block_hash_for_id(at) .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(at))?; @@ -262,7 +262,7 @@ pub trait LoadState: let (cfg, mut block_env, _) = self.evm_env_at(header.parent_hash.into()).await?; let after_merge = cfg.handler_cfg.spec_id >= SpecId::MERGE; - LoadPendingBlock::evm_config(self).fill_block_env(&mut block_env, header, after_merge); + self.evm_config().fill_block_env(&mut block_env, header, after_merge); Ok((cfg, block_env)) } diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index 9ce9ee1ebcbf..ec1a43c7548e 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -130,12 +130,12 @@ where } else if cfg.handler_cfg.spec_id.is_enabled_in(SpecId::LONDON) { let parent_block = block_env.number.saturating_to::(); // here we need to fetch the _next_ block's basefee based on the parent block - let parent = LoadPendingBlock::provider(self.eth_api()) + let parent = RpcNodeCore::provider(self.eth_api()) .header_by_number(parent_block) .map_err(Eth::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(parent_block.into()))?; if let Some(base_fee) = parent.next_block_base_fee( - LoadPendingBlock::provider(self.eth_api()) + RpcNodeCore::provider(self.eth_api()) .chain_spec() .base_fee_params_at_block(parent_block), ) { diff --git a/crates/rpc/rpc/src/eth/helpers/pending_block.rs b/crates/rpc/rpc/src/eth/helpers/pending_block.rs index 69d55f58bfa2..6b28947df358 100644 --- a/crates/rpc/rpc/src/eth/helpers/pending_block.rs +++ b/crates/rpc/rpc/src/eth/helpers/pending_block.rs @@ -1,10 +1,13 @@ //! Support for building a pending block with transactions from local view of mempool. -use reth_chainspec::EthereumHardforks; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_evm::ConfigureEvm; use reth_primitives::Header; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; -use reth_rpc_eth_api::helpers::{LoadPendingBlock, SpawnBlocking}; +use reth_rpc_eth_api::{ + helpers::{LoadPendingBlock, SpawnBlocking}, + RpcNodeCore, +}; use reth_rpc_eth_types::PendingBlock; use reth_transaction_pool::TransactionPool; @@ -13,36 +16,18 @@ use crate::EthApi; impl LoadPendingBlock for EthApi where - Self: SpawnBlocking, - Provider: BlockReaderIdExt - + EvmEnvProvider - + ChainSpecProvider - + StateProviderFactory, - Pool: TransactionPool, - EvmConfig: ConfigureEvm
, + Self: SpawnBlocking + + RpcNodeCore< + Provider: BlockReaderIdExt + + EvmEnvProvider + + ChainSpecProvider + + StateProviderFactory, + Pool: TransactionPool, + Evm: ConfigureEvm
, + >, { - #[inline] - fn provider( - &self, - ) -> impl BlockReaderIdExt - + EvmEnvProvider - + ChainSpecProvider - + StateProviderFactory { - self.inner.provider() - } - - #[inline] - fn pool(&self) -> impl TransactionPool { - self.inner.pool() - } - #[inline] fn pending_block(&self) -> &tokio::sync::Mutex> { self.inner.pending_block() } - - #[inline] - fn evm_config(&self) -> &impl ConfigureEvm
{ - self.inner.evm_config() - } } From 8eb1742284964de1a7a32df523a2d0a4b6df1b4f Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sun, 27 Oct 2024 02:59:57 +0100 Subject: [PATCH 118/239] refactor(tx-pool): small refactor (#12107) --- crates/transaction-pool/src/maintain.rs | 2 +- crates/transaction-pool/src/pool/best.rs | 2 +- crates/transaction-pool/src/pool/blob.rs | 15 +++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 23a8d0dc66a5..09c042ae66dc 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -575,7 +575,7 @@ where // Filter out errors ::try_from_consensus(tx.into()).ok() }) - .collect::>(); + .collect(); let outcome = pool.add_transactions(crate::TransactionOrigin::Local, pool_transactions).await; diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 77cd35d8a6be..268e3e262c6c 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -196,7 +196,7 @@ impl Iterator for BestTransactions { } } -/// A[`BestTransactions`](crate::traits::BestTransactions) implementation that filters the +/// A [`BestTransactions`](crate::traits::BestTransactions) implementation that filters the /// transactions of iter with predicate. /// /// Filter out transactions are marked as invalid: diff --git a/crates/transaction-pool/src/pool/blob.rs b/crates/transaction-pool/src/pool/blob.rs index cb09e8234095..ac39c6ab781a 100644 --- a/crates/transaction-pool/src/pool/blob.rs +++ b/crates/transaction-pool/src/pool/blob.rs @@ -11,7 +11,7 @@ use std::{ /// A set of validated blob transactions in the pool that are __not pending__. /// -/// The purpose of this pool is keep track of blob transactions that are queued and to evict the +/// The purpose of this pool is to keep track of blob transactions that are queued and to evict the /// worst blob transactions once the sub-pool is full. /// /// This expects that certain constraints are met: @@ -198,14 +198,13 @@ impl BlobTransactions { &mut self, pending_fees: &PendingFees, ) -> Vec>> { - let to_remove = self.satisfy_pending_fee_ids(pending_fees); - - let mut removed = Vec::with_capacity(to_remove.len()); - for id in to_remove { - removed.push(self.remove_transaction(&id).expect("transaction exists")); - } + let removed = self + .satisfy_pending_fee_ids(pending_fees) + .into_iter() + .map(|id| self.remove_transaction(&id).expect("transaction exists")) + .collect(); - // set pending fees and reprioritize / resort + // Update pending fees and reprioritize self.pending_fees = pending_fees.clone(); self.reprioritize(); From fae36bd25ff1efa01f0c66d5c73606f63c1f0794 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sun, 27 Oct 2024 03:00:32 +0100 Subject: [PATCH 119/239] refactor(storage): small refactor (#12106) --- crates/storage/db-api/src/cursor.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/storage/db-api/src/cursor.rs b/crates/storage/db-api/src/cursor.rs index 585aa4947a28..9297f738ab5a 100644 --- a/crates/storage/db-api/src/cursor.rs +++ b/crates/storage/db-api/src/cursor.rs @@ -152,12 +152,7 @@ where impl> Iterator for Walker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { - let start = self.start.take(); - if start.is_some() { - return start - } - - self.cursor.next().transpose() + self.start.take().or_else(|| self.cursor.next().transpose()) } } From e158369a68102661fdfdb8ecf1ee205360e1104c Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sun, 27 Oct 2024 15:35:24 +0800 Subject: [PATCH 120/239] chore(rpc): remove redundant trait method `LoadBlock::provider` (#12100) --- crates/optimism/rpc/src/eth/block.rs | 7 +--- crates/optimism/rpc/src/eth/transaction.rs | 15 ++------- crates/rpc/rpc-eth-api/src/helpers/block.rs | 25 +++++++------- crates/rpc/rpc-eth-api/src/helpers/call.rs | 3 +- .../rpc-eth-api/src/helpers/transaction.rs | 33 ++++++++----------- crates/rpc/rpc/src/eth/helpers/block.rs | 5 --- crates/rpc/rpc/src/eth/helpers/transaction.rs | 20 +++-------- 7 files changed, 34 insertions(+), 74 deletions(-) diff --git a/crates/optimism/rpc/src/eth/block.rs b/crates/optimism/rpc/src/eth/block.rs index dfdd09608563..42c4789d44cc 100644 --- a/crates/optimism/rpc/src/eth/block.rs +++ b/crates/optimism/rpc/src/eth/block.rs @@ -7,7 +7,7 @@ use reth_chainspec::ChainSpecProvider; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_optimism_chainspec::OpChainSpec; use reth_primitives::TransactionMeta; -use reth_provider::{BlockReaderIdExt, HeaderProvider}; +use reth_provider::HeaderProvider; use reth_rpc_eth_api::{ helpers::{EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, SpawnBlocking}, RpcReceipt, @@ -87,11 +87,6 @@ where Self: LoadPendingBlock + SpawnBlocking, N: FullNodeComponents, { - #[inline] - fn provider(&self) -> impl BlockReaderIdExt { - self.inner.provider() - } - #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 4ac2d7e6b74f..451c8a805fb3 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -10,7 +10,7 @@ use reth_provider::{BlockReaderIdExt, ReceiptProvider, TransactionsProvider}; use reth_rpc::eth::EthTxBuilder; use reth_rpc_eth_api::{ helpers::{EthSigner, EthTransactions, LoadTransaction, SpawnBlocking}, - FromEthApiError, FullEthApiTypes, TransactionCompat, + FromEthApiError, FullEthApiTypes, RpcNodeCore, TransactionCompat, }; use reth_rpc_eth_types::{utils::recover_raw_transaction, EthStateCache}; use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; @@ -61,21 +61,12 @@ where impl LoadTransaction for OpEthApi where Self: SpawnBlocking + FullEthApiTypes, - N: FullNodeComponents, + N: RpcNodeCore, { - type Pool = N::Pool; - - fn provider(&self) -> impl TransactionsProvider { - self.inner.provider() - } - + #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() } - - fn pool(&self) -> &Self::Pool { - self.inner.pool() - } } impl OpEthApi diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 7599cbb599d1..839b48919145 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -85,13 +85,13 @@ pub trait EthBlocks: LoadBlock { async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching - return Ok(LoadBlock::provider(self) + return Ok(RpcNodeCore::provider(self) .pending_block() .map_err(Self::Error::from_eth_err)? .map(|block| block.body.transactions.len())) } - let block_hash = match LoadBlock::provider(self) + let block_hash = match RpcNodeCore::provider(self) .block_hash_for_id(block_id) .map_err(Self::Error::from_eth_err)? { @@ -132,7 +132,7 @@ pub trait EthBlocks: LoadBlock { if block_id.is_pending() { // First, try to get the pending block from the provider, in case we already // received the actual pending block from the CL. - if let Some((block, receipts)) = LoadBlock::provider(self) + if let Some((block, receipts)) = RpcNodeCore::provider(self) .pending_block_and_receipts() .map_err(Self::Error::from_eth_err)? { @@ -145,7 +145,7 @@ pub trait EthBlocks: LoadBlock { } } - if let Some(block_hash) = LoadBlock::provider(self) + if let Some(block_hash) = RpcNodeCore::provider(self) .block_hash_for_id(block_id) .map_err(Self::Error::from_eth_err)? { @@ -167,7 +167,7 @@ pub trait EthBlocks: LoadBlock { &self, block_id: BlockId, ) -> Result>, Self::Error> { - LoadBlock::provider(self).ommers_by_id(block_id).map_err(Self::Error::from_eth_err) + RpcNodeCore::provider(self).ommers_by_id(block_id).map_err(Self::Error::from_eth_err) } /// Returns uncle block at given index in given block. @@ -182,12 +182,12 @@ pub trait EthBlocks: LoadBlock { async move { let uncles = if block_id.is_pending() { // Pending block can be fetched directly without need for caching - LoadBlock::provider(self) + RpcNodeCore::provider(self) .pending_block() .map_err(Self::Error::from_eth_err)? .map(|block| block.body.ommers) } else { - LoadBlock::provider(self) + RpcNodeCore::provider(self) .ommers_by_id(block_id) .map_err(Self::Error::from_eth_err)? } @@ -202,11 +202,6 @@ pub trait EthBlocks: LoadBlock { /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` blocks RPC methods. pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { - // Returns a handle for reading data from disk. - /// - /// Data access in default (L1) trait method implementations. - fn provider(&self) -> impl BlockReaderIdExt; - /// Returns a handle for reading data from memory. /// /// Data access in default (L1) trait method implementations. @@ -220,7 +215,8 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching - if let Some(pending_block) = RpcNodeCore::provider(self) + if let Some(pending_block) = self + .provider() .pending_block_with_senders() .map_err(Self::Error::from_eth_err)? { @@ -234,7 +230,8 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { }; } - let block_hash = match RpcNodeCore::provider(self) + let block_hash = match self + .provider() .block_hash_for_id(block_id) .map_err(Self::Error::from_eth_err)? { diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index c2b6524b3efe..b90577c14867 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -259,7 +259,8 @@ pub trait EthCall: Call + LoadPendingBlock { // if it's not pending, we should always use block_hash over block_number to ensure that // different provider calls query data related to the same block. if !is_block_target_pending { - target_block = LoadBlock::provider(self) + target_block = self + .provider() .block_hash_for_id(target_block) .map_err(|_| EthApiError::HeaderNotFound(target_block))? .ok_or_else(|| EthApiError::HeaderNotFound(target_block))? diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index a91e4e6faef7..c693945cacac 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -111,7 +111,7 @@ pub trait EthTransactions: LoadTransaction { } self.spawn_blocking_io(move |ref this| { - Ok(LoadTransaction::provider(this) + Ok(RpcNodeCore::provider(this) .transaction_by_hash(hash) .map_err(Self::Error::from_eth_err)? .map(|tx| tx.encoded_2718().into())) @@ -166,7 +166,7 @@ pub trait EthTransactions: LoadTransaction { { let this = self.clone(); self.spawn_blocking_io(move |_| { - let (tx, meta) = match LoadTransaction::provider(&this) + let (tx, meta) = match RpcNodeCore::provider(&this) .transaction_by_hash_with_meta(hash) .map_err(Self::Error::from_eth_err)? { @@ -257,7 +257,7 @@ pub trait EthTransactions: LoadTransaction { return Ok(None); } - let Ok(high) = LoadBlock::provider(self).best_block_number() else { + let Ok(high) = RpcNodeCore::provider(self).best_block_number() else { return Err(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()).into()); }; @@ -383,10 +383,15 @@ pub trait EthTransactions: LoadTransaction { let transaction = self.sign_request(&from, request).await?.with_signer(from); - let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(transaction.into()).map_err(|_| EthApiError::TransactionConversionError)?; + let pool_transaction = + <::Pool as TransactionPool>::Transaction::try_from_consensus( + transaction.into(), + ) + .map_err(|_| EthApiError::TransactionConversionError)?; // submit the transaction to the pool with a `Local` origin - let hash = LoadTransaction::pool(self) + let hash = self + .pool() .add_transaction(TransactionOrigin::Local, pool_transaction) .await .map_err(Self::Error::from_eth_err)?; @@ -460,26 +465,14 @@ pub trait EthTransactions: LoadTransaction { /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` transactions RPC /// methods. -pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes { - /// Transaction pool with pending transactions. [`TransactionPool::Transaction`] is the - /// supported transaction type. - type Pool: TransactionPool; - - /// Returns a handle for reading data from disk. - /// - /// Data access in default (L1) trait method implementations. - fn provider(&self) -> impl TransactionsProvider; - +pub trait LoadTransaction: + SpawnBlocking + FullEthApiTypes + RpcNodeCore +{ /// Returns a handle for reading data from memory. /// /// Data access in default (L1) trait method implementations. fn cache(&self) -> &EthStateCache; - /// Returns a handle for reading data from pool. - /// - /// Data access in default (L1) trait method implementations. - fn pool(&self) -> &Self::Pool; - /// Returns the transaction by hash. /// /// Checks the pool and state. diff --git a/crates/rpc/rpc/src/eth/helpers/block.rs b/crates/rpc/rpc/src/eth/helpers/block.rs index b29a24c38c4c..22853f263576 100644 --- a/crates/rpc/rpc/src/eth/helpers/block.rs +++ b/crates/rpc/rpc/src/eth/helpers/block.rs @@ -73,11 +73,6 @@ where Self: LoadPendingBlock + SpawnBlocking, Provider: BlockReaderIdExt, { - #[inline] - fn provider(&self) -> impl BlockReaderIdExt { - self.inner.provider() - } - #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 24a13cb8062c..c4505bef09d8 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -3,7 +3,7 @@ use reth_provider::{BlockReaderIdExt, TransactionsProvider}; use reth_rpc_eth_api::{ helpers::{EthSigner, EthTransactions, LoadTransaction, SpawnBlocking}, - FullEthApiTypes, + FullEthApiTypes, RpcNodeCore, }; use reth_rpc_eth_types::EthStateCache; use reth_transaction_pool::TransactionPool; @@ -31,26 +31,14 @@ where impl LoadTransaction for EthApi where - Self: SpawnBlocking + FullEthApiTypes, - Provider: TransactionsProvider, - Pool: TransactionPool, + Self: SpawnBlocking + + FullEthApiTypes + + RpcNodeCore, { - type Pool = Pool; - - #[inline] - fn provider(&self) -> impl TransactionsProvider { - self.inner.provider() - } - #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() } - - #[inline] - fn pool(&self) -> &Self::Pool { - self.inner.pool() - } } #[cfg(test)] From 768404c59e7b673d055faf1791bb376b97466ef4 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sun, 27 Oct 2024 16:30:02 +0800 Subject: [PATCH 121/239] chore(rpc): remove redundant trait bounds in eth api (#12105) --- crates/e2e-test-utils/src/node.rs | 7 ++----- crates/rpc/rpc-eth-api/src/helpers/transaction.rs | 2 +- crates/rpc/rpc/src/eth/filter.rs | 3 +-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 07df36a33e1a..8b385115b3eb 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -10,10 +10,7 @@ use reth::{ network::PeersHandleProvider, providers::{BlockReader, BlockReaderIdExt, CanonStateSubscriptions, StageCheckpointReader}, rpc::{ - api::eth::{ - helpers::{EthApiSpec, EthTransactions, TraceExt}, - FullEthApiTypes, - }, + api::eth::helpers::{EthApiSpec, EthTransactions, TraceExt}, types::engine::PayloadStatusEnum, }, }; @@ -97,7 +94,7 @@ where where Engine::ExecutionPayloadEnvelopeV3: From + PayloadEnvelopeExt, Engine::ExecutionPayloadEnvelopeV4: From + PayloadEnvelopeExt, - AddOns::EthApi: EthApiSpec + EthTransactions + TraceExt + FullEthApiTypes, + AddOns::EthApi: EthApiSpec + EthTransactions + TraceExt, { let mut chain = Vec::with_capacity(length as usize); for i in 0..length { diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index c693945cacac..791cb2ae1eb8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -231,7 +231,7 @@ pub trait EthTransactions: LoadTransaction { include_pending: bool, ) -> impl Future>, Self::Error>> + Send where - Self: LoadBlock + LoadState + FullEthApiTypes, + Self: LoadBlock + LoadState, { async move { // Check the pool first diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 5ef224609c5b..3d05cdc727fe 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -145,8 +145,7 @@ where impl EthFilter where Provider: BlockReader + BlockIdReader + EvmEnvProvider + 'static, - Pool: TransactionPool + 'static, - ::Transaction: 'static, + Pool: TransactionPool + 'static, Eth: FullEthApiTypes, { /// Returns all the filter changes for the given id, if any From 131cc5175ebf8c5714e090b230b4926678ef7d97 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sun, 27 Oct 2024 17:21:34 +0800 Subject: [PATCH 122/239] chore(rpc): remove redundant `EthBlocks::provider` (#12109) --- crates/optimism/rpc/src/eth/block.rs | 11 ++---- crates/rpc/rpc-eth-api/src/helpers/block.rs | 39 +++++++++------------ crates/rpc/rpc/src/eth/helpers/block.rs | 7 +--- 3 files changed, 21 insertions(+), 36 deletions(-) diff --git a/crates/optimism/rpc/src/eth/block.rs b/crates/optimism/rpc/src/eth/block.rs index 42c4789d44cc..ed31a7509490 100644 --- a/crates/optimism/rpc/src/eth/block.rs +++ b/crates/optimism/rpc/src/eth/block.rs @@ -4,13 +4,13 @@ use alloy_rpc_types::BlockId; use op_alloy_network::Network; use op_alloy_rpc_types::OpTransactionReceipt; use reth_chainspec::ChainSpecProvider; -use reth_node_api::{FullNodeComponents, NodeTypes}; +use reth_node_api::FullNodeComponents; use reth_optimism_chainspec::OpChainSpec; use reth_primitives::TransactionMeta; use reth_provider::HeaderProvider; use reth_rpc_eth_api::{ helpers::{EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, SpawnBlocking}, - RpcReceipt, + RpcNodeCore, RpcReceipt, }; use reth_rpc_eth_types::EthStateCache; @@ -22,13 +22,8 @@ where Error = OpEthApiError, NetworkTypes: Network, >, - N: FullNodeComponents>, + N: RpcNodeCore + HeaderProvider>, { - #[inline] - fn provider(&self) -> impl HeaderProvider { - self.inner.provider() - } - async fn block_receipts( &self, block_id: BlockId, diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 839b48919145..861afb3ad261 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -9,7 +9,7 @@ use reth_provider::{BlockIdReader, BlockReader, BlockReaderIdExt, HeaderProvider use reth_rpc_eth_types::EthStateCache; use reth_rpc_types_compat::block::{from_block, uncle_block_from_header}; -use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcNodeCore, RpcReceipt}; +use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; use super::{LoadPendingBlock, LoadReceipt, SpawnBlocking}; @@ -20,12 +20,7 @@ pub type BlockAndReceiptsResult = Result impl HeaderProvider; - +pub trait EthBlocks: LoadBlock { /// Returns the block header for the given block id. fn rpc_block_header( &self, @@ -52,15 +47,15 @@ pub trait EthBlocks: LoadBlock { async move { let Some(block) = self.block_with_senders(block_id).await? else { return Ok(None) }; let block_hash = block.hash(); - let mut total_difficulty = EthBlocks::provider(self) + let mut total_difficulty = self + .provider() .header_td_by_number(block.number) .map_err(Self::Error::from_eth_err)?; if total_difficulty.is_none() { // if we failed to find td after we successfully loaded the block, try again using // the hash this only matters if the chain is currently transitioning the merge block and there's a reorg: - total_difficulty = EthBlocks::provider(self) - .header_td(&block.hash()) - .map_err(Self::Error::from_eth_err)?; + total_difficulty = + self.provider().header_td(&block.hash()).map_err(Self::Error::from_eth_err)?; } let block = from_block( @@ -85,13 +80,15 @@ pub trait EthBlocks: LoadBlock { async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching - return Ok(RpcNodeCore::provider(self) + return Ok(self + .provider() .pending_block() .map_err(Self::Error::from_eth_err)? .map(|block| block.body.transactions.len())) } - let block_hash = match RpcNodeCore::provider(self) + let block_hash = match self + .provider() .block_hash_for_id(block_id) .map_err(Self::Error::from_eth_err)? { @@ -132,7 +129,8 @@ pub trait EthBlocks: LoadBlock { if block_id.is_pending() { // First, try to get the pending block from the provider, in case we already // received the actual pending block from the CL. - if let Some((block, receipts)) = RpcNodeCore::provider(self) + if let Some((block, receipts)) = self + .provider() .pending_block_and_receipts() .map_err(Self::Error::from_eth_err)? { @@ -145,9 +143,8 @@ pub trait EthBlocks: LoadBlock { } } - if let Some(block_hash) = RpcNodeCore::provider(self) - .block_hash_for_id(block_id) - .map_err(Self::Error::from_eth_err)? + if let Some(block_hash) = + self.provider().block_hash_for_id(block_id).map_err(Self::Error::from_eth_err)? { return LoadReceipt::cache(self) .get_block_and_receipts(block_hash) @@ -167,7 +164,7 @@ pub trait EthBlocks: LoadBlock { &self, block_id: BlockId, ) -> Result>, Self::Error> { - RpcNodeCore::provider(self).ommers_by_id(block_id).map_err(Self::Error::from_eth_err) + self.provider().ommers_by_id(block_id).map_err(Self::Error::from_eth_err) } /// Returns uncle block at given index in given block. @@ -182,14 +179,12 @@ pub trait EthBlocks: LoadBlock { async move { let uncles = if block_id.is_pending() { // Pending block can be fetched directly without need for caching - RpcNodeCore::provider(self) + self.provider() .pending_block() .map_err(Self::Error::from_eth_err)? .map(|block| block.body.ommers) } else { - RpcNodeCore::provider(self) - .ommers_by_id(block_id) - .map_err(Self::Error::from_eth_err)? + self.provider().ommers_by_id(block_id).map_err(Self::Error::from_eth_err)? } .unwrap_or_default(); diff --git a/crates/rpc/rpc/src/eth/helpers/block.rs b/crates/rpc/rpc/src/eth/helpers/block.rs index 22853f263576..a869cbd54035 100644 --- a/crates/rpc/rpc/src/eth/helpers/block.rs +++ b/crates/rpc/rpc/src/eth/helpers/block.rs @@ -17,14 +17,9 @@ where Self: LoadBlock< Error = EthApiError, NetworkTypes: alloy_network::Network, + Provider: HeaderProvider, >, - Provider: HeaderProvider, { - #[inline] - fn provider(&self) -> impl HeaderProvider { - self.inner.provider() - } - async fn block_receipts( &self, block_id: BlockId, From b7b3f8149c8f91c1b075f5d610cf3210bc23bd2e Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sun, 27 Oct 2024 22:24:21 +0800 Subject: [PATCH 123/239] chore(rpc): remove redundant `Trace::evm_config` (#12102) --- crates/optimism/rpc/src/eth/mod.rs | 8 ++------ crates/rpc/rpc-eth-api/src/helpers/trace.rs | 15 +++++---------- crates/rpc/rpc/src/eth/helpers/trace.rs | 10 ++-------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index bc1692dff4ea..d12a2dd02cd3 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -243,13 +243,9 @@ where impl Trace for OpEthApi where - Self: LoadState, - N: FullNodeComponents, + Self: LoadState>, + N: RpcNodeCore, { - #[inline] - fn evm_config(&self) -> &impl ConfigureEvm
{ - self.inner.evm_config() - } } impl AddDevSigners for OpEthApi diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 6c7dd0f6f8d9..da1d1cdb919b 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -21,12 +21,7 @@ use revm_primitives::{EnvWithHandlerCfg, EvmState, ExecutionResult, ResultAndSta use super::{Call, LoadBlock, LoadPendingBlock, LoadState, LoadTransaction}; /// Executes CPU heavy tasks. -pub trait Trace: LoadState { - /// Returns a handle for reading evm config. - /// - /// Data access in default (L1) trait method implementations. - fn evm_config(&self) -> &impl ConfigureEvm
; - +pub trait Trace: LoadState> { /// Executes the [`EnvWithHandlerCfg`] against the given [Database] without committing state /// changes. fn inspect( @@ -60,7 +55,7 @@ pub trait Trace: LoadState { I: GetInspector, { - let mut evm = Trace::evm_config(self).evm_with_env_and_inspector(db, env, inspector); + let mut evm = self.evm_config().evm_with_env_and_inspector(db, env, inspector); let res = evm.transact().map_err(Self::Error::from_evm_err)?; let (db, env) = evm.into_db_and_env_with_handler_cfg(); Ok((res, env, db)) @@ -201,7 +196,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( - Trace::evm_config(&this).clone(), + this.evm_config().clone(), RpcNodeCore::provider(&this).chain_spec(), ); system_caller @@ -344,7 +339,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( - Trace::evm_config(&this).clone(), + this.evm_config().clone(), RpcNodeCore::provider(&this).chain_spec(), ); system_caller @@ -379,7 +374,7 @@ pub trait Trace: LoadState { block_number: Some(block_number), base_fee: Some(base_fee), }; - let tx_env = Trace::evm_config(&this).tx_env(tx, *signer); + let tx_env = this.evm_config().tx_env(tx, *signer); (tx_info, tx_env) }) .peekable(); diff --git a/crates/rpc/rpc/src/eth/helpers/trace.rs b/crates/rpc/rpc/src/eth/helpers/trace.rs index c40b7acf50d1..b270ed1b2ad1 100644 --- a/crates/rpc/rpc/src/eth/helpers/trace.rs +++ b/crates/rpc/rpc/src/eth/helpers/trace.rs @@ -6,13 +6,7 @@ use reth_rpc_eth_api::helpers::{LoadState, Trace}; use crate::EthApi; -impl Trace for EthApi -where - Self: LoadState, - EvmConfig: ConfigureEvm
, +impl Trace for EthApi where + Self: LoadState> { - #[inline] - fn evm_config(&self) -> &impl ConfigureEvm
{ - self.inner.evm_config() - } } From 0c516091b8f45f07cf4d76879153d3cd90015363 Mon Sep 17 00:00:00 2001 From: Parikalp Bhardwaj <53660958+Parikalp-Bhardwaj@users.noreply.github.com> Date: Sun, 27 Oct 2024 19:11:03 +0400 Subject: [PATCH 124/239] TransactionsHandle propagation commands should not adhere to caching (#12079) Co-authored-by: Matthias Seitz --- crates/net/network/src/transactions/mod.rs | 113 +++++++++++++++++---- 1 file changed, 93 insertions(+), 20 deletions(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 439f92bada9a..b6e589c6ff79 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -416,6 +416,7 @@ where fn propagate_all(&mut self, hashes: Vec) { let propagated = self.propagate_transactions( self.pool.get_all(hashes).into_iter().map(PropagateTransaction::new).collect(), + PropagationMode::Basic, ); // notify pool so events get fired @@ -431,6 +432,7 @@ where fn propagate_transactions( &mut self, to_propagate: Vec, + propagation_mode: PropagationMode, ) -> PropagatedTransactions { let mut propagated = PropagatedTransactions::default(); if self.network.tx_gossip_disabled() { @@ -449,14 +451,18 @@ where PropagateTransactionsBuilder::full(peer.version) }; - // Iterate through the transactions to propagate and fill the hashes and full - // transaction lists, before deciding whether or not to send full transactions to the - // peer. - for tx in &to_propagate { - // Only proceed if the transaction is not in the peer's list of seen transactions - if !peer.seen_transactions.contains(&tx.hash()) { - // add transaction to the list of hashes to propagate - builder.push(tx); + if propagation_mode.is_forced() { + builder.extend(to_propagate.iter()); + } else { + // Iterate through the transactions to propagate and fill the hashes and full + // transaction lists, before deciding whether or not to send full transactions to + // the peer. + for tx in &to_propagate { + // Only proceed if the transaction is not in the peer's list of seen + // transactions + if !peer.seen_transactions.contains(&tx.hash()) { + builder.push(tx); + } } } @@ -514,6 +520,7 @@ where &mut self, txs: Vec, peer_id: PeerId, + propagation_mode: PropagationMode, ) -> Option { trace!(target: "net::tx", ?peer_id, "Propagating transactions to peer"); @@ -525,10 +532,17 @@ where let to_propagate = self.pool.get_all(txs).into_iter().map(PropagateTransaction::new); - // Iterate through the transactions to propagate and fill the hashes and full transaction - for tx in to_propagate { - if !peer.seen_transactions.contains(&tx.hash()) { - full_transactions.push(&tx); + if propagation_mode.is_forced() { + // skip cache check if forced + full_transactions.extend(to_propagate); + } else { + // Iterate through the transactions to propagate and fill the hashes and full + // transaction + for tx in to_propagate { + if !peer.seen_transactions.contains(&tx.hash()) { + // Only include if the peer hasn't seen the transaction + full_transactions.push(&tx); + } } } @@ -546,6 +560,7 @@ where // mark transaction as seen by peer peer.seen_transactions.insert(hash); } + // send hashes of transactions self.network.send_transactions_hashes(peer_id, new_pooled_hashes); } @@ -557,6 +572,7 @@ where // mark transaction as seen by peer peer.seen_transactions.insert(tx.hash()); } + // send full transactions self.network.send_transactions(peer_id, new_full_transactions); } @@ -570,7 +586,12 @@ where /// Propagate the transaction hashes to the given peer /// /// Note: This will only send the hashes for transactions that exist in the pool. - fn propagate_hashes_to(&mut self, hashes: Vec, peer_id: PeerId) { + fn propagate_hashes_to( + &mut self, + hashes: Vec, + peer_id: PeerId, + propagation_mode: PropagationMode, + ) { trace!(target: "net::tx", "Start propagating transactions as hashes"); // This fetches a transactions from the pool, including the blob transactions, which are @@ -589,9 +610,14 @@ where // check if transaction is known to peer let mut hashes = PooledTransactionsHashesBuilder::new(peer.version); - for tx in to_propagate { - if !peer.seen_transactions.insert(tx.hash()) { - hashes.push(&tx); + if propagation_mode.is_forced() { + hashes.extend(to_propagate) + } else { + for tx in to_propagate { + if !peer.seen_transactions.contains(&tx.hash()) { + // Include if the peer hasn't seen it + hashes.push(&tx); + } } } @@ -880,14 +906,16 @@ where self.on_new_pending_transactions(vec![hash]) } TransactionsCommand::PropagateHashesTo(hashes, peer) => { - self.propagate_hashes_to(hashes, peer) + self.propagate_hashes_to(hashes, peer, PropagationMode::Forced) } TransactionsCommand::GetActivePeers(tx) => { let peers = self.peers.keys().copied().collect::>(); tx.send(peers).ok(); } TransactionsCommand::PropagateTransactionsTo(txs, peer) => { - if let Some(propagated) = self.propagate_full_transactions_to_peer(txs, peer) { + if let Some(propagated) = + self.propagate_full_transactions_to_peer(txs, peer, PropagationMode::Forced) + { self.pool.on_propagated(propagated); } } @@ -1395,6 +1423,29 @@ where } } +/// Represents the different modes of transaction propagation. +/// +/// This enum is used to determine how transactions are propagated to peers in the network. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +enum PropagationMode { + /// Default propagation mode. + /// + /// Transactions are only sent to peers that haven't seen them yet. + Basic, + /// Forced propagation mode. + /// + /// Transactions are sent to all peers regardless of whether they have been sent or received + /// before. + Forced, +} + +impl PropagationMode { + /// Returns `true` if the propagation kind is `Forced`. + const fn is_forced(self) -> bool { + matches!(self, Self::Forced) + } +} + /// A transaction that's about to be propagated to multiple peers. #[derive(Debug, Clone)] struct PropagateTransaction { @@ -1441,6 +1492,13 @@ impl PropagateTransactionsBuilder { Self::Full(FullTransactionsBuilder::new(version)) } + /// Appends all transactions + fn extend<'a>(&mut self, txs: impl IntoIterator) { + for tx in txs { + self.push(tx); + } + } + /// Appends a transaction to the list. fn push(&mut self, transaction: &PropagateTransaction) { match self { @@ -1502,6 +1560,13 @@ impl FullTransactionsBuilder { } } + /// Appends all transactions. + fn extend(&mut self, txs: impl IntoIterator) { + for tx in txs { + self.push(&tx) + } + } + /// Append a transaction to the list of full transaction if the total message bytes size doesn't /// exceed the soft maximum target byte size. The limit is soft, meaning if one single /// transaction goes over the limit, it will be broadcasted in its own [`Transactions`] @@ -1581,6 +1646,13 @@ impl PooledTransactionsHashesBuilder { } } + /// Appends all hashes + fn extend(&mut self, txs: impl IntoIterator) { + for tx in txs { + self.push(&tx); + } + } + fn push(&mut self, tx: &PropagateTransaction) { match self { Self::Eth66(msg) => msg.0.push(tx.hash()), @@ -2388,7 +2460,8 @@ mod tests { let eip4844_tx = Arc::new(factory.create_eip4844()); propagate.push(PropagateTransaction::new(eip4844_tx.clone())); - let propagated = tx_manager.propagate_transactions(propagate.clone()); + let propagated = + tx_manager.propagate_transactions(propagate.clone(), PropagationMode::Basic); assert_eq!(propagated.0.len(), 2); let prop_txs = propagated.0.get(eip1559_tx.transaction.hash()).unwrap(); assert_eq!(prop_txs.len(), 1); @@ -2404,7 +2477,7 @@ mod tests { peer.seen_transactions.contains(eip4844_tx.transaction.hash()); // propagate again - let propagated = tx_manager.propagate_transactions(propagate); + let propagated = tx_manager.propagate_transactions(propagate, PropagationMode::Basic); assert!(propagated.0.is_empty()); } } From 1c36b7161216a76382a082cd91cdf8c006f88609 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 28 Oct 2024 08:13:47 +0100 Subject: [PATCH 125/239] docs: small fix in payload doc (#12116) --- crates/payload/basic/src/lib.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index b8eab3c0fea0..d0bb29502eac 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -95,10 +95,11 @@ impl BasicPayloadJobGenerator Client software SHOULD stop the updating process when either a call to engine_getPayload - // > with the build process's payloadId is made or SECONDS_PER_SLOT (12s in the Mainnet - // > configuration) have passed since the point in time identified by the timestamp parameter. - // See also + /// > Client software SHOULD stop the updating process when either a call to engine_getPayload + /// > with the build process's payloadId is made or SECONDS_PER_SLOT (12s in the Mainnet + /// > configuration) have passed since the point in time identified by the timestamp parameter. + /// + /// See also #[inline] fn max_job_duration(&self, unix_timestamp: u64) -> Duration { let duration_until_timestamp = duration_until(unix_timestamp); From 8605d04a09679904ef25594729ac6b83dfcacfcb Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:30:06 +0100 Subject: [PATCH 126/239] refactor: rm re-exports of alloy eip 4844 constants (#12120) --- Cargo.lock | 2 +- crates/consensus/common/src/validation.rs | 6 ++---- crates/ethereum/payload/src/lib.rs | 3 +-- crates/node/events/Cargo.toml | 2 +- crates/node/events/src/node.rs | 5 ++--- crates/primitives/src/constants/eip4844.rs | 7 ------- crates/primitives/src/constants/mod.rs | 3 --- crates/primitives/src/transaction/mod.rs | 2 +- crates/primitives/src/transaction/pooled.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/fee.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/pending_block.rs | 5 +++-- crates/rpc/rpc-eth-types/src/fee_history.rs | 2 +- crates/transaction-pool/src/pool/txpool.rs | 6 ++++-- crates/transaction-pool/src/test_utils/mock.rs | 8 ++++---- crates/transaction-pool/src/traits.rs | 3 ++- crates/transaction-pool/src/validate/eth.rs | 5 ++--- 16 files changed, 26 insertions(+), 37 deletions(-) delete mode 100644 crates/primitives/src/constants/eip4844.rs diff --git a/Cargo.lock b/Cargo.lock index b483fd6641db..6762da237cd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8033,6 +8033,7 @@ name = "reth-node-events" version = "1.1.0" dependencies = [ "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "futures", @@ -8041,7 +8042,6 @@ dependencies = [ "reth-beacon-consensus", "reth-network", "reth-network-api", - "reth-primitives", "reth-primitives-traits", "reth-provider", "reth-prune", diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index 1070bbdbc0f5..c6539cdcf71c 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -1,12 +1,10 @@ //! Collection of methods for block validation. use alloy_consensus::constants::MAXIMUM_EXTRA_DATA_SIZE; +use alloy_eips::eip4844::{DATA_GAS_PER_BLOB, MAX_DATA_GAS_PER_BLOCK}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_consensus::ConsensusError; -use reth_primitives::{ - constants::eip4844::{DATA_GAS_PER_BLOB, MAX_DATA_GAS_PER_BLOCK}, - EthereumHardfork, GotExpected, Header, SealedBlock, SealedHeader, -}; +use reth_primitives::{EthereumHardfork, GotExpected, Header, SealedBlock, SealedHeader}; use revm_primitives::calc_excess_blob_gas; /// Gas used needs to be less than gas limit. Gas used is going to be checked after execution. diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index f14c145889c9..73b22efac401 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -10,7 +10,7 @@ #![allow(clippy::useless_let_if_seq)] use alloy_consensus::EMPTY_OMMER_ROOT_HASH; -use alloy_eips::{eip7685::Requests, merge::BEACON_NONCE}; +use alloy_eips::{eip4844::MAX_DATA_GAS_PER_BLOCK, eip7685::Requests, merge::BEACON_NONCE}; use alloy_primitives::U256; use reth_basic_payload_builder::{ commit_withdrawals, is_better_payload, BuildArguments, BuildOutcome, PayloadBuilder, @@ -25,7 +25,6 @@ use reth_execution_types::ExecutionOutcome; use reth_payload_builder::{EthBuiltPayload, EthPayloadBuilderAttributes}; use reth_payload_primitives::{PayloadBuilderAttributes, PayloadBuilderError}; use reth_primitives::{ - constants::eip4844::MAX_DATA_GAS_PER_BLOCK, proofs::{self}, revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg}, Block, BlockBody, EthereumHardforks, Header, Receipt, diff --git a/crates/node/events/Cargo.toml b/crates/node/events/Cargo.toml index 3b515b8ab5e9..6af3d8cbeb40 100644 --- a/crates/node/events/Cargo.toml +++ b/crates/node/events/Cargo.toml @@ -19,13 +19,13 @@ reth-network-api.workspace = true reth-stages.workspace = true reth-prune.workspace = true reth-static-file.workspace = true -reth-primitives.workspace = true reth-primitives-traits.workspace = true # ethereum alloy-primitives.workspace = true alloy-rpc-types-engine.workspace = true alloy-consensus.workspace = true +alloy-eips.workspace = true # async tokio.workspace = true diff --git a/crates/node/events/src/node.rs b/crates/node/events/src/node.rs index 92f8cb5e0fe7..fb0f4d48d77f 100644 --- a/crates/node/events/src/node.rs +++ b/crates/node/events/src/node.rs @@ -10,7 +10,6 @@ use reth_beacon_consensus::{ }; use reth_network::NetworkEvent; use reth_network_api::PeersInfo; -use reth_primitives::constants; use reth_primitives_traits::{format_gas, format_gas_throughput}; use reth_prune::PrunerEvent; use reth_stages::{EntitiesCheckpoint, ExecOutput, PipelineEvent, StageCheckpoint, StageId}; @@ -265,8 +264,8 @@ impl NodeState { gas_throughput=%format_gas_throughput(block.header.gas_used, elapsed), full=%format!("{:.1}%", block.header.gas_used as f64 * 100.0 / block.header.gas_limit as f64), base_fee=%format!("{:.2}gwei", block.header.base_fee_per_gas.unwrap_or(0) as f64 / GWEI_TO_WEI as f64), - blobs=block.header.blob_gas_used.unwrap_or(0) / constants::eip4844::DATA_GAS_PER_BLOB, - excess_blobs=block.header.excess_blob_gas.unwrap_or(0) / constants::eip4844::DATA_GAS_PER_BLOB, + blobs=block.header.blob_gas_used.unwrap_or(0) / alloy_eips::eip4844::DATA_GAS_PER_BLOB, + excess_blobs=block.header.excess_blob_gas.unwrap_or(0) / alloy_eips::eip4844::DATA_GAS_PER_BLOB, ?elapsed, "Block added to canonical chain" ); diff --git a/crates/primitives/src/constants/eip4844.rs b/crates/primitives/src/constants/eip4844.rs deleted file mode 100644 index 14e892adfbeb..000000000000 --- a/crates/primitives/src/constants/eip4844.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844#parameters) protocol constants and utils for shard Blob Transactions. - -pub use alloy_eips::eip4844::{ - BLOB_GASPRICE_UPDATE_FRACTION, BLOB_TX_MIN_BLOB_GASPRICE, DATA_GAS_PER_BLOB, - FIELD_ELEMENTS_PER_BLOB, FIELD_ELEMENT_BYTES, MAX_BLOBS_PER_BLOCK, MAX_DATA_GAS_PER_BLOCK, - TARGET_BLOBS_PER_BLOCK, TARGET_DATA_GAS_PER_BLOCK, VERSIONED_HASH_VERSION_KZG, -}; diff --git a/crates/primitives/src/constants/mod.rs b/crates/primitives/src/constants/mod.rs index fd1dc1586248..09c488cc25ad 100644 --- a/crates/primitives/src/constants/mod.rs +++ b/crates/primitives/src/constants/mod.rs @@ -1,6 +1,3 @@ //! Ethereum protocol-related constants pub use reth_primitives_traits::constants::*; - -/// [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844#parameters) constants. -pub mod eip4844; diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index b09fff9e2b6a..3a5c3674166c 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -297,7 +297,7 @@ impl Transaction { /// transaction. /// /// This is the number of blobs times the - /// [`DATA_GAS_PER_BLOB`](crate::constants::eip4844::DATA_GAS_PER_BLOB) a single blob consumes. + /// [`DATA_GAS_PER_BLOB`](alloy_eips::eip4844::DATA_GAS_PER_BLOB) a single blob consumes. pub fn blob_gas_used(&self) -> Option { self.as_eip4844().map(TxEip4844::blob_gas) } diff --git a/crates/primitives/src/transaction/pooled.rs b/crates/primitives/src/transaction/pooled.rs index 32d4da65980e..000ff41fe523 100644 --- a/crates/primitives/src/transaction/pooled.rs +++ b/crates/primitives/src/transaction/pooled.rs @@ -264,7 +264,7 @@ impl PooledTransactionsElement { /// transaction. /// /// This is the number of blobs times the - /// [`DATA_GAS_PER_BLOB`](crate::constants::eip4844::DATA_GAS_PER_BLOB) a single blob consumes. + /// [`DATA_GAS_PER_BLOB`](alloy_eips::eip4844::DATA_GAS_PER_BLOB) a single blob consumes. pub fn blob_gas_used(&self) -> Option { self.as_eip4844().map(TxEip4844::blob_gas) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index 34ba6dc7e4ec..20e847a8cc94 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -167,7 +167,7 @@ pub trait EthFees: LoadFee { base_fee_per_blob_gas.push(header.blob_fee().unwrap_or_default()); blob_gas_used_ratio.push( header.blob_gas_used.unwrap_or_default() as f64 - / reth_primitives::constants::eip4844::MAX_DATA_GAS_PER_BLOCK as f64, + / alloy_eips::eip4844::MAX_DATA_GAS_PER_BLOCK as f64, ); // Percentiles were specified, so we need to collect reward percentile ino diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 2c04f3beb3b3..f2d1416139b8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -6,7 +6,9 @@ use std::time::{Duration, Instant}; use crate::{EthApiTypes, FromEthApiError, FromEvmError, RpcNodeCore}; use alloy_consensus::EMPTY_OMMER_ROOT_HASH; -use alloy_eips::{eip7685::EMPTY_REQUESTS_HASH, merge::BEACON_NONCE}; +use alloy_eips::{ + eip4844::MAX_DATA_GAS_PER_BLOCK, eip7685::EMPTY_REQUESTS_HASH, merge::BEACON_NONCE, +}; use alloy_primitives::{BlockNumber, B256, U256}; use alloy_rpc_types::BlockNumberOrTag; use futures::Future; @@ -17,7 +19,6 @@ use reth_evm::{ }; use reth_execution_types::ExecutionOutcome; use reth_primitives::{ - constants::eip4844::MAX_DATA_GAS_PER_BLOCK, proofs::calculate_transaction_root, revm_primitives::{ BlockEnv, CfgEnv, CfgEnvWithHandlerCfg, EVMError, Env, ExecutionResult, InvalidTransaction, diff --git a/crates/rpc/rpc-eth-types/src/fee_history.rs b/crates/rpc/rpc-eth-types/src/fee_history.rs index c845d9683870..7692d47de998 100644 --- a/crates/rpc/rpc-eth-types/src/fee_history.rs +++ b/crates/rpc/rpc-eth-types/src/fee_history.rs @@ -366,7 +366,7 @@ impl FeeHistoryEntry { gas_used_ratio: block.gas_used as f64 / block.gas_limit as f64, base_fee_per_blob_gas: block.blob_fee(), blob_gas_used_ratio: block.blob_gas_used() as f64 / - reth_primitives::constants::eip4844::MAX_DATA_GAS_PER_BLOCK as f64, + alloy_eips::eip4844::MAX_DATA_GAS_PER_BLOCK as f64, excess_blob_gas: block.excess_blob_gas, blob_gas_used: block.blob_gas_used, gas_used: block.gas_used, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 42de860db794..c6369c98a7fc 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -22,9 +22,11 @@ use alloy_consensus::constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; -use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}; +use alloy_eips::{ + eip1559::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}, + eip4844::BLOB_TX_MIN_BLOB_GASPRICE, +}; use alloy_primitives::{Address, TxHash, B256}; -use reth_primitives::constants::eip4844::BLOB_TX_MIN_BLOB_GASPRICE; use rustc_hash::FxHashMap; use smallvec::SmallVec; use std::{ diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 99b0caaf48a3..a3cddaf0a71b 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -11,7 +11,7 @@ use alloy_consensus::{ constants::{EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, LEGACY_TX_TYPE_ID}, TxEip1559, TxEip2930, TxEip4844, TxLegacy, }; -use alloy_eips::{eip1559::MIN_PROTOCOL_BASE_FEE, eip2930::AccessList}; +use alloy_eips::{eip1559::MIN_PROTOCOL_BASE_FEE, eip2930::AccessList, eip4844::DATA_GAS_PER_BLOB}; use alloy_primitives::{Address, Bytes, ChainId, TxHash, TxKind, B256, U256}; use paste::paste; use rand::{ @@ -19,9 +19,9 @@ use rand::{ prelude::Distribution, }; use reth_primitives::{ - constants::eip4844::DATA_GAS_PER_BLOB, transaction::TryFromRecoveredTransactionError, - BlobTransactionSidecar, BlobTransactionValidationError, PooledTransactionsElementEcRecovered, - Signature, Transaction, TransactionSigned, TransactionSignedEcRecovered, TxType, + transaction::TryFromRecoveredTransactionError, BlobTransactionSidecar, + BlobTransactionValidationError, PooledTransactionsElementEcRecovered, Signature, Transaction, + TransactionSigned, TransactionSignedEcRecovered, TxType, }; use std::{ops::Range, sync::Arc, time::Instant, vec::IntoIter}; diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index c21a7a4ea75a..9db9c53d3873 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1484,7 +1484,8 @@ impl Stream for NewSubpoolTransactionStream { mod tests { use super::*; use alloy_consensus::{TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy}; - use reth_primitives::{constants::eip4844::DATA_GAS_PER_BLOB, Signature, TransactionSigned}; + use alloy_eips::eip4844::DATA_GAS_PER_BLOB; + use reth_primitives::{Signature, TransactionSigned}; #[test] fn test_pool_size_invariants() { diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index bf7749fb85c3..62e9f3f2917d 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -15,10 +15,9 @@ use alloy_consensus::constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; +use alloy_eips::eip4844::MAX_BLOBS_PER_BLOCK; use reth_chainspec::{ChainSpec, EthereumHardforks}; -use reth_primitives::{ - constants::eip4844::MAX_BLOBS_PER_BLOCK, GotExpected, InvalidTransactionError, SealedBlock, -}; +use reth_primitives::{GotExpected, InvalidTransactionError, SealedBlock}; use reth_storage_api::{AccountReader, StateProviderFactory}; use reth_tasks::TaskSpawner; use revm::{ From fbdebe08e02214eb5df7a13d6277878a337c1b86 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:16:33 +0100 Subject: [PATCH 127/239] chain-state: fix typo (#12112) --- crates/chain-state/src/in_memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index a850e66521a6..6bef197bea98 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -777,7 +777,7 @@ pub struct ExecutedBlock { pub senders: Arc>, /// Block's execution outcome. pub execution_output: Arc, - /// Block's hashedst state. + /// Block's hashed state. pub hashed_state: Arc, /// Trie updates that result of applying the block. pub trie: Arc, From 0f86287b65ac11113c47a6a0627a737ff9719106 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 28 Oct 2024 10:09:47 +0000 Subject: [PATCH 128/239] fix(trie): sparse trie walk should be done in a sorted manner (#12087) --- crates/trie/sparse/src/trie.rs | 49 ++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 8d65378f614e..8b214d5f7ba9 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -11,7 +11,7 @@ use reth_trie_common::{ EMPTY_ROOT_HASH, }; use smallvec::SmallVec; -use std::{collections::HashSet, fmt}; +use std::fmt; /// Inner representation of the sparse trie. /// Sparse trie is blind by default until nodes are revealed. @@ -579,19 +579,19 @@ impl RevealedSparseTrie { /// Returns a list of paths to the nodes that are located at the provided depth when counting /// from the root node. If there's a leaf at a depth less than the provided depth, it will be /// included in the result. - fn get_nodes_at_depth(&self, depth: usize) -> HashSet { + fn get_nodes_at_depth(&self, depth: usize) -> Vec { let mut paths = Vec::from([(Nibbles::default(), 0)]); - let mut targets = HashSet::::default(); + let mut targets = Vec::new(); while let Some((mut path, level)) = paths.pop() { match self.nodes.get(&path).unwrap() { SparseNode::Empty | SparseNode::Hash(_) => {} SparseNode::Leaf { .. } => { - targets.insert(path); + targets.push(path); } SparseNode::Extension { key, .. } => { if level >= depth { - targets.insert(path); + targets.push(path); } else { path.extend_from_slice_unchecked(key); paths.push((path, level + 1)); @@ -599,9 +599,9 @@ impl RevealedSparseTrie { } SparseNode::Branch { state_mask, .. } => { if level >= depth { - targets.insert(path); + targets.push(path); } else { - for bit in CHILD_INDEX_RANGE { + for bit in CHILD_INDEX_RANGE.rev() { if state_mask.is_bit_set(bit) { let mut child_path = path.clone(); child_path.push_unchecked(bit); @@ -666,7 +666,9 @@ impl RevealedSparseTrie { } branch_child_buf.clear(); - for bit in CHILD_INDEX_RANGE { + // Walk children in a reverse order from `f` to `0`, so we pop the `0` first + // from the stack. + for bit in CHILD_INDEX_RANGE.rev() { if state_mask.is_bit_set(bit) { let mut child = path.clone(); child.push_unchecked(bit); @@ -674,13 +676,17 @@ impl RevealedSparseTrie { } } - branch_value_stack_buf.clear(); - for child_path in &branch_child_buf { + branch_value_stack_buf.resize(branch_child_buf.len(), Default::default()); + let mut added_children = false; + for (i, child_path) in branch_child_buf.iter().enumerate() { if rlp_node_stack.last().map_or(false, |e| &e.0 == child_path) { let (_, child) = rlp_node_stack.pop().unwrap(); - branch_value_stack_buf.push(child); + // Insert children in the resulting buffer in a normal order, because + // initially we iterated in reverse. + branch_value_stack_buf[branch_child_buf.len() - i - 1] = child; + added_children = true; } else { - debug_assert!(branch_value_stack_buf.is_empty()); + debug_assert!(!added_children); path_stack.push(path); path_stack.extend(branch_child_buf.drain(..)); continue 'main @@ -1568,38 +1574,35 @@ mod tests { .unwrap(); sparse.update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value).unwrap(); - assert_eq!(sparse.get_nodes_at_depth(0), HashSet::from([Nibbles::default()])); - assert_eq!( - sparse.get_nodes_at_depth(1), - HashSet::from([Nibbles::from_nibbles_unchecked([0x5])]) - ); + assert_eq!(sparse.get_nodes_at_depth(0), vec![Nibbles::default()]); + assert_eq!(sparse.get_nodes_at_depth(1), vec![Nibbles::from_nibbles_unchecked([0x5])]); assert_eq!( sparse.get_nodes_at_depth(2), - HashSet::from([ + vec![ Nibbles::from_nibbles_unchecked([0x5, 0x0]), Nibbles::from_nibbles_unchecked([0x5, 0x2]), Nibbles::from_nibbles_unchecked([0x5, 0x3]) - ]) + ] ); assert_eq!( sparse.get_nodes_at_depth(3), - HashSet::from([ + vec![ Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3]), Nibbles::from_nibbles_unchecked([0x5, 0x2]), Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x1]), Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x3]) - ]) + ] ); assert_eq!( sparse.get_nodes_at_depth(4), - HashSet::from([ + vec![ Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3, 0x1]), Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3, 0x3]), Nibbles::from_nibbles_unchecked([0x5, 0x2]), Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x1]), Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x3, 0x0]), Nibbles::from_nibbles_unchecked([0x5, 0x3, 0x3, 0x2]) - ]) + ] ); } } From e4bd13534df447e5da190c7216ffcd7232ff8e8f Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 28 Oct 2024 19:13:43 +0900 Subject: [PATCH 129/239] fix(ci): remove import path from type names on `compact-codec` (#12125) --- .github/workflows/compact.yml | 4 +++- crates/cli/commands/src/test_vectors/compact.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 63f6f282fa26..c5d39f72aeca 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -36,7 +36,9 @@ jobs: ref: ${{ github.base_ref || 'main' }} # On `main` branch, generates test vectors and serializes them to disk using `Compact`. - name: Generate compact vectors - run: ${{ matrix.bin }} -- test-vectors compact --write + run: | + ${{ matrix.bin }} -- test-vectors compact --write && + for f in ./testdata/micro/compact/*; do mv "$f" "$(dirname "$f")/$(basename "$f" | awk -F '__' '{print $NF}')"; done - name: Checkout PR uses: actions/checkout@v4 with: diff --git a/crates/cli/commands/src/test_vectors/compact.rs b/crates/cli/commands/src/test_vectors/compact.rs index 162ee1ceaa49..94552d5c2158 100644 --- a/crates/cli/commands/src/test_vectors/compact.rs +++ b/crates/cli/commands/src/test_vectors/compact.rs @@ -267,5 +267,5 @@ where } pub fn type_name() -> String { - std::any::type_name::().replace("::", "__") + std::any::type_name::().split("::").last().unwrap_or(std::any::type_name::()).to_string() } From 77e5748124ffd68c646bf775fe086227f7603129 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 28 Oct 2024 18:14:11 +0800 Subject: [PATCH 130/239] chore(rpc): remove redundant `LoadFee::provider` (#12122) --- crates/optimism/rpc/src/eth/mod.rs | 24 ++++++++++----------- crates/rpc/rpc-eth-api/src/helpers/block.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/fee.rs | 23 +++++++------------- crates/rpc/rpc/src/eth/helpers/fees.rs | 21 +++++++----------- 4 files changed, 28 insertions(+), 42 deletions(-) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index d12a2dd02cd3..9b04e1c730a5 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -14,15 +14,15 @@ use std::{fmt, sync::Arc}; use alloy_primitives::U256; use derive_more::Deref; use op_alloy_network::Optimism; -use reth_chainspec::EthereumHardforks; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_evm::ConfigureEvm; use reth_network_api::NetworkInfo; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_node_builder::EthApiBuilderCtx; use reth_primitives::Header; use reth_provider::{ - BlockIdReader, BlockNumReader, BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, - HeaderProvider, StageCheckpointReader, StateProviderFactory, + BlockNumReader, BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider, EvmEnvProvider, + StageCheckpointReader, StateProviderFactory, }; use reth_rpc::eth::{core::EthApiInner, DevSigner}; use reth_rpc_eth_api::{ @@ -184,23 +184,21 @@ where impl LoadFee for OpEthApi where - Self: LoadBlock, - N: FullNodeComponents>, + Self: LoadBlock, + N: RpcNodeCore< + Provider: BlockReaderIdExt + + EvmEnvProvider + + ChainSpecProvider + + StateProviderFactory, + >, { - #[inline] - fn provider( - &self, - ) -> impl BlockIdReader + HeaderProvider + ChainSpecProvider { - self.inner.provider() - } - #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() } #[inline] - fn gas_oracle(&self) -> &GasPriceOracle { + fn gas_oracle(&self) -> &GasPriceOracle { self.inner.gas_oracle() } diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 861afb3ad261..217e84a4754e 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -20,7 +20,7 @@ pub type BlockAndReceiptsResult = Result { +pub trait EthBlocks: LoadBlock { /// Returns the block header for the given block id. fn rpc_block_header( &self, diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index 20e847a8cc94..dcde2214a5d3 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -3,8 +3,8 @@ use alloy_primitives::U256; use alloy_rpc_types::{BlockNumberOrTag, FeeHistory}; use futures::Future; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_provider::{BlockIdReader, BlockReaderIdExt, ChainSpecProvider, HeaderProvider}; +use reth_chainspec::EthChainSpec; +use reth_provider::{BlockIdReader, ChainSpecProvider, HeaderProvider}; use reth_rpc_eth_types::{ fee_history::calculate_reward_percentiles_for_block, EthApiError, EthStateCache, FeeHistoryCache, FeeHistoryEntry, GasPriceOracle, RpcInvalidTransactionError, @@ -82,7 +82,8 @@ pub trait EthFees: LoadFee { block_count = block_count.saturating_sub(1); } - let end_block = LoadFee::provider(self) + let end_block = self + .provider() .block_number_for_id(newest_block.into()) .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(newest_block.into()))?; @@ -147,13 +148,12 @@ pub trait EthFees: LoadFee { // Also need to include the `base_fee_per_gas` and `base_fee_per_blob_gas` for the // next block base_fee_per_gas - .push(last_entry.next_block_base_fee(LoadFee::provider(self).chain_spec()) - as u128); + .push(last_entry.next_block_base_fee(self.provider().chain_spec()) as u128); base_fee_per_blob_gas.push(last_entry.next_block_blob_fee().unwrap_or_default()); } else { // read the requested header range - let headers = LoadFee::provider(self) + let headers = self.provider() .sealed_headers_range(start_block..=end_block) .map_err(Self::Error::from_eth_err)?; if headers.len() != block_count as usize { @@ -197,7 +197,7 @@ pub trait EthFees: LoadFee { // The unwrap is safe since we checked earlier that we got at least 1 header. let last_header = headers.last().expect("is present"); base_fee_per_gas.push( - LoadFee::provider(self) + self.provider() .chain_spec() .base_fee_params_at_timestamp(last_header.timestamp) .next_block_base_fee( @@ -242,13 +242,6 @@ pub trait EthFees: LoadFee { /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` fees RPC methods. pub trait LoadFee: LoadBlock { - // Returns a handle for reading data from disk. - /// - /// Data access in default (L1) trait method implementations. - fn provider( - &self, - ) -> impl BlockIdReader + HeaderProvider + ChainSpecProvider; - /// Returns a handle for reading data from memory. /// /// Data access in default (L1) trait method implementations. @@ -257,7 +250,7 @@ pub trait LoadFee: LoadBlock { /// Returns a handle for reading gas price. /// /// Data access in default (L1) trait method implementations. - fn gas_oracle(&self) -> &GasPriceOracle; + fn gas_oracle(&self) -> &GasPriceOracle; /// Returns a handle for reading fee history data from memory. /// diff --git a/crates/rpc/rpc/src/eth/helpers/fees.rs b/crates/rpc/rpc/src/eth/helpers/fees.rs index a792f7289514..2c5db5bacbad 100644 --- a/crates/rpc/rpc/src/eth/helpers/fees.rs +++ b/crates/rpc/rpc/src/eth/helpers/fees.rs @@ -1,8 +1,7 @@ //! Contains RPC handler implementations for fee history. -use reth_chainspec::EthereumHardforks; -use reth_provider::{BlockIdReader, BlockReaderIdExt, ChainSpecProvider, HeaderProvider}; - +use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_eth_api::helpers::{EthFees, LoadBlock, LoadFee}; use reth_rpc_eth_types::{EthStateCache, FeeHistoryCache, GasPriceOracle}; @@ -15,23 +14,19 @@ impl EthFees for EthApi LoadFee for EthApi where - Self: LoadBlock, - Provider: BlockReaderIdExt + HeaderProvider + ChainSpecProvider, + Self: LoadBlock, + Provider: BlockReaderIdExt + + EvmEnvProvider + + ChainSpecProvider + + StateProviderFactory, { - #[inline] - fn provider( - &self, - ) -> impl BlockIdReader + HeaderProvider + ChainSpecProvider { - self.inner.provider() - } - #[inline] fn cache(&self) -> &EthStateCache { self.inner.cache() } #[inline] - fn gas_oracle(&self) -> &GasPriceOracle { + fn gas_oracle(&self) -> &GasPriceOracle { self.inner.gas_oracle() } From 8f5fd1d70c1670e576b3103a12b9027fd91fac70 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 28 Oct 2024 18:14:40 +0800 Subject: [PATCH 131/239] chore(rpc): remove redundant `EthTransactions::provider` (#12121) --- crates/optimism/rpc/src/eth/transaction.rs | 10 +++---- .../rpc-eth-api/src/helpers/transaction.rs | 27 ++++++++----------- crates/rpc/rpc/src/eth/helpers/transaction.rs | 9 +------ 3 files changed, 15 insertions(+), 31 deletions(-) diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 451c8a805fb3..5135b13a2ded 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -19,13 +19,9 @@ use crate::{OpEthApi, SequencerClient}; impl EthTransactions for OpEthApi where - Self: LoadTransaction, - N: FullNodeComponents, + Self: LoadTransaction, + N: RpcNodeCore, { - fn provider(&self) -> impl BlockReaderIdExt { - self.inner.provider() - } - fn signers(&self) -> &parking_lot::RwLock>> { self.inner.signers() } @@ -71,7 +67,7 @@ where impl OpEthApi where - N: FullNodeComponents, + N: RpcNodeCore, { /// Returns the [`SequencerClient`] if one is set. pub fn raw_tx_forwarder(&self) -> Option { diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 791cb2ae1eb8..af647fedf2c4 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -52,12 +52,7 @@ use super::{ /// See also /// /// This implementation follows the behaviour of Geth and disables the basefee check for tracing. -pub trait EthTransactions: LoadTransaction { - /// Returns a handle for reading data from disk. - /// - /// Data access in default (L1) trait method implementations. - fn provider(&self) -> impl BlockReaderIdExt; - +pub trait EthTransactions: LoadTransaction { /// Returns a handle for signing data. /// /// Singer access in default (L1) trait method implementations. @@ -111,7 +106,8 @@ pub trait EthTransactions: LoadTransaction { } self.spawn_blocking_io(move |ref this| { - Ok(RpcNodeCore::provider(this) + Ok(this + .provider() .transaction_by_hash(hash) .map_err(Self::Error::from_eth_err)? .map(|tx| tx.encoded_2718().into())) @@ -166,7 +162,8 @@ pub trait EthTransactions: LoadTransaction { { let this = self.clone(); self.spawn_blocking_io(move |_| { - let (tx, meta) = match RpcNodeCore::provider(&this) + let (tx, meta) = match this + .provider() .transaction_by_hash_with_meta(hash) .map_err(Self::Error::from_eth_err)? { @@ -174,13 +171,11 @@ pub trait EthTransactions: LoadTransaction { None => return Ok(None), }; - let receipt = match EthTransactions::provider(&this) - .receipt_by_hash(hash) - .map_err(Self::Error::from_eth_err)? - { - Some(recpt) => recpt, - None => return Ok(None), - }; + let receipt = + match this.provider().receipt_by_hash(hash).map_err(Self::Error::from_eth_err)? { + Some(recpt) => recpt, + None => return Ok(None), + }; Ok(Some((tx, meta, receipt))) }) @@ -257,7 +252,7 @@ pub trait EthTransactions: LoadTransaction { return Ok(None); } - let Ok(high) = RpcNodeCore::provider(self).best_block_number() else { + let Ok(high) = self.provider().best_block_number() else { return Err(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()).into()); }; diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index c4505bef09d8..623db35e5ade 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -13,15 +13,8 @@ use crate::EthApi; impl EthTransactions for EthApi where - Self: LoadTransaction, - Pool: TransactionPool + 'static, - Provider: BlockReaderIdExt, + Self: LoadTransaction, { - #[inline] - fn provider(&self) -> impl BlockReaderIdExt { - self.inner.provider() - } - #[inline] fn signers(&self) -> &parking_lot::RwLock>> { self.inner.signers() From 268090e879f40bf388e3b0f1dac9983b76c2255b Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 28 Oct 2024 10:44:36 +0000 Subject: [PATCH 132/239] bench(trie): `RevealedSparseTrie::update_rlp_node_level` (#12046) --- crates/trie/sparse/Cargo.toml | 4 ++ crates/trie/sparse/benches/rlp_node.rs | 78 ++++++++++++++++++++++++++ crates/trie/sparse/benches/root.rs | 1 + crates/trie/sparse/src/trie.rs | 2 +- 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 crates/trie/sparse/benches/rlp_node.rs diff --git a/crates/trie/sparse/Cargo.toml b/crates/trie/sparse/Cargo.toml index 26d036f57ff9..1c5bb7d8a33e 100644 --- a/crates/trie/sparse/Cargo.toml +++ b/crates/trie/sparse/Cargo.toml @@ -41,3 +41,7 @@ rand.workspace = true [[bench]] name = "root" harness = false + +[[bench]] +name = "rlp_node" +harness = false diff --git a/crates/trie/sparse/benches/rlp_node.rs b/crates/trie/sparse/benches/rlp_node.rs new file mode 100644 index 000000000000..57ab52978b64 --- /dev/null +++ b/crates/trie/sparse/benches/rlp_node.rs @@ -0,0 +1,78 @@ +#![allow(missing_docs, unreachable_pub)] + +use std::time::{Duration, Instant}; + +use alloy_primitives::{B256, U256}; +use criterion::{criterion_group, criterion_main, Criterion}; +use prop::strategy::ValueTree; +use proptest::{prelude::*, test_runner::TestRunner}; +use rand::seq::IteratorRandom; +use reth_testing_utils::generators; +use reth_trie::Nibbles; +use reth_trie_sparse::RevealedSparseTrie; + +pub fn update_rlp_node_level(c: &mut Criterion) { + let mut rng = generators::rng(); + + let mut group = c.benchmark_group("update rlp node level"); + group.sample_size(20); + + for size in [100_000] { + let mut runner = TestRunner::new(ProptestConfig::default()); + let state = proptest::collection::hash_map(any::(), any::(), size) + .new_tree(&mut runner) + .unwrap() + .current(); + + // Create a sparse trie with `size` leaves + let mut sparse = RevealedSparseTrie::default(); + for (key, value) in &state { + sparse + .update_leaf(Nibbles::unpack(key), alloy_rlp::encode_fixed_size(value).to_vec()) + .unwrap(); + } + sparse.root(); + + for updated_leaves in [0.1, 1.0] { + for key in state + .keys() + .choose_multiple(&mut rng, (size as f64 * (updated_leaves / 100.0)) as usize) + { + sparse + .update_leaf( + Nibbles::unpack(key), + alloy_rlp::encode_fixed_size(&rng.gen::()).to_vec(), + ) + .unwrap(); + } + + // Calculate the maximum depth of the trie for the given number of leaves + let max_depth = (size as f64).log(16.0).ceil() as usize; + + for depth in 0..=max_depth { + group.bench_function( + format!("size {size} | updated {updated_leaves}% | depth {depth}"), + |b| { + // Use `iter_custom` to avoid measuring clones and drops + b.iter_custom(|iters| { + let mut elapsed = Duration::ZERO; + + let mut cloned = sparse.clone(); + for _ in 0..iters { + let start = Instant::now(); + cloned.update_rlp_node_level(depth); + elapsed += start.elapsed(); + cloned = sparse.clone(); + } + + elapsed + }) + }, + ); + } + } + } +} + +criterion_group!(rlp_node, update_rlp_node_level); +criterion_main!(rlp_node); diff --git a/crates/trie/sparse/benches/root.rs b/crates/trie/sparse/benches/root.rs index bc221a8f8313..30ce566fb5f6 100644 --- a/crates/trie/sparse/benches/root.rs +++ b/crates/trie/sparse/benches/root.rs @@ -1,4 +1,5 @@ #![allow(missing_docs, unreachable_pub)] + use alloy_primitives::{map::HashMap, B256, U256}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use itertools::Itertools; diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 8b214d5f7ba9..e7ee66c54005 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -77,7 +77,7 @@ impl SparseTrie { /// - Each leaf entry in `nodes` collection must have a corresponding entry in `values` collection. /// The opposite is also true. /// - All keys in `values` collection are full leaf paths. -#[derive(PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct RevealedSparseTrie { /// All trie nodes. nodes: HashMap, From 1b0f625f1d315473b86fc38153a7c5ed27d27ffa Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 28 Oct 2024 11:06:26 +0000 Subject: [PATCH 133/239] perf(trie): collect only changed sparse nodes at a depth (#12093) --- crates/trie/sparse/src/trie.rs | 47 ++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index e7ee66c54005..91362eed527f 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -569,27 +569,36 @@ impl RevealedSparseTrie { /// Update hashes of the nodes that are located at a level deeper than or equal to the provided /// depth. Root node has a level of 0. pub fn update_rlp_node_level(&mut self, depth: usize) { - let targets = self.get_nodes_at_depth(depth); let mut prefix_set = self.prefix_set.clone().freeze(); + + let targets = self.get_changed_nodes_at_depth(&mut prefix_set, depth); for target in targets { self.rlp_node(target, &mut prefix_set); } } - /// Returns a list of paths to the nodes that are located at the provided depth when counting - /// from the root node. If there's a leaf at a depth less than the provided depth, it will be - /// included in the result. - fn get_nodes_at_depth(&self, depth: usize) -> Vec { + /// Returns a list of paths to the nodes that were changed according to the prefix set and are + /// located at the provided depth when counting from the root node. If there's a leaf at a + /// depth less than the provided depth, it will be included in the result. + fn get_changed_nodes_at_depth(&self, prefix_set: &mut PrefixSet, depth: usize) -> Vec { let mut paths = Vec::from([(Nibbles::default(), 0)]); let mut targets = Vec::new(); while let Some((mut path, level)) = paths.pop() { match self.nodes.get(&path).unwrap() { SparseNode::Empty | SparseNode::Hash(_) => {} - SparseNode::Leaf { .. } => { + SparseNode::Leaf { hash, .. } => { + if hash.is_some() && !prefix_set.contains(&path) { + continue + } + targets.push(path); } - SparseNode::Extension { key, .. } => { + SparseNode::Extension { key, hash } => { + if hash.is_some() && !prefix_set.contains(&path) { + continue + } + if level >= depth { targets.push(path); } else { @@ -597,7 +606,11 @@ impl RevealedSparseTrie { paths.push((path, level + 1)); } } - SparseNode::Branch { state_mask, .. } => { + SparseNode::Branch { state_mask, hash } => { + if hash.is_some() && !prefix_set.contains(&path) { + continue + } + if level >= depth { targets.push(path); } else { @@ -1540,7 +1553,7 @@ mod tests { } #[test] - fn sparse_trie_get_nodes_at_depth() { + fn sparse_trie_get_changed_nodes_at_depth() { let mut sparse = RevealedSparseTrie::default(); let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); @@ -1574,10 +1587,16 @@ mod tests { .unwrap(); sparse.update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value).unwrap(); - assert_eq!(sparse.get_nodes_at_depth(0), vec![Nibbles::default()]); - assert_eq!(sparse.get_nodes_at_depth(1), vec![Nibbles::from_nibbles_unchecked([0x5])]); assert_eq!( - sparse.get_nodes_at_depth(2), + sparse.get_changed_nodes_at_depth(&mut PrefixSet::default(), 0), + vec![Nibbles::default()] + ); + assert_eq!( + sparse.get_changed_nodes_at_depth(&mut PrefixSet::default(), 1), + vec![Nibbles::from_nibbles_unchecked([0x5])] + ); + assert_eq!( + sparse.get_changed_nodes_at_depth(&mut PrefixSet::default(), 2), vec![ Nibbles::from_nibbles_unchecked([0x5, 0x0]), Nibbles::from_nibbles_unchecked([0x5, 0x2]), @@ -1585,7 +1604,7 @@ mod tests { ] ); assert_eq!( - sparse.get_nodes_at_depth(3), + sparse.get_changed_nodes_at_depth(&mut PrefixSet::default(), 3), vec![ Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3]), Nibbles::from_nibbles_unchecked([0x5, 0x2]), @@ -1594,7 +1613,7 @@ mod tests { ] ); assert_eq!( - sparse.get_nodes_at_depth(4), + sparse.get_changed_nodes_at_depth(&mut PrefixSet::default(), 4), vec![ Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3, 0x1]), Nibbles::from_nibbles_unchecked([0x5, 0x0, 0x2, 0x3, 0x3]), From 0d07d27f3c5eea7b3ab9dedecf786fcc78954b1f Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Mon, 28 Oct 2024 05:00:36 -0700 Subject: [PATCH 134/239] feat: Add version to `BeaconEngineMessage` FCU (#12089) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- .../src/commands/debug_cmd/replay_engine.rs | 13 ++++++++--- crates/consensus/auto-seal/src/task.rs | 3 ++- crates/consensus/beacon/src/engine/handle.rs | 7 ++++-- crates/consensus/beacon/src/engine/message.rs | 4 +++- crates/consensus/beacon/src/engine/mod.rs | 17 +++++++++++--- .../consensus/beacon/src/engine/test_utils.rs | 10 ++++++-- crates/engine/local/src/miner.rs | 4 +++- crates/engine/tree/src/tree/mod.rs | 23 ++++++++++++++----- crates/engine/util/src/engine_store.rs | 7 +++++- crates/engine/util/src/reorg.rs | 20 +++++++++++++--- crates/engine/util/src/skip_fcu.rs | 14 +++++++++-- crates/payload/primitives/src/lib.rs | 11 +++++---- crates/payload/primitives/src/traits.rs | 2 +- crates/rpc/rpc-engine-api/src/engine_api.rs | 5 ++-- 14 files changed, 107 insertions(+), 33 deletions(-) diff --git a/bin/reth/src/commands/debug_cmd/replay_engine.rs b/bin/reth/src/commands/debug_cmd/replay_engine.rs index e7b3de6b6c13..9314a439265d 100644 --- a/bin/reth/src/commands/debug_cmd/replay_engine.rs +++ b/bin/reth/src/commands/debug_cmd/replay_engine.rs @@ -18,7 +18,9 @@ use reth_engine_util::engine_store::{EngineMessageStore, StoredEngineApiMessage} use reth_fs_util as fs; use reth_network::{BlockDownloaderProvider, NetworkHandle}; use reth_network_api::NetworkInfo; -use reth_node_api::{NodeTypesWithDB, NodeTypesWithDBAdapter, NodeTypesWithEngine}; +use reth_node_api::{ + EngineApiMessageVersion, NodeTypesWithDB, NodeTypesWithDBAdapter, NodeTypesWithEngine, +}; use reth_node_ethereum::{EthEngineTypes, EthEvmConfig, EthExecutorProvider}; use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; use reth_provider::{ @@ -166,8 +168,13 @@ impl> Command { debug!(target: "reth::cli", filepath = %filepath.display(), ?message, "Forwarding Engine API message"); match message { StoredEngineApiMessage::ForkchoiceUpdated { state, payload_attrs } => { - let response = - beacon_engine_handle.fork_choice_updated(state, payload_attrs).await?; + let response = beacon_engine_handle + .fork_choice_updated( + state, + payload_attrs, + EngineApiMessageVersion::default(), + ) + .await?; debug!(target: "reth::cli", ?response, "Received for forkchoice updated"); } StoredEngineApiMessage::NewPayload { payload, sidecar } => { diff --git a/crates/consensus/auto-seal/src/task.rs b/crates/consensus/auto-seal/src/task.rs index cb0586d44407..75ddda90861c 100644 --- a/crates/consensus/auto-seal/src/task.rs +++ b/crates/consensus/auto-seal/src/task.rs @@ -3,7 +3,7 @@ use alloy_rpc_types_engine::ForkchoiceState; use futures_util::{future::BoxFuture, FutureExt}; use reth_beacon_consensus::{BeaconEngineMessage, ForkchoiceStatus}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_engine_primitives::EngineTypes; +use reth_engine_primitives::{EngineApiMessageVersion, EngineTypes}; use reth_evm::execute::BlockExecutorProvider; use reth_provider::{CanonChainTracker, StateProviderFactory}; use reth_stages_api::PipelineEvent; @@ -155,6 +155,7 @@ where state, payload_attrs: None, tx, + version: EngineApiMessageVersion::default(), }); debug!(target: "consensus::auto", ?state, "Sent fork choice update"); diff --git a/crates/consensus/beacon/src/engine/handle.rs b/crates/consensus/beacon/src/engine/handle.rs index 4aafc6e07c1c..f8840cf78abb 100644 --- a/crates/consensus/beacon/src/engine/handle.rs +++ b/crates/consensus/beacon/src/engine/handle.rs @@ -8,7 +8,7 @@ use alloy_rpc_types_engine::{ ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState, ForkchoiceUpdated, PayloadStatus, }; use futures::TryFutureExt; -use reth_engine_primitives::EngineTypes; +use reth_engine_primitives::{EngineApiMessageVersion, EngineTypes}; use reth_errors::RethResult; use reth_tokio_util::{EventSender, EventStream}; use tokio::sync::{mpsc::UnboundedSender, oneshot}; @@ -60,9 +60,10 @@ where &self, state: ForkchoiceState, payload_attrs: Option, + version: EngineApiMessageVersion, ) -> Result { Ok(self - .send_fork_choice_updated(state, payload_attrs) + .send_fork_choice_updated(state, payload_attrs, version) .map_err(|_| BeaconForkChoiceUpdateError::EngineUnavailable) .await?? .await?) @@ -74,12 +75,14 @@ where &self, state: ForkchoiceState, payload_attrs: Option, + version: EngineApiMessageVersion, ) -> oneshot::Receiver> { let (tx, rx) = oneshot::channel(); let _ = self.to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx, + version, }); rx } diff --git a/crates/consensus/beacon/src/engine/message.rs b/crates/consensus/beacon/src/engine/message.rs index e33decbd848f..fa7457c1225d 100644 --- a/crates/consensus/beacon/src/engine/message.rs +++ b/crates/consensus/beacon/src/engine/message.rs @@ -4,7 +4,7 @@ use alloy_rpc_types_engine::{ ForkchoiceUpdateError, ForkchoiceUpdated, PayloadId, PayloadStatus, PayloadStatusEnum, }; use futures::{future::Either, FutureExt}; -use reth_engine_primitives::EngineTypes; +use reth_engine_primitives::{EngineApiMessageVersion, EngineTypes}; use reth_errors::RethResult; use reth_payload_primitives::PayloadBuilderError; use std::{ @@ -156,6 +156,8 @@ pub enum BeaconEngineMessage { state: ForkchoiceState, /// The payload attributes for block building. payload_attrs: Option, + /// The Engine API Version. + version: EngineApiMessageVersion, /// The sender for returning forkchoice updated result. tx: oneshot::Sender>, }, diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 2363b9078408..770821de749e 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -10,7 +10,7 @@ use reth_blockchain_tree_api::{ error::{BlockchainTreeError, CanonicalError, InsertBlockError, InsertBlockErrorKind}, BlockStatus, BlockValidationKind, BlockchainTreeEngine, CanonicalOutcome, InsertPayloadOk, }; -use reth_engine_primitives::{EngineTypes, PayloadTypes}; +use reth_engine_primitives::{EngineApiMessageVersion, EngineTypes, PayloadTypes}; use reth_errors::{BlockValidationError, ProviderResult, RethError, RethResult}; use reth_network_p2p::{ sync::{NetworkSyncUpdater, SyncState}, @@ -428,7 +428,12 @@ where } else if let Some(attrs) = attrs { // the CL requested to build a new payload on top of this new VALID head let head = outcome.into_header().unseal(); - self.process_payload_attributes(attrs, head, state) + self.process_payload_attributes( + attrs, + head, + state, + EngineApiMessageVersion::default(), + ) } else { OnForkChoiceUpdated::valid(PayloadStatus::new( PayloadStatusEnum::Valid, @@ -1160,6 +1165,7 @@ where attrs: ::PayloadAttributes, head: Header, state: ForkchoiceState, + _version: EngineApiMessageVersion, ) -> OnForkChoiceUpdated { // 7. Client software MUST ensure that payloadAttributes.timestamp is greater than timestamp // of a block referenced by forkchoiceState.headBlockHash. If this condition isn't held @@ -1855,7 +1861,12 @@ where // sensitive, hence they are polled first. if let Poll::Ready(Some(msg)) = this.engine_message_stream.poll_next_unpin(cx) { match msg { - BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx } => { + BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs, + tx, + version: _version, + } => { this.on_forkchoice_updated(state, payload_attrs, tx); } BeaconEngineMessage::NewPayload { payload, sidecar, tx } => { diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 912f0a871bf2..6e03aebfa8db 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -19,6 +19,7 @@ use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, headers::reverse_headers::ReverseHeadersDownloaderBuilder, }; +use reth_engine_primitives::EngineApiMessageVersion; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_evm::{either::Either, test_utils::MockExecutorProvider}; use reth_evm_ethereum::execute::EthExecutorProvider; @@ -93,7 +94,9 @@ impl TestEnv { &self, state: ForkchoiceState, ) -> Result { - self.engine_handle.fork_choice_updated(state, None).await + self.engine_handle + .fork_choice_updated(state, None, EngineApiMessageVersion::default()) + .await } /// Sends the `ForkchoiceUpdated` message to the consensus engine and retries if the engine @@ -103,7 +106,10 @@ impl TestEnv { state: ForkchoiceState, ) -> Result { loop { - let result = self.engine_handle.fork_choice_updated(state, None).await?; + let result = self + .engine_handle + .fork_choice_updated(state, None, EngineApiMessageVersion::default()) + .await?; if !result.is_syncing() { return Ok(result) } diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index 706ddc43de3f..7cebd3063097 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -6,7 +6,7 @@ use eyre::OptionExt; use futures_util::{stream::Fuse, StreamExt}; use reth_beacon_consensus::BeaconEngineMessage; use reth_chainspec::EthereumHardforks; -use reth_engine_primitives::EngineTypes; +use reth_engine_primitives::{EngineApiMessageVersion, EngineTypes}; use reth_payload_builder::PayloadBuilderHandle; use reth_payload_primitives::{ BuiltPayload, PayloadAttributesBuilder, PayloadBuilder, PayloadKind, PayloadTypes, @@ -167,6 +167,7 @@ where state: self.forkchoice_state(), payload_attrs: None, tx, + version: EngineApiMessageVersion::default(), })?; let res = rx.await??; @@ -193,6 +194,7 @@ where state: self.forkchoice_state(), payload_attrs: Some(self.payload_attributes_builder.build(timestamp)), tx, + version: EngineApiMessageVersion::default(), })?; let res = rx.await??.await?; diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 555cf89164f9..dd2f67916afd 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -26,7 +26,7 @@ use reth_chain_state::{ }; use reth_chainspec::EthereumHardforks; use reth_consensus::{Consensus, PostExecutionInput}; -use reth_engine_primitives::EngineTypes; +use reth_engine_primitives::{EngineApiMessageVersion, EngineTypes}; use reth_errors::{ConsensusError, ProviderResult}; use reth_evm::execute::BlockExecutorProvider; use reth_payload_builder::PayloadBuilderHandle; @@ -969,6 +969,7 @@ where &mut self, state: ForkchoiceState, attrs: Option, + version: EngineApiMessageVersion, ) -> ProviderResult> { trace!(target: "engine::tree", ?attrs, "invoked forkchoice update"); self.metrics.engine.forkchoice_updated_messages.increment(1); @@ -1018,7 +1019,7 @@ where // to return an error ProviderError::HeaderNotFound(state.head_block_hash.into()) })?; - let updated = self.process_payload_attributes(attr, &tip, state); + let updated = self.process_payload_attributes(attr, &tip, state, version); return Ok(TreeOutcome::new(updated)) } @@ -1038,7 +1039,7 @@ where } if let Some(attr) = attrs { - let updated = self.process_payload_attributes(attr, &tip, state); + let updated = self.process_payload_attributes(attr, &tip, state, version); return Ok(TreeOutcome::new(updated)) } @@ -1054,7 +1055,8 @@ where if self.engine_kind.is_opstack() { if let Some(attr) = attrs { debug!(target: "engine::tree", head = canonical_header.number, "handling payload attributes for canonical head"); - let updated = self.process_payload_attributes(attr, &canonical_header, state); + let updated = + self.process_payload_attributes(attr, &canonical_header, state, version); return Ok(TreeOutcome::new(updated)) } } @@ -1206,8 +1208,14 @@ where } EngineApiRequest::Beacon(request) => { match request { - BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx } => { - let mut output = self.on_forkchoice_updated(state, payload_attrs); + BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs, + tx, + version, + } => { + let mut output = + self.on_forkchoice_updated(state, payload_attrs, version); if let Ok(res) = &mut output { // track last received forkchoice state @@ -2484,6 +2492,7 @@ where attrs: T::PayloadAttributes, head: &Header, state: ForkchoiceState, + _version: EngineApiMessageVersion, ) -> OnForkChoiceUpdated { // 7. Client software MUST ensure that payloadAttributes.timestamp is greater than timestamp // of a block referenced by forkchoiceState.headBlockHash. If this condition isn't held @@ -2808,6 +2817,7 @@ mod tests { state: fcu_state, payload_attrs: None, tx, + version: EngineApiMessageVersion::default(), } .into(), )) @@ -3097,6 +3107,7 @@ mod tests { }, payload_attrs: None, tx, + version: EngineApiMessageVersion::default(), } .into(), )) diff --git a/crates/engine/util/src/engine_store.rs b/crates/engine/util/src/engine_store.rs index 85c5e126fa44..6b584f0c1f55 100644 --- a/crates/engine/util/src/engine_store.rs +++ b/crates/engine/util/src/engine_store.rs @@ -64,7 +64,12 @@ impl EngineMessageStore { fs::create_dir_all(&self.path)?; // ensure that store path had been created let timestamp = received_at.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_millis(); match msg { - BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx: _tx } => { + BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs, + tx: _tx, + version: _version, + } => { let filename = format!("{}-fcu-{}.json", timestamp, state.head_block_hash); fs::write( self.path.join(filename), diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index d109fb9e94ae..0d51d2dfab6f 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -8,7 +8,7 @@ use alloy_rpc_types_engine::{ use futures::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt}; use itertools::Either; use reth_beacon_consensus::{BeaconEngineMessage, BeaconOnNewPayloadError, OnForkChoiceUpdated}; -use reth_engine_primitives::EngineTypes; +use reth_engine_primitives::{EngineApiMessageVersion, EngineTypes}; use reth_errors::{BlockExecutionError, BlockValidationError, RethError, RethResult}; use reth_ethereum_forks::EthereumHardforks; use reth_evm::{ @@ -211,18 +211,32 @@ where state: reorg_forkchoice_state, payload_attrs: None, tx: reorg_fcu_tx, + version: EngineApiMessageVersion::default(), }, ]); *this.state = EngineReorgState::Reorg { queue }; continue } - (Some(BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx }), _) => { + ( + Some(BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs, + tx, + version, + }), + _, + ) => { // Record last forkchoice state forwarded to the engine. // We do not care if it's valid since engine should be able to handle // reorgs that rely on invalid forkchoice state. *this.last_forkchoice_state = Some(state); *this.forkchoice_states_forwarded += 1; - Some(BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx }) + Some(BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs, + tx, + version, + }) } (item, _) => item, }; diff --git a/crates/engine/util/src/skip_fcu.rs b/crates/engine/util/src/skip_fcu.rs index e110cecedc89..adadfb595f89 100644 --- a/crates/engine/util/src/skip_fcu.rs +++ b/crates/engine/util/src/skip_fcu.rs @@ -45,7 +45,12 @@ where loop { let next = ready!(this.stream.poll_next_unpin(cx)); let item = match next { - Some(BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx }) => { + Some(BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs, + tx, + version, + }) => { if this.skipped < this.threshold { *this.skipped += 1; tracing::warn!(target: "engine::stream::skip_fcu", ?state, ?payload_attrs, threshold=this.threshold, skipped=this.skipped, "Skipping FCU"); @@ -53,7 +58,12 @@ where continue } *this.skipped = 0; - Some(BeaconEngineMessage::ForkchoiceUpdated { state, payload_attrs, tx }) + Some(BeaconEngineMessage::ForkchoiceUpdated { + state, + payload_attrs, + tx, + version, + }) } next => next, }; diff --git a/crates/payload/primitives/src/lib.rs b/crates/payload/primitives/src/lib.rs index 08aa428000ed..7013d9fd913c 100644 --- a/crates/payload/primitives/src/lib.rs +++ b/crates/payload/primitives/src/lib.rs @@ -324,22 +324,23 @@ where } /// The version of Engine API message. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default)] pub enum EngineApiMessageVersion { /// Version 1 - V1, + V1 = 1, /// Version 2 /// /// Added in the Shanghai hardfork. - V2, + V2 = 2, /// Version 3 /// /// Added in the Cancun hardfork. - V3, + #[default] + V3 = 3, /// Version 4 /// /// Added in the Prague hardfork. - V4, + V4 = 4, } /// Determines how we should choose the payload to return. diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index df76149028aa..f6a043755363 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -84,7 +84,7 @@ pub trait PayloadBuilderAttributes: Send + Sync + std::fmt::Debug { /// Creates a new payload builder for the given parent block and the attributes. /// - /// Derives the unique [`PayloadId`] for the given parent and attributes + /// Derives the unique [`PayloadId`] for the given parent, attributes and version. fn try_new( parent: B256, rpc_payload_attributes: Self::RpcPayloadAttributes, diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index eb280408ecdb..cca9f5d6b64c 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -616,7 +616,8 @@ where // To do this, we set the payload attrs to `None` if attribute validation failed, but // we still apply the forkchoice update. if let Err(err) = attr_validation_res { - let fcu_res = self.inner.beacon_consensus.fork_choice_updated(state, None).await?; + let fcu_res = + self.inner.beacon_consensus.fork_choice_updated(state, None, version).await?; // TODO: decide if we want this branch - the FCU INVALID response might be more // useful than the payload attributes INVALID response if fcu_res.is_invalid() { @@ -626,7 +627,7 @@ where } } - Ok(self.inner.beacon_consensus.fork_choice_updated(state, payload_attrs).await?) + Ok(self.inner.beacon_consensus.fork_choice_updated(state, payload_attrs, version).await?) } } From b5c0a46363ad4d32c5f6fbb704c4da3e98e0495b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Mon, 28 Oct 2024 19:03:20 +0700 Subject: [PATCH 135/239] feat: add `pending|queued` txs pool helpers (#12128) --- crates/transaction-pool/src/lib.rs | 14 +++++++++++++ crates/transaction-pool/src/noop.rs | 14 +++++++++++++ crates/transaction-pool/src/pool/mod.rs | 18 +++++++++++++++++ crates/transaction-pool/src/pool/txpool.rs | 23 ++++++++++++++++++++++ crates/transaction-pool/src/traits.rs | 14 ++++++++++++- 5 files changed, 82 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 2cffcd33fa8b..609ab987f502 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -503,6 +503,20 @@ where self.pool.get_transactions_by_sender(sender) } + fn get_pending_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>> { + self.pool.get_pending_transactions_by_sender(sender) + } + + fn get_queued_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>> { + self.pool.get_queued_transactions_by_sender(sender) + } + fn get_highest_transaction_by_sender( &self, sender: Address, diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 11c5e7eea29b..817ea7bad7a0 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -220,6 +220,20 @@ impl TransactionPool for NoopTransactionPool { vec![] } + fn get_pending_transactions_by_sender( + &self, + _sender: Address, + ) -> Vec>> { + vec![] + } + + fn get_queued_transactions_by_sender( + &self, + _sender: Address, + ) -> Vec>> { + vec![] + } + fn get_highest_transaction_by_sender( &self, _sender: Address, diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 600a8da934ec..a408c7684104 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -776,6 +776,24 @@ where self.get_pool_data().get_transactions_by_sender(sender_id) } + /// Returns all queued transactions of the address by sender + pub(crate) fn get_queued_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>> { + let sender_id = self.get_sender_id(sender); + self.get_pool_data().pending_txs_by_sender(sender_id) + } + + /// Returns all pending transactions of the address by sender + pub(crate) fn get_pending_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>> { + let sender_id = self.get_sender_id(sender); + self.get_pool_data().queued_txs_by_sender(sender_id) + } + /// Returns the highest transaction of the address pub(crate) fn get_highest_transaction_by_sender( &self, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index c6369c98a7fc..b11815fc4b54 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -364,11 +364,26 @@ impl TxPool { self.pending_pool.all() } + /// Returns all pending transactions for the specified sender + pub(crate) fn pending_txs_by_sender( + &self, + sender: SenderId, + ) -> Vec>> { + self.pending_transactions_iter().filter(|tx| tx.sender_id() == sender).collect() + } + /// Returns all transactions from parked pools pub(crate) fn queued_transactions(&self) -> Vec>> { self.basefee_pool.all().chain(self.queued_pool.all()).collect() } + /// Returns an iterator over all transactions from parked pools + pub(crate) fn queued_transactions_iter( + &self, + ) -> impl Iterator>> + '_ { + self.basefee_pool.all().chain(self.queued_pool.all()) + } + /// Returns queued and pending transactions for the specified sender pub fn queued_and_pending_txs_by_sender( &self, @@ -377,6 +392,14 @@ impl TxPool { (self.queued_pool.get_txs_by_sender(sender), self.pending_pool.get_txs_by_sender(sender)) } + /// Returns all queued transactions for the specified sender + pub(crate) fn queued_txs_by_sender( + &self, + sender: SenderId, + ) -> Vec>> { + self.queued_transactions_iter().filter(|tx| tx.sender_id() == sender).collect() + } + /// Returns `true` if the transaction with the given hash is already included in this pool. pub(crate) fn contains(&self, tx_hash: &TxHash) -> bool { self.all_transactions.contains(tx_hash) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 9db9c53d3873..ff6e8855efa6 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -352,6 +352,18 @@ pub trait TransactionPool: Send + Sync + Clone { sender: Address, ) -> Vec>>; + /// Returns all pending transactions sent by a given user + fn get_pending_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>>; + + /// Returns all queued transactions sent by a given user + fn get_queued_transactions_by_sender( + &self, + sender: Address, + ) -> Vec>>; + /// Returns the highest transaction sent by a given user fn get_highest_transaction_by_sender( &self, @@ -1332,7 +1344,7 @@ impl TryFrom for EthPooledTransaction { } EIP4844_TX_TYPE_ID => { // doesn't have a blob sidecar - return Err(TryFromRecoveredTransactionError::BlobSidecarMissing) + return Err(TryFromRecoveredTransactionError::BlobSidecarMissing); } unsupported => { // unsupported transaction type From 87a615fe265137fab0831fb9047268addbbe4d2c Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:28:52 +0900 Subject: [PATCH 136/239] fix(ci): remove renaming from `compact-codec` (#12133) --- .github/workflows/compact.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index c5d39f72aeca..484b27c820d0 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -37,8 +37,7 @@ jobs: # On `main` branch, generates test vectors and serializes them to disk using `Compact`. - name: Generate compact vectors run: | - ${{ matrix.bin }} -- test-vectors compact --write && - for f in ./testdata/micro/compact/*; do mv "$f" "$(dirname "$f")/$(basename "$f" | awk -F '__' '{print $NF}')"; done + ${{ matrix.bin }} -- test-vectors compact --write - name: Checkout PR uses: actions/checkout@v4 with: From d74730af3b8fbbf67d8db25ae43916e67a4b89b6 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Mon, 28 Oct 2024 16:31:08 +0400 Subject: [PATCH 137/239] feat: add a wrapper for `BestTransactions` prioritizing given senders (#12123) --- crates/transaction-pool/src/pool/best.rs | 86 +++++++++++++++++++++++- crates/transaction-pool/src/pool/mod.rs | 2 +- crates/transaction-pool/src/traits.rs | 4 +- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 268e3e262c6c..763572e7e82e 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -2,10 +2,10 @@ use crate::{ identifier::TransactionId, pool::pending::PendingTransaction, PoolTransaction, TransactionOrdering, ValidPoolTransaction, }; -use alloy_primitives::B256 as TxHash; +use alloy_primitives::{Address, B256 as TxHash}; use core::fmt; use std::{ - collections::{BTreeMap, BTreeSet, HashSet}, + collections::{BTreeMap, BTreeSet, HashSet, VecDeque}, sync::Arc, }; @@ -259,6 +259,88 @@ impl fmt::Debug for BestTransactionFilter { } } +/// Wrapper over [`crate::traits::BestTransactions`] that prioritizes transactions of certain +/// senders capping total gas used by such transactions. +#[derive(Debug)] +pub struct BestTransactionsWithPrioritizedSenders { + /// Inner iterator + inner: I, + /// A set of senders which transactions should be prioritized + prioritized_senders: HashSet
, + /// Maximum total gas limit of prioritized transactions + max_prioritized_gas: u64, + /// Buffer with transactions that are not being prioritized. Those will be the first to be + /// included after the prioritized transactions + buffer: VecDeque, + /// Tracker of total gas limit of prioritized transactions. Once it reaches + /// `max_prioritized_gas` no more transactions will be prioritized + prioritized_gas: u64, +} + +impl BestTransactionsWithPrioritizedSenders { + /// Constructs a new [`BestTransactionsWithPrioritizedSenders`]. + pub fn new(prioritized_senders: HashSet
, max_prioritized_gas: u64, inner: I) -> Self { + Self { + inner, + prioritized_senders, + max_prioritized_gas, + buffer: Default::default(), + prioritized_gas: Default::default(), + } + } +} + +impl Iterator for BestTransactionsWithPrioritizedSenders +where + I: crate::traits::BestTransactions>>, + T: PoolTransaction, +{ + type Item = ::Item; + + fn next(&mut self) -> Option { + // If we have space, try prioritizing transactions + if self.prioritized_gas < self.max_prioritized_gas { + for item in &mut self.inner { + if self.prioritized_senders.contains(&item.transaction.sender()) && + self.prioritized_gas + item.transaction.gas_limit() <= + self.max_prioritized_gas + { + self.prioritized_gas += item.transaction.gas_limit(); + return Some(item) + } + self.buffer.push_back(item); + } + } + + if let Some(item) = self.buffer.pop_front() { + Some(item) + } else { + self.inner.next() + } + } +} + +impl crate::traits::BestTransactions for BestTransactionsWithPrioritizedSenders +where + I: crate::traits::BestTransactions>>, + T: PoolTransaction, +{ + fn mark_invalid(&mut self, tx: &Self::Item) { + self.inner.mark_invalid(tx) + } + + fn no_updates(&mut self) { + self.inner.no_updates() + } + + fn set_skip_blobs(&mut self, skip_blobs: bool) { + if skip_blobs { + self.buffer.retain(|tx| !tx.transaction.is_eip4844()) + } + self.inner.set_skip_blobs(skip_blobs) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index a408c7684104..69f17504f79f 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -106,7 +106,7 @@ use crate::{ traits::{GetPooledTransactionLimit, NewBlobSidecar, TransactionListenerKind}, validate::ValidTransaction, }; -pub use best::BestTransactionFilter; +pub use best::{BestTransactionFilter, BestTransactionsWithPrioritizedSenders}; pub use blob::{blob_tx_priority, fee_delta}; pub use events::{FullTransactionEvent, TransactionEvent}; pub use listener::{AllTransactionsEvents, TransactionEvents}; diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index ff6e8855efa6..fbbddb98f43c 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -776,7 +776,9 @@ pub trait BestTransactions: Iterator + Send { /// If called then the iterator will no longer yield blob transactions. /// /// Note: this will also exclude any transactions that depend on blob transactions. - fn skip_blobs(&mut self); + fn skip_blobs(&mut self) { + self.set_skip_blobs(true); + } /// Controls whether the iterator skips blob transactions or not. /// From 719ca3a68280947dce04de3fea6a58bf246e923a Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:56:28 +0100 Subject: [PATCH 138/239] chain-spec: use alloy `MAINNET_DEPOSIT_CONTRACT_ADDRESS` constant (#12113) --- crates/chainspec/src/constants.rs | 5 +++-- crates/chainspec/src/spec.rs | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/chainspec/src/constants.rs b/crates/chainspec/src/constants.rs index 2e22b2299a4b..3f46fb6b746a 100644 --- a/crates/chainspec/src/constants.rs +++ b/crates/chainspec/src/constants.rs @@ -1,11 +1,12 @@ use crate::spec::DepositContract; -use alloy_primitives::{address, b256}; +use alloy_eips::eip6110::MAINNET_DEPOSIT_CONTRACT_ADDRESS; +use alloy_primitives::b256; /// Gas per transaction not creating a contract. pub const MIN_TRANSACTION_GAS: u64 = 21_000u64; /// Deposit contract address: `0x00000000219ab540356cbb839cbe05303d7705fa` pub(crate) const MAINNET_DEPOSIT_CONTRACT: DepositContract = DepositContract::new( - address!("00000000219ab540356cbb839cbe05303d7705fa"), + MAINNET_DEPOSIT_CONTRACT_ADDRESS, 11052984, b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), ); diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index 02f4b5ca983e..b0958d4fbb57 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -3,7 +3,10 @@ pub use alloy_eips::eip1559::BaseFeeParams; use alloc::{boxed::Box, sync::Arc, vec::Vec}; use alloy_chains::{Chain, NamedChain}; use alloy_consensus::constants::EMPTY_WITHDRAWALS; -use alloy_eips::{eip1559::INITIAL_BASE_FEE, eip7685::EMPTY_REQUESTS_HASH}; +use alloy_eips::{ + eip1559::INITIAL_BASE_FEE, eip6110::MAINNET_DEPOSIT_CONTRACT_ADDRESS, + eip7685::EMPTY_REQUESTS_HASH, +}; use alloy_genesis::Genesis; use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256}; use derive_more::From; @@ -39,7 +42,7 @@ pub static MAINNET: LazyLock> = LazyLock::new(|| { hardforks: EthereumHardfork::mainnet().into(), // https://etherscan.io/tx/0xe75fb554e433e03763a1560646ee22dcb74e5274b34c5ad644e7c0f619a7e1d0 deposit_contract: Some(DepositContract::new( - address!("00000000219ab540356cbb839cbe05303d7705fa"), + MAINNET_DEPOSIT_CONTRACT_ADDRESS, 11052984, b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), )), From 72096221dfecd611128848f3d4dea0970542e056 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:08:57 +0100 Subject: [PATCH 139/239] refactor(chainspec): refac and improved doc for `last_block_fork_before_merge_or_timestamp` (#12114) --- crates/chainspec/src/spec.rs | 44 +++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index b0958d4fbb57..779eb8a37577 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -512,34 +512,36 @@ impl ChainSpec { } } - /// An internal helper function that returns the block number of the last block-based - /// fork that occurs before any existing TTD (merge)/timestamp based forks. + /// This internal helper function retrieves the block number of the last block-based fork + /// that occurs before: + /// - Any existing Total Terminal Difficulty (TTD) or + /// - Timestamp-based forks in the current [`ChainSpec`]. /// - /// Note: this returns None if the `ChainSpec` is not configured with a TTD/Timestamp fork. + /// The function operates by examining the configured hard forks in the chain. It iterates + /// through the fork conditions and identifies the most recent block-based fork that + /// precedes any TTD or timestamp-based conditions. + /// + /// If there are no block-based forks found before these conditions, or if the [`ChainSpec`] + /// is not configured with a TTD or timestamp fork, this function will return `None`. pub(crate) fn last_block_fork_before_merge_or_timestamp(&self) -> Option { let mut hardforks_iter = self.hardforks.forks_iter().peekable(); while let Some((_, curr_cond)) = hardforks_iter.next() { if let Some((_, next_cond)) = hardforks_iter.peek() { - // peek and find the first occurrence of ForkCondition::TTD (merge) , or in - // custom ChainSpecs, the first occurrence of - // ForkCondition::Timestamp. If curr_cond is ForkCondition::Block at - // this point, which it should be in most "normal" ChainSpecs, - // return its block_num + // Match against the `next_cond` to see if it represents: + // - A TTD (merge) + // - A timestamp-based fork match next_cond { - ForkCondition::TTD { fork_block, .. } => { - // handle Sepolia merge netsplit case - if fork_block.is_some() { - return *fork_block - } - // ensure curr_cond is indeed ForkCondition::Block and return block_num - if let ForkCondition::Block(block_num) = curr_cond { - return Some(block_num) - } - } - ForkCondition::Timestamp(_) => { - // ensure curr_cond is indeed ForkCondition::Block and return block_num + // If the next fork is TTD and specifies a specific block, return that block + // number + ForkCondition::TTD { fork_block: Some(block), .. } => return Some(*block), + + // If the next fork is TTD without a specific block or is timestamp-based, + // return the block number of the current condition if it is block-based. + ForkCondition::TTD { .. } | ForkCondition::Timestamp(_) => { + // Check if `curr_cond` is a block-based fork and return its block number if + // true. if let ForkCondition::Block(block_num) = curr_cond { - return Some(block_num) + return Some(block_num); } } ForkCondition::Block(_) | ForkCondition::Never => continue, From 1f1c68d65e5534e2b351a3af8fc56711d56ee129 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 28 Oct 2024 12:36:12 +0000 Subject: [PATCH 140/239] perf(trie): cache prefix set lookups in sparse trie (#12088) --- crates/trie/sparse/src/trie.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 91362eed527f..035eeaf73e81 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -631,7 +631,7 @@ impl RevealedSparseTrie { fn rlp_node(&mut self, path: Nibbles, prefix_set: &mut PrefixSet) -> RlpNode { // stack of paths we need rlp nodes for - let mut path_stack = Vec::from([path]); + let mut path_stack = Vec::from([(path, None)]); // stack of rlp nodes let mut rlp_node_stack = Vec::<(Nibbles, RlpNode)>::new(); // reusable branch child path @@ -639,7 +639,13 @@ impl RevealedSparseTrie { // reusable branch value stack let mut branch_value_stack_buf = SmallVec::<[RlpNode; 16]>::new_const(); - 'main: while let Some(path) = path_stack.pop() { + 'main: while let Some((path, mut is_in_prefix_set)) = path_stack.pop() { + // Check if the path is in the prefix set. + // First, check the cached value. If it's `None`, then check the prefix set, and update + // the cached value. + let mut prefix_set_contains = + |path: &Nibbles| *is_in_prefix_set.get_or_insert_with(|| prefix_set.contains(path)); + let rlp_node = match self.nodes.get_mut(&path).unwrap() { SparseNode::Empty => RlpNode::word_rlp(&EMPTY_ROOT_HASH), SparseNode::Hash(hash) => RlpNode::word_rlp(hash), @@ -647,7 +653,7 @@ impl RevealedSparseTrie { self.rlp_buf.clear(); let mut path = path.clone(); path.extend_from_slice_unchecked(key); - if let Some(hash) = hash.filter(|_| !prefix_set.contains(&path)) { + if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path)) { RlpNode::word_rlp(&hash) } else { let value = self.values.get(&path).unwrap(); @@ -659,7 +665,7 @@ impl RevealedSparseTrie { SparseNode::Extension { key, hash } => { let mut child_path = path.clone(); child_path.extend_from_slice_unchecked(key); - if let Some(hash) = hash.filter(|_| !prefix_set.contains(&path)) { + if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path)) { RlpNode::word_rlp(&hash) } else if rlp_node_stack.last().map_or(false, |e| e.0 == child_path) { let (_, child) = rlp_node_stack.pop().unwrap(); @@ -668,12 +674,13 @@ impl RevealedSparseTrie { *hash = rlp_node.as_hash(); rlp_node } else { - path_stack.extend([path, child_path]); // need to get rlp node for child first + // need to get rlp node for child first + path_stack.extend([(path, is_in_prefix_set), (child_path, None)]); continue } } SparseNode::Branch { state_mask, hash } => { - if let Some(hash) = hash.filter(|_| !prefix_set.contains(&path)) { + if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path)) { rlp_node_stack.push((path, RlpNode::word_rlp(&hash))); continue } @@ -700,8 +707,8 @@ impl RevealedSparseTrie { added_children = true; } else { debug_assert!(!added_children); - path_stack.push(path); - path_stack.extend(branch_child_buf.drain(..)); + path_stack.push((path, is_in_prefix_set)); + path_stack.extend(branch_child_buf.drain(..).map(|p| (p, None))); continue 'main } } From e446feb116049c74a9ff21701fe14141e4f8e675 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:44:11 +0000 Subject: [PATCH 141/239] chore(deps): weekly `cargo update` (#12108) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com> --- Cargo.lock | 319 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 6 +- 2 files changed, 159 insertions(+), 166 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6762da237cd6..c7a47e8f81ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.40" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4932d790c723181807738cf1ac68198ab581cd699545b155601332541ee47bd" +checksum = "dca4a1469a3e572e9ba362920ff145f5d0a00a3e71a64ddcb4a3659cf64c76a7" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -150,9 +150,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6228abfc751a29cde117b0879b805a3e0b3b641358f063272c83ca459a56886" +checksum = "5647fce5a168f9630f935bf7821c4207b1755184edaeba783cb4e11d35058484" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -229,9 +229,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d46eb5871592c216d39192499c95a99f7175cb94104f88c307e6dc960676d9f1" +checksum = "4b5671117c38b1c2306891f97ad3828d85487087f54ebe2c7591a055ea5bcea7" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a968c063fcfcb937736665c865a71fc2242b68916156f5ffa41fee7b44bb695" +checksum = "514f70ee2a953db21631cd817b13a1571474ec77ddc03d47616d5e8203489fde" dependencies = [ "alloy-consensus", "alloy-eips", @@ -398,9 +398,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26154390b1d205a4a7ac7352aa2eb4f81f391399d4e2f546fb81a2f8bb383f62" +checksum = "da0822426598f95e45dd1ea32a738dac057529a709ee645fcc516ffa4cbde08f" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -409,13 +409,13 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d0f2d905ebd295e7effec65e5f6868d153936130ae718352771de3e7d03c75c" +checksum = "2b09cae092c27b6f1bde952653a22708691802e57bfef4a2973b80bea21efd3f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -639,7 +639,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -655,7 +655,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "syn-solidity", "tiny-keccak", ] @@ -671,15 +671,15 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "syn-solidity", ] [[package]] name = "alloy-sol-type-parser" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f631f0bd9a9d79619b27c91b6b1ab2c4ef4e606a65192369a1ee05d40dcf81cc" +checksum = "45d1fbee9e698f3ba176b6e7a145f4aefe6d2b746b611e8bb246fe11a0e9f6c4" dependencies = [ "serde", "winnow", @@ -687,9 +687,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2841af22d99e2c0f82a78fe107b6481be3dd20b89bfb067290092794734343a" +checksum = "086f41bc6ebcd8cb15f38ba20e47be38dd03692149681ce8061c35d960dbf850" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -813,9 +813,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" dependencies = [ "anstyle", "anstyle-parse", @@ -828,43 +828,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.90" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" +checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" [[package]] name = "aquamarine" @@ -877,7 +877,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -1053,9 +1053,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "103db485efc3e41214fe4fda9f3dbeae2eb9082f48fd236e6095627a9422066e" +checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" dependencies = [ "brotli", "flate2", @@ -1100,7 +1100,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -1111,7 +1111,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -1149,7 +1149,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -1255,7 +1255,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -1437,7 +1437,7 @@ checksum = "240f4126219a83519bad05c9a40bfc0303921eeb571fc2d7e44c17ffac99d3f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "synstructure", ] @@ -1559,7 +1559,7 @@ checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -1570,9 +1570,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" dependencies = [ "serde", ] @@ -1771,7 +1771,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -1833,9 +1833,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "combine" @@ -2228,7 +2228,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2252,7 +2252,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2263,7 +2263,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2385,7 +2385,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2396,7 +2396,7 @@ checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2417,7 +2417,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "unicode-xid", ] @@ -2531,7 +2531,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2679,7 +2679,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2690,7 +2690,7 @@ checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -2747,7 +2747,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -3302,7 +3302,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -3828,7 +3828,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -3978,7 +3978,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -4146,7 +4146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -4394,7 +4394,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -4570,9 +4570,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "a00419de735aac21d53b0de5ce2c03bd3627277cf471300f27ebc89f7d828047" [[package]] name = "libp2p-identity" @@ -4795,9 +4795,9 @@ dependencies = [ [[package]] name = "metrics" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884adb57038347dfbaf2d5065887b6cf4312330dc8e94bc30a1a839bd79d3261" +checksum = "8ae428771d17306715c5091d446327d1cfdedc82185c65ba8423ab404e45bf10" dependencies = [ "ahash", "portable-atomic", @@ -4812,14 +4812,14 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.15.3" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" +checksum = "85b6f8152da6d7892ff1b7a1c0fa3f435e92b5918ad67035c3bb432111d9a29b" dependencies = [ "base64 0.22.1", "indexmap 2.6.0", @@ -4831,9 +4831,9 @@ dependencies = [ [[package]] name = "metrics-process" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e69e6ced169644e186e060ddc15f3923fdf06862c811a867bb1e5e7c7824f4d0" +checksum = "57ca8ecd85575fbb143b2678cb123bb818779391ec0f745b1c4a9dbabadde407" dependencies = [ "libc", "libproc", @@ -4847,15 +4847,14 @@ dependencies = [ [[package]] name = "metrics-util" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4259040465c955f9f2f1a4a8a16dc46726169bca0f88e8fb2dbeced487c3e828" +checksum = "15b482df36c13dd1869d73d14d28cd4855fbd6cfc32294bee109908a9f4a4ed7" dependencies = [ "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "metrics", - "num_cpus", "quanta", "sketches-ddsketch", ] @@ -4959,7 +4958,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -5004,7 +5003,7 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint 0.8.0", + "unsigned-varint", "url", ] @@ -5021,12 +5020,12 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +checksum = "cc41f430805af9d1cf4adae4ed2149c759b877b01d909a1f40256188d09345d2" dependencies = [ "core2", - "unsigned-varint 0.7.2", + "unsigned-varint", ] [[package]] @@ -5207,7 +5206,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -5260,9 +5259,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "op-alloy-consensus" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d49163f952491820088dd0e66f3a35d63337c3066eceff0a931bf83a8e2101" +checksum = "ba7c98055fd048073738df0cc6d6537e992a0d8828f39d99a469e870db126dbd" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5278,9 +5277,9 @@ dependencies = [ [[package]] name = "op-alloy-genesis" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e46c2ab105f679f0cbfbc3fb762f3456d4b8556c841e667fc8f3c2226eb6c1e" +checksum = "d631e8113cf88d30e621022677209caa148a9ca3ccb590fd34bbd1c731e3aff3" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5292,9 +5291,9 @@ dependencies = [ [[package]] name = "op-alloy-network" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ff1ea317441b9eb6317b24d13f9088e3b14ef48b15bfb6a125ca404df036d8" +checksum = "1eabe7683d7e19c7cc5171d664e49fc449176cf1334ffff82808e2a7eea5933a" dependencies = [ "alloy-consensus", "alloy-network", @@ -5306,9 +5305,9 @@ dependencies = [ [[package]] name = "op-alloy-protocol" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c439457b2a1791325603fc18a94cc175e0b4b1127f11ff8a45071f05d044dcb" +checksum = "9b39574acb1873315e6bd89df174f6223e897188fb87eeea2ad1eda04f7d28eb" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5323,9 +5322,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9556293835232b019ec9c6fd84e4265a3151111af60ea09b5b513e3dbed41c" +checksum = "919e9b69212d61f3c8932bfb717c7ad458ea3fc52072b3433d99994f8223d555" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5341,9 +5340,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types-engine" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a42a5ac4e07ed226b6a2aeefaad9b2cc7ec160e372ba626a4214d681a355fc2" +checksum = "0e3a47ea24cee189b4351be247fd138c68571704ee57060cf5a722502f44412c" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -5561,7 +5560,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -5575,29 +5574,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -5757,12 +5756,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904afd36257cdb6ce0bee88b7981847bd7b955e5e216bb32f466b302923ad446" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -5813,14 +5812,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] name = "proc-macro2" -version = "1.0.88" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -5911,7 +5910,7 @@ checksum = "6ff7ff745a347b87471d859a377a9a404361e7efc2a971d73424a6d183c0fc77" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -6175,9 +6174,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -6731,7 +6730,7 @@ dependencies = [ "proc-macro2", "quote", "similar-asserts", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -9303,9 +9302,9 @@ dependencies = [ [[package]] name = "revm" -version = "17.0.0" +version = "17.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eab16cb0a8cd5ac88b11230b20df588b7e8aae7dfab4b3f830e98aebeb4b365" +checksum = "055bee6a81aaeee8c2389ae31f0d4de87f44df24f4444a1116f9755fd87a76ad" dependencies = [ "auto_impl", "cfg-if", @@ -9578,9 +9577,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags 2.6.0", "errno", @@ -9724,9 +9723,9 @@ dependencies = [ [[package]] name = "scc" -version = "2.2.2" +version = "2.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c1f7fc6deb21665a9060dfc7d271be784669295a31babdcd4dd2c79ae8cbfb" +checksum = "d8d25269dd3a12467afe2e510f69fb0b46b698e5afb296b59f2145259deaf8e8" dependencies = [ "sdd", ] @@ -9868,22 +9867,22 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -9918,7 +9917,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -9969,7 +9968,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -9992,7 +9991,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -10155,9 +10154,9 @@ checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "sketches-ddsketch" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" [[package]] name = "slab" @@ -10278,7 +10277,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -10336,9 +10335,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.80" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e185e337f816bc8da115b8afcb3324006ccc82eeaddf35113888d3bd8e44ac" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -10354,7 +10353,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -10380,7 +10379,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -10457,7 +10456,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -10481,22 +10480,22 @@ checksum = "a38c90d48152c236a3ab59271da4f4ae63d678c5d7ad6b7714d7cb9760be5e4b" [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -10649,9 +10648,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -10673,7 +10672,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -10874,7 +10873,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11174,12 +11173,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unsigned-varint" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" - [[package]] name = "unsigned-varint" version = "0.8.0" @@ -11271,7 +11264,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11342,7 +11335,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "wasm-bindgen-shared", ] @@ -11376,7 +11369,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -11389,9 +11382,9 @@ checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -11532,7 +11525,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11543,7 +11536,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11554,7 +11547,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11565,7 +11558,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11840,7 +11833,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "synstructure", ] @@ -11862,7 +11855,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11882,7 +11875,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", "synstructure", ] @@ -11903,7 +11896,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] @@ -11925,7 +11918,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.80", + "syn 2.0.85", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d01ee01ce5cf..c83d76318e15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -527,11 +527,11 @@ url = "2.3" zstd = "0.13" # metrics -metrics = "0.23.0" +metrics = "0.24.0" metrics-derive = "0.1" -metrics-exporter-prometheus = { version = "0.15.0", default-features = false } +metrics-exporter-prometheus = { version = "0.16.0", default-features = false } metrics-process = "2.1.0" -metrics-util = { default-features = false, version = "0.17.0" } +metrics-util = { default-features = false, version = "0.18.0" } # proc-macros proc-macro2 = "1.0" From 380e237257e4b6005f2d4146bc73d0f4907a5769 Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Mon, 28 Oct 2024 20:48:32 +0700 Subject: [PATCH 142/239] refactor: replace receipt envelope encoded with trait (#11742) Co-authored-by: Tuan Tran Co-authored-by: Matthias Seitz Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- crates/primitives/src/receipt.rs | 82 ++++++++++++++++++++++++++++---- crates/rpc/rpc/src/debug.rs | 2 +- 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 940b491e3351..21443f482c91 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -5,7 +5,8 @@ use alloc::{vec, vec::Vec}; use alloy_consensus::constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, }; -use alloy_primitives::{Bloom, Bytes, Log, B256}; +use alloy_eips::eip2718::Encodable2718; +use alloy_primitives::{Bloom, Log, B256}; use alloy_rlp::{length_of_length, Decodable, Encodable, RlpDecodable, RlpEncodable}; use bytes::{Buf, BufMut}; use core::{cmp::Ordering, ops::Deref}; @@ -204,14 +205,20 @@ impl<'a> arbitrary::Arbitrary<'a> for Receipt { } } -impl ReceiptWithBloom { - /// Returns the enveloped encoded receipt. - /// - /// See also [`ReceiptWithBloom::encode_enveloped`] - pub fn envelope_encoded(&self) -> Bytes { - let mut buf = Vec::new(); - self.encode_enveloped(&mut buf); - buf.into() +impl Encodable2718 for ReceiptWithBloom { + fn type_flag(&self) -> Option { + match self.receipt.tx_type { + TxType::Legacy => None, + tx_type => Some(tx_type as u8), + } + } + + fn encode_2718_len(&self) -> usize { + let encoder = self.as_encoder(); + match self.receipt.tx_type { + TxType::Legacy => encoder.receipt_length(), + _ => 1 + encoder.receipt_length(), // 1 byte for the type prefix + } } /// Encodes the receipt into its "raw" format. @@ -223,10 +230,18 @@ impl ReceiptWithBloom { /// of the receipt: /// - EIP-1559, 2930 and 4844 transactions: `tx-type || rlp([status, cumulativeGasUsed, /// logsBloom, logs])` - pub fn encode_enveloped(&self, out: &mut dyn bytes::BufMut) { + fn encode_2718(&self, out: &mut dyn BufMut) { self.encode_inner(out, false) } + fn encoded_2718(&self) -> Vec { + let mut out = vec![]; + self.encode_2718(&mut out); + out + } +} + +impl ReceiptWithBloom { /// Encode receipt with or without the header data. pub fn encode_inner(&self, out: &mut dyn BufMut, with_header: bool) { self.as_encoder().encode_inner(out, with_header) @@ -501,6 +516,7 @@ impl Encodable for ReceiptWithBloomEncoder<'_> { #[cfg(test)] mod tests { use super::*; + use crate::revm_primitives::Bytes; use alloy_primitives::{address, b256, bytes, hex_literal::hex}; #[test] @@ -661,4 +677,50 @@ mod tests { let (decoded, _) = Receipt::from_compact(&data[..], data.len()); assert_eq!(decoded, receipt); } + + #[test] + fn test_encode_2718_length() { + let receipt = ReceiptWithBloom { + receipt: Receipt { + tx_type: TxType::Eip1559, + success: true, + cumulative_gas_used: 21000, + logs: vec![], + #[cfg(feature = "optimism")] + deposit_nonce: None, + #[cfg(feature = "optimism")] + deposit_receipt_version: None, + }, + bloom: Bloom::default(), + }; + + let encoded = receipt.encoded_2718(); + assert_eq!( + encoded.len(), + receipt.encode_2718_len(), + "Encoded length should match the actual encoded data length" + ); + + // Test for legacy receipt as well + let legacy_receipt = ReceiptWithBloom { + receipt: Receipt { + tx_type: TxType::Legacy, + success: true, + cumulative_gas_used: 21000, + logs: vec![], + #[cfg(feature = "optimism")] + deposit_nonce: None, + #[cfg(feature = "optimism")] + deposit_receipt_version: None, + }, + bloom: Bloom::default(), + }; + + let legacy_encoded = legacy_receipt.encoded_2718(); + assert_eq!( + legacy_encoded.len(), + legacy_receipt.encode_2718_len(), + "Encoded length for legacy receipt should match the actual encoded data length" + ); + } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index dd1cd9739ed1..6da03b046758 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -940,7 +940,7 @@ where .to_rpc_result()? .unwrap_or_default() .into_iter() - .map(|receipt| receipt.with_bloom().envelope_encoded()) + .map(|receipt| receipt.with_bloom().encoded_2718().into()) .collect()) } From af5ae5a792762d2b02abc31a99807ed185c3b617 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 28 Oct 2024 13:55:31 +0000 Subject: [PATCH 143/239] perf(trie): reduce allocations in sparse trie rlp node calculation (#12092) --- crates/trie/sparse/src/trie.rs | 86 ++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 035eeaf73e81..11ca100791ec 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -558,7 +558,7 @@ impl RevealedSparseTrie { pub fn root(&mut self) -> B256 { // take the current prefix set. let mut prefix_set = std::mem::take(&mut self.prefix_set).freeze(); - let root_rlp = self.rlp_node(Nibbles::default(), &mut prefix_set); + let root_rlp = self.rlp_node_allocate(Nibbles::default(), &mut prefix_set); if let Some(root_hash) = root_rlp.as_hash() { root_hash } else { @@ -570,10 +570,12 @@ impl RevealedSparseTrie { /// depth. Root node has a level of 0. pub fn update_rlp_node_level(&mut self, depth: usize) { let mut prefix_set = self.prefix_set.clone().freeze(); + let mut buffers = RlpNodeBuffers::default(); let targets = self.get_changed_nodes_at_depth(&mut prefix_set, depth); for target in targets { - self.rlp_node(target, &mut prefix_set); + buffers.path_stack.push((target, Some(true))); + self.rlp_node(&mut prefix_set, &mut buffers); } } @@ -629,17 +631,13 @@ impl RevealedSparseTrie { targets } - fn rlp_node(&mut self, path: Nibbles, prefix_set: &mut PrefixSet) -> RlpNode { - // stack of paths we need rlp nodes for - let mut path_stack = Vec::from([(path, None)]); - // stack of rlp nodes - let mut rlp_node_stack = Vec::<(Nibbles, RlpNode)>::new(); - // reusable branch child path - let mut branch_child_buf = SmallVec::<[Nibbles; 16]>::new_const(); - // reusable branch value stack - let mut branch_value_stack_buf = SmallVec::<[RlpNode; 16]>::new_const(); - - 'main: while let Some((path, mut is_in_prefix_set)) = path_stack.pop() { + fn rlp_node_allocate(&mut self, path: Nibbles, prefix_set: &mut PrefixSet) -> RlpNode { + let mut buffers = RlpNodeBuffers::new_with_path(path); + self.rlp_node(prefix_set, &mut buffers) + } + + fn rlp_node(&mut self, prefix_set: &mut PrefixSet, buffers: &mut RlpNodeBuffers) -> RlpNode { + 'main: while let Some((path, mut is_in_prefix_set)) = buffers.path_stack.pop() { // Check if the path is in the prefix set. // First, check the cached value. If it's `None`, then check the prefix set, and update // the cached value. @@ -667,63 +665,68 @@ impl RevealedSparseTrie { child_path.extend_from_slice_unchecked(key); if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path)) { RlpNode::word_rlp(&hash) - } else if rlp_node_stack.last().map_or(false, |e| e.0 == child_path) { - let (_, child) = rlp_node_stack.pop().unwrap(); + } else if buffers.rlp_node_stack.last().map_or(false, |e| e.0 == child_path) { + let (_, child) = buffers.rlp_node_stack.pop().unwrap(); self.rlp_buf.clear(); let rlp_node = ExtensionNodeRef::new(key, &child).rlp(&mut self.rlp_buf); *hash = rlp_node.as_hash(); rlp_node } else { // need to get rlp node for child first - path_stack.extend([(path, is_in_prefix_set), (child_path, None)]); + buffers.path_stack.extend([(path, is_in_prefix_set), (child_path, None)]); continue } } SparseNode::Branch { state_mask, hash } => { if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path)) { - rlp_node_stack.push((path, RlpNode::word_rlp(&hash))); + buffers.rlp_node_stack.push((path, RlpNode::word_rlp(&hash))); continue } - branch_child_buf.clear(); + buffers.branch_child_buf.clear(); // Walk children in a reverse order from `f` to `0`, so we pop the `0` first // from the stack. for bit in CHILD_INDEX_RANGE.rev() { if state_mask.is_bit_set(bit) { let mut child = path.clone(); child.push_unchecked(bit); - branch_child_buf.push(child); + buffers.branch_child_buf.push(child); } } - branch_value_stack_buf.resize(branch_child_buf.len(), Default::default()); + buffers + .branch_value_stack_buf + .resize(buffers.branch_child_buf.len(), Default::default()); let mut added_children = false; - for (i, child_path) in branch_child_buf.iter().enumerate() { - if rlp_node_stack.last().map_or(false, |e| &e.0 == child_path) { - let (_, child) = rlp_node_stack.pop().unwrap(); + for (i, child_path) in buffers.branch_child_buf.iter().enumerate() { + if buffers.rlp_node_stack.last().map_or(false, |e| &e.0 == child_path) { + let (_, child) = buffers.rlp_node_stack.pop().unwrap(); // Insert children in the resulting buffer in a normal order, because // initially we iterated in reverse. - branch_value_stack_buf[branch_child_buf.len() - i - 1] = child; + buffers.branch_value_stack_buf + [buffers.branch_child_buf.len() - i - 1] = child; added_children = true; } else { debug_assert!(!added_children); - path_stack.push((path, is_in_prefix_set)); - path_stack.extend(branch_child_buf.drain(..).map(|p| (p, None))); + buffers.path_stack.push((path, is_in_prefix_set)); + buffers + .path_stack + .extend(buffers.branch_child_buf.drain(..).map(|p| (p, None))); continue 'main } } self.rlp_buf.clear(); - let rlp_node = BranchNodeRef::new(&branch_value_stack_buf, *state_mask) + let rlp_node = BranchNodeRef::new(&buffers.branch_value_stack_buf, *state_mask) .rlp(&mut self.rlp_buf); *hash = rlp_node.as_hash(); rlp_node } }; - rlp_node_stack.push((path, rlp_node)); + buffers.rlp_node_stack.push((path, rlp_node)); } - rlp_node_stack.pop().unwrap().1 + buffers.rlp_node_stack.pop().unwrap().1 } } @@ -803,6 +806,31 @@ struct RemovedSparseNode { unset_branch_nibble: Option, } +/// Collection of reusable buffers for [`RevealedSparseTrie::rlp_node`]. +#[derive(Debug, Default)] +struct RlpNodeBuffers { + /// Stack of paths we need rlp nodes for and whether the path is in the prefix set. + path_stack: Vec<(Nibbles, Option)>, + /// Stack of rlp nodes + rlp_node_stack: Vec<(Nibbles, RlpNode)>, + /// Reusable branch child path + branch_child_buf: SmallVec<[Nibbles; 16]>, + /// Reusable branch value stack + branch_value_stack_buf: SmallVec<[RlpNode; 16]>, +} + +impl RlpNodeBuffers { + /// Creates a new instance of buffers with the given path on the stack. + fn new_with_path(path: Nibbles) -> Self { + Self { + path_stack: vec![(path, None)], + rlp_node_stack: Vec::new(), + branch_child_buf: SmallVec::<[Nibbles; 16]>::new_const(), + branch_value_stack_buf: SmallVec::<[RlpNode; 16]>::new_const(), + } + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; From 3f4634ccbc4cbf64720627872ead3155259ce9cc Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Mon, 28 Oct 2024 07:37:36 -0700 Subject: [PATCH 144/239] chore: add version to PayloadBuilderAttributes::try_new (#12137) --- bin/reth/src/commands/debug_cmd/build_block.rs | 5 ++++- crates/consensus/beacon/src/engine/mod.rs | 3 ++- crates/engine/tree/src/tree/mod.rs | 3 ++- crates/ethereum/engine-primitives/src/payload.rs | 6 +++++- crates/optimism/payload/src/payload.rs | 6 +++++- crates/payload/primitives/src/traits.rs | 1 + examples/custom-engine-types/src/main.rs | 6 +++++- 7 files changed, 24 insertions(+), 6 deletions(-) diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index 455d8356aff9..272f107d3c44 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -22,7 +22,9 @@ use reth_errors::RethResult; use reth_evm::execute::{BlockExecutorProvider, Executor}; use reth_execution_types::ExecutionOutcome; use reth_fs_util as fs; -use reth_node_api::{NodeTypesWithDB, NodeTypesWithEngine, PayloadBuilderAttributes}; +use reth_node_api::{ + EngineApiMessageVersion, NodeTypesWithDB, NodeTypesWithEngine, PayloadBuilderAttributes, +}; use reth_node_ethereum::{EthEvmConfig, EthExecutorProvider}; use reth_payload_builder::database::CachedReads; use reth_primitives::{ @@ -227,6 +229,7 @@ impl> Command { reth_payload_builder::EthPayloadBuilderAttributes::try_new( best_block.hash(), payload_attrs, + EngineApiMessageVersion::default() as u8, )?, ); diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 770821de749e..a00f507dbd99 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -1165,7 +1165,7 @@ where attrs: ::PayloadAttributes, head: Header, state: ForkchoiceState, - _version: EngineApiMessageVersion, + version: EngineApiMessageVersion, ) -> OnForkChoiceUpdated { // 7. Client software MUST ensure that payloadAttributes.timestamp is greater than timestamp // of a block referenced by forkchoiceState.headBlockHash. If this condition isn't held @@ -1183,6 +1183,7 @@ where match <::PayloadBuilderAttributes as PayloadBuilderAttributes>::try_new( state.head_block_hash, attrs, + version as u8 ) { Ok(attributes) => { // send the payload to the builder and return the receiver for the pending payload diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index dd2f67916afd..bc070d87345e 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2492,7 +2492,7 @@ where attrs: T::PayloadAttributes, head: &Header, state: ForkchoiceState, - _version: EngineApiMessageVersion, + version: EngineApiMessageVersion, ) -> OnForkChoiceUpdated { // 7. Client software MUST ensure that payloadAttributes.timestamp is greater than timestamp // of a block referenced by forkchoiceState.headBlockHash. If this condition isn't held @@ -2510,6 +2510,7 @@ where match ::try_new( state.head_block_hash, attrs, + version as u8, ) { Ok(attributes) => { // send the payload to the builder and return the receiver for the pending payload diff --git a/crates/ethereum/engine-primitives/src/payload.rs b/crates/ethereum/engine-primitives/src/payload.rs index 420352cf2b98..2d162ef15050 100644 --- a/crates/ethereum/engine-primitives/src/payload.rs +++ b/crates/ethereum/engine-primitives/src/payload.rs @@ -237,7 +237,11 @@ impl PayloadBuilderAttributes for EthPayloadBuilderAttributes { /// Creates a new payload builder for the given parent block and the attributes. /// /// Derives the unique [`PayloadId`] for the given parent and attributes - fn try_new(parent: B256, attributes: PayloadAttributes) -> Result { + fn try_new( + parent: B256, + attributes: PayloadAttributes, + _version: u8, + ) -> Result { Ok(Self::new(parent, attributes)) } diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index 98b0e41b0f57..f46891fdc449 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -43,7 +43,11 @@ impl PayloadBuilderAttributes for OptimismPayloadBuilderAttributes { /// Creates a new payload builder for the given parent block and the attributes. /// /// Derives the unique [`PayloadId`] for the given parent and attributes - fn try_new(parent: B256, attributes: OpPayloadAttributes) -> Result { + fn try_new( + parent: B256, + attributes: OpPayloadAttributes, + _version: u8, + ) -> Result { let id = payload_id_optimism(&parent, &attributes); let transactions = attributes diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index f6a043755363..a78dc8c13227 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -88,6 +88,7 @@ pub trait PayloadBuilderAttributes: Send + Sync + std::fmt::Debug { fn try_new( parent: B256, rpc_payload_attributes: Self::RpcPayloadAttributes, + version: u8, ) -> Result where Self: Sized; diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index 46a7d7d9af93..0fa2b1658f85 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -110,7 +110,11 @@ impl PayloadBuilderAttributes for CustomPayloadBuilderAttributes { type RpcPayloadAttributes = CustomPayloadAttributes; type Error = Infallible; - fn try_new(parent: B256, attributes: CustomPayloadAttributes) -> Result { + fn try_new( + parent: B256, + attributes: CustomPayloadAttributes, + _version: u8, + ) -> Result { Ok(Self(EthPayloadBuilderAttributes::new(parent, attributes.inner))) } From 12762775686c6429d13107b69a2708acdcece21f Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 28 Oct 2024 15:57:58 +0000 Subject: [PATCH 145/239] test(trie): use proptest to generate random values (#12140) --- crates/trie/sparse/src/trie.rs | 126 ++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 57 deletions(-) diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 11ca100791ec..9db1dff53131 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -836,13 +836,12 @@ mod tests { use std::collections::BTreeMap; use super::*; - use alloy_primitives::U256; + use alloy_primitives::{map::HashSet, U256}; use assert_matches::assert_matches; use itertools::Itertools; use prop::sample::SizeRange; use proptest::prelude::*; use rand::seq::IteratorRandom; - use reth_testing_utils::generators; use reth_trie::{BranchNode, ExtensionNode, LeafNode}; use reth_trie_common::{ proof::{ProofNodes, ProofRetainer}, @@ -1304,6 +1303,7 @@ mod tests { ); } + #[allow(clippy::type_complexity)] #[test] fn sparse_trie_fuzz() { // Having only the first 3 nibbles set, we narrow down the range of keys @@ -1311,63 +1311,51 @@ mod tests { // to test the sparse trie updates. const KEY_NIBBLES_LEN: usize = 3; - fn test(updates: I) - where - I: IntoIterator, - T: IntoIterator)> + Clone, - { - let mut rng = generators::rng(); + fn test(updates: Vec<(HashMap>, HashSet)>) { + { + let mut state = BTreeMap::default(); + let mut sparse = RevealedSparseTrie::default(); - let mut state = BTreeMap::default(); - let mut sparse = RevealedSparseTrie::default(); + for (update, keys_to_delete) in updates { + // Insert state updates into the sparse trie and calculate the root + for (key, value) in update.clone() { + sparse.update_leaf(key, value).unwrap(); + } + let sparse_root = sparse.root(); + + // Insert state updates into the hash builder and calculate the root + state.extend(update); + let (hash_builder_root, hash_builder_proof_nodes) = + hash_builder_root_with_proofs( + state.clone(), + state.keys().cloned().collect::>(), + ); - for update in updates { - let mut count = 0; - // Insert state updates into the sparse trie and calculate the root - for (key, value) in update.clone() { - sparse.update_leaf(key, value).unwrap(); - count += 1; - } - let keys_to_delete_len = count / 2; - let sparse_root = sparse.root(); - - // Insert state updates into the hash builder and calculate the root - state.extend(update); - let (hash_builder_root, hash_builder_proof_nodes) = hash_builder_root_with_proofs( - state.clone(), - state.keys().cloned().collect::>(), - ); - - // Assert that the sparse trie root matches the hash builder root - assert_eq!(sparse_root, hash_builder_root); - // Assert that the sparse trie nodes match the hash builder proof nodes - assert_eq_sparse_trie_proof_nodes(&sparse, hash_builder_proof_nodes); - - // Delete some keys from both the hash builder and the sparse trie and check - // that the sparse trie root still matches the hash builder root - - let keys_to_delete = state - .keys() - .choose_multiple(&mut rng, keys_to_delete_len) - .into_iter() - .cloned() - .collect::>(); - for key in keys_to_delete { - state.remove(&key).unwrap(); - sparse.remove_leaf(&key).unwrap(); - } + // Assert that the sparse trie root matches the hash builder root + assert_eq!(sparse_root, hash_builder_root); + // Assert that the sparse trie nodes match the hash builder proof nodes + assert_eq_sparse_trie_proof_nodes(&sparse, hash_builder_proof_nodes); + + // Delete some keys from both the hash builder and the sparse trie and check + // that the sparse trie root still matches the hash builder root + for key in keys_to_delete { + state.remove(&key).unwrap(); + sparse.remove_leaf(&key).unwrap(); + } - let sparse_root = sparse.root(); + let sparse_root = sparse.root(); - let (hash_builder_root, hash_builder_proof_nodes) = hash_builder_root_with_proofs( - state.clone(), - state.keys().cloned().collect::>(), - ); + let (hash_builder_root, hash_builder_proof_nodes) = + hash_builder_root_with_proofs( + state.clone(), + state.keys().cloned().collect::>(), + ); - // Assert that the sparse trie root matches the hash builder root - assert_eq!(sparse_root, hash_builder_root); - // Assert that the sparse trie nodes match the hash builder proof nodes - assert_eq_sparse_trie_proof_nodes(&sparse, hash_builder_proof_nodes); + // Assert that the sparse trie root matches the hash builder root + assert_eq!(sparse_root, hash_builder_root); + // Assert that the sparse trie nodes match the hash builder proof nodes + assert_eq_sparse_trie_proof_nodes(&sparse, hash_builder_proof_nodes); + } } } @@ -1379,17 +1367,41 @@ mod tests { base } + fn transform_updates( + updates: Vec>>, + mut rng: impl Rng, + ) -> Vec<(HashMap>, HashSet)> { + let mut keys = HashSet::new(); + updates + .into_iter() + .map(|update| { + keys.extend(update.keys().cloned()); + + let keys_to_delete_len = update.len() / 2; + let keys_to_delete = (0..keys_to_delete_len) + .map(|_| { + let key = keys.iter().choose(&mut rng).unwrap().clone(); + keys.take(&key).unwrap() + }) + .collect(); + + (update, keys_to_delete) + }) + .collect::>() + } + proptest!(ProptestConfig::with_cases(10), |( updates in proptest::collection::vec( proptest::collection::hash_map( any_with::(SizeRange::new(KEY_NIBBLES_LEN..=KEY_NIBBLES_LEN)).prop_map(pad_nibbles), any::>(), 1..100, - ), + ).prop_map(HashMap::from_iter), 1..100, - ) + ).prop_perturb(transform_updates) )| { - test(updates) }); + test(updates) + }); } /// We have three leaves that share the same prefix: 0x00, 0x01 and 0x02. Hash builder trie has From b36b021aa27ee8fa931e1833d39806c6773ac63d Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 28 Oct 2024 23:59:26 +0800 Subject: [PATCH 146/239] chore(rpc): define trait `RpcNodeCoreExt` and replace `LoadBlock::cache` (#12141) --- crates/optimism/rpc/src/eth/block.rs | 8 +------- crates/optimism/rpc/src/eth/mod.rs | 13 +++++++++++-- crates/rpc/rpc-eth-api/src/helpers/block.rs | 10 ++-------- crates/rpc/rpc-eth-api/src/lib.rs | 2 +- crates/rpc/rpc-eth-api/src/node.rs | 8 ++++++++ crates/rpc/rpc/src/eth/core.rs | 12 ++++++++++++ crates/rpc/rpc/src/eth/helpers/block.rs | 6 +----- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/crates/optimism/rpc/src/eth/block.rs b/crates/optimism/rpc/src/eth/block.rs index ed31a7509490..85f36570f2e5 100644 --- a/crates/optimism/rpc/src/eth/block.rs +++ b/crates/optimism/rpc/src/eth/block.rs @@ -4,7 +4,6 @@ use alloy_rpc_types::BlockId; use op_alloy_network::Network; use op_alloy_rpc_types::OpTransactionReceipt; use reth_chainspec::ChainSpecProvider; -use reth_node_api::FullNodeComponents; use reth_optimism_chainspec::OpChainSpec; use reth_primitives::TransactionMeta; use reth_provider::HeaderProvider; @@ -12,7 +11,6 @@ use reth_rpc_eth_api::{ helpers::{EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, SpawnBlocking}, RpcNodeCore, RpcReceipt, }; -use reth_rpc_eth_types::EthStateCache; use crate::{OpEthApi, OpEthApiError, OpReceiptBuilder}; @@ -80,10 +78,6 @@ where impl LoadBlock for OpEthApi where Self: LoadPendingBlock + SpawnBlocking, - N: FullNodeComponents, + N: RpcNodeCore, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } } diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index 9b04e1c730a5..b69b5ef28215 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -30,7 +30,7 @@ use reth_rpc_eth_api::{ AddDevSigners, EthApiSpec, EthFees, EthSigner, EthState, LoadBlock, LoadFee, LoadState, SpawnBlocking, Trace, }, - EthApiTypes, RpcNodeCore, + EthApiTypes, RpcNodeCore, RpcNodeCoreExt, }; use reth_rpc_eth_types::{EthStateCache, FeeHistoryCache, GasPriceOracle}; use reth_tasks::{ @@ -116,7 +116,6 @@ where impl RpcNodeCore for OpEthApi where - Self: Clone, N: RpcNodeCore, { type Provider = N::Provider; @@ -141,6 +140,16 @@ where } } +impl RpcNodeCoreExt for OpEthApi +where + N: RpcNodeCore, +{ + #[inline] + fn cache(&self) -> &EthStateCache { + self.inner.cache() + } +} + impl EthApiSpec for OpEthApi where N: RpcNodeCore< diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 217e84a4754e..c777c64d420d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -6,10 +6,9 @@ use alloy_rpc_types::{Header, Index}; use futures::Future; use reth_primitives::{BlockId, Receipt, SealedBlock, SealedBlockWithSenders}; use reth_provider::{BlockIdReader, BlockReader, BlockReaderIdExt, HeaderProvider}; -use reth_rpc_eth_types::EthStateCache; use reth_rpc_types_compat::block::{from_block, uncle_block_from_header}; -use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; +use crate::{node::RpcNodeCoreExt, FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; use super::{LoadPendingBlock, LoadReceipt, SpawnBlocking}; @@ -196,12 +195,7 @@ pub trait EthBlocks: LoadBlock { /// Loads a block from database. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` blocks RPC methods. -pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { - /// Returns a handle for reading data from memory. - /// - /// Data access in default (L1) trait method implementations. - fn cache(&self) -> &EthStateCache; - +pub trait LoadBlock: LoadPendingBlock + SpawnBlocking + RpcNodeCoreExt { /// Returns the block object for the given block id. fn block_with_senders( &self, diff --git a/crates/rpc/rpc-eth-api/src/lib.rs b/crates/rpc/rpc-eth-api/src/lib.rs index bc46d526c6f0..fa9737f84f0d 100644 --- a/crates/rpc/rpc-eth-api/src/lib.rs +++ b/crates/rpc/rpc-eth-api/src/lib.rs @@ -26,7 +26,7 @@ pub use bundle::{EthBundleApiServer, EthCallBundleApiServer}; pub use core::{EthApiServer, FullEthApiServer}; pub use filter::EthFilterApiServer; pub use helpers::error::{AsEthApiError, FromEthApiError, FromEvmError, IntoEthApiError}; -pub use node::RpcNodeCore; +pub use node::{RpcNodeCore, RpcNodeCoreExt}; pub use pubsub::EthPubSubApiServer; pub use types::{EthApiTypes, FullEthApiTypes, RpcBlock, RpcReceipt, RpcTransaction}; diff --git a/crates/rpc/rpc-eth-api/src/node.rs b/crates/rpc/rpc-eth-api/src/node.rs index 950271dfcb17..463f508f7ef9 100644 --- a/crates/rpc/rpc-eth-api/src/node.rs +++ b/crates/rpc/rpc-eth-api/src/node.rs @@ -1,6 +1,7 @@ //! Helper trait for interfacing with [`FullNodeComponents`]. use reth_node_api::FullNodeComponents; +use reth_rpc_eth_types::EthStateCache; /// Helper trait to relax trait bounds on [`FullNodeComponents`]. /// @@ -56,3 +57,10 @@ where FullNodeComponents::provider(self) } } + +/// Additional components, asides the core node components, needed to run `eth_` namespace API +/// server. +pub trait RpcNodeCoreExt: RpcNodeCore { + /// Returns handle to RPC cache service. + fn cache(&self) -> &EthStateCache; +} diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index 3fca76e8b0cc..339f2200c67a 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -10,6 +10,7 @@ use reth_primitives::BlockNumberOrTag; use reth_provider::{BlockReaderIdExt, CanonStateSubscriptions, ChainSpecProvider}; use reth_rpc_eth_api::{ helpers::{EthSigner, SpawnBlocking}, + node::RpcNodeCoreExt, EthApiTypes, RpcNodeCore, }; use reth_rpc_eth_types::{ @@ -169,6 +170,17 @@ where } } +impl RpcNodeCoreExt + for EthApi +where + Self: RpcNodeCore, +{ + #[inline] + fn cache(&self) -> &EthStateCache { + self.inner.cache() + } +} + impl std::fmt::Debug for EthApi { diff --git a/crates/rpc/rpc/src/eth/helpers/block.rs b/crates/rpc/rpc/src/eth/helpers/block.rs index a869cbd54035..d5341d0b22b4 100644 --- a/crates/rpc/rpc/src/eth/helpers/block.rs +++ b/crates/rpc/rpc/src/eth/helpers/block.rs @@ -8,7 +8,7 @@ use reth_rpc_eth_api::{ helpers::{EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, SpawnBlocking}, RpcReceipt, }; -use reth_rpc_eth_types::{EthApiError, EthStateCache, ReceiptBuilder}; +use reth_rpc_eth_types::{EthApiError, ReceiptBuilder}; use crate::EthApi; @@ -68,8 +68,4 @@ where Self: LoadPendingBlock + SpawnBlocking, Provider: BlockReaderIdExt, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } } From 06d73eec8aa4068e039b2c3ec28fe0b7682fd9e2 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Tue, 29 Oct 2024 00:35:26 +0800 Subject: [PATCH 147/239] chore(rpc): inline trait methods of `RpcNodeCore` impl (#12144) --- crates/optimism/rpc/src/eth/mod.rs | 4 ++++ crates/rpc/rpc-eth-api/src/node.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index b69b5ef28215..ae463d7158bc 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -123,18 +123,22 @@ where type Network = ::Network; type Evm = ::Evm; + #[inline] fn pool(&self) -> &Self::Pool { self.inner.pool() } + #[inline] fn evm_config(&self) -> &Self::Evm { self.inner.evm_config() } + #[inline] fn network(&self) -> &Self::Network { self.inner.network() } + #[inline] fn provider(&self) -> &Self::Provider { self.inner.provider() } diff --git a/crates/rpc/rpc-eth-api/src/node.rs b/crates/rpc/rpc-eth-api/src/node.rs index 463f508f7ef9..851b26b72b99 100644 --- a/crates/rpc/rpc-eth-api/src/node.rs +++ b/crates/rpc/rpc-eth-api/src/node.rs @@ -41,18 +41,22 @@ where type Network = ::Network; type Evm = ::Evm; + #[inline] fn pool(&self) -> &Self::Pool { FullNodeComponents::pool(self) } + #[inline] fn evm_config(&self) -> &Self::Evm { FullNodeComponents::evm_config(self) } + #[inline] fn network(&self) -> &Self::Network { FullNodeComponents::network(self) } + #[inline] fn provider(&self) -> &Self::Provider { FullNodeComponents::provider(self) } From 0733da9e12d1daa8febbc6e9fc4131c2acfe6798 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Tue, 29 Oct 2024 00:37:11 +0800 Subject: [PATCH 148/239] chore(rpc): relax `FullNodeComponents` trait bound on `OpEthApi` to `RpcNodeCore` (#12142) --- crates/optimism/rpc/src/eth/call.rs | 2 +- crates/optimism/rpc/src/eth/mod.rs | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/optimism/rpc/src/eth/call.rs b/crates/optimism/rpc/src/eth/call.rs index c8f8200bc69c..9ddf7b3855b6 100644 --- a/crates/optimism/rpc/src/eth/call.rs +++ b/crates/optimism/rpc/src/eth/call.rs @@ -22,9 +22,9 @@ where impl Call for OpEthApi where - N: RpcNodeCore, Self: LoadState> + SpawnBlocking, Self::Error: From, + N: RpcNodeCore, { #[inline] fn call_gas_limit(&self) -> u64 { diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index ae463d7158bc..83e79078c250 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -17,7 +17,6 @@ use op_alloy_network::Optimism; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_evm::ConfigureEvm; use reth_network_api::NetworkInfo; -use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_node_builder::EthApiBuilderCtx; use reth_primitives::Header; use reth_provider::{ @@ -177,7 +176,7 @@ where impl SpawnBlocking for OpEthApi where Self: Send + Sync + Clone + 'static, - N: FullNodeComponents, + N: RpcNodeCore, { #[inline] fn io_task_spawner(&self) -> impl TaskSpawner { @@ -237,7 +236,7 @@ where impl EthState for OpEthApi where Self: LoadState + SpawnBlocking, - N: FullNodeComponents, + N: RpcNodeCore, { #[inline] fn max_proof_window(&self) -> u64 { @@ -248,7 +247,7 @@ where impl EthFees for OpEthApi where Self: LoadFee, - N: FullNodeComponents, + N: RpcNodeCore, { } @@ -261,10 +260,10 @@ where impl AddDevSigners for OpEthApi where - N: FullNodeComponents>, + N: RpcNodeCore, { fn with_dev_accounts(&self) { - *self.signers().write() = DevSigner::random_signers(20) + *self.inner.signers().write() = DevSigner::random_signers(20) } } From 473026f40a665cbdfb1125d50012d6403601f2af Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Tue, 29 Oct 2024 00:51:45 +0800 Subject: [PATCH 149/239] chore(rpc): remove redundant `LoadFee::cache` (#12146) --- crates/optimism/rpc/src/eth/mod.rs | 5 ----- crates/rpc/rpc-eth-api/src/helpers/fee.rs | 11 +++-------- crates/rpc/rpc/src/eth/helpers/fees.rs | 7 +------ 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index 83e79078c250..52acf4240352 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -204,11 +204,6 @@ where + StateProviderFactory, >, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } - #[inline] fn gas_oracle(&self) -> &GasPriceOracle { self.inner.gas_oracle() diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index dcde2214a5d3..18d2d6311488 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -6,8 +6,8 @@ use futures::Future; use reth_chainspec::EthChainSpec; use reth_provider::{BlockIdReader, ChainSpecProvider, HeaderProvider}; use reth_rpc_eth_types::{ - fee_history::calculate_reward_percentiles_for_block, EthApiError, EthStateCache, - FeeHistoryCache, FeeHistoryEntry, GasPriceOracle, RpcInvalidTransactionError, + fee_history::calculate_reward_percentiles_for_block, EthApiError, FeeHistoryCache, + FeeHistoryEntry, GasPriceOracle, RpcInvalidTransactionError, }; use tracing::debug; @@ -172,7 +172,7 @@ pub trait EthFees: LoadFee { // Percentiles were specified, so we need to collect reward percentile ino if let Some(percentiles) = &reward_percentiles { - let (block, receipts) = LoadFee::cache(self) + let (block, receipts) = self.cache() .get_block_and_receipts(header.hash()) .await .map_err(Self::Error::from_eth_err)? @@ -242,11 +242,6 @@ pub trait EthFees: LoadFee { /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` fees RPC methods. pub trait LoadFee: LoadBlock { - /// Returns a handle for reading data from memory. - /// - /// Data access in default (L1) trait method implementations. - fn cache(&self) -> &EthStateCache; - /// Returns a handle for reading gas price. /// /// Data access in default (L1) trait method implementations. diff --git a/crates/rpc/rpc/src/eth/helpers/fees.rs b/crates/rpc/rpc/src/eth/helpers/fees.rs index 2c5db5bacbad..e1a17ef647c5 100644 --- a/crates/rpc/rpc/src/eth/helpers/fees.rs +++ b/crates/rpc/rpc/src/eth/helpers/fees.rs @@ -3,7 +3,7 @@ use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_eth_api::helpers::{EthFees, LoadBlock, LoadFee}; -use reth_rpc_eth_types::{EthStateCache, FeeHistoryCache, GasPriceOracle}; +use reth_rpc_eth_types::{FeeHistoryCache, GasPriceOracle}; use crate::EthApi; @@ -20,11 +20,6 @@ where + ChainSpecProvider + StateProviderFactory, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } - #[inline] fn gas_oracle(&self) -> &GasPriceOracle { self.inner.gas_oracle() From 3d62bfde14c469f3fea089b90dc0559ef4e096ae Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Tue, 29 Oct 2024 02:09:58 +0800 Subject: [PATCH 150/239] chore(rpc): add super trait `RpcNodeCoreExt` to `LoadReceipt` (#12149) --- crates/optimism/rpc/src/eth/receipt.rs | 10 +++------- crates/rpc/rpc-eth-api/src/helpers/block.rs | 3 ++- crates/rpc/rpc-eth-api/src/helpers/receipt.rs | 10 ++-------- crates/rpc/rpc/src/eth/helpers/receipt.rs | 11 +++-------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index f2f09cdc7ff4..3f2a81573e2f 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -11,7 +11,7 @@ use reth_optimism_forks::OptimismHardforks; use reth_primitives::{Receipt, TransactionMeta, TransactionSigned, TxType}; use reth_provider::ChainSpecProvider; use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcReceipt}; -use reth_rpc_eth_types::{EthApiError, EthStateCache, ReceiptBuilder}; +use reth_rpc_eth_types::{EthApiError, ReceiptBuilder}; use crate::{OpEthApi, OpEthApiError}; @@ -20,18 +20,14 @@ where Self: Send + Sync, N: FullNodeComponents>, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } - async fn build_transaction_receipt( &self, tx: TransactionSigned, meta: TransactionMeta, receipt: Receipt, ) -> Result, Self::Error> { - let (block, receipts) = LoadReceipt::cache(self) + let (block, receipts) = self + .cache() .get_block_and_receipts(meta.block_hash) .await .map_err(Self::Error::from_eth_err)? diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index c777c64d420d..fa397db35e02 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -145,7 +145,8 @@ pub trait EthBlocks: LoadBlock { if let Some(block_hash) = self.provider().block_hash_for_id(block_id).map_err(Self::Error::from_eth_err)? { - return LoadReceipt::cache(self) + return self + .cache() .get_block_and_receipts(block_hash) .await .map_err(Self::Error::from_eth_err) diff --git a/crates/rpc/rpc-eth-api/src/helpers/receipt.rs b/crates/rpc/rpc-eth-api/src/helpers/receipt.rs index eae99bbe45d8..48394f1cd6bb 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/receipt.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/receipt.rs @@ -3,19 +3,13 @@ use futures::Future; use reth_primitives::{Receipt, TransactionMeta, TransactionSigned}; -use reth_rpc_eth_types::EthStateCache; -use crate::{EthApiTypes, RpcReceipt}; +use crate::{EthApiTypes, RpcNodeCoreExt, RpcReceipt}; /// Assembles transaction receipt data w.r.t to network. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` receipts RPC methods. -pub trait LoadReceipt: EthApiTypes + Send + Sync { - /// Returns a handle for reading data from memory. - /// - /// Data access in default (L1) trait method implementations. - fn cache(&self) -> &EthStateCache; - +pub trait LoadReceipt: EthApiTypes + RpcNodeCoreExt + Send + Sync { /// Helper method for `eth_getBlockReceipts` and `eth_getTransactionReceipt`. fn build_transaction_receipt( &self, diff --git a/crates/rpc/rpc/src/eth/helpers/receipt.rs b/crates/rpc/rpc/src/eth/helpers/receipt.rs index 570ec4fa3c0f..d0cb5867eac4 100644 --- a/crates/rpc/rpc/src/eth/helpers/receipt.rs +++ b/crates/rpc/rpc/src/eth/helpers/receipt.rs @@ -2,20 +2,15 @@ use alloy_serde::WithOtherFields; use reth_primitives::{Receipt, TransactionMeta, TransactionSigned}; -use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcReceipt}; -use reth_rpc_eth_types::{EthApiError, EthStateCache, ReceiptBuilder}; +use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcNodeCoreExt, RpcReceipt}; +use reth_rpc_eth_types::{EthApiError, ReceiptBuilder}; use crate::EthApi; impl LoadReceipt for EthApi where - Self: Send + Sync, + Self: RpcNodeCoreExt, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } - async fn build_transaction_receipt( &self, tx: TransactionSigned, From 28f8c47dc0642e0bbd5f2abd162e003c22349eec Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Tue, 29 Oct 2024 02:10:18 +0800 Subject: [PATCH 151/239] chore(rpc): remove redundant `LoadTransaction::cache` (#12148) --- crates/optimism/rpc/src/eth/transaction.rs | 6 +----- crates/rpc/rpc-eth-api/src/helpers/transaction.rs | 14 ++++++-------- crates/rpc/rpc/src/eth/helpers/transaction.rs | 5 ----- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 5135b13a2ded..3345ac5d4521 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -12,7 +12,7 @@ use reth_rpc_eth_api::{ helpers::{EthSigner, EthTransactions, LoadTransaction, SpawnBlocking}, FromEthApiError, FullEthApiTypes, RpcNodeCore, TransactionCompat, }; -use reth_rpc_eth_types::{utils::recover_raw_transaction, EthStateCache}; +use reth_rpc_eth_types::utils::recover_raw_transaction; use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; use crate::{OpEthApi, SequencerClient}; @@ -59,10 +59,6 @@ where Self: SpawnBlocking + FullEthApiTypes, N: RpcNodeCore, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } } impl OpEthApi diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index af647fedf2c4..3c526cbb0253 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -15,14 +15,15 @@ use reth_primitives::{ use reth_provider::{BlockNumReader, BlockReaderIdExt, ReceiptProvider, TransactionsProvider}; use reth_rpc_eth_types::{ utils::{binary_search, recover_raw_transaction}, - EthApiError, EthStateCache, SignError, TransactionSource, + EthApiError, SignError, TransactionSource, }; use reth_rpc_types_compat::transaction::{from_recovered, from_recovered_with_block_context}; use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; use std::sync::Arc; use crate::{ - FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcNodeCore, RpcReceipt, RpcTransaction, + FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcNodeCore, RpcNodeCoreExt, RpcReceipt, + RpcTransaction, }; use super::{ @@ -461,13 +462,10 @@ pub trait EthTransactions: LoadTransaction { /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` transactions RPC /// methods. pub trait LoadTransaction: - SpawnBlocking + FullEthApiTypes + RpcNodeCore + SpawnBlocking + + FullEthApiTypes + + RpcNodeCoreExt { - /// Returns a handle for reading data from memory. - /// - /// Data access in default (L1) trait method implementations. - fn cache(&self) -> &EthStateCache; - /// Returns the transaction by hash. /// /// Checks the pool and state. diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 623db35e5ade..8ac0785b2620 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -5,7 +5,6 @@ use reth_rpc_eth_api::{ helpers::{EthSigner, EthTransactions, LoadTransaction, SpawnBlocking}, FullEthApiTypes, RpcNodeCore, }; -use reth_rpc_eth_types::EthStateCache; use reth_transaction_pool::TransactionPool; use crate::EthApi; @@ -28,10 +27,6 @@ where + FullEthApiTypes + RpcNodeCore, { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } } #[cfg(test)] From 37d96436073134136f625cd8f33862b39480af0f Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Tue, 29 Oct 2024 02:10:30 +0800 Subject: [PATCH 152/239] chore(rpc): remove redundant `LoadState::cache` (#12147) --- crates/optimism/rpc/src/eth/mod.rs | 9 ++------- crates/rpc/rpc-eth-api/src/helpers/state.rs | 11 +++-------- crates/rpc/rpc/src/eth/helpers/state.rs | 10 ++-------- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index 52acf4240352..7b427467b2c5 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -215,17 +215,12 @@ where } } -impl LoadState for OpEthApi -where +impl LoadState for OpEthApi where N: RpcNodeCore< Provider: StateProviderFactory + ChainSpecProvider, Pool: TransactionPool, - >, + > { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } } impl EthState for OpEthApi diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 87e66cb74817..97c94b949323 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -13,12 +13,12 @@ use reth_provider::{ BlockIdReader, BlockNumReader, ChainSpecProvider, StateProvider, StateProviderBox, StateProviderFactory, }; -use reth_rpc_eth_types::{EthApiError, EthStateCache, PendingBlockEnv, RpcInvalidTransactionError}; +use reth_rpc_eth_types::{EthApiError, PendingBlockEnv, RpcInvalidTransactionError}; use reth_rpc_types_compat::proof::from_primitive_account_proof; use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg, SpecId}; -use crate::{EthApiTypes, FromEthApiError, RpcNodeCore}; +use crate::{EthApiTypes, FromEthApiError, RpcNodeCore, RpcNodeCoreExt}; use super::{EthApiSpec, LoadPendingBlock, SpawnBlocking}; @@ -170,17 +170,12 @@ pub trait EthState: LoadState + SpawnBlocking { /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` state RPC methods. pub trait LoadState: EthApiTypes - + RpcNodeCore< + + RpcNodeCoreExt< Provider: StateProviderFactory + ChainSpecProvider, Pool: TransactionPool, > { - /// Returns a handle for reading data from memory. - /// - /// Data access in default (L1) trait method implementations. - fn cache(&self) -> &EthStateCache; - /// Returns the state at the given block number fn state_at_hash(&self, block_hash: B256) -> Result { self.provider().history_by_block_hash(block_hash).map_err(Self::Error::from_eth_err) diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 8c958ea2ae2d..a3e909cf6f6f 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -8,7 +8,6 @@ use reth_rpc_eth_api::{ helpers::{EthState, LoadState, SpawnBlocking}, RpcNodeCore, }; -use reth_rpc_eth_types::EthStateCache; use crate::EthApi; @@ -21,17 +20,12 @@ where } } -impl LoadState for EthApi -where +impl LoadState for EthApi where Self: RpcNodeCore< Provider: StateProviderFactory + ChainSpecProvider, Pool: TransactionPool, - >, + > { - #[inline] - fn cache(&self) -> &EthStateCache { - self.inner.cache() - } } #[cfg(test)] From ddc9bda315c0815369794f51bc232dff0ac9f43e Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:27:56 -0400 Subject: [PATCH 153/239] fix(op): fix payload id calculation (#11730) Co-authored-by: Matthias Seitz --- crates/optimism/payload/src/payload.rs | 76 +++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 8 deletions(-) diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index f46891fdc449..7f95d04ad9fa 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -3,7 +3,7 @@ //! Optimism builder support use alloy_eips::{eip2718::Decodable2718, eip7685::Requests}; -use alloy_primitives::{Address, B256, U256}; +use alloy_primitives::{keccak256, Address, B256, U256}; use alloy_rlp::Encodable; use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV2, ExecutionPayloadV1, PayloadId}; /// Re-export for use in downstream arguments. @@ -46,9 +46,9 @@ impl PayloadBuilderAttributes for OptimismPayloadBuilderAttributes { fn try_new( parent: B256, attributes: OpPayloadAttributes, - _version: u8, + version: u8, ) -> Result { - let id = payload_id_optimism(&parent, &attributes); + let id = payload_id_optimism(&parent, &attributes, version); let transactions = attributes .transactions @@ -281,7 +281,11 @@ impl From for OpExecutionPayloadEnvelopeV4 { /// Generates the payload id for the configured payload from the [`OpPayloadAttributes`]. /// /// Returns an 8-byte identifier by hashing the payload components with sha256 hash. -pub(crate) fn payload_id_optimism(parent: &B256, attributes: &OpPayloadAttributes) -> PayloadId { +pub(crate) fn payload_id_optimism( + parent: &B256, + attributes: &OpPayloadAttributes, + payload_version: u8, +) -> PayloadId { use sha2::Digest; let mut hasher = sha2::Sha256::new(); hasher.update(parent.as_slice()); @@ -299,15 +303,71 @@ pub(crate) fn payload_id_optimism(parent: &B256, attributes: &OpPayloadAttribute } let no_tx_pool = attributes.no_tx_pool.unwrap_or_default(); - hasher.update([no_tx_pool as u8]); - if let Some(txs) = &attributes.transactions { - txs.iter().for_each(|tx| hasher.update(tx)); + if no_tx_pool || attributes.transactions.as_ref().is_some_and(|txs| !txs.is_empty()) { + hasher.update([no_tx_pool as u8]); + let txs_len = attributes.transactions.as_ref().map(|txs| txs.len()).unwrap_or_default(); + hasher.update(&txs_len.to_be_bytes()[..]); + if let Some(txs) = &attributes.transactions { + for tx in txs { + // we have to just hash the bytes here because otherwise we would need to decode + // the transactions here which really isn't ideal + let tx_hash = keccak256(tx); + // maybe we can try just taking the hash and not decoding + hasher.update(tx_hash) + } + } } if let Some(gas_limit) = attributes.gas_limit { hasher.update(gas_limit.to_be_bytes()); } - let out = hasher.finalize(); + if let Some(eip_1559_params) = attributes.eip_1559_params { + hasher.update(eip_1559_params.as_slice()); + } + + let mut out = hasher.finalize(); + out[0] = payload_version; PayloadId::new(out.as_slice()[..8].try_into().expect("sufficient length")) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::OpPayloadAttributes; + use alloy_primitives::{address, b256, bytes, FixedBytes}; + use alloy_rpc_types_engine::PayloadAttributes; + use reth_payload_primitives::EngineApiMessageVersion; + use std::str::FromStr; + + #[test] + fn test_payload_id_parity_op_geth() { + // INFO rollup_boost::server:received fork_choice_updated_v3 from builder and l2_client + // payload_id_builder="0x6ef26ca02318dcf9" payload_id_l2="0x03d2dae446d2a86a" + let expected = + PayloadId::new(FixedBytes::<8>::from_str("0x03d2dae446d2a86a").unwrap().into()); + let attrs = OpPayloadAttributes { + payload_attributes: PayloadAttributes { + timestamp: 1728933301, + prev_randao: b256!("9158595abbdab2c90635087619aa7042bbebe47642dfab3c9bfb934f6b082765"), + suggested_fee_recipient: address!("4200000000000000000000000000000000000011"), + withdrawals: Some([].into()), + parent_beacon_block_root: b256!("8fe0193b9bf83cb7e5a08538e494fecc23046aab9a497af3704f4afdae3250ff").into() + }, + transactions: Some([bytes!("7ef8f8a0dc19cfa777d90980e4875d0a548a881baaa3f83f14d1bc0d3038bc329350e54194deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e20000f424000000000000000000000000300000000670d6d890000000000000125000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000014bf9181db6e381d4384bbf69c48b0ee0eed23c6ca26143c6d2544f9d39997a590000000000000000000000007f83d659683caf2767fd3c720981d51f5bc365bc")].into()), + no_tx_pool: None, + gas_limit: Some(30000000), + eip_1559_params: None, + }; + + // Reth's `PayloadId` should match op-geth's `PayloadId`. This fails + assert_eq!( + expected, + payload_id_optimism( + &b256!("3533bf30edaf9505d0810bf475cbe4e5f4b9889904b9845e83efdeab4e92eb1e"), + &attrs, + EngineApiMessageVersion::V3 as u8 + ) + ); + } +} From 05ee75f32c95600f0010455ccaf66ed6187b2c59 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 28 Oct 2024 23:31:10 +0100 Subject: [PATCH 154/239] fix: restrict concurrent incoming connections (#12150) --- crates/net/network-types/src/peers/state.rs | 6 ++ crates/net/network/src/peers.rs | 62 +++++++++++++++++++-- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/crates/net/network-types/src/peers/state.rs b/crates/net/network-types/src/peers/state.rs index f6ab1a39f853..1e2466c805a1 100644 --- a/crates/net/network-types/src/peers/state.rs +++ b/crates/net/network-types/src/peers/state.rs @@ -31,6 +31,12 @@ impl PeerConnectionState { } } + /// Returns true if this is the idle state. + #[inline] + pub const fn is_idle(&self) -> bool { + matches!(self, Self::Idle) + } + /// Returns true if this is an active incoming connection. #[inline] pub const fn is_incoming(&self) -> bool { diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index 3d5ff7a0d43a..016e7fa8d92e 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -218,6 +218,11 @@ impl PeersManager { self.backed_off_peers.len() } + /// Returns the number of idle trusted peers. + fn num_idle_trusted_peers(&self) -> usize { + self.peers.iter().filter(|(_, peer)| peer.kind.is_trusted() && peer.state.is_idle()).count() + } + /// Invoked when a new _incoming_ tcp connection is accepted. /// /// returns an error if the inbound ip address is on the ban list @@ -229,9 +234,34 @@ impl PeersManager { return Err(InboundConnectionError::IpBanned) } - if !self.connection_info.has_in_capacity() && self.trusted_peer_ids.is_empty() { - // if we don't have any inbound slots and no trusted peers, we don't accept any new - // connections + // check if we even have slots for a new incoming connection + if !self.connection_info.has_in_capacity() { + if self.trusted_peer_ids.is_empty() { + // if we don't have any incoming slots and no trusted peers, we don't accept any new + // connections + return Err(InboundConnectionError::ExceedsCapacity) + } + + // there's an edge case here where no incoming connections besides from trusted peers + // are allowed (max_inbound == 0), in which case we still need to allow new pending + // incoming connections until all trusted peers are connected. + let num_idle_trusted_peers = self.num_idle_trusted_peers(); + if num_idle_trusted_peers <= self.trusted_peer_ids.len() { + // we still want to limit concurrent pending connections + let max_inbound = + self.trusted_peer_ids.len().max(self.connection_info.config.max_inbound); + if self.connection_info.num_pending_in <= max_inbound { + self.connection_info.inc_pending_in(); + } + return Ok(()) + } + + // all trusted peers are either connected or connecting + return Err(InboundConnectionError::ExceedsCapacity) + } + + // also cap the incoming connections we can process at once + if !self.connection_info.has_in_pending_capacity() { return Err(InboundConnectionError::ExceedsCapacity) } @@ -968,17 +998,22 @@ impl ConnectionInfo { Self { config, num_outbound: 0, num_pending_out: 0, num_inbound: 0, num_pending_in: 0 } } - /// Returns `true` if there's still capacity for a new outgoing connection. + /// Returns `true` if there's still capacity to perform an outgoing connection. const fn has_out_capacity(&self) -> bool { self.num_pending_out < self.config.max_concurrent_outbound_dials && self.num_outbound < self.config.max_outbound } - /// Returns `true` if there's still capacity for a new incoming connection. + /// Returns `true` if there's still capacity to accept a new incoming connection. const fn has_in_capacity(&self) -> bool { self.num_inbound < self.config.max_inbound } + /// Returns `true` if we can handle an additional incoming pending connection. + const fn has_in_pending_capacity(&self) -> bool { + self.num_pending_in < self.config.max_inbound + } + fn decr_state(&mut self, state: PeerConnectionState) { match state { PeerConnectionState::Idle => {} @@ -1597,6 +1632,23 @@ mod tests { assert_eq!(peers.connection_info.num_pending_in, 0); } + #[tokio::test] + async fn test_reject_incoming_at_pending_capacity() { + let mut peers = PeersManager::default(); + + for count in 1..=peers.connection_info.config.max_inbound { + let socket_addr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, count as u8)), 8008); + assert!(peers.on_incoming_pending_session(socket_addr.ip()).is_ok()); + assert_eq!(peers.connection_info.num_pending_in, count); + } + assert!(peers.connection_info.has_in_capacity()); + assert!(!peers.connection_info.has_in_pending_capacity()); + + let socket_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 100)), 8008); + assert!(peers.on_incoming_pending_session(socket_addr.ip()).is_err()); + } + #[tokio::test] async fn test_closed_incoming() { let socket_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 1, 2)), 8008); From 367eb44177b3cba8dae4c98acf0cdc93273e745e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 29 Oct 2024 00:18:37 +0100 Subject: [PATCH 155/239] chore: remove one unwrap (#12152) --- crates/net/ecies/src/algorithm.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/net/ecies/src/algorithm.rs b/crates/net/ecies/src/algorithm.rs index 83dcc657bce9..6bf9717fe52c 100644 --- a/crates/net/ecies/src/algorithm.rs +++ b/crates/net/ecies/src/algorithm.rs @@ -650,6 +650,7 @@ impl ECIES { out.extend_from_slice(tag.as_slice()); } + /// Extracts the header from slice and returns the body size. pub fn read_header(&mut self, data: &mut [u8]) -> Result { // If the data is not large enough to fit the header and mac bytes, return an error // @@ -677,7 +678,7 @@ impl ECIES { self.body_size = Some(body_size); - Ok(self.body_size.unwrap()) + Ok(body_size) } pub const fn header_len() -> usize { From 462157880c5cf27408c304d9ce8dd32e10201800 Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Tue, 29 Oct 2024 00:51:20 +0100 Subject: [PATCH 156/239] dev: track invalid transactions by sender in pool (#12138) --- crates/transaction-pool/src/pool/best.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 763572e7e82e..6ade15be7d8a 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -1,14 +1,14 @@ use crate::{ - identifier::TransactionId, pool::pending::PendingTransaction, PoolTransaction, - TransactionOrdering, ValidPoolTransaction, + identifier::{SenderId, TransactionId}, + pool::pending::PendingTransaction, + PoolTransaction, TransactionOrdering, ValidPoolTransaction, }; -use alloy_primitives::{Address, B256 as TxHash}; +use alloy_primitives::Address; use core::fmt; use std::{ collections::{BTreeMap, BTreeSet, HashSet, VecDeque}, sync::Arc, }; - use tokio::sync::broadcast::{error::TryRecvError, Receiver}; use tracing::debug; @@ -80,7 +80,7 @@ pub(crate) struct BestTransactions { /// then can be moved from the `all` set to the `independent` set. pub(crate) independent: BTreeSet>, /// There might be the case where a yielded transactions is invalid, this will track it. - pub(crate) invalid: HashSet, + pub(crate) invalid: HashSet, /// Used to receive any new pending transactions that have been added to the pool after this /// iterator was static fileted /// @@ -94,7 +94,7 @@ pub(crate) struct BestTransactions { impl BestTransactions { /// Mark the transaction and it's descendants as invalid. pub(crate) fn mark_invalid(&mut self, tx: &Arc>) { - self.invalid.insert(*tx.hash()); + self.invalid.insert(tx.sender_id()); } /// Returns the ancestor the given transaction, the transaction with `nonce - 1`. @@ -168,14 +168,14 @@ impl Iterator for BestTransactions { self.add_new_transactions(); // Remove the next independent tx with the highest priority let best = self.independent.pop_last()?; - let hash = best.transaction.hash(); + let sender_id = best.transaction.sender_id(); - // skip transactions that were marked as invalid - if self.invalid.contains(hash) { + // skip transactions for which sender was marked as invalid + if self.invalid.contains(&sender_id) { debug!( target: "txpool", "[{:?}] skipping invalid transaction", - hash + best.transaction.hash() ); continue } @@ -186,7 +186,7 @@ impl Iterator for BestTransactions { } if self.skip_blobs && best.transaction.transaction.is_eip4844() { - // blobs should be skipped, marking the as invalid will ensure that no dependent + // blobs should be skipped, marking them as invalid will ensure that no dependent // transactions are returned self.mark_invalid(&best.transaction) } else { From 0297b8f6949d2354c1d68df5360cc375dc889879 Mon Sep 17 00:00:00 2001 From: zilayo <84344709+zilayo@users.noreply.github.com> Date: Mon, 28 Oct 2024 23:59:52 +0000 Subject: [PATCH 157/239] fix: use net::discv5 for reth's discv5 tracing target namespace (reverts #12045) (#12151) --- crates/net/discv5/src/config.rs | 4 ++-- crates/net/discv5/src/lib.rs | 30 +++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/crates/net/discv5/src/config.rs b/crates/net/discv5/src/config.rs index 0684c263b8c7..203ef76134be 100644 --- a/crates/net/discv5/src/config.rs +++ b/crates/net/discv5/src/config.rs @@ -412,7 +412,7 @@ pub fn discv5_sockets_wrt_rlpx_addr( discv5_addr_ipv6.map(|ip| SocketAddrV6::new(ip, discv5_port_ipv6, 0, 0)); if let Some(discv5_addr) = discv5_addr_ipv4 { - warn!(target: "discv5", + warn!(target: "net::discv5", %discv5_addr, %rlpx_addr, "Overwriting discv5 IPv4 address with RLPx IPv4 address, limited to one advertised IP address per IP version" @@ -429,7 +429,7 @@ pub fn discv5_sockets_wrt_rlpx_addr( discv5_addr_ipv4.map(|ip| SocketAddrV4::new(ip, discv5_port_ipv4)); if let Some(discv5_addr) = discv5_addr_ipv6 { - warn!(target: "discv5", + warn!(target: "net::discv5", %discv5_addr, %rlpx_addr, "Overwriting discv5 IPv6 address with RLPx IPv6 address, limited to one advertised IP address per IP version" diff --git a/crates/net/discv5/src/lib.rs b/crates/net/discv5/src/lib.rs index a8154b7bd19a..da54d0b52663 100644 --- a/crates/net/discv5/src/lib.rs +++ b/crates/net/discv5/src/lib.rs @@ -95,14 +95,14 @@ impl Discv5 { /// CAUTION: The value **must** be rlp encoded pub fn set_eip868_in_local_enr(&self, key: Vec, rlp: Bytes) { let Ok(key_str) = std::str::from_utf8(&key) else { - error!(target: "discv5", + error!(target: "net::discv5", err="key not utf-8", "failed to update local enr" ); return }; if let Err(err) = self.discv5.enr_insert(key_str, &rlp) { - error!(target: "discv5", + error!(target: "net::discv5", %err, "failed to update local enr" ); @@ -131,7 +131,7 @@ impl Discv5 { self.discv5.ban_node(&node_id, None); self.ban_ip(ip); } - Err(err) => error!(target: "discv5", + Err(err) => error!(target: "net::discv5", %err, "failed to ban peer" ), @@ -167,7 +167,7 @@ impl Discv5 { // let (enr, bc_enr, fork_key, rlpx_ip_mode) = build_local_enr(sk, &discv5_config); - trace!(target: "discv5", + trace!(target: "net::discv5", ?enr, "local ENR" ); @@ -271,7 +271,7 @@ impl Discv5 { // to them over RLPx, to be compatible with EL discv5 implementations that don't // enforce this security measure. - trace!(target: "discv5", + trace!(target: "net::discv5", ?enr, %socket, "discovered unverifiable enr, source socket doesn't match socket advertised in ENR" @@ -296,7 +296,7 @@ impl Discv5 { let node_record = match self.try_into_reachable(enr, socket) { Ok(enr_bc) => enr_bc, Err(err) => { - trace!(target: "discv5", + trace!(target: "net::discv5", %err, ?enr, "discovered peer is unreachable" @@ -308,7 +308,7 @@ impl Discv5 { } }; if let FilterOutcome::Ignore { reason } = self.filter_discovered_peer(enr) { - trace!(target: "discv5", + trace!(target: "net::discv5", ?enr, reason, "filtered out discovered peer" @@ -324,7 +324,7 @@ impl Discv5 { .then(|| self.get_fork_id(enr).ok()) .flatten(); - trace!(target: "discv5", + trace!(target: "net::discv5", ?fork_id, ?enr, "discovered peer" @@ -491,7 +491,7 @@ pub async fn bootstrap( bootstrap_nodes: HashSet, discv5: &Arc, ) -> Result<(), Error> { - trace!(target: "discv5", + trace!(target: "net::discv5", ?bootstrap_nodes, "adding bootstrap nodes .." ); @@ -508,7 +508,7 @@ pub async fn bootstrap( let discv5 = discv5.clone(); enr_requests.push(async move { if let Err(err) = discv5.request_enr(enode.to_string()).await { - debug!(target: "discv5", + debug!(target: "net::discv5", ?enode, %err, "failed adding boot node" @@ -545,7 +545,7 @@ pub fn spawn_populate_kbuckets_bg( for i in (0..bootstrap_lookup_countdown).rev() { let target = discv5::enr::NodeId::random(); - trace!(target: "discv5", + trace!(target: "net::discv5", %target, bootstrap_boost_runs_countdown=i, lookup_interval=format!("{:#?}", pulse_lookup_interval), @@ -563,7 +563,7 @@ pub fn spawn_populate_kbuckets_bg( // selection (ref kademlia) let target = get_lookup_target(kbucket_index, local_node_id); - trace!(target: "discv5", + trace!(target: "net::discv5", %target, lookup_interval=format!("{:#?}", lookup_interval), "starting periodic lookup query" @@ -628,11 +628,11 @@ pub async fn lookup( ); match discv5.find_node(target).await { - Err(err) => trace!(target: "discv5", + Err(err) => trace!(target: "net::discv5", %err, "lookup query failed" ), - Ok(peers) => trace!(target: "discv5", + Ok(peers) => trace!(target: "net::discv5", target=format!("{:#?}", target), peers_count=peers.len(), peers=format!("[{:#}]", peers.iter() @@ -645,7 +645,7 @@ pub async fn lookup( // `Discv5::connected_peers` can be subset of sessions, not all peers make it // into kbuckets, e.g. incoming sessions from peers with // unreachable enrs - debug!(target: "discv5", + debug!(target: "net::discv5", connected_peers=discv5.connected_peers(), "connected peers in routing table" ); From fbe04625b9c488e76d41b82cc28bbce268ebc4c1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 29 Oct 2024 01:04:24 +0100 Subject: [PATCH 158/239] test: use port0 in tests (#12154) --- crates/net/network/tests/it/startup.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/net/network/tests/it/startup.rs b/crates/net/network/tests/it/startup.rs index 89889a869460..d84ff492e5e7 100644 --- a/crates/net/network/tests/it/startup.rs +++ b/crates/net/network/tests/it/startup.rs @@ -113,6 +113,7 @@ async fn test_node_record_address_with_nat() { .add_nat(Some(NatResolver::ExternalIp("10.1.1.1".parse().unwrap()))) .disable_discv4_discovery() .disable_dns_discovery() + .listener_port(0) .build_with_noop_provider(MAINNET.clone()); let network = NetworkManager::new(config).await.unwrap(); @@ -127,6 +128,7 @@ async fn test_node_record_address_with_nat_disable_discovery() { let config = NetworkConfigBuilder::new(secret_key) .add_nat(Some(NatResolver::ExternalIp("10.1.1.1".parse().unwrap()))) .disable_discovery() + .listener_port(0) .build_with_noop_provider(MAINNET.clone()); let network = NetworkManager::new(config).await.unwrap(); From cc2a33cfc0c5f06b0d44fef72ffa987e1c97f025 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 29 Oct 2024 03:09:11 +0100 Subject: [PATCH 159/239] feat: rate limit incoming ips (#12153) --- crates/net/network-types/src/peers/config.rs | 9 +++ crates/net/network/src/peers.rs | 63 +++++++++++++++++--- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/crates/net/network-types/src/peers/config.rs b/crates/net/network-types/src/peers/config.rs index 97a8bb3cac39..890679f5d34b 100644 --- a/crates/net/network-types/src/peers/config.rs +++ b/crates/net/network-types/src/peers/config.rs @@ -24,6 +24,9 @@ pub const DEFAULT_MAX_COUNT_PEERS_INBOUND: u32 = 30; /// This restricts how many outbound dials can be performed concurrently. pub const DEFAULT_MAX_COUNT_CONCURRENT_OUTBOUND_DIALS: usize = 15; +/// A temporary timeout for ips on incoming connection attempts. +pub const INBOUND_IP_THROTTLE_DURATION: Duration = Duration::from_secs(30); + /// The durations to use when a backoff should be applied to a peer. /// /// See also [`BackoffKind`]. @@ -155,6 +158,11 @@ pub struct PeersConfig { /// /// The backoff duration increases with number of backoff attempts. pub backoff_durations: PeerBackoffDurations, + /// How long to temporarily ban ips on incoming connection attempts. + /// + /// This acts as an IP based rate limit. + #[cfg_attr(feature = "serde", serde(default, with = "humantime_serde"))] + pub incoming_ip_throttle_duration: Duration, } impl Default for PeersConfig { @@ -171,6 +179,7 @@ impl Default for PeersConfig { trusted_nodes_only: false, basic_nodes: Default::default(), max_backoff_count: 5, + incoming_ip_throttle_duration: INBOUND_IP_THROTTLE_DURATION, } } } diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index 016e7fa8d92e..4855ff5e7431 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -84,6 +84,8 @@ pub struct PeersManager { max_backoff_count: u8, /// Tracks the connection state of the node net_connection_state: NetworkConnectionState, + /// How long to temporarily ban ip on an incoming connection attempt. + incoming_ip_throttle_duration: Duration, } impl PeersManager { @@ -100,6 +102,7 @@ impl PeersManager { trusted_nodes_only, basic_nodes, max_backoff_count, + incoming_ip_throttle_duration, } = config; let (manager_tx, handle_rx) = mpsc::unbounded_channel(); let now = Instant::now(); @@ -148,6 +151,7 @@ impl PeersManager { last_tick: Instant::now(), max_backoff_count, net_connection_state: NetworkConnectionState::default(), + incoming_ip_throttle_duration, } } @@ -265,6 +269,9 @@ impl PeersManager { return Err(InboundConnectionError::ExceedsCapacity) } + // apply the rate limit + self.throttle_incoming_ip(addr); + self.connection_info.inc_pending_in(); Ok(()) } @@ -383,6 +390,12 @@ impl PeersManager { self.ban_list.ban_ip_until(ip, std::time::Instant::now() + self.ban_duration); } + /// Bans the IP temporarily to rate limit inbound connection attempts per IP. + fn throttle_incoming_ip(&mut self, ip: IpAddr) { + self.ban_list + .ban_ip_until(ip, std::time::Instant::now() + self.incoming_ip_throttle_duration); + } + /// Temporarily puts the peer in timeout by inserting it into the backedoff peers set fn backoff_peer_until(&mut self, peer_id: PeerId, until: std::time::Instant) { trace!(target: "net::peers", ?peer_id, "backing off"); @@ -1129,15 +1142,6 @@ impl Display for InboundConnectionError { #[cfg(test)] mod tests { - use std::{ - future::{poll_fn, Future}, - io, - net::{IpAddr, Ipv4Addr, SocketAddr}, - pin::Pin, - task::{Context, Poll}, - time::Duration, - }; - use alloy_primitives::B512; use reth_eth_wire::{ errors::{EthHandshakeError, EthStreamError, P2PHandshakeError, P2PStreamError}, @@ -1149,6 +1153,14 @@ mod tests { use reth_network_types::{ peers::reputation::DEFAULT_REPUTATION, BackoffKind, ReputationChangeKind, }; + use std::{ + future::{poll_fn, Future}, + io, + net::{IpAddr, Ipv4Addr, SocketAddr}, + pin::Pin, + task::{Context, Poll}, + time::Duration, + }; use url::Host; use super::PeersManager; @@ -2330,6 +2342,39 @@ mod tests { ); } + #[tokio::test] + async fn test_incoming_rate_limit() { + let config = PeersConfig { + incoming_ip_throttle_duration: Duration::from_millis(100), + ..PeersConfig::test() + }; + let mut peers = PeersManager::new(config); + + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(168, 0, 1, 2)), 8009); + assert!(peers.on_incoming_pending_session(addr.ip()).is_ok()); + assert_eq!( + peers.on_incoming_pending_session(addr.ip()).unwrap_err(), + InboundConnectionError::IpBanned + ); + + peers.release_interval.reset_immediately(); + tokio::time::sleep(peers.incoming_ip_throttle_duration).await; + + // await unban + poll_fn(|cx| loop { + if peers.poll(cx).is_pending() { + return Poll::Ready(()); + } + }) + .await; + + assert!(peers.on_incoming_pending_session(addr.ip()).is_ok()); + assert_eq!( + peers.on_incoming_pending_session(addr.ip()).unwrap_err(), + InboundConnectionError::IpBanned + ); + } + #[tokio::test] async fn test_tick() { let ip = IpAddr::V4(Ipv4Addr::new(127, 0, 1, 2)); From b48fa68f65a606b53ec19eb1dd2cbc090c3cd777 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 29 Oct 2024 01:14:34 -0400 Subject: [PATCH 160/239] fix(ecies): ecies typo (#12155) --- crates/net/network/src/session/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index 74f303df7b88..3522aa6a75b2 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -868,7 +868,7 @@ async fn authenticate( extra_handlers: RlpxSubProtocolHandlers, ) { let local_addr = stream.local_addr().ok(); - let stream = match get_eciess_stream(stream, secret_key, direction).await { + let stream = match get_ecies_stream(stream, secret_key, direction).await { Ok(stream) => stream, Err(error) => { let _ = events @@ -917,7 +917,7 @@ async fn authenticate( /// Returns an [`ECIESStream`] if it can be built. If not, send a /// [`PendingSessionEvent::EciesAuthError`] and returns `None` -async fn get_eciess_stream( +async fn get_ecies_stream( stream: Io, secret_key: SecretKey, direction: Direction, From 7880d4ddb016c9befc16c5a8f2c70601d9dd0fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:52:00 +0700 Subject: [PATCH 161/239] refactor: change `PayloadConfig` to use parent header instead of parent block (#12159) --- .../src/commands/debug_cmd/build_block.rs | 5 ++-- crates/ethereum/payload/src/lib.rs | 28 +++++++++---------- crates/optimism/payload/src/builder.rs | 18 ++++++------ crates/payload/basic/src/lib.rs | 20 +++++++------ examples/custom-engine-types/src/main.rs | 8 +++--- .../custom-payload-builder/src/generator.rs | 7 +++-- 6 files changed, 47 insertions(+), 39 deletions(-) diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index 272f107d3c44..a2dfb5ab3eaa 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -29,7 +29,8 @@ use reth_node_ethereum::{EthEvmConfig, EthExecutorProvider}; use reth_payload_builder::database::CachedReads; use reth_primitives::{ revm_primitives::KzgSettings, BlobTransaction, BlobTransactionSidecar, - PooledTransactionsElement, SealedBlock, SealedBlockWithSenders, Transaction, TransactionSigned, + PooledTransactionsElement, SealedBlock, SealedBlockWithSenders, SealedHeader, Transaction, + TransactionSigned, }; use reth_provider::{ providers::BlockchainProvider, BlockHashReader, BlockReader, BlockWriter, ChainSpecProvider, @@ -224,7 +225,7 @@ impl> Command { withdrawals: None, }; let payload_config = PayloadConfig::new( - Arc::clone(&best_block), + Arc::new(SealedHeader::new(best_block.header().clone(), best_block.hash())), Bytes::default(), reth_payload_builder::EthPayloadBuilderAttributes::try_new( best_block.hash(), diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 73b22efac401..8e188f890fd9 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -97,7 +97,7 @@ where &self, args: BuildArguments, ) -> Result, PayloadBuilderError> { - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); + let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); let pool = args.pool.clone(); default_ethereum_payload(self.evm_config.clone(), args, cfg_env, block_env, |attributes| { @@ -120,7 +120,7 @@ where None, ); - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); + let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); let pool = args.pool.clone(); @@ -154,13 +154,13 @@ where let BuildArguments { client, pool, mut cached_reads, config, cancel, best_payload } = args; let chain_spec = client.chain_spec(); - let state_provider = client.state_by_block_hash(config.parent_block.hash())?; + let state_provider = client.state_by_block_hash(config.parent_header.hash())?; let state = StateProviderDatabase::new(state_provider); let mut db = State::builder().with_database_ref(cached_reads.as_db(state)).with_bundle_update().build(); - let PayloadConfig { parent_block, extra_data, attributes } = config; + let PayloadConfig { parent_header, extra_data, attributes } = config; - debug!(target: "payload_builder", id=%attributes.id, parent_hash = ?parent_block.hash(), parent_number = parent_block.number, "building new payload"); + debug!(target: "payload_builder", id=%attributes.id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload"); let mut cumulative_gas_used = 0; let mut sum_blob_gas_used = 0; let block_gas_limit: u64 = initialized_block_env.gas_limit.to::(); @@ -189,7 +189,7 @@ where ) .map_err(|err| { warn!(target: "payload_builder", - parent_hash=%parent_block.hash(), + parent_hash=%parent_header.hash(), %err, "failed to apply beacon root contract call for payload" ); @@ -201,10 +201,10 @@ where &mut db, &initialized_cfg, &initialized_block_env, - parent_block.hash(), + parent_header.hash(), ) .map_err(|err| { - warn!(target: "payload_builder", parent_hash=%parent_block.hash(), %err, "failed to update blockhashes for payload"); + warn!(target: "payload_builder", parent_hash=%parent_header.hash(), %err, "failed to update parent header blockhashes for payload"); PayloadBuilderError::Internal(err.into()) })?; @@ -371,7 +371,7 @@ where let state_provider = db.database.0.inner.borrow_mut(); state_provider.db.state_root_with_updates(hashed_state.clone()).inspect_err(|err| { warn!(target: "payload_builder", - parent_hash=%parent_block.hash(), + parent_hash=%parent_header.hash(), %err, "failed to calculate state root for payload" ); @@ -393,9 +393,9 @@ where executed_txs.iter().filter(|tx| tx.is_eip4844()).map(|tx| tx.hash).collect(), )?; - excess_blob_gas = if chain_spec.is_cancun_active_at_timestamp(parent_block.timestamp) { - let parent_excess_blob_gas = parent_block.excess_blob_gas.unwrap_or_default(); - let parent_blob_gas_used = parent_block.blob_gas_used.unwrap_or_default(); + excess_blob_gas = if chain_spec.is_cancun_active_at_timestamp(parent_header.timestamp) { + let parent_excess_blob_gas = parent_header.excess_blob_gas.unwrap_or_default(); + let parent_blob_gas_used = parent_header.blob_gas_used.unwrap_or_default(); Some(calc_excess_blob_gas(parent_excess_blob_gas, parent_blob_gas_used)) } else { // for the first post-fork block, both parent.blob_gas_used and @@ -407,7 +407,7 @@ where } let header = Header { - parent_hash: parent_block.hash(), + parent_hash: parent_header.hash(), ommers_hash: EMPTY_OMMER_ROOT_HASH, beneficiary: initialized_block_env.coinbase, state_root, @@ -419,7 +419,7 @@ where mix_hash: attributes.prev_randao, nonce: BEACON_NONCE.into(), base_fee_per_gas: Some(base_fee), - number: parent_block.number + 1, + number: parent_header.number + 1, gas_limit: block_gas_limit, difficulty: U256::ZERO, gas_used: cumulative_gas_used, diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 36b7d17a07d8..a1536ccf8fad 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -103,7 +103,7 @@ where &self, args: BuildArguments, ) -> Result, PayloadBuilderError> { - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); + let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); optimism_payload(&self.evm_config, args, cfg_env, block_env, self.compute_pending_block) } @@ -132,7 +132,7 @@ where cancel: Default::default(), best_payload: None, }; - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_block); + let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); optimism_payload(&self.evm_config, args, cfg_env, block_env, false)? .into_payload() .ok_or_else(|| PayloadBuilderError::MissingPayload) @@ -163,13 +163,13 @@ where let BuildArguments { client, pool, mut cached_reads, config, cancel, best_payload } = args; let chain_spec = client.chain_spec(); - let state_provider = client.state_by_block_hash(config.parent_block.hash())?; + let state_provider = client.state_by_block_hash(config.parent_header.hash())?; let state = StateProviderDatabase::new(state_provider); let mut db = State::builder().with_database_ref(cached_reads.as_db(state)).with_bundle_update().build(); - let PayloadConfig { parent_block, attributes, extra_data } = config; + let PayloadConfig { parent_header, attributes, extra_data } = config; - debug!(target: "payload_builder", id=%attributes.payload_attributes.payload_id(), parent_hash = ?parent_block.hash(), parent_number = parent_block.number, "building new payload"); + debug!(target: "payload_builder", id=%attributes.payload_attributes.payload_id(), parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload"); let mut cumulative_gas_used = 0; let block_gas_limit: u64 = attributes.gas_limit.unwrap_or_else(|| { @@ -206,7 +206,7 @@ where ) .map_err(|err| { warn!(target: "payload_builder", - parent_hash=%parent_block.hash(), + parent_header=%parent_header.hash(), %err, "failed to apply beacon root contract call for payload" ); @@ -449,7 +449,7 @@ where let state_provider = db.database.0.inner.borrow_mut(); state_provider.db.state_root_with_updates(hashed_state.clone()).inspect_err(|err| { warn!(target: "payload_builder", - parent_hash=%parent_block.hash(), + parent_header=%parent_header.hash(), %err, "failed to calculate state root for payload" ); @@ -470,7 +470,7 @@ where }; let header = Header { - parent_hash: parent_block.hash(), + parent_hash: parent_header.hash(), ommers_hash: EMPTY_OMMER_ROOT_HASH, beneficiary: initialized_block_env.coinbase, state_root, @@ -482,7 +482,7 @@ where mix_hash: attributes.payload_attributes.prev_randao, nonce: BEACON_NONCE.into(), base_fee_per_gas: Some(base_fee), - number: parent_block.number + 1, + number: parent_header.number + 1, gas_limit: block_gas_limit, difficulty: U256::ZERO, gas_used: cumulative_gas_used, diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index d0bb29502eac..bb8dc0ef66aa 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -23,7 +23,7 @@ use reth_payload_primitives::{ BuiltPayload, PayloadBuilderAttributes, PayloadBuilderError, PayloadKind, }; use reth_primitives::{ - constants::RETH_CLIENT_VERSION, proofs, BlockNumberOrTag, SealedBlock, Withdrawals, + constants::RETH_CLIENT_VERSION, proofs, BlockNumberOrTag, SealedHeader, Withdrawals, }; use reth_provider::{ BlockReaderIdExt, BlockSource, CanonStateNotification, ProviderError, StateProviderFactory, @@ -122,7 +122,7 @@ impl BasicPayloadJobGenerator Option { self.pre_cached.as_ref().filter(|pc| pc.block == parent).map(|pc| pc.cached.clone()) @@ -163,13 +163,17 @@ where block.seal(attributes.parent()) }; + let hash = parent_block.hash(); + let parent_header = parent_block.header(); + let header = SealedHeader::new(parent_header.clone(), hash); + let config = - PayloadConfig::new(Arc::new(parent_block), self.config.extradata.clone(), attributes); + PayloadConfig::new(Arc::new(header), self.config.extradata.clone(), attributes); let until = self.job_deadline(config.attributes.timestamp()); let deadline = Box::pin(tokio::time::sleep_until(until)); - let cached_reads = self.maybe_pre_cached(config.parent_block.hash()); + let cached_reads = self.maybe_pre_cached(hash); let mut job = BasicPayloadJob { config, @@ -706,8 +710,8 @@ impl Drop for Cancelled { /// Static config for how to build a payload. #[derive(Clone, Debug)] pub struct PayloadConfig { - /// The parent block. - pub parent_block: Arc, + /// The parent header. + pub parent_header: Arc, /// Block extra data. pub extra_data: Bytes, /// Requested attributes for the payload. @@ -727,11 +731,11 @@ where { /// Create new payload config. pub const fn new( - parent_block: Arc, + parent_header: Arc, extra_data: Bytes, attributes: Attributes, ) -> Self { - Self { parent_block, extra_data, attributes } + Self { parent_header, extra_data, attributes } } /// Returns the payload id. diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index 0fa2b1658f85..30f89a0b9d78 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -341,7 +341,7 @@ where args: BuildArguments, ) -> Result, PayloadBuilderError> { let BuildArguments { client, pool, cached_reads, config, cancel, best_payload } = args; - let PayloadConfig { parent_block, extra_data, attributes } = config; + let PayloadConfig { parent_header, extra_data, attributes } = config; let chain_spec = client.chain_spec(); @@ -354,7 +354,7 @@ where client, pool, cached_reads, - config: PayloadConfig { parent_block, extra_data, attributes: attributes.0 }, + config: PayloadConfig { parent_header, extra_data, attributes: attributes.0 }, cancel, best_payload, }) @@ -365,10 +365,10 @@ where client: &Client, config: PayloadConfig, ) -> Result { - let PayloadConfig { parent_block, extra_data, attributes } = config; + let PayloadConfig { parent_header, extra_data, attributes } = config; let chain_spec = client.chain_spec(); >::build_empty_payload(&reth_ethereum_payload_builder::EthereumPayloadBuilder::new(EthEvmConfig::new(chain_spec.clone())),client, - PayloadConfig { parent_block, extra_data, attributes: attributes.0}) + PayloadConfig { parent_header, extra_data, attributes: attributes.0}) } } diff --git a/examples/custom-payload-builder/src/generator.rs b/examples/custom-payload-builder/src/generator.rs index f5d64e41cd09..7341428872fa 100644 --- a/examples/custom-payload-builder/src/generator.rs +++ b/examples/custom-payload-builder/src/generator.rs @@ -8,7 +8,7 @@ use reth::{ use reth_basic_payload_builder::{BasicPayloadJobGeneratorConfig, PayloadBuilder, PayloadConfig}; use reth_node_api::PayloadBuilderAttributes; use reth_payload_builder::{PayloadBuilderError, PayloadJobGenerator}; -use reth_primitives::BlockNumberOrTag; +use reth_primitives::{BlockNumberOrTag, SealedHeader}; use std::sync::Arc; /// The generator type that creates new jobs that builds empty blocks. @@ -77,7 +77,10 @@ where // we already know the hash, so we can seal it block.seal(attributes.parent()) }; - let config = PayloadConfig::new(Arc::new(parent_block), Bytes::default(), attributes); + let hash = parent_block.hash(); + let header = SealedHeader::new(parent_block.header().clone(), hash); + + let config = PayloadConfig::new(Arc::new(header), Bytes::default(), attributes); Ok(EmptyBlockPayloadJob { client: self.client.clone(), _pool: self.pool.clone(), From 2dbbd152cbc77f12e0056b8ded5bbe7141d1f6f7 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 29 Oct 2024 20:00:22 +0900 Subject: [PATCH 162/239] chore(provider): remove unused `BlockExecutionReader` trait (#12156) --- .../src/providers/database/provider.rs | 180 +----------------- .../storage/provider/src/test_utils/mock.rs | 20 +- crates/storage/provider/src/traits/block.rs | 11 -- 3 files changed, 13 insertions(+), 198 deletions(-) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 308fa364a3d3..e59a4f5635f4 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -6,15 +6,14 @@ use crate::{ AccountExtReader, BlockSource, ChangeSetReader, ReceiptProvider, StageCheckpointWriter, }, writer::UnifiedStorageWriter, - AccountReader, BlockExecutionReader, BlockExecutionWriter, BlockHashReader, BlockNumReader, - BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, - DBProvider, EvmEnvProvider, HashingWriter, HeaderProvider, HeaderSyncGap, - HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, - LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, - PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, StageCheckpointReader, - StateChangeWriter, StateProviderBox, StateReader, StateWriter, StaticFileProviderFactory, - StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, TransactionsProvider, - TransactionsProviderExt, TrieWriter, WithdrawalsProvider, + AccountReader, BlockExecutionWriter, BlockHashReader, BlockNumReader, BlockReader, BlockWriter, + BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, DBProvider, EvmEnvProvider, + HashingWriter, HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider, HistoricalStateProvider, + HistoricalStateProviderRef, HistoryWriter, LatestStateProvider, LatestStateProviderRef, + OriginalValuesKnown, ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, + StageCheckpointReader, StateChangeWriter, StateProviderBox, StateReader, StateWriter, + StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, + TransactionsProvider, TransactionsProviderExt, TrieWriter, WithdrawalsProvider, }; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; @@ -684,152 +683,6 @@ impl DatabaseProvider { }) } - /// Get requested blocks transaction with senders - pub(crate) fn get_block_transaction_range( - &self, - range: impl RangeBounds + Clone, - ) -> ProviderResult)>> { - // Raad range of block bodies to get all transactions id's of this range. - let block_bodies = self.get::(range)?; - - if block_bodies.is_empty() { - return Ok(Vec::new()) - } - - // Compute the first and last tx ID in the range - let first_transaction = block_bodies.first().expect("If we have headers").1.first_tx_num(); - let last_transaction = block_bodies.last().expect("Not empty").1.last_tx_num(); - - // If this is the case then all of the blocks in the range are empty - if last_transaction < first_transaction { - return Ok(block_bodies.into_iter().map(|(n, _)| (n, Vec::new())).collect()) - } - - // Get transactions and senders - let transactions = self - .get::(first_transaction..=last_transaction)? - .into_iter() - .map(|(id, tx)| (id, tx.into())) - .collect::>(); - - let mut senders = - self.get::(first_transaction..=last_transaction)?; - - recover_block_senders(&mut senders, &transactions, first_transaction, last_transaction)?; - - // Merge transaction into blocks - let mut block_tx = Vec::with_capacity(block_bodies.len()); - let mut senders = senders.into_iter(); - let mut transactions = transactions.into_iter(); - for (block_number, block_body) in block_bodies { - let mut one_block_tx = Vec::with_capacity(block_body.tx_count as usize); - for _ in block_body.tx_num_range() { - let tx = transactions.next(); - let sender = senders.next(); - - let recovered = match (tx, sender) { - (Some((tx_id, tx)), Some((sender_tx_id, sender))) => { - if tx_id == sender_tx_id { - Ok(TransactionSignedEcRecovered::from_signed_transaction(tx, sender)) - } else { - Err(ProviderError::MismatchOfTransactionAndSenderId { tx_id }) - } - } - (Some((tx_id, _)), _) | (_, Some((tx_id, _))) => { - Err(ProviderError::MismatchOfTransactionAndSenderId { tx_id }) - } - (None, None) => Err(ProviderError::BlockBodyTransactionCount), - }?; - one_block_tx.push(recovered) - } - block_tx.push((block_number, one_block_tx)); - } - - Ok(block_tx) - } - - /// Get the given range of blocks. - pub fn get_block_range( - &self, - range: impl RangeBounds + Clone, - ) -> ProviderResult> - where - Spec: EthereumHardforks, - { - // For blocks we need: - // - // - Headers - // - Bodies (transactions) - // - Uncles/ommers - // - Withdrawals - // - Signers - - let block_headers = self.get::(range.clone())?; - if block_headers.is_empty() { - return Ok(Vec::new()) - } - - let block_header_hashes = self.get::(range.clone())?; - let block_ommers = self.get::(range.clone())?; - let block_withdrawals = self.get::(range.clone())?; - - let block_tx = self.get_block_transaction_range(range)?; - let mut blocks = Vec::with_capacity(block_headers.len()); - - // merge all into block - let block_header_iter = block_headers.into_iter(); - let block_header_hashes_iter = block_header_hashes.into_iter(); - let block_tx_iter = block_tx.into_iter(); - - // Ommers can be empty for some blocks - let mut block_ommers_iter = block_ommers.into_iter(); - let mut block_withdrawals_iter = block_withdrawals.into_iter(); - let mut block_ommers = block_ommers_iter.next(); - let mut block_withdrawals = block_withdrawals_iter.next(); - - for ((main_block_number, header), (_, header_hash), (_, tx)) in - izip!(block_header_iter, block_header_hashes_iter, block_tx_iter) - { - let header = SealedHeader::new(header, header_hash); - - let (transactions, senders) = tx.into_iter().map(|tx| tx.to_components()).unzip(); - - // Ommers can be missing - let mut ommers = Vec::new(); - if let Some((block_number, _)) = block_ommers.as_ref() { - if *block_number == main_block_number { - ommers = block_ommers.take().unwrap().1.ommers; - block_ommers = block_ommers_iter.next(); - } - }; - - // withdrawal can be missing - let shanghai_is_active = - self.chain_spec.is_shanghai_active_at_timestamp(header.timestamp); - let mut withdrawals = Some(Withdrawals::default()); - if shanghai_is_active { - if let Some((block_number, _)) = block_withdrawals.as_ref() { - if *block_number == main_block_number { - withdrawals = Some(block_withdrawals.take().unwrap().1.withdrawals); - block_withdrawals = block_withdrawals_iter.next(); - } - } - } else { - withdrawals = None - } - - blocks.push(SealedBlockWithSenders { - block: SealedBlock { - header, - body: BlockBody { transactions, ommers, withdrawals }, - }, - senders, - }) - } - - Ok(blocks) - } - /// Return the last N blocks of state, recreating the [`ExecutionOutcome`]. /// /// 1. Iterate over the [`BlockBodyIndices`][tables::BlockBodyIndices] table to get all the @@ -3107,23 +2960,6 @@ impl HistoryWriter for DatabaseProvider BlockExecutionReader - for DatabaseProvider -{ - fn get_block_and_execution_range( - &self, - range: RangeInclusive, - ) -> ProviderResult { - // get blocks - let blocks = self.get_block_range(range.clone())?; - - // get execution res - let execution_state = self.get_state(range)?.unwrap_or_default(); - - Ok(Chain::new(blocks, execution_state, None)) - } -} - impl StateReader for DatabaseProvider { fn get_state(&self, block: BlockNumber) -> ProviderResult> { self.get_state(block..=block) diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 08530acf0a7f..ed861f5f1820 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -1,10 +1,9 @@ use crate::{ traits::{BlockSource, ReceiptProvider}, - AccountReader, BlockExecutionReader, BlockHashReader, BlockIdReader, BlockNumReader, - BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, DatabaseProvider, - EvmEnvProvider, HeaderProvider, ReceiptProviderIdExt, StateProvider, StateProviderBox, - StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, TransactionsProvider, - WithdrawalsProvider, + AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, + ChainSpecProvider, ChangeSetReader, DatabaseProvider, EvmEnvProvider, HeaderProvider, + ReceiptProviderIdExt, StateProvider, StateProviderBox, StateProviderFactory, StateReader, + StateRootProvider, TransactionVariant, TransactionsProvider, WithdrawalsProvider, }; use alloy_consensus::constants::EMPTY_ROOT_HASH; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; @@ -19,7 +18,7 @@ use reth_chainspec::{ChainInfo, ChainSpec}; use reth_db::mock::{DatabaseMock, TxMock}; use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_evm::ConfigureEvmEnv; -use reth_execution_types::{Chain, ExecutionOutcome}; +use reth_execution_types::ExecutionOutcome; use reth_primitives::{ Account, Block, BlockWithSenders, Bytecode, GotExpected, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, TransactionMeta, TransactionSigned, @@ -818,15 +817,6 @@ impl ChangeSetReader for MockEthProvider { } } -impl BlockExecutionReader for MockEthProvider { - fn get_block_and_execution_range( - &self, - _range: RangeInclusive, - ) -> ProviderResult { - Ok(Chain::default()) - } -} - impl StateReader for MockEthProvider { fn get_state(&self, _block: BlockNumber) -> ProviderResult> { Ok(None) diff --git a/crates/storage/provider/src/traits/block.rs b/crates/storage/provider/src/traits/block.rs index 8e3a54d86b9c..7202c405f068 100644 --- a/crates/storage/provider/src/traits/block.rs +++ b/crates/storage/provider/src/traits/block.rs @@ -2,7 +2,6 @@ use alloy_primitives::BlockNumber; use reth_db_api::models::StoredBlockBodyIndices; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::SealedBlockWithSenders; -use reth_storage_api::BlockReader; use reth_storage_errors::provider::ProviderResult; use reth_trie::{updates::TrieUpdates, HashedPostStateSorted}; use std::ops::RangeInclusive; @@ -23,16 +22,6 @@ pub trait BlockExecutionWriter: BlockWriter + Send + Sync { ) -> ProviderResult<()>; } -/// BlockExecution Reader -#[auto_impl::auto_impl(&, Arc, Box)] -pub trait BlockExecutionReader: BlockReader + Send + Sync { - /// Get range of blocks and its execution result - fn get_block_and_execution_range( - &self, - range: RangeInclusive, - ) -> ProviderResult; -} - /// This just receives state, or [`ExecutionOutcome`], from the provider #[auto_impl::auto_impl(&, Arc, Box)] pub trait StateReader: Send + Sync { From dd18af1f1617ed855a889fe208ea4ba772409fb5 Mon Sep 17 00:00:00 2001 From: Debjit Bhowal Date: Tue, 29 Oct 2024 16:45:20 +0530 Subject: [PATCH 163/239] feat: without-evm cli option in reth (#12134) Co-authored-by: Matthias Seitz --- Cargo.lock | 1 + book/cli/reth/init-state.md | 16 +++ crates/cli/commands/Cargo.toml | 1 + crates/cli/commands/src/init_state.rs | 79 ----------- crates/cli/commands/src/init_state/mod.rs | 132 ++++++++++++++++++ .../commands/src/init_state/without_evm.rs} | 65 +++++---- .../{init_state/mod.rs => init_state.rs} | 12 +- 7 files changed, 194 insertions(+), 112 deletions(-) delete mode 100644 crates/cli/commands/src/init_state.rs create mode 100644 crates/cli/commands/src/init_state/mod.rs rename crates/{optimism/cli/src/commands/init_state/bedrock.rs => cli/commands/src/init_state/without_evm.rs} (70%) rename crates/optimism/cli/src/commands/{init_state/mod.rs => init_state.rs} (87%) diff --git a/Cargo.lock b/Cargo.lock index c7a47e8f81ff..8febcffa52c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6616,6 +6616,7 @@ dependencies = [ "ahash", "alloy-eips", "alloy-primitives", + "alloy-rlp", "arbitrary", "backon", "clap", diff --git a/book/cli/reth/init-state.md b/book/cli/reth/init-state.md index ddcd3cece378..3e073516778d 100644 --- a/book/cli/reth/init-state.md +++ b/book/cli/reth/init-state.md @@ -72,6 +72,22 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --without-evm + Specifies whether to initialize the state without relying on EVM historical data. + + When enabled, and before inserting the state, it creates a dummy chain up to the last EVM block specified. It then, appends the first block provided block. + + - **Note**: **Do not** import receipts and blocks beforehand, or this will fail or be ignored. + + --header + Header file containing the header in an RLP encoded format. + + --total-difficulty + Total difficulty of the header. + + --header-hash + Hash of the header. + JSONL file with state dump. diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index ef66a99410f0..a0bc5147700d 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -51,6 +51,7 @@ reth-trie-common = { workspace = true, optional = true } # ethereum alloy-eips.workspace = true alloy-primitives.workspace = true +alloy-rlp.workspace = true itertools.workspace = true futures.workspace = true diff --git a/crates/cli/commands/src/init_state.rs b/crates/cli/commands/src/init_state.rs deleted file mode 100644 index 16e99f8fe976..000000000000 --- a/crates/cli/commands/src/init_state.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Command that initializes the node from a genesis file. - -use crate::common::{AccessRights, Environment, EnvironmentArgs}; -use alloy_primitives::B256; -use clap::Parser; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_cli::chainspec::ChainSpecParser; -use reth_config::config::EtlConfig; -use reth_db_common::init::init_from_state_dump; -use reth_node_builder::NodeTypesWithEngine; -use reth_provider::{providers::ProviderNodeTypes, ProviderFactory}; - -use std::{fs::File, io::BufReader, path::PathBuf}; -use tracing::info; - -/// Initializes the database with the genesis block. -#[derive(Debug, Parser)] -pub struct InitStateCommand { - #[command(flatten)] - pub env: EnvironmentArgs, - - /// JSONL file with state dump. - /// - /// Must contain accounts in following format, additional account fields are ignored. Must - /// also contain { "root": \ } as first line. - /// { - /// "balance": "\", - /// "nonce": \, - /// "code": "\", - /// "storage": { - /// "\": "\", - /// .. - /// }, - /// "address": "\", - /// } - /// - /// Allows init at a non-genesis block. Caution! Blocks must be manually imported up until - /// and including the non-genesis block to init chain at. See 'import' command. - #[arg(value_name = "STATE_DUMP_FILE", verbatim_doc_comment)] - pub state: PathBuf, -} - -impl> InitStateCommand { - /// Execute the `init` command - pub async fn execute>( - self, - ) -> eyre::Result<()> { - info!(target: "reth::cli", "Reth init-state starting"); - - let Environment { config, provider_factory, .. } = self.env.init::(AccessRights::RW)?; - - info!(target: "reth::cli", "Initiating state dump"); - - let hash = init_at_state(self.state, provider_factory, config.stages.etl)?; - - info!(target: "reth::cli", hash = ?hash, "Genesis block written"); - Ok(()) - } -} - -/// Initialize chain with state at specific block, from a file with state dump. -pub fn init_at_state( - state_dump_path: PathBuf, - factory: ProviderFactory, - etl_config: EtlConfig, -) -> eyre::Result { - info!(target: "reth::cli", - path=?state_dump_path, - "Opening state dump"); - - let file = File::open(state_dump_path)?; - let reader = BufReader::new(file); - - let provider_rw = factory.provider_rw()?; - let hash = init_from_state_dump(reader, &provider_rw.0, etl_config)?; - provider_rw.commit()?; - - Ok(hash) -} diff --git a/crates/cli/commands/src/init_state/mod.rs b/crates/cli/commands/src/init_state/mod.rs new file mode 100644 index 000000000000..adaec3e8be37 --- /dev/null +++ b/crates/cli/commands/src/init_state/mod.rs @@ -0,0 +1,132 @@ +//! Command that initializes the node from a genesis file. + +use crate::common::{AccessRights, Environment, EnvironmentArgs}; +use alloy_primitives::{B256, U256}; +use clap::Parser; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_cli::chainspec::ChainSpecParser; +use reth_db_common::init::init_from_state_dump; +use reth_node_builder::NodeTypesWithEngine; +use reth_primitives::SealedHeader; +use reth_provider::{ + BlockNumReader, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, +}; + +use std::{fs::File, io::BufReader, path::PathBuf, str::FromStr}; +use tracing::info; + +pub mod without_evm; + +/// Initializes the database with the genesis block. +#[derive(Debug, Parser)] +pub struct InitStateCommand { + #[command(flatten)] + pub env: EnvironmentArgs, + + /// JSONL file with state dump. + /// + /// Must contain accounts in following format, additional account fields are ignored. Must + /// also contain { "root": \ } as first line. + /// { + /// "balance": "\", + /// "nonce": \, + /// "code": "\", + /// "storage": { + /// "\": "\", + /// .. + /// }, + /// "address": "\", + /// } + /// + /// Allows init at a non-genesis block. Caution! Blocks must be manually imported up until + /// and including the non-genesis block to init chain at. See 'import' command. + #[arg(value_name = "STATE_DUMP_FILE", verbatim_doc_comment)] + pub state: PathBuf, + + /// Specifies whether to initialize the state without relying on EVM historical data. + /// + /// When enabled, and before inserting the state, it creates a dummy chain up to the last EVM + /// block specified. It then, appends the first block provided block. + /// + /// - **Note**: **Do not** import receipts and blocks beforehand, or this will fail or be + /// ignored. + #[arg(long, default_value = "false")] + pub without_evm: bool, + + /// Header file containing the header in an RLP encoded format. + #[arg(long, value_name = "HEADER_FILE", verbatim_doc_comment)] + pub header: Option, + + /// Total difficulty of the header. + #[arg(long, value_name = "TOTAL_DIFFICULTY", verbatim_doc_comment)] + pub total_difficulty: Option, + + /// Hash of the header. + #[arg(long, value_name = "HEADER_HASH", verbatim_doc_comment)] + pub header_hash: Option, +} + +impl> InitStateCommand { + /// Execute the `init` command + pub async fn execute>( + self, + ) -> eyre::Result<()> { + info!(target: "reth::cli", "Reth init-state starting"); + + let Environment { config, provider_factory, .. } = self.env.init::(AccessRights::RW)?; + + let static_file_provider = provider_factory.static_file_provider(); + let provider_rw = provider_factory.database_provider_rw()?; + + if self.without_evm { + // ensure header, total difficulty and header hash are provided + let header = self.header.ok_or_else(|| eyre::eyre!("Header file must be provided"))?; + let header = without_evm::read_header_from_file(header)?; + + let header_hash = + self.header_hash.ok_or_else(|| eyre::eyre!("Header hash must be provided"))?; + let header_hash = B256::from_str(&header_hash)?; + + let total_difficulty = self + .total_difficulty + .ok_or_else(|| eyre::eyre!("Total difficulty must be provided"))?; + let total_difficulty = U256::from_str(&total_difficulty)?; + + let last_block_number = provider_rw.last_block_number()?; + + if last_block_number == 0 { + without_evm::setup_without_evm( + &provider_rw, + &static_file_provider, + // &header, + // header_hash, + SealedHeader::new(header, header_hash), + total_difficulty, + )?; + + // SAFETY: it's safe to commit static files, since in the event of a crash, they + // will be unwinded according to database checkpoints. + // + // Necessary to commit, so the header is accessible to provider_rw and + // init_state_dump + static_file_provider.commit()?; + } else if last_block_number > 0 && last_block_number < header.number { + return Err(eyre::eyre!( + "Data directory should be empty when calling init-state with --without-evm-history." + )); + } + } + + info!(target: "reth::cli", "Initiating state dump"); + + let file = File::open(self.state)?; + let reader = BufReader::new(file); + + let hash = init_from_state_dump(reader, &provider_rw, config.stages.etl)?; + + provider_rw.commit()?; + + info!(target: "reth::cli", hash = ?hash, "Genesis block written"); + Ok(()) + } +} diff --git a/crates/optimism/cli/src/commands/init_state/bedrock.rs b/crates/cli/commands/src/init_state/without_evm.rs similarity index 70% rename from crates/optimism/cli/src/commands/init_state/bedrock.rs rename to crates/cli/commands/src/init_state/without_evm.rs index efff065e5051..187996653c30 100644 --- a/crates/optimism/cli/src/commands/init_state/bedrock.rs +++ b/crates/cli/commands/src/init_state/without_evm.rs @@ -1,5 +1,6 @@ use alloy_primitives::{BlockNumber, B256, U256}; -use reth_optimism_primitives::bedrock::{BEDROCK_HEADER, BEDROCK_HEADER_HASH, BEDROCK_HEADER_TTD}; +use alloy_rlp::Decodable; + use reth_primitives::{ BlockBody, Header, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment, }; @@ -7,28 +8,42 @@ use reth_provider::{ providers::StaticFileProvider, BlockWriter, StageCheckpointWriter, StaticFileWriter, }; use reth_stages::{StageCheckpoint, StageId}; + +use std::{fs::File, io::Read, path::PathBuf}; use tracing::info; -/// Creates a dummy chain (with no transactions) up to the last OVM block and appends the -/// first valid Bedrock block. -pub(crate) fn setup_op_mainnet_without_ovm( +/// Reads the header RLP from a file and returns the Header. +pub(crate) fn read_header_from_file(path: PathBuf) -> Result { + let mut file = File::open(path)?; + let mut buf = Vec::new(); + file.read_to_end(&mut buf)?; + + let header = Header::decode(&mut &buf[..])?; + Ok(header) +} + +/// Creates a dummy chain (with no transactions) up to the last EVM block and appends the +/// first valid block. +pub fn setup_without_evm( provider_rw: &Provider, static_file_provider: &StaticFileProvider, + header: SealedHeader, + total_difficulty: U256, ) -> Result<(), eyre::Error> where Provider: StageCheckpointWriter + BlockWriter, { - info!(target: "reth::cli", "Setting up dummy OVM chain before importing state."); + info!(target: "reth::cli", "Setting up dummy EVM chain before importing state."); - // Write OVM dummy data up to `BEDROCK_HEADER - 1` block - append_dummy_chain(static_file_provider, BEDROCK_HEADER.number - 1)?; + // Write EVM dummy data up to `header - 1` block + append_dummy_chain(static_file_provider, header.number - 1)?; - info!(target: "reth::cli", "Appending Bedrock block."); + info!(target: "reth::cli", "Appending first valid block."); - append_bedrock_block(provider_rw, static_file_provider)?; + append_first_block(provider_rw, static_file_provider, &header, total_difficulty)?; for stage in StageId::ALL { - provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(BEDROCK_HEADER.number))?; + provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(header.number))?; } info!(target: "reth::cli", "Set up finished."); @@ -36,38 +51,30 @@ where Ok(()) } -/// Appends the first bedrock block. +/// Appends the first block. /// /// By appending it, static file writer also verifies that all segments are at the same /// height. -fn append_bedrock_block( +fn append_first_block( provider_rw: impl BlockWriter, sf_provider: &StaticFileProvider, + header: &SealedHeader, + total_difficulty: U256, ) -> Result<(), eyre::Error> { provider_rw.insert_block( - SealedBlockWithSenders::new( - SealedBlock::new( - SealedHeader::new(BEDROCK_HEADER, BEDROCK_HEADER_HASH), - BlockBody::default(), - ), - vec![], - ) - .expect("no senders or txes"), + SealedBlockWithSenders::new(SealedBlock::new(header.clone(), BlockBody::default()), vec![]) + .expect("no senders or txes"), )?; sf_provider.latest_writer(StaticFileSegment::Headers)?.append_header( - &BEDROCK_HEADER, - BEDROCK_HEADER_TTD, - &BEDROCK_HEADER_HASH, + header, + total_difficulty, + &header.hash(), )?; - sf_provider - .latest_writer(StaticFileSegment::Receipts)? - .increment_block(BEDROCK_HEADER.number)?; + sf_provider.latest_writer(StaticFileSegment::Receipts)?.increment_block(header.number)?; - sf_provider - .latest_writer(StaticFileSegment::Transactions)? - .increment_block(BEDROCK_HEADER.number)?; + sf_provider.latest_writer(StaticFileSegment::Transactions)?.increment_block(header.number)?; Ok(()) } diff --git a/crates/optimism/cli/src/commands/init_state/mod.rs b/crates/optimism/cli/src/commands/init_state.rs similarity index 87% rename from crates/optimism/cli/src/commands/init_state/mod.rs rename to crates/optimism/cli/src/commands/init_state.rs index 3537f89e7519..68f5d9a585f6 100644 --- a/crates/optimism/cli/src/commands/init_state/mod.rs +++ b/crates/optimism/cli/src/commands/init_state.rs @@ -6,7 +6,8 @@ use reth_cli_commands::common::{AccessRights, Environment}; use reth_db_common::init::init_from_state_dump; use reth_node_builder::NodeTypesWithEngine; use reth_optimism_chainspec::OpChainSpec; -use reth_optimism_primitives::bedrock::BEDROCK_HEADER; +use reth_optimism_primitives::bedrock::{BEDROCK_HEADER, BEDROCK_HEADER_HASH, BEDROCK_HEADER_TTD}; +use reth_primitives::SealedHeader; use reth_provider::{ BlockNumReader, ChainSpecProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, @@ -14,8 +15,6 @@ use reth_provider::{ use std::{fs::File, io::BufReader}; use tracing::info; -mod bedrock; - /// Initializes the database with the genesis block. #[derive(Debug, Parser)] pub struct InitStateCommandOp { @@ -53,7 +52,12 @@ impl> InitStateCommandOp { let last_block_number = provider_rw.last_block_number()?; if last_block_number == 0 { - bedrock::setup_op_mainnet_without_ovm(&provider_rw, &static_file_provider)?; + reth_cli_commands::init_state::without_evm::setup_without_evm( + &provider_rw, + &static_file_provider, + SealedHeader::new(BEDROCK_HEADER, BEDROCK_HEADER_HASH), + BEDROCK_HEADER_TTD, + )?; // SAFETY: it's safe to commit static files, since in the event of a crash, they // will be unwinded according to database checkpoints. From 3c6077812678e1a63896162db0bab225a809771d Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:01:59 +0100 Subject: [PATCH 164/239] storage: `into_iter` with `self` by value (#12115) --- crates/storage/libmdbx-rs/src/cursor.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/crates/storage/libmdbx-rs/src/cursor.rs b/crates/storage/libmdbx-rs/src/cursor.rs index 3deff0c249bc..26cfef54d8d5 100644 --- a/crates/storage/libmdbx-rs/src/cursor.rs +++ b/crates/storage/libmdbx-rs/src/cursor.rs @@ -59,19 +59,18 @@ where } /// Returns an iterator over the raw key value slices. - #[allow(clippy::needless_lifetimes)] - pub fn iter_slices<'a>(&'a self) -> IntoIter<'a, K, Cow<'a, [u8]>, Cow<'a, [u8]>> { + pub fn iter_slices<'a>(self) -> IntoIter, Cow<'a, [u8]>> { self.into_iter() } /// Returns an iterator over database items. #[allow(clippy::should_implement_trait)] - pub fn into_iter(&self) -> IntoIter<'_, K, Key, Value> + pub fn into_iter(self) -> IntoIter where Key: TableObject, Value: TableObject, { - IntoIter::new(self.clone(), MDBX_NEXT, MDBX_NEXT) + IntoIter::new(self, MDBX_NEXT, MDBX_NEXT) } /// Retrieves a key/data pair from the cursor. Depending on the cursor op, @@ -508,7 +507,7 @@ unsafe impl Sync for Cursor where K: TransactionKind {} /// An iterator over the key/value pairs in an MDBX database. #[derive(Debug)] -pub enum IntoIter<'cur, K, Key, Value> +pub enum IntoIter where K: TransactionKind, Key: TableObject, @@ -535,11 +534,11 @@ where /// The next and subsequent operations to perform. next_op: ffi::MDBX_cursor_op, - _marker: PhantomData<(&'cur (), Key, Value)>, + _marker: PhantomData<(Key, Value)>, }, } -impl IntoIter<'_, K, Key, Value> +impl IntoIter where K: TransactionKind, Key: TableObject, @@ -547,11 +546,11 @@ where { /// Creates a new iterator backed by the given cursor. fn new(cursor: Cursor, op: ffi::MDBX_cursor_op, next_op: ffi::MDBX_cursor_op) -> Self { - IntoIter::Ok { cursor, op, next_op, _marker: Default::default() } + Self::Ok { cursor, op, next_op, _marker: Default::default() } } } -impl Iterator for IntoIter<'_, K, Key, Value> +impl Iterator for IntoIter where K: TransactionKind, Key: TableObject, @@ -747,13 +746,13 @@ where } } -impl<'cur, K, Key, Value> Iterator for IterDup<'cur, K, Key, Value> +impl Iterator for IterDup<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, Value: TableObject, { - type Item = IntoIter<'cur, K, Key, Value>; + type Item = IntoIter; fn next(&mut self) -> Option { match self { From 1653877ed50b7a3d8d31104e18fd39f1e0ba66c9 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:14:41 +0900 Subject: [PATCH 165/239] chore(ci): try to read all vectors on `compact-codec` before exiting in error (#12160) --- .../cli/commands/src/test_vectors/compact.rs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/cli/commands/src/test_vectors/compact.rs b/crates/cli/commands/src/test_vectors/compact.rs index 94552d5c2158..8def25fa39be 100644 --- a/crates/cli/commands/src/test_vectors/compact.rs +++ b/crates/cli/commands/src/test_vectors/compact.rs @@ -167,9 +167,22 @@ pub fn generate_vectors_with(gen: &[fn(&mut TestRunner) -> eyre::Result<()>]) -> /// re-encoding. pub fn read_vectors_with(read: &[fn() -> eyre::Result<()>]) -> Result<()> { fs::create_dir_all(VECTORS_FOLDER)?; + let mut errors = None; for read_fn in read { - read_fn()?; + if let Err(err) = read_fn() { + errors.get_or_insert_with(Vec::new).push(err); + } + } + + if let Some(err_list) = errors { + for error in err_list { + eprintln!("{:?}", error); + } + return Err(eyre::eyre!( + "If there are missing types, make sure to run `reth test-vectors compact --write` first.\n + If it happened during CI, ignore IF it's a new proposed type that `main` branch does not have." + )); } Ok(()) @@ -238,9 +251,8 @@ where // Read the file where the vectors are stored let file_path = format!("{VECTORS_FOLDER}/{}.json", &type_name); - let file = File::open(&file_path).wrap_err_with(|| { - "Failed to open vector. Make sure to run `reth test-vectors compact --write` first." - })?; + let file = + File::open(&file_path).wrap_err_with(|| format!("Failed to open vector {type_name}."))?; let reader = BufReader::new(file); let stored_values: Vec = serde_json::from_reader(reader)?; From 0f9ba64e954136a5507419ced37d1de96ff5b262 Mon Sep 17 00:00:00 2001 From: Evan Chipman <42247026+evchip@users.noreply.github.com> Date: Tue, 29 Oct 2024 19:17:04 +0700 Subject: [PATCH 166/239] feat: add geometry to database args (#11828) Co-authored-by: Matthias Seitz --- book/cli/reth/db.md | 6 + book/cli/reth/db/diff.md | 6 + book/cli/reth/debug/build-block.md | 6 + book/cli/reth/debug/execution.md | 6 + book/cli/reth/debug/in-memory-merkle.md | 6 + book/cli/reth/debug/merkle.md | 6 + book/cli/reth/debug/replay-engine.md | 6 + book/cli/reth/import.md | 6 + book/cli/reth/init-state.md | 6 + book/cli/reth/init.md | 6 + book/cli/reth/node.md | 6 + book/cli/reth/p2p.md | 6 + book/cli/reth/prune.md | 6 + book/cli/reth/recover/storage-tries.md | 6 + book/cli/reth/stage/drop.md | 6 + book/cli/reth/stage/dump.md | 6 + book/cli/reth/stage/run.md | 6 + book/cli/reth/stage/unwind.md | 6 + crates/node/core/src/args/database.rs | 183 +++++++++++++++++- .../storage/db/src/implementation/mdbx/mod.rs | 48 +++-- 20 files changed, 323 insertions(+), 16 deletions(-) diff --git a/book/cli/reth/db.md b/book/cli/reth/db.md index f9a8a158adc8..17a6de4e607e 100644 --- a/book/cli/reth/db.md +++ b/book/cli/reth/db.md @@ -81,6 +81,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/db/diff.md b/book/cli/reth/db/diff.md index f57c6ac364fe..efb9e7d32e30 100644 --- a/book/cli/reth/db/diff.md +++ b/book/cli/reth/db/diff.md @@ -45,6 +45,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/build-block.md b/book/cli/reth/debug/build-block.md index 2e6d637d52c2..7bceb62b940a 100644 --- a/book/cli/reth/debug/build-block.md +++ b/book/cli/reth/debug/build-block.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/execution.md b/book/cli/reth/debug/execution.md index 9ca74897c5ea..b8e1ce05d171 100644 --- a/book/cli/reth/debug/execution.md +++ b/book/cli/reth/debug/execution.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/in-memory-merkle.md b/book/cli/reth/debug/in-memory-merkle.md index 3e322a6913dd..a183db997e9d 100644 --- a/book/cli/reth/debug/in-memory-merkle.md +++ b/book/cli/reth/debug/in-memory-merkle.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/merkle.md b/book/cli/reth/debug/merkle.md index d701803b81ca..d9a72794ef2c 100644 --- a/book/cli/reth/debug/merkle.md +++ b/book/cli/reth/debug/merkle.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/replay-engine.md b/book/cli/reth/debug/replay-engine.md index dd587620a868..b7a1266d3993 100644 --- a/book/cli/reth/debug/replay-engine.md +++ b/book/cli/reth/debug/replay-engine.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/import.md b/book/cli/reth/import.md index 28e085bda718..82a521ac0ab2 100644 --- a/book/cli/reth/import.md +++ b/book/cli/reth/import.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/init-state.md b/book/cli/reth/init-state.md index 3e073516778d..533c0f8f8889 100644 --- a/book/cli/reth/init-state.md +++ b/book/cli/reth/init-state.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/init.md b/book/cli/reth/init.md index cd01accc0471..ebe2a8386cf7 100644 --- a/book/cli/reth/init.md +++ b/book/cli/reth/init.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index a3ff8f6a57b1..52f597279f06 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -590,6 +590,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/p2p.md b/book/cli/reth/p2p.md index 603b451d9405..33639042a1de 100644 --- a/book/cli/reth/p2p.md +++ b/book/cli/reth/p2p.md @@ -247,6 +247,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/prune.md b/book/cli/reth/prune.md index ed16197a76c3..41684ecd9e0e 100644 --- a/book/cli/reth/prune.md +++ b/book/cli/reth/prune.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/recover/storage-tries.md b/book/cli/reth/recover/storage-tries.md index ecdaabe77817..1afe94f55dbc 100644 --- a/book/cli/reth/recover/storage-tries.md +++ b/book/cli/reth/recover/storage-tries.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/drop.md b/book/cli/reth/stage/drop.md index 399b3818c28f..c22d6be66805 100644 --- a/book/cli/reth/stage/drop.md +++ b/book/cli/reth/stage/drop.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/dump.md b/book/cli/reth/stage/dump.md index 4b3de3fb1cb5..e3df5bf2df7c 100644 --- a/book/cli/reth/stage/dump.md +++ b/book/cli/reth/stage/dump.md @@ -76,6 +76,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/run.md b/book/cli/reth/stage/run.md index 9da3ce0deb6d..204efc9685bc 100644 --- a/book/cli/reth/stage/run.md +++ b/book/cli/reth/stage/run.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index 700ab3d7e7ce..cb72b9313c0f 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -74,6 +74,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/crates/node/core/src/args/database.rs b/crates/node/core/src/args/database.rs index 0eec6639a117..16ba61935525 100644 --- a/crates/node/core/src/args/database.rs +++ b/crates/node/core/src/args/database.rs @@ -1,6 +1,6 @@ //! clap [Args](clap::Args) for database configuration -use std::time::Duration; +use std::{fmt, str::FromStr, time::Duration}; use crate::version::default_client_version; use clap::{ @@ -22,6 +22,12 @@ pub struct DatabaseArgs { /// NFS volume. #[arg(long = "db.exclusive")] pub exclusive: Option, + /// Maximum database size (e.g., 4TB, 8MB) + #[arg(long = "db.max-size", value_parser = parse_byte_size)] + pub max_size: Option, + /// Database growth step (e.g., 4GB, 4KB) + #[arg(long = "db.growth-step", value_parser = parse_byte_size)] + pub growth_step: Option, /// Read transaction timeout in seconds, 0 means no timeout. #[arg(long = "db.read-transaction-timeout")] pub read_transaction_timeout: Option, @@ -33,8 +39,9 @@ impl DatabaseArgs { self.get_database_args(default_client_version()) } - /// Returns the database arguments with configured log level and given client version. - pub const fn get_database_args( + /// Returns the database arguments with configured log level, client version, + /// max read transaction duration, and geometry. + pub fn get_database_args( &self, client_version: ClientVersion, ) -> reth_db::mdbx::DatabaseArguments { @@ -48,6 +55,7 @@ impl DatabaseArgs { .with_log_level(self.log_level) .with_exclusive(self.exclusive) .with_max_read_transaction_duration(max_read_transaction_duration) + .with_geometry(self.max_size, self.growth_step) } } @@ -89,10 +97,84 @@ impl TypedValueParser for LogLevelValueParser { Some(Box::new(values)) } } + +/// Size in bytes. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct ByteSize(pub usize); + +impl From for usize { + fn from(s: ByteSize) -> Self { + s.0 + } +} + +impl FromStr for ByteSize { + type Err = String; + + fn from_str(s: &str) -> Result { + let s = s.trim().to_uppercase(); + let parts: Vec<&str> = s.split_whitespace().collect(); + + let (num_str, unit) = match parts.len() { + 1 => { + let (num, unit) = + s.split_at(s.find(|c: char| c.is_alphabetic()).unwrap_or(s.len())); + (num, unit) + } + 2 => (parts[0], parts[1]), + _ => { + return Err("Invalid format. Use '' or ' '.".to_string()) + } + }; + + let num: usize = num_str.parse().map_err(|_| "Invalid number".to_string())?; + + let multiplier = match unit { + "B" | "" => 1, // Assume bytes if no unit is specified + "KB" => 1024, + "MB" => 1024 * 1024, + "GB" => 1024 * 1024 * 1024, + "TB" => 1024 * 1024 * 1024 * 1024, + _ => return Err(format!("Invalid unit: {}. Use B, KB, MB, GB, or TB.", unit)), + }; + + Ok(Self(num * multiplier)) + } +} + +impl fmt::Display for ByteSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + const KB: usize = 1024; + const MB: usize = KB * 1024; + const GB: usize = MB * 1024; + const TB: usize = GB * 1024; + + let (size, unit) = if self.0 >= TB { + (self.0 as f64 / TB as f64, "TB") + } else if self.0 >= GB { + (self.0 as f64 / GB as f64, "GB") + } else if self.0 >= MB { + (self.0 as f64 / MB as f64, "MB") + } else if self.0 >= KB { + (self.0 as f64 / KB as f64, "KB") + } else { + (self.0 as f64, "B") + }; + + write!(f, "{:.2}{}", size, unit) + } +} + +/// Value parser function that supports various formats. +fn parse_byte_size(s: &str) -> Result { + s.parse::().map(Into::into) +} + #[cfg(test)] mod tests { use super::*; use clap::Parser; + use reth_db::mdbx::{GIGABYTE, KILOBYTE, MEGABYTE, TERABYTE}; /// A helper type to parse Args more easily #[derive(Parser)] @@ -108,6 +190,101 @@ mod tests { assert_eq!(args, default_args); } + #[test] + fn test_command_parser_with_valid_max_size() { + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "4398046511104", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(TERABYTE * 4)); + } + + #[test] + fn test_command_parser_with_invalid_max_size() { + let result = + CommandParser::::try_parse_from(["reth", "--db.max-size", "invalid"]); + assert!(result.is_err()); + } + + #[test] + fn test_command_parser_with_valid_growth_step() { + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.growth-step", + "4294967296", + ]) + .unwrap(); + assert_eq!(cmd.args.growth_step, Some(GIGABYTE * 4)); + } + + #[test] + fn test_command_parser_with_invalid_growth_step() { + let result = + CommandParser::::try_parse_from(["reth", "--db.growth-step", "invalid"]); + assert!(result.is_err()); + } + + #[test] + fn test_command_parser_with_valid_max_size_and_growth_step_from_str() { + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "2TB", + "--db.growth-step", + "1GB", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(TERABYTE * 2)); + assert_eq!(cmd.args.growth_step, Some(GIGABYTE)); + + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "12MB", + "--db.growth-step", + "2KB", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(MEGABYTE * 12)); + assert_eq!(cmd.args.growth_step, Some(KILOBYTE * 2)); + + // with spaces + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "12 MB", + "--db.growth-step", + "2 KB", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(MEGABYTE * 12)); + assert_eq!(cmd.args.growth_step, Some(KILOBYTE * 2)); + + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "1073741824", + "--db.growth-step", + "1048576", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(GIGABYTE)); + assert_eq!(cmd.args.growth_step, Some(MEGABYTE)); + } + + #[test] + fn test_command_parser_max_size_and_growth_step_from_str_invalid_unit() { + let result = + CommandParser::::try_parse_from(["reth", "--db.growth-step", "1 PB"]); + assert!(result.is_err()); + + let result = + CommandParser::::try_parse_from(["reth", "--db.max-size", "2PB"]); + assert!(result.is_err()); + } + #[test] fn test_possible_values() { // Initialize the LogLevelValueParser diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 65b804e6a58d..92ad20272761 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -23,7 +23,7 @@ use reth_libmdbx::{ use reth_storage_errors::db::LogLevel; use reth_tracing::tracing::error; use std::{ - ops::Deref, + ops::{Deref, Range}, path::Path, sync::Arc, time::{SystemTime, UNIX_EPOCH}, @@ -33,8 +33,14 @@ use tx::Tx; pub mod cursor; pub mod tx; -const GIGABYTE: usize = 1024 * 1024 * 1024; -const TERABYTE: usize = GIGABYTE * 1024; +/// 1 KB in bytes +pub const KILOBYTE: usize = 1024; +/// 1 MB in bytes +pub const MEGABYTE: usize = KILOBYTE * 1024; +/// 1 GB in bytes +pub const GIGABYTE: usize = MEGABYTE * 1024; +/// 1 TB in bytes +pub const TERABYTE: usize = GIGABYTE * 1024; /// MDBX allows up to 32767 readers (`MDBX_READERS_LIMIT`), but we limit it to slightly below that const DEFAULT_MAX_READERS: u64 = 32_000; @@ -64,6 +70,8 @@ impl DatabaseEnvKind { pub struct DatabaseArguments { /// Client version that accesses the database. client_version: ClientVersion, + /// Database geometry settings. + geometry: Geometry>, /// Database log level. If [None], the default value is used. log_level: Option, /// Maximum duration of a read transaction. If [None], the default value is used. @@ -93,15 +101,37 @@ pub struct DatabaseArguments { impl DatabaseArguments { /// Create new database arguments with given client version. - pub const fn new(client_version: ClientVersion) -> Self { + pub fn new(client_version: ClientVersion) -> Self { Self { client_version, + geometry: Geometry { + size: Some(0..(4 * TERABYTE)), + growth_step: Some(4 * GIGABYTE as isize), + shrink_threshold: Some(0), + page_size: Some(PageSize::Set(default_page_size())), + }, log_level: None, max_read_transaction_duration: None, exclusive: None, } } + /// Set the geometry. + /// + /// # Arguments + /// + /// * `max_size` - Maximum database size in bytes + /// * `growth_step` - Database growth step in bytes + pub fn with_geometry(mut self, max_size: Option, growth_step: Option) -> Self { + self.geometry = Geometry { + size: max_size.map(|size| 0..size), + growth_step: growth_step.map(|growth_step| growth_step as isize), + shrink_threshold: Some(0), + page_size: Some(PageSize::Set(default_page_size())), + }; + self + } + /// Set the log level. pub const fn with_log_level(mut self, log_level: Option) -> Self { self.log_level = log_level; @@ -278,15 +308,7 @@ impl DatabaseEnv { // environment creation. debug_assert!(Tables::ALL.len() <= 256, "number of tables exceed max dbs"); inner_env.set_max_dbs(256); - inner_env.set_geometry(Geometry { - // Maximum database size of 4 terabytes - size: Some(0..(4 * TERABYTE)), - // We grow the database in increments of 4 gigabytes - growth_step: Some(4 * GIGABYTE as isize), - // The database never shrinks - shrink_threshold: Some(0), - page_size: Some(PageSize::Set(default_page_size())), - }); + inner_env.set_geometry(args.geometry); fn is_current_process(id: u32) -> bool { #[cfg(unix)] From 4a8799f98bba7aa80a8c333f9b32f23221cb0cb8 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:17:48 +0100 Subject: [PATCH 167/239] consensus: add unit tests for `ForkchoiceStateTracker` and `ForkchoiceStateHash` (#12077) --- .../consensus/beacon/src/engine/forkchoice.rs | 269 ++++++++++++++++-- 1 file changed, 243 insertions(+), 26 deletions(-) diff --git a/crates/consensus/beacon/src/engine/forkchoice.rs b/crates/consensus/beacon/src/engine/forkchoice.rs index 7e49714ba375..a9d9301738f7 100644 --- a/crates/consensus/beacon/src/engine/forkchoice.rs +++ b/crates/consensus/beacon/src/engine/forkchoice.rs @@ -8,7 +8,6 @@ pub struct ForkchoiceStateTracker { /// /// Caution: this can be invalid. latest: Option, - /// Tracks the latest forkchoice state that we received to which we need to sync. last_syncing: Option, /// The latest valid forkchoice state that we received and processed as valid. @@ -48,19 +47,19 @@ impl ForkchoiceStateTracker { /// Returns whether the latest received FCU is valid: [`ForkchoiceStatus::Valid`] #[allow(dead_code)] pub(crate) fn is_latest_valid(&self) -> bool { - self.latest_status().map(|s| s.is_valid()).unwrap_or(false) + self.latest_status().map_or(false, |s| s.is_valid()) } /// Returns whether the latest received FCU is syncing: [`ForkchoiceStatus::Syncing`] #[allow(dead_code)] pub(crate) fn is_latest_syncing(&self) -> bool { - self.latest_status().map(|s| s.is_syncing()).unwrap_or(false) + self.latest_status().map_or(false, |s| s.is_syncing()) } /// Returns whether the latest received FCU is syncing: [`ForkchoiceStatus::Invalid`] #[allow(dead_code)] pub(crate) fn is_latest_invalid(&self) -> bool { - self.latest_status().map(|s| s.is_invalid()).unwrap_or(false) + self.latest_status().map_or(false, |s| s.is_invalid()) } /// Returns the last valid head hash. @@ -75,32 +74,28 @@ impl ForkchoiceStateTracker { self.last_syncing.as_ref().map(|s| s.head_block_hash) } - /// Returns the latest received `ForkchoiceState`. + /// Returns the latest received [`ForkchoiceState`]. /// /// Caution: this can be invalid. pub const fn latest_state(&self) -> Option { self.last_valid } - /// Returns the last valid `ForkchoiceState`. + /// Returns the last valid [`ForkchoiceState`]. pub const fn last_valid_state(&self) -> Option { self.last_valid } /// Returns the last valid finalized hash. /// - /// This will return [`None`], if either there is no valid finalized forkchoice state, or the - /// finalized hash for the latest valid forkchoice state is zero. + /// This will return [`None`]: + /// - If either there is no valid finalized forkchoice state, + /// - Or the finalized hash for the latest valid forkchoice state is zero. #[inline] pub fn last_valid_finalized(&self) -> Option { - self.last_valid.and_then(|state| { - // if the hash is zero then we should act like there is no finalized hash - if state.finalized_block_hash.is_zero() { - None - } else { - Some(state.finalized_block_hash) - } - }) + self.last_valid + .filter(|state| !state.finalized_block_hash.is_zero()) + .map(|state| state.finalized_block_hash) } /// Returns the last received `ForkchoiceState` to which we need to sync. @@ -110,18 +105,14 @@ impl ForkchoiceStateTracker { /// Returns the sync target finalized hash. /// - /// This will return [`None`], if either there is no sync target forkchoice state, or the - /// finalized hash for the sync target forkchoice state is zero. + /// This will return [`None`]: + /// - If either there is no sync target forkchoice state, + /// - Or the finalized hash for the sync target forkchoice state is zero. #[inline] pub fn sync_target_finalized(&self) -> Option { - self.last_syncing.and_then(|state| { - // if the hash is zero then we should act like there is no finalized hash - if state.finalized_block_hash.is_zero() { - None - } else { - Some(state.finalized_block_hash) - } - }) + self.last_syncing + .filter(|state| !state.finalized_block_hash.is_zero()) + .map(|state| state.finalized_block_hash) } /// Returns true if no forkchoice state has been received yet. @@ -222,3 +213,229 @@ impl AsRef for ForkchoiceStateHash { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_forkchoice_state_tracker_set_latest_valid() { + let mut tracker = ForkchoiceStateTracker::default(); + + // Latest state is None + assert!(tracker.latest_status().is_none()); + + // Create a valid ForkchoiceState + let state = ForkchoiceState { + head_block_hash: B256::from_slice(&[1; 32]), + safe_block_hash: B256::from_slice(&[2; 32]), + finalized_block_hash: B256::from_slice(&[3; 32]), + }; + let status = ForkchoiceStatus::Valid; + + tracker.set_latest(state, status); + + // Assert that the latest state is set + assert!(tracker.latest.is_some()); + assert_eq!(tracker.latest.as_ref().unwrap().state, state); + + // Assert that last valid state is updated + assert!(tracker.last_valid.is_some()); + assert_eq!(tracker.last_valid.as_ref().unwrap(), &state); + + // Assert that last syncing state is None + assert!(tracker.last_syncing.is_none()); + + // Test when there is a latest status and it is valid + assert_eq!(tracker.latest_status(), Some(ForkchoiceStatus::Valid)); + } + + #[test] + fn test_forkchoice_state_tracker_set_latest_syncing() { + let mut tracker = ForkchoiceStateTracker::default(); + + // Create a syncing ForkchoiceState + let state = ForkchoiceState { + head_block_hash: B256::from_slice(&[1; 32]), + safe_block_hash: B256::from_slice(&[2; 32]), + finalized_block_hash: B256::from_slice(&[0; 32]), // Zero to simulate not finalized + }; + let status = ForkchoiceStatus::Syncing; + + tracker.set_latest(state, status); + + // Assert that the latest state is set + assert!(tracker.latest.is_some()); + assert_eq!(tracker.latest.as_ref().unwrap().state, state); + + // Assert that last valid state is None since the status is syncing + assert!(tracker.last_valid.is_none()); + + // Assert that last syncing state is updated + assert!(tracker.last_syncing.is_some()); + assert_eq!(tracker.last_syncing.as_ref().unwrap(), &state); + + // Test when there is a latest status and it is syncing + assert_eq!(tracker.latest_status(), Some(ForkchoiceStatus::Syncing)); + } + + #[test] + fn test_forkchoice_state_tracker_set_latest_invalid() { + let mut tracker = ForkchoiceStateTracker::default(); + + // Create an invalid ForkchoiceState + let state = ForkchoiceState { + head_block_hash: B256::from_slice(&[1; 32]), + safe_block_hash: B256::from_slice(&[2; 32]), + finalized_block_hash: B256::from_slice(&[3; 32]), + }; + let status = ForkchoiceStatus::Invalid; + + tracker.set_latest(state, status); + + // Assert that the latest state is set + assert!(tracker.latest.is_some()); + assert_eq!(tracker.latest.as_ref().unwrap().state, state); + + // Assert that last valid state is None since the status is invalid + assert!(tracker.last_valid.is_none()); + + // Assert that last syncing state is None since the status is invalid + assert!(tracker.last_syncing.is_none()); + + // Test when there is a latest status and it is invalid + assert_eq!(tracker.latest_status(), Some(ForkchoiceStatus::Invalid)); + } + + #[test] + fn test_forkchoice_state_tracker_sync_target() { + let mut tracker = ForkchoiceStateTracker::default(); + + // Test when there is no last syncing state (should return None) + assert!(tracker.sync_target().is_none()); + + // Set a last syncing forkchoice state + let state = ForkchoiceState { + head_block_hash: B256::from_slice(&[1; 32]), + safe_block_hash: B256::from_slice(&[2; 32]), + finalized_block_hash: B256::from_slice(&[3; 32]), + }; + tracker.last_syncing = Some(state); + + // Test when the last syncing state is set (should return the head block hash) + assert_eq!(tracker.sync_target(), Some(B256::from_slice(&[1; 32]))); + } + + #[test] + fn test_forkchoice_state_tracker_last_valid_finalized() { + let mut tracker = ForkchoiceStateTracker::default(); + + // No valid finalized state (should return None) + assert!(tracker.last_valid_finalized().is_none()); + + // Valid finalized state, but finalized hash is zero (should return None) + let zero_finalized_state = ForkchoiceState { + head_block_hash: B256::ZERO, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, // Zero finalized hash + }; + tracker.last_valid = Some(zero_finalized_state); + assert!(tracker.last_valid_finalized().is_none()); + + // Valid finalized state with non-zero finalized hash (should return finalized hash) + let valid_finalized_state = ForkchoiceState { + head_block_hash: B256::from_slice(&[1; 32]), + safe_block_hash: B256::from_slice(&[2; 32]), + finalized_block_hash: B256::from_slice(&[123; 32]), // Non-zero finalized hash + }; + tracker.last_valid = Some(valid_finalized_state); + assert_eq!(tracker.last_valid_finalized(), Some(B256::from_slice(&[123; 32]))); + + // Reset the last valid state to None + tracker.last_valid = None; + assert!(tracker.last_valid_finalized().is_none()); + } + + #[test] + fn test_forkchoice_state_tracker_sync_target_finalized() { + let mut tracker = ForkchoiceStateTracker::default(); + + // No sync target state (should return None) + assert!(tracker.sync_target_finalized().is_none()); + + // Sync target state with finalized hash as zero (should return None) + let zero_finalized_sync_target = ForkchoiceState { + head_block_hash: B256::from_slice(&[1; 32]), + safe_block_hash: B256::from_slice(&[2; 32]), + finalized_block_hash: B256::ZERO, // Zero finalized hash + }; + tracker.last_syncing = Some(zero_finalized_sync_target); + assert!(tracker.sync_target_finalized().is_none()); + + // Sync target state with non-zero finalized hash (should return the hash) + let valid_sync_target = ForkchoiceState { + head_block_hash: B256::from_slice(&[1; 32]), + safe_block_hash: B256::from_slice(&[2; 32]), + finalized_block_hash: B256::from_slice(&[22; 32]), // Non-zero finalized hash + }; + tracker.last_syncing = Some(valid_sync_target); + assert_eq!(tracker.sync_target_finalized(), Some(B256::from_slice(&[22; 32]))); + + // Reset the last sync target state to None + tracker.last_syncing = None; + assert!(tracker.sync_target_finalized().is_none()); + } + + #[test] + fn test_forkchoice_state_tracker_is_empty() { + let mut forkchoice = ForkchoiceStateTracker::default(); + + // Initially, no forkchoice state has been received, so it should be empty. + assert!(forkchoice.is_empty()); + + // After setting a forkchoice state, it should no longer be empty. + forkchoice.set_latest(ForkchoiceState::default(), ForkchoiceStatus::Valid); + assert!(!forkchoice.is_empty()); + + // Reset the forkchoice latest, it should be empty again. + forkchoice.latest = None; + assert!(forkchoice.is_empty()); + } + + #[test] + fn test_forkchoice_state_hash_find() { + // Define example hashes + let head_hash = B256::random(); + let safe_hash = B256::random(); + let finalized_hash = B256::random(); + let non_matching_hash = B256::random(); + + // Create a ForkchoiceState with specific hashes + let state = ForkchoiceState { + head_block_hash: head_hash, + safe_block_hash: safe_hash, + finalized_block_hash: finalized_hash, + }; + + // Test finding the head hash + assert_eq!( + ForkchoiceStateHash::find(&state, head_hash), + Some(ForkchoiceStateHash::Head(head_hash)) + ); + + // Test finding the safe hash + assert_eq!( + ForkchoiceStateHash::find(&state, safe_hash), + Some(ForkchoiceStateHash::Safe(safe_hash)) + ); + + // Test finding the finalized hash + assert_eq!( + ForkchoiceStateHash::find(&state, finalized_hash), + Some(ForkchoiceStateHash::Finalized(finalized_hash)) + ); + + // Test with a hash that doesn't match any of the hashes in ForkchoiceState + assert_eq!(ForkchoiceStateHash::find(&state, non_matching_hash), None); + } +} From 6f3600dc38f7548a40999cc4c479f84d69326b76 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 29 Oct 2024 13:58:57 +0100 Subject: [PATCH 168/239] Revert "feat: add geometry to database args" (#12164) --- book/cli/reth/db.md | 6 - book/cli/reth/db/diff.md | 6 - book/cli/reth/debug/build-block.md | 6 - book/cli/reth/debug/execution.md | 6 - book/cli/reth/debug/in-memory-merkle.md | 6 - book/cli/reth/debug/merkle.md | 6 - book/cli/reth/debug/replay-engine.md | 6 - book/cli/reth/import.md | 6 - book/cli/reth/init-state.md | 6 - book/cli/reth/init.md | 6 - book/cli/reth/node.md | 6 - book/cli/reth/p2p.md | 6 - book/cli/reth/prune.md | 6 - book/cli/reth/recover/storage-tries.md | 6 - book/cli/reth/stage/drop.md | 6 - book/cli/reth/stage/dump.md | 6 - book/cli/reth/stage/run.md | 6 - book/cli/reth/stage/unwind.md | 6 - crates/node/core/src/args/database.rs | 183 +----------------- .../storage/db/src/implementation/mdbx/mod.rs | 48 ++--- 20 files changed, 16 insertions(+), 323 deletions(-) diff --git a/book/cli/reth/db.md b/book/cli/reth/db.md index 17a6de4e607e..f9a8a158adc8 100644 --- a/book/cli/reth/db.md +++ b/book/cli/reth/db.md @@ -81,12 +81,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/db/diff.md b/book/cli/reth/db/diff.md index efb9e7d32e30..f57c6ac364fe 100644 --- a/book/cli/reth/db/diff.md +++ b/book/cli/reth/db/diff.md @@ -45,12 +45,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/build-block.md b/book/cli/reth/debug/build-block.md index 7bceb62b940a..2e6d637d52c2 100644 --- a/book/cli/reth/debug/build-block.md +++ b/book/cli/reth/debug/build-block.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/execution.md b/book/cli/reth/debug/execution.md index b8e1ce05d171..9ca74897c5ea 100644 --- a/book/cli/reth/debug/execution.md +++ b/book/cli/reth/debug/execution.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/in-memory-merkle.md b/book/cli/reth/debug/in-memory-merkle.md index a183db997e9d..3e322a6913dd 100644 --- a/book/cli/reth/debug/in-memory-merkle.md +++ b/book/cli/reth/debug/in-memory-merkle.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/merkle.md b/book/cli/reth/debug/merkle.md index d9a72794ef2c..d701803b81ca 100644 --- a/book/cli/reth/debug/merkle.md +++ b/book/cli/reth/debug/merkle.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/replay-engine.md b/book/cli/reth/debug/replay-engine.md index b7a1266d3993..dd587620a868 100644 --- a/book/cli/reth/debug/replay-engine.md +++ b/book/cli/reth/debug/replay-engine.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/import.md b/book/cli/reth/import.md index 82a521ac0ab2..28e085bda718 100644 --- a/book/cli/reth/import.md +++ b/book/cli/reth/import.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/init-state.md b/book/cli/reth/init-state.md index 533c0f8f8889..3e073516778d 100644 --- a/book/cli/reth/init-state.md +++ b/book/cli/reth/init-state.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/init.md b/book/cli/reth/init.md index ebe2a8386cf7..cd01accc0471 100644 --- a/book/cli/reth/init.md +++ b/book/cli/reth/init.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 52f597279f06..a3ff8f6a57b1 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -590,12 +590,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/p2p.md b/book/cli/reth/p2p.md index 33639042a1de..603b451d9405 100644 --- a/book/cli/reth/p2p.md +++ b/book/cli/reth/p2p.md @@ -247,12 +247,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/prune.md b/book/cli/reth/prune.md index 41684ecd9e0e..ed16197a76c3 100644 --- a/book/cli/reth/prune.md +++ b/book/cli/reth/prune.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/recover/storage-tries.md b/book/cli/reth/recover/storage-tries.md index 1afe94f55dbc..ecdaabe77817 100644 --- a/book/cli/reth/recover/storage-tries.md +++ b/book/cli/reth/recover/storage-tries.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/drop.md b/book/cli/reth/stage/drop.md index c22d6be66805..399b3818c28f 100644 --- a/book/cli/reth/stage/drop.md +++ b/book/cli/reth/stage/drop.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/dump.md b/book/cli/reth/stage/dump.md index e3df5bf2df7c..4b3de3fb1cb5 100644 --- a/book/cli/reth/stage/dump.md +++ b/book/cli/reth/stage/dump.md @@ -76,12 +76,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/run.md b/book/cli/reth/stage/run.md index 204efc9685bc..9da3ce0deb6d 100644 --- a/book/cli/reth/stage/run.md +++ b/book/cli/reth/stage/run.md @@ -69,12 +69,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index cb72b9313c0f..700ab3d7e7ce 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -74,12 +74,6 @@ Database: [possible values: true, false] - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/crates/node/core/src/args/database.rs b/crates/node/core/src/args/database.rs index 16ba61935525..0eec6639a117 100644 --- a/crates/node/core/src/args/database.rs +++ b/crates/node/core/src/args/database.rs @@ -1,6 +1,6 @@ //! clap [Args](clap::Args) for database configuration -use std::{fmt, str::FromStr, time::Duration}; +use std::time::Duration; use crate::version::default_client_version; use clap::{ @@ -22,12 +22,6 @@ pub struct DatabaseArgs { /// NFS volume. #[arg(long = "db.exclusive")] pub exclusive: Option, - /// Maximum database size (e.g., 4TB, 8MB) - #[arg(long = "db.max-size", value_parser = parse_byte_size)] - pub max_size: Option, - /// Database growth step (e.g., 4GB, 4KB) - #[arg(long = "db.growth-step", value_parser = parse_byte_size)] - pub growth_step: Option, /// Read transaction timeout in seconds, 0 means no timeout. #[arg(long = "db.read-transaction-timeout")] pub read_transaction_timeout: Option, @@ -39,9 +33,8 @@ impl DatabaseArgs { self.get_database_args(default_client_version()) } - /// Returns the database arguments with configured log level, client version, - /// max read transaction duration, and geometry. - pub fn get_database_args( + /// Returns the database arguments with configured log level and given client version. + pub const fn get_database_args( &self, client_version: ClientVersion, ) -> reth_db::mdbx::DatabaseArguments { @@ -55,7 +48,6 @@ impl DatabaseArgs { .with_log_level(self.log_level) .with_exclusive(self.exclusive) .with_max_read_transaction_duration(max_read_transaction_duration) - .with_geometry(self.max_size, self.growth_step) } } @@ -97,84 +89,10 @@ impl TypedValueParser for LogLevelValueParser { Some(Box::new(values)) } } - -/// Size in bytes. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct ByteSize(pub usize); - -impl From for usize { - fn from(s: ByteSize) -> Self { - s.0 - } -} - -impl FromStr for ByteSize { - type Err = String; - - fn from_str(s: &str) -> Result { - let s = s.trim().to_uppercase(); - let parts: Vec<&str> = s.split_whitespace().collect(); - - let (num_str, unit) = match parts.len() { - 1 => { - let (num, unit) = - s.split_at(s.find(|c: char| c.is_alphabetic()).unwrap_or(s.len())); - (num, unit) - } - 2 => (parts[0], parts[1]), - _ => { - return Err("Invalid format. Use '' or ' '.".to_string()) - } - }; - - let num: usize = num_str.parse().map_err(|_| "Invalid number".to_string())?; - - let multiplier = match unit { - "B" | "" => 1, // Assume bytes if no unit is specified - "KB" => 1024, - "MB" => 1024 * 1024, - "GB" => 1024 * 1024 * 1024, - "TB" => 1024 * 1024 * 1024 * 1024, - _ => return Err(format!("Invalid unit: {}. Use B, KB, MB, GB, or TB.", unit)), - }; - - Ok(Self(num * multiplier)) - } -} - -impl fmt::Display for ByteSize { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - const KB: usize = 1024; - const MB: usize = KB * 1024; - const GB: usize = MB * 1024; - const TB: usize = GB * 1024; - - let (size, unit) = if self.0 >= TB { - (self.0 as f64 / TB as f64, "TB") - } else if self.0 >= GB { - (self.0 as f64 / GB as f64, "GB") - } else if self.0 >= MB { - (self.0 as f64 / MB as f64, "MB") - } else if self.0 >= KB { - (self.0 as f64 / KB as f64, "KB") - } else { - (self.0 as f64, "B") - }; - - write!(f, "{:.2}{}", size, unit) - } -} - -/// Value parser function that supports various formats. -fn parse_byte_size(s: &str) -> Result { - s.parse::().map(Into::into) -} - #[cfg(test)] mod tests { use super::*; use clap::Parser; - use reth_db::mdbx::{GIGABYTE, KILOBYTE, MEGABYTE, TERABYTE}; /// A helper type to parse Args more easily #[derive(Parser)] @@ -190,101 +108,6 @@ mod tests { assert_eq!(args, default_args); } - #[test] - fn test_command_parser_with_valid_max_size() { - let cmd = CommandParser::::try_parse_from([ - "reth", - "--db.max-size", - "4398046511104", - ]) - .unwrap(); - assert_eq!(cmd.args.max_size, Some(TERABYTE * 4)); - } - - #[test] - fn test_command_parser_with_invalid_max_size() { - let result = - CommandParser::::try_parse_from(["reth", "--db.max-size", "invalid"]); - assert!(result.is_err()); - } - - #[test] - fn test_command_parser_with_valid_growth_step() { - let cmd = CommandParser::::try_parse_from([ - "reth", - "--db.growth-step", - "4294967296", - ]) - .unwrap(); - assert_eq!(cmd.args.growth_step, Some(GIGABYTE * 4)); - } - - #[test] - fn test_command_parser_with_invalid_growth_step() { - let result = - CommandParser::::try_parse_from(["reth", "--db.growth-step", "invalid"]); - assert!(result.is_err()); - } - - #[test] - fn test_command_parser_with_valid_max_size_and_growth_step_from_str() { - let cmd = CommandParser::::try_parse_from([ - "reth", - "--db.max-size", - "2TB", - "--db.growth-step", - "1GB", - ]) - .unwrap(); - assert_eq!(cmd.args.max_size, Some(TERABYTE * 2)); - assert_eq!(cmd.args.growth_step, Some(GIGABYTE)); - - let cmd = CommandParser::::try_parse_from([ - "reth", - "--db.max-size", - "12MB", - "--db.growth-step", - "2KB", - ]) - .unwrap(); - assert_eq!(cmd.args.max_size, Some(MEGABYTE * 12)); - assert_eq!(cmd.args.growth_step, Some(KILOBYTE * 2)); - - // with spaces - let cmd = CommandParser::::try_parse_from([ - "reth", - "--db.max-size", - "12 MB", - "--db.growth-step", - "2 KB", - ]) - .unwrap(); - assert_eq!(cmd.args.max_size, Some(MEGABYTE * 12)); - assert_eq!(cmd.args.growth_step, Some(KILOBYTE * 2)); - - let cmd = CommandParser::::try_parse_from([ - "reth", - "--db.max-size", - "1073741824", - "--db.growth-step", - "1048576", - ]) - .unwrap(); - assert_eq!(cmd.args.max_size, Some(GIGABYTE)); - assert_eq!(cmd.args.growth_step, Some(MEGABYTE)); - } - - #[test] - fn test_command_parser_max_size_and_growth_step_from_str_invalid_unit() { - let result = - CommandParser::::try_parse_from(["reth", "--db.growth-step", "1 PB"]); - assert!(result.is_err()); - - let result = - CommandParser::::try_parse_from(["reth", "--db.max-size", "2PB"]); - assert!(result.is_err()); - } - #[test] fn test_possible_values() { // Initialize the LogLevelValueParser diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 92ad20272761..65b804e6a58d 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -23,7 +23,7 @@ use reth_libmdbx::{ use reth_storage_errors::db::LogLevel; use reth_tracing::tracing::error; use std::{ - ops::{Deref, Range}, + ops::Deref, path::Path, sync::Arc, time::{SystemTime, UNIX_EPOCH}, @@ -33,14 +33,8 @@ use tx::Tx; pub mod cursor; pub mod tx; -/// 1 KB in bytes -pub const KILOBYTE: usize = 1024; -/// 1 MB in bytes -pub const MEGABYTE: usize = KILOBYTE * 1024; -/// 1 GB in bytes -pub const GIGABYTE: usize = MEGABYTE * 1024; -/// 1 TB in bytes -pub const TERABYTE: usize = GIGABYTE * 1024; +const GIGABYTE: usize = 1024 * 1024 * 1024; +const TERABYTE: usize = GIGABYTE * 1024; /// MDBX allows up to 32767 readers (`MDBX_READERS_LIMIT`), but we limit it to slightly below that const DEFAULT_MAX_READERS: u64 = 32_000; @@ -70,8 +64,6 @@ impl DatabaseEnvKind { pub struct DatabaseArguments { /// Client version that accesses the database. client_version: ClientVersion, - /// Database geometry settings. - geometry: Geometry>, /// Database log level. If [None], the default value is used. log_level: Option, /// Maximum duration of a read transaction. If [None], the default value is used. @@ -101,37 +93,15 @@ pub struct DatabaseArguments { impl DatabaseArguments { /// Create new database arguments with given client version. - pub fn new(client_version: ClientVersion) -> Self { + pub const fn new(client_version: ClientVersion) -> Self { Self { client_version, - geometry: Geometry { - size: Some(0..(4 * TERABYTE)), - growth_step: Some(4 * GIGABYTE as isize), - shrink_threshold: Some(0), - page_size: Some(PageSize::Set(default_page_size())), - }, log_level: None, max_read_transaction_duration: None, exclusive: None, } } - /// Set the geometry. - /// - /// # Arguments - /// - /// * `max_size` - Maximum database size in bytes - /// * `growth_step` - Database growth step in bytes - pub fn with_geometry(mut self, max_size: Option, growth_step: Option) -> Self { - self.geometry = Geometry { - size: max_size.map(|size| 0..size), - growth_step: growth_step.map(|growth_step| growth_step as isize), - shrink_threshold: Some(0), - page_size: Some(PageSize::Set(default_page_size())), - }; - self - } - /// Set the log level. pub const fn with_log_level(mut self, log_level: Option) -> Self { self.log_level = log_level; @@ -308,7 +278,15 @@ impl DatabaseEnv { // environment creation. debug_assert!(Tables::ALL.len() <= 256, "number of tables exceed max dbs"); inner_env.set_max_dbs(256); - inner_env.set_geometry(args.geometry); + inner_env.set_geometry(Geometry { + // Maximum database size of 4 terabytes + size: Some(0..(4 * TERABYTE)), + // We grow the database in increments of 4 gigabytes + growth_step: Some(4 * GIGABYTE as isize), + // The database never shrinks + shrink_threshold: Some(0), + page_size: Some(PageSize::Set(default_page_size())), + }); fn is_current_process(id: u32) -> bool { #[cfg(unix)] From 52328422aad608705e68e118e911362c9adf761d Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 29 Oct 2024 13:18:12 +0000 Subject: [PATCH 169/239] feat(storage): pass changesets to unwind methods (#7879) --- .../stages/src/stages/hashing_account.rs | 2 +- .../stages/src/stages/hashing_storage.rs | 2 +- .../src/stages/index_account_history.rs | 2 +- .../src/stages/index_storage_history.rs | 2 +- .../src/providers/database/provider.rs | 149 ++++++++++++------ crates/storage/provider/src/traits/hashing.rs | 30 +++- crates/storage/provider/src/traits/history.rs | 27 +++- 7 files changed, 155 insertions(+), 59 deletions(-) diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 14afb37d81db..5b4f720972f1 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -234,7 +234,7 @@ where input.unwind_block_range_with_threshold(self.commit_threshold); // Aggregate all transition changesets and make a list of accounts that have been changed. - provider.unwind_account_hashing(range)?; + provider.unwind_account_hashing_range(range)?; let mut stage_checkpoint = input.checkpoint.account_hashing_stage_checkpoint().unwrap_or_default(); diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index ef070d30c6d6..dcabbe83ee64 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -169,7 +169,7 @@ where let (range, unwind_progress, _) = input.unwind_block_range_with_threshold(self.commit_threshold); - provider.unwind_storage_hashing(BlockNumberAddress::range(range))?; + provider.unwind_storage_hashing_range(BlockNumberAddress::range(range))?; let mut stage_checkpoint = input.checkpoint.storage_hashing_stage_checkpoint().unwrap_or_default(); diff --git a/crates/stages/stages/src/stages/index_account_history.rs b/crates/stages/stages/src/stages/index_account_history.rs index 8b10283fb4b7..38c238e5d988 100644 --- a/crates/stages/stages/src/stages/index_account_history.rs +++ b/crates/stages/stages/src/stages/index_account_history.rs @@ -134,7 +134,7 @@ where let (range, unwind_progress, _) = input.unwind_block_range_with_threshold(self.commit_threshold); - provider.unwind_account_history_indices(range)?; + provider.unwind_account_history_indices_range(range)?; // from HistoryIndex higher than that number. Ok(UnwindOutput { checkpoint: StageCheckpoint::new(unwind_progress) }) diff --git a/crates/stages/stages/src/stages/index_storage_history.rs b/crates/stages/stages/src/stages/index_storage_history.rs index ac645b8dd754..ba61e6312302 100644 --- a/crates/stages/stages/src/stages/index_storage_history.rs +++ b/crates/stages/stages/src/stages/index_storage_history.rs @@ -140,7 +140,7 @@ where let (range, unwind_progress, _) = input.unwind_block_range_with_threshold(self.commit_threshold); - provider.unwind_storage_history_indices(BlockNumberAddress::range(range))?; + provider.unwind_storage_history_indices_range(BlockNumberAddress::range(range))?; Ok(UnwindOutput { checkpoint: StageCheckpoint::new(unwind_progress) }) } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index e59a4f5635f4..2dcc3f92d705 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -2639,19 +2639,17 @@ impl StorageTrieWriter for DatabaseProvid } impl HashingWriter for DatabaseProvider { - fn unwind_account_hashing( + fn unwind_account_hashing<'a>( &self, - range: RangeInclusive, + changesets: impl Iterator, ) -> ProviderResult>> { // Aggregate all block changesets and make a list of accounts that have been changed. // Note that collecting and then reversing the order is necessary to ensure that the // changes are applied in the correct order. - let hashed_accounts = self - .tx - .cursor_read::()? - .walk_range(range)? - .map(|entry| entry.map(|(_, e)| (keccak256(e.address), e.info))) - .collect::, _>>()? + let hashed_accounts = changesets + .into_iter() + .map(|(_, e)| (keccak256(e.address), e.info)) + .collect::>() .into_iter() .rev() .collect::>(); @@ -2669,13 +2667,25 @@ impl HashingWriter for DatabaseProvider, + ) -> ProviderResult>> { + let changesets = self + .tx + .cursor_read::()? + .walk_range(range)? + .collect::, _>>()?; + self.unwind_account_hashing(changesets.iter()) + } + fn insert_account_for_hashing( &self, - accounts: impl IntoIterator)>, + changesets: impl IntoIterator)>, ) -> ProviderResult>> { let mut hashed_accounts_cursor = self.tx.cursor_write::()?; let hashed_accounts = - accounts.into_iter().map(|(ad, ac)| (keccak256(ad), ac)).collect::>(); + changesets.into_iter().map(|(ad, ac)| (keccak256(ad), ac)).collect::>(); for (hashed_address, account) in &hashed_accounts { if let Some(account) = account { hashed_accounts_cursor.upsert(*hashed_address, *account)?; @@ -2688,18 +2698,15 @@ impl HashingWriter for DatabaseProvider, + changesets: impl Iterator, ) -> ProviderResult>> { // Aggregate all block changesets and make list of accounts that have been changed. - let mut changesets = self.tx.cursor_read::()?; let mut hashed_storages = changesets - .walk_range(range)? - .map(|entry| { - entry.map(|(BlockNumberAddress((_, address)), storage_entry)| { - (keccak256(address), keccak256(storage_entry.key), storage_entry.value) - }) + .into_iter() + .map(|(BlockNumberAddress((_, address)), storage_entry)| { + (keccak256(address), keccak256(storage_entry.key), storage_entry.value) }) - .collect::, _>>()?; + .collect::>(); hashed_storages.sort_by_key(|(ha, hk, _)| (*ha, *hk)); // Apply values to HashedState, and remove the account if it's None. @@ -2724,6 +2731,18 @@ impl HashingWriter for DatabaseProvider, + ) -> ProviderResult>> { + let changesets = self + .tx + .cursor_read::()? + .walk_range(range)? + .collect::, _>>()?; + self.unwind_storage_hashing(changesets.into_iter()) + } + fn insert_storage_for_hashing( &self, storages: impl IntoIterator)>, @@ -2845,16 +2864,14 @@ impl HashingWriter for DatabaseProvider HistoryWriter for DatabaseProvider { - fn unwind_account_history_indices( + fn unwind_account_history_indices<'a>( &self, - range: RangeInclusive, + changesets: impl Iterator, ) -> ProviderResult { - let mut last_indices = self - .tx - .cursor_read::()? - .walk_range(range)? - .map(|entry| entry.map(|(index, account)| (account.address, index))) - .collect::, _>>()?; + let mut last_indices = changesets + .into_iter() + .map(|(index, account)| (account.address, *index)) + .collect::>(); last_indices.sort_by_key(|(a, _)| *a); // Unwind the account history index. @@ -2881,6 +2898,18 @@ impl HistoryWriter for DatabaseProvider, + ) -> ProviderResult { + let changesets = self + .tx + .cursor_read::()? + .walk_range(range)? + .collect::, _>>()?; + self.unwind_account_history_indices(changesets.iter()) + } + fn insert_account_history_index( &self, account_transitions: impl IntoIterator)>, @@ -2893,16 +2922,12 @@ impl HistoryWriter for DatabaseProvider, + changesets: impl Iterator, ) -> ProviderResult { - let mut storage_changesets = self - .tx - .cursor_read::()? - .walk_range(range)? - .map(|entry| { - entry.map(|(BlockNumberAddress((bn, address)), storage)| (address, storage.key, bn)) - }) - .collect::, _>>()?; + let mut storage_changesets = changesets + .into_iter() + .map(|(BlockNumberAddress((bn, address)), storage)| (address, storage.key, bn)) + .collect::>(); storage_changesets.sort_by_key(|(address, key, _)| (*address, *key)); let mut cursor = self.tx.cursor_write::()?; @@ -2931,6 +2956,18 @@ impl HistoryWriter for DatabaseProvider, + ) -> ProviderResult { + let changesets = self + .tx + .cursor_read::()? + .walk_range(range)? + .collect::, _>>()?; + self.unwind_storage_history_indices(changesets.into_iter()) + } + fn insert_storage_history_index( &self, storage_transitions: impl IntoIterator)>, @@ -2973,10 +3010,14 @@ impl, ) -> ProviderResult { - let storage_range = BlockNumberAddress::range(range.clone()); + let changed_accounts = self + .tx + .cursor_read::()? + .walk_range(range.clone())? + .collect::, _>>()?; // Unwind account hashes. Add changed accounts to account prefix set. - let hashed_addresses = self.unwind_account_hashing(range.clone())?; + let hashed_addresses = self.unwind_account_hashing(changed_accounts.iter())?; let mut account_prefix_set = PrefixSetMut::with_capacity(hashed_addresses.len()); let mut destroyed_accounts = HashSet::default(); for (hashed_address, account) in hashed_addresses { @@ -2987,12 +3028,19 @@ impl()? + .walk_range(storage_range)? + .collect::, _>>()?; // Unwind storage hashes. Add changed account and storage keys to corresponding prefix // sets. let mut storage_prefix_sets = HashMap::::default(); - let storage_entries = self.unwind_storage_hashing(storage_range.clone())?; + let storage_entries = self.unwind_storage_hashing(changed_storages.iter().copied())?; for (hashed_address, hashed_slots) in storage_entries { account_prefix_set.insert(Nibbles::unpack(hashed_address)); let mut storage_prefix_set = PrefixSetMut::with_capacity(hashed_slots.len()); @@ -3003,7 +3051,7 @@ impl, ) -> ProviderResult<()> { - let storage_range = BlockNumberAddress::range(range.clone()); + let changed_accounts = self + .tx + .cursor_read::()? + .walk_range(range.clone())? + .collect::, _>>()?; // Unwind account hashes. Add changed accounts to account prefix set. - let hashed_addresses = self.unwind_account_hashing(range.clone())?; + let hashed_addresses = self.unwind_account_hashing(changed_accounts.iter())?; let mut account_prefix_set = PrefixSetMut::with_capacity(hashed_addresses.len()); let mut destroyed_accounts = HashSet::default(); for (hashed_address, account) in hashed_addresses { @@ -3075,12 +3127,19 @@ impl()? + .walk_range(storage_range)? + .collect::, _>>()?; // Unwind storage hashes. Add changed account and storage keys to corresponding prefix // sets. let mut storage_prefix_sets = HashMap::::default(); - let storage_entries = self.unwind_storage_hashing(storage_range.clone())?; + let storage_entries = self.unwind_storage_hashing(changed_storages.iter().copied())?; for (hashed_address, hashed_slots) in storage_entries { account_prefix_set.insert(Nibbles::unpack(hashed_address)); let mut storage_prefix_set = PrefixSetMut::with_capacity(hashed_slots.len()); @@ -3091,7 +3150,7 @@ impl( &self, - range: RangeInclusive, + changesets: impl Iterator, + ) -> ProviderResult>>; + + /// Unwind and clear account hashing in a given block range. + /// + /// # Returns + /// + /// Set of hashed keys of updated accounts. + fn unwind_account_hashing_range( + &self, + range: impl RangeBounds, ) -> ProviderResult>>; /// Inserts all accounts into [reth_db::tables::AccountsHistory] table. @@ -38,7 +48,17 @@ pub trait HashingWriter: Send + Sync { /// Mapping of hashed keys of updated accounts to their respective updated hashed slots. fn unwind_storage_hashing( &self, - range: Range, + changesets: impl Iterator, + ) -> ProviderResult>>; + + /// Unwind and clear storage hashing in a given block range. + /// + /// # Returns + /// + /// Mapping of hashed keys of updated accounts to their respective updated hashed slots. + fn unwind_storage_hashing_range( + &self, + range: impl RangeBounds, ) -> ProviderResult>>; /// Iterates over storages and inserts them to hashing table. diff --git a/crates/storage/provider/src/traits/history.rs b/crates/storage/provider/src/traits/history.rs index cbf9bece4b94..4eadd6031c35 100644 --- a/crates/storage/provider/src/traits/history.rs +++ b/crates/storage/provider/src/traits/history.rs @@ -1,8 +1,9 @@ use alloy_primitives::{Address, BlockNumber, B256}; use auto_impl::auto_impl; -use reth_db_api::models::BlockNumberAddress; +use reth_db::models::{AccountBeforeTx, BlockNumberAddress}; +use reth_primitives::StorageEntry; use reth_storage_errors::provider::ProviderResult; -use std::ops::{Range, RangeInclusive}; +use std::ops::{RangeBounds, RangeInclusive}; /// History Writer #[auto_impl(&, Arc, Box)] @@ -10,9 +11,17 @@ pub trait HistoryWriter: Send + Sync { /// Unwind and clear account history indices. /// /// Returns number of changesets walked. - fn unwind_account_history_indices( + fn unwind_account_history_indices<'a>( &self, - range: RangeInclusive, + changesets: impl Iterator, + ) -> ProviderResult; + + /// Unwind and clear account history indices in a given block range. + /// + /// Returns number of changesets walked. + fn unwind_account_history_indices_range( + &self, + range: impl RangeBounds, ) -> ProviderResult; /// Insert account change index to database. Used inside AccountHistoryIndex stage @@ -26,7 +35,15 @@ pub trait HistoryWriter: Send + Sync { /// Returns number of changesets walked. fn unwind_storage_history_indices( &self, - range: Range, + changesets: impl Iterator, + ) -> ProviderResult; + + /// Unwind and clear storage history indices in a given block range. + /// + /// Returns number of changesets walked. + fn unwind_storage_history_indices_range( + &self, + range: impl RangeBounds, ) -> ProviderResult; /// Insert storage change index to database. Used inside StorageHistoryIndex stage From f545877bb82724f60092ec87dbd71b668ab0132e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 29 Oct 2024 14:23:54 +0100 Subject: [PATCH 170/239] Revert "Revert "feat: add geometry to database args"" (#12165) --- book/cli/reth/db.md | 6 + book/cli/reth/db/diff.md | 6 + book/cli/reth/debug/build-block.md | 6 + book/cli/reth/debug/execution.md | 6 + book/cli/reth/debug/in-memory-merkle.md | 6 + book/cli/reth/debug/merkle.md | 6 + book/cli/reth/debug/replay-engine.md | 6 + book/cli/reth/import.md | 6 + book/cli/reth/init-state.md | 6 + book/cli/reth/init.md | 6 + book/cli/reth/node.md | 6 + book/cli/reth/p2p.md | 6 + book/cli/reth/prune.md | 6 + book/cli/reth/recover/storage-tries.md | 6 + book/cli/reth/stage/drop.md | 6 + book/cli/reth/stage/dump.md | 6 + book/cli/reth/stage/run.md | 6 + book/cli/reth/stage/unwind.md | 6 + crates/node/core/src/args/database.rs | 184 +++++++++++++++++- .../storage/db/src/implementation/mdbx/mod.rs | 48 +++-- crates/storage/libmdbx-rs/src/environment.rs | 2 + 21 files changed, 326 insertions(+), 16 deletions(-) diff --git a/book/cli/reth/db.md b/book/cli/reth/db.md index f9a8a158adc8..17a6de4e607e 100644 --- a/book/cli/reth/db.md +++ b/book/cli/reth/db.md @@ -81,6 +81,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/db/diff.md b/book/cli/reth/db/diff.md index f57c6ac364fe..efb9e7d32e30 100644 --- a/book/cli/reth/db/diff.md +++ b/book/cli/reth/db/diff.md @@ -45,6 +45,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/build-block.md b/book/cli/reth/debug/build-block.md index 2e6d637d52c2..7bceb62b940a 100644 --- a/book/cli/reth/debug/build-block.md +++ b/book/cli/reth/debug/build-block.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/execution.md b/book/cli/reth/debug/execution.md index 9ca74897c5ea..b8e1ce05d171 100644 --- a/book/cli/reth/debug/execution.md +++ b/book/cli/reth/debug/execution.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/in-memory-merkle.md b/book/cli/reth/debug/in-memory-merkle.md index 3e322a6913dd..a183db997e9d 100644 --- a/book/cli/reth/debug/in-memory-merkle.md +++ b/book/cli/reth/debug/in-memory-merkle.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/merkle.md b/book/cli/reth/debug/merkle.md index d701803b81ca..d9a72794ef2c 100644 --- a/book/cli/reth/debug/merkle.md +++ b/book/cli/reth/debug/merkle.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/debug/replay-engine.md b/book/cli/reth/debug/replay-engine.md index dd587620a868..b7a1266d3993 100644 --- a/book/cli/reth/debug/replay-engine.md +++ b/book/cli/reth/debug/replay-engine.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/import.md b/book/cli/reth/import.md index 28e085bda718..82a521ac0ab2 100644 --- a/book/cli/reth/import.md +++ b/book/cli/reth/import.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/init-state.md b/book/cli/reth/init-state.md index 3e073516778d..533c0f8f8889 100644 --- a/book/cli/reth/init-state.md +++ b/book/cli/reth/init-state.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/init.md b/book/cli/reth/init.md index cd01accc0471..ebe2a8386cf7 100644 --- a/book/cli/reth/init.md +++ b/book/cli/reth/init.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index a3ff8f6a57b1..52f597279f06 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -590,6 +590,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/p2p.md b/book/cli/reth/p2p.md index 603b451d9405..33639042a1de 100644 --- a/book/cli/reth/p2p.md +++ b/book/cli/reth/p2p.md @@ -247,6 +247,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/prune.md b/book/cli/reth/prune.md index ed16197a76c3..41684ecd9e0e 100644 --- a/book/cli/reth/prune.md +++ b/book/cli/reth/prune.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/recover/storage-tries.md b/book/cli/reth/recover/storage-tries.md index ecdaabe77817..1afe94f55dbc 100644 --- a/book/cli/reth/recover/storage-tries.md +++ b/book/cli/reth/recover/storage-tries.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/drop.md b/book/cli/reth/stage/drop.md index 399b3818c28f..c22d6be66805 100644 --- a/book/cli/reth/stage/drop.md +++ b/book/cli/reth/stage/drop.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/dump.md b/book/cli/reth/stage/dump.md index 4b3de3fb1cb5..e3df5bf2df7c 100644 --- a/book/cli/reth/stage/dump.md +++ b/book/cli/reth/stage/dump.md @@ -76,6 +76,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/run.md b/book/cli/reth/stage/run.md index 9da3ce0deb6d..204efc9685bc 100644 --- a/book/cli/reth/stage/run.md +++ b/book/cli/reth/stage/run.md @@ -69,6 +69,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index 700ab3d7e7ce..cb72b9313c0f 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -74,6 +74,12 @@ Database: [possible values: true, false] + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout diff --git a/crates/node/core/src/args/database.rs b/crates/node/core/src/args/database.rs index 0eec6639a117..5b9d6ae61e21 100644 --- a/crates/node/core/src/args/database.rs +++ b/crates/node/core/src/args/database.rs @@ -1,6 +1,6 @@ //! clap [Args](clap::Args) for database configuration -use std::time::Duration; +use std::{fmt, str::FromStr, time::Duration}; use crate::version::default_client_version; use clap::{ @@ -22,6 +22,12 @@ pub struct DatabaseArgs { /// NFS volume. #[arg(long = "db.exclusive")] pub exclusive: Option, + /// Maximum database size (e.g., 4TB, 8MB) + #[arg(long = "db.max-size", value_parser = parse_byte_size)] + pub max_size: Option, + /// Database growth step (e.g., 4GB, 4KB) + #[arg(long = "db.growth-step", value_parser = parse_byte_size)] + pub growth_step: Option, /// Read transaction timeout in seconds, 0 means no timeout. #[arg(long = "db.read-transaction-timeout")] pub read_transaction_timeout: Option, @@ -33,8 +39,9 @@ impl DatabaseArgs { self.get_database_args(default_client_version()) } - /// Returns the database arguments with configured log level and given client version. - pub const fn get_database_args( + /// Returns the database arguments with configured log level, client version, + /// max read transaction duration, and geometry. + pub fn get_database_args( &self, client_version: ClientVersion, ) -> reth_db::mdbx::DatabaseArguments { @@ -48,6 +55,8 @@ impl DatabaseArgs { .with_log_level(self.log_level) .with_exclusive(self.exclusive) .with_max_read_transaction_duration(max_read_transaction_duration) + .with_geometry_max_size(self.max_size) + .with_growth_step(self.growth_step) } } @@ -89,10 +98,84 @@ impl TypedValueParser for LogLevelValueParser { Some(Box::new(values)) } } + +/// Size in bytes. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct ByteSize(pub usize); + +impl From for usize { + fn from(s: ByteSize) -> Self { + s.0 + } +} + +impl FromStr for ByteSize { + type Err = String; + + fn from_str(s: &str) -> Result { + let s = s.trim().to_uppercase(); + let parts: Vec<&str> = s.split_whitespace().collect(); + + let (num_str, unit) = match parts.len() { + 1 => { + let (num, unit) = + s.split_at(s.find(|c: char| c.is_alphabetic()).unwrap_or(s.len())); + (num, unit) + } + 2 => (parts[0], parts[1]), + _ => { + return Err("Invalid format. Use '' or ' '.".to_string()) + } + }; + + let num: usize = num_str.parse().map_err(|_| "Invalid number".to_string())?; + + let multiplier = match unit { + "B" | "" => 1, // Assume bytes if no unit is specified + "KB" => 1024, + "MB" => 1024 * 1024, + "GB" => 1024 * 1024 * 1024, + "TB" => 1024 * 1024 * 1024 * 1024, + _ => return Err(format!("Invalid unit: {}. Use B, KB, MB, GB, or TB.", unit)), + }; + + Ok(Self(num * multiplier)) + } +} + +impl fmt::Display for ByteSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + const KB: usize = 1024; + const MB: usize = KB * 1024; + const GB: usize = MB * 1024; + const TB: usize = GB * 1024; + + let (size, unit) = if self.0 >= TB { + (self.0 as f64 / TB as f64, "TB") + } else if self.0 >= GB { + (self.0 as f64 / GB as f64, "GB") + } else if self.0 >= MB { + (self.0 as f64 / MB as f64, "MB") + } else if self.0 >= KB { + (self.0 as f64 / KB as f64, "KB") + } else { + (self.0 as f64, "B") + }; + + write!(f, "{:.2}{}", size, unit) + } +} + +/// Value parser function that supports various formats. +fn parse_byte_size(s: &str) -> Result { + s.parse::().map(Into::into) +} + #[cfg(test)] mod tests { use super::*; use clap::Parser; + use reth_db::mdbx::{GIGABYTE, KILOBYTE, MEGABYTE, TERABYTE}; /// A helper type to parse Args more easily #[derive(Parser)] @@ -108,6 +191,101 @@ mod tests { assert_eq!(args, default_args); } + #[test] + fn test_command_parser_with_valid_max_size() { + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "4398046511104", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(TERABYTE * 4)); + } + + #[test] + fn test_command_parser_with_invalid_max_size() { + let result = + CommandParser::::try_parse_from(["reth", "--db.max-size", "invalid"]); + assert!(result.is_err()); + } + + #[test] + fn test_command_parser_with_valid_growth_step() { + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.growth-step", + "4294967296", + ]) + .unwrap(); + assert_eq!(cmd.args.growth_step, Some(GIGABYTE * 4)); + } + + #[test] + fn test_command_parser_with_invalid_growth_step() { + let result = + CommandParser::::try_parse_from(["reth", "--db.growth-step", "invalid"]); + assert!(result.is_err()); + } + + #[test] + fn test_command_parser_with_valid_max_size_and_growth_step_from_str() { + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "2TB", + "--db.growth-step", + "1GB", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(TERABYTE * 2)); + assert_eq!(cmd.args.growth_step, Some(GIGABYTE)); + + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "12MB", + "--db.growth-step", + "2KB", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(MEGABYTE * 12)); + assert_eq!(cmd.args.growth_step, Some(KILOBYTE * 2)); + + // with spaces + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "12 MB", + "--db.growth-step", + "2 KB", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(MEGABYTE * 12)); + assert_eq!(cmd.args.growth_step, Some(KILOBYTE * 2)); + + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.max-size", + "1073741824", + "--db.growth-step", + "1048576", + ]) + .unwrap(); + assert_eq!(cmd.args.max_size, Some(GIGABYTE)); + assert_eq!(cmd.args.growth_step, Some(MEGABYTE)); + } + + #[test] + fn test_command_parser_max_size_and_growth_step_from_str_invalid_unit() { + let result = + CommandParser::::try_parse_from(["reth", "--db.growth-step", "1 PB"]); + assert!(result.is_err()); + + let result = + CommandParser::::try_parse_from(["reth", "--db.max-size", "2PB"]); + assert!(result.is_err()); + } + #[test] fn test_possible_values() { // Initialize the LogLevelValueParser diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 65b804e6a58d..78a3f7971dab 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -23,7 +23,7 @@ use reth_libmdbx::{ use reth_storage_errors::db::LogLevel; use reth_tracing::tracing::error; use std::{ - ops::Deref, + ops::{Deref, Range}, path::Path, sync::Arc, time::{SystemTime, UNIX_EPOCH}, @@ -33,8 +33,14 @@ use tx::Tx; pub mod cursor; pub mod tx; -const GIGABYTE: usize = 1024 * 1024 * 1024; -const TERABYTE: usize = GIGABYTE * 1024; +/// 1 KB in bytes +pub const KILOBYTE: usize = 1024; +/// 1 MB in bytes +pub const MEGABYTE: usize = KILOBYTE * 1024; +/// 1 GB in bytes +pub const GIGABYTE: usize = MEGABYTE * 1024; +/// 1 TB in bytes +pub const TERABYTE: usize = GIGABYTE * 1024; /// MDBX allows up to 32767 readers (`MDBX_READERS_LIMIT`), but we limit it to slightly below that const DEFAULT_MAX_READERS: u64 = 32_000; @@ -64,6 +70,8 @@ impl DatabaseEnvKind { pub struct DatabaseArguments { /// Client version that accesses the database. client_version: ClientVersion, + /// Database geometry settings. + geometry: Geometry>, /// Database log level. If [None], the default value is used. log_level: Option, /// Maximum duration of a read transaction. If [None], the default value is used. @@ -93,15 +101,37 @@ pub struct DatabaseArguments { impl DatabaseArguments { /// Create new database arguments with given client version. - pub const fn new(client_version: ClientVersion) -> Self { + pub fn new(client_version: ClientVersion) -> Self { Self { client_version, + geometry: Geometry { + size: Some(0..(4 * TERABYTE)), + growth_step: Some(4 * GIGABYTE as isize), + shrink_threshold: Some(0), + page_size: Some(PageSize::Set(default_page_size())), + }, log_level: None, max_read_transaction_duration: None, exclusive: None, } } + /// Sets the upper size limit of the db environment, the maximum database size in bytes. + pub const fn with_geometry_max_size(mut self, max_size: Option) -> Self { + if let Some(max_size) = max_size { + self.geometry.size = Some(0..max_size); + } + self + } + + /// Configures the database growth step in bytes. + pub const fn with_growth_step(mut self, growth_step: Option) -> Self { + if let Some(growth_step) = growth_step { + self.geometry.growth_step = Some(growth_step as isize); + } + self + } + /// Set the log level. pub const fn with_log_level(mut self, log_level: Option) -> Self { self.log_level = log_level; @@ -278,15 +308,7 @@ impl DatabaseEnv { // environment creation. debug_assert!(Tables::ALL.len() <= 256, "number of tables exceed max dbs"); inner_env.set_max_dbs(256); - inner_env.set_geometry(Geometry { - // Maximum database size of 4 terabytes - size: Some(0..(4 * TERABYTE)), - // We grow the database in increments of 4 gigabytes - growth_step: Some(4 * GIGABYTE as isize), - // The database never shrinks - shrink_threshold: Some(0), - page_size: Some(PageSize::Set(default_page_size())), - }); + inner_env.set_geometry(args.geometry); fn is_current_process(id: u32) -> bool { #[cfg(unix)] diff --git a/crates/storage/libmdbx-rs/src/environment.rs b/crates/storage/libmdbx-rs/src/environment.rs index 480f5aaab65c..6a0b210401e4 100644 --- a/crates/storage/libmdbx-rs/src/environment.rs +++ b/crates/storage/libmdbx-rs/src/environment.rs @@ -489,8 +489,10 @@ pub struct PageOps { pub mincore: u64, } +/// Represents the geometry settings for the database environment #[derive(Clone, Debug, PartialEq, Eq)] pub struct Geometry { + /// The size range in bytes. pub size: Option, pub growth_step: Option, pub shrink_threshold: Option, From e92ecfbc220929f6bddb283b32896c8257a82748 Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Tue, 29 Oct 2024 21:34:12 +0700 Subject: [PATCH 171/239] feat: Introduce trait for OpTransaction (#11745) Co-authored-by: Matthias Seitz --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/optimism/evm/src/execute.rs | 1 + crates/optimism/payload/Cargo.toml | 5 +- crates/optimism/payload/src/builder.rs | 1 + crates/optimism/rpc/src/eth/receipt.rs | 4 +- crates/optimism/rpc/src/eth/transaction.rs | 1 + crates/primitives/src/transaction/compat.rs | 2 + crates/primitives/src/transaction/mod.rs | 63 +++++++++------------ 9 files changed, 40 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8febcffa52c1..5b62244576ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8260,6 +8260,7 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", + "op-alloy-consensus", "op-alloy-rpc-types-engine", "reth-basic-payload-builder", "reth-chain-state", diff --git a/Cargo.toml b/Cargo.toml index c83d76318e15..8c5cf7b0cd5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -633,4 +633,4 @@ tracy-client = "0.17.3" #op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", rev = "6a042e7681b1" } #op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", rev = "6a042e7681b1" } #op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", rev = "6a042e7681b1" } -#op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", rev = "6a042e7681b1" } +#op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", rev = "6a042e7681b1" } \ No newline at end of file diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 1cd92409847e..9c5db9d4b614 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -5,6 +5,7 @@ use alloc::{boxed::Box, sync::Arc, vec::Vec}; use alloy_consensus::Transaction as _; use alloy_eips::eip7685::Requests; use core::fmt::Display; +use op_alloy_consensus::DepositTransaction; use reth_chainspec::EthereumHardforks; use reth_consensus::ConsensusError; use reth_evm::{ diff --git a/crates/optimism/payload/Cargo.toml b/crates/optimism/payload/Cargo.toml index ba0b105e8329..4b8e64f2dba3 100644 --- a/crates/optimism/payload/Cargo.toml +++ b/crates/optimism/payload/Cargo.toml @@ -39,6 +39,7 @@ alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true op-alloy-rpc-types-engine.workspace = true +op-alloy-consensus.workspace = true alloy-rpc-types-engine.workspace = true alloy-consensus.workspace = true @@ -54,5 +55,5 @@ optimism = [ "reth-optimism-evm/optimism", "revm/optimism", "reth-execution-types/optimism", - "reth-optimism-consensus/optimism", -] + "reth-optimism-consensus/optimism" +] \ No newline at end of file diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index a1536ccf8fad..0550adeaa8d5 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -36,6 +36,7 @@ use crate::{ error::OptimismPayloadBuilderError, payload::{OptimismBuiltPayload, OptimismPayloadBuilderAttributes}, }; +use op_alloy_consensus::DepositTransaction; /// Optimism's payload builder #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index 3f2a81573e2f..2734fb5458c8 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -2,7 +2,9 @@ use alloy_eips::eip2718::Encodable2718; use alloy_rpc_types::{AnyReceiptEnvelope, Log, TransactionReceipt}; -use op_alloy_consensus::{OpDepositReceipt, OpDepositReceiptWithBloom, OpReceiptEnvelope}; +use op_alloy_consensus::{ + DepositTransaction, OpDepositReceipt, OpDepositReceiptWithBloom, OpReceiptEnvelope, +}; use op_alloy_rpc_types::{receipt::L1BlockInfo, OpTransactionReceipt, OpTransactionReceiptFields}; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_optimism_chainspec::OpChainSpec; diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 3345ac5d4521..3994afe19841 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -3,6 +3,7 @@ use alloy_consensus::Transaction as _; use alloy_primitives::{Bytes, B256}; use alloy_rpc_types::TransactionInfo; +use op_alloy_consensus::DepositTransaction; use op_alloy_rpc_types::Transaction; use reth_node_api::FullNodeComponents; use reth_primitives::TransactionSignedEcRecovered; diff --git a/crates/primitives/src/transaction/compat.rs b/crates/primitives/src/transaction/compat.rs index 81281186f64c..883c89c45f51 100644 --- a/crates/primitives/src/transaction/compat.rs +++ b/crates/primitives/src/transaction/compat.rs @@ -1,5 +1,7 @@ use crate::{Transaction, TransactionSigned}; use alloy_primitives::{Address, TxKind, U256}; +#[cfg(feature = "optimism")] +use op_alloy_consensus::DepositTransaction; use revm_primitives::{AuthorizationList, TxEnv}; /// Implements behaviour to fill a [`TxEnv`] from another transaction. diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 3a5c3674166c..e81c5ad332c3 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -18,6 +18,8 @@ use derive_more::{AsRef, Deref}; use once_cell as _; #[cfg(not(feature = "std"))] use once_cell::sync::Lazy as LazyLock; +#[cfg(feature = "optimism")] +use op_alloy_consensus::DepositTransaction; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use signature::{decode_with_eip155_chain_id, with_eip155_parity}; @@ -136,6 +138,31 @@ pub enum Transaction { Deposit(TxDeposit), } +#[cfg(feature = "optimism")] +impl DepositTransaction for Transaction { + fn source_hash(&self) -> Option { + match self { + Self::Deposit(tx) => tx.source_hash(), + _ => None, + } + } + fn mint(&self) -> Option { + match self { + Self::Deposit(tx) => tx.mint(), + _ => None, + } + } + fn is_system_transaction(&self) -> bool { + match self { + Self::Deposit(tx) => tx.is_system_transaction(), + _ => false, + } + } + fn is_deposit(&self) -> bool { + matches!(self, Self::Deposit(_)) + } +} + #[cfg(any(test, feature = "arbitrary"))] impl<'a> arbitrary::Arbitrary<'a> for Transaction { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { @@ -361,42 +388,6 @@ impl Transaction { } } - /// Returns the source hash of the transaction, which uniquely identifies its source. - /// If not a deposit transaction, this will always return `None`. - #[cfg(feature = "optimism")] - pub const fn source_hash(&self) -> Option { - match self { - Self::Deposit(TxDeposit { source_hash, .. }) => Some(*source_hash), - _ => None, - } - } - - /// Returns the amount of ETH locked up on L1 that will be minted on L2. If the transaction - /// is not a deposit transaction, this will always return `None`. - #[cfg(feature = "optimism")] - pub const fn mint(&self) -> Option { - match self { - Self::Deposit(TxDeposit { mint, .. }) => *mint, - _ => None, - } - } - - /// Returns whether or not the transaction is a system transaction. If the transaction - /// is not a deposit transaction, this will always return `false`. - #[cfg(feature = "optimism")] - pub const fn is_system_transaction(&self) -> bool { - match self { - Self::Deposit(TxDeposit { is_system_transaction, .. }) => *is_system_transaction, - _ => false, - } - } - - /// Returns whether or not the transaction is an Optimism Deposited transaction. - #[cfg(feature = "optimism")] - pub const fn is_deposit(&self) -> bool { - matches!(self, Self::Deposit(_)) - } - /// This encodes the transaction _without_ the signature, and is only suitable for creating a /// hash intended for signing. pub fn encode_without_signature(&self, out: &mut dyn bytes::BufMut) { From 58f24e0056b8113b4762796c005b2a73a75a42c9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 29 Oct 2024 15:35:42 +0100 Subject: [PATCH 172/239] chore: remove generate sidecar fn (#12167) --- crates/primitives/src/transaction/mod.rs | 2 -- crates/primitives/src/transaction/sidecar.rs | 37 +++----------------- crates/transaction-pool/src/pool/mod.rs | 5 +-- 3 files changed, 7 insertions(+), 37 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index e81c5ad332c3..2e0a786fc1a8 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -31,8 +31,6 @@ pub use error::{ }; pub use meta::TransactionMeta; pub use pooled::{PooledTransactionsElement, PooledTransactionsElementEcRecovered}; -#[cfg(all(feature = "c-kzg", any(test, feature = "arbitrary")))] -pub use sidecar::generate_blob_sidecar; #[cfg(feature = "c-kzg")] pub use sidecar::BlobTransactionValidationError; pub use sidecar::{BlobTransaction, BlobTransactionSidecar}; diff --git a/crates/primitives/src/transaction/sidecar.rs b/crates/primitives/src/transaction/sidecar.rs index e901cbfc08dc..1e6560e152be 100644 --- a/crates/primitives/src/transaction/sidecar.rs +++ b/crates/primitives/src/transaction/sidecar.rs @@ -191,35 +191,6 @@ impl BlobTransaction { } } -/// Generates a [`BlobTransactionSidecar`] structure containing blobs, commitments, and proofs. -#[cfg(all(feature = "c-kzg", any(test, feature = "arbitrary")))] -pub fn generate_blob_sidecar(blobs: Vec) -> BlobTransactionSidecar { - use alloc::vec::Vec; - use alloy_eips::eip4844::env_settings::EnvKzgSettings; - use c_kzg::{KzgCommitment, KzgProof}; - - let kzg_settings = EnvKzgSettings::Default; - - let commitments: Vec = blobs - .iter() - .map(|blob| { - KzgCommitment::blob_to_kzg_commitment(&blob.clone(), kzg_settings.get()).unwrap() - }) - .map(|commitment| commitment.to_bytes()) - .collect(); - - let proofs: Vec = blobs - .iter() - .zip(commitments.iter()) - .map(|(blob, commitment)| { - KzgProof::compute_blob_kzg_proof(blob, commitment, kzg_settings.get()).unwrap() - }) - .map(|proof| proof.to_bytes()) - .collect(); - - BlobTransactionSidecar::from_kzg(blobs, commitments, proofs) -} - #[cfg(all(test, feature = "c-kzg"))] mod tests { use super::*; @@ -251,7 +222,7 @@ mod tests { .unwrap()]; // Generate a BlobTransactionSidecar from the blobs - let sidecar = generate_blob_sidecar(blobs); + let sidecar = BlobTransactionSidecar::try_from_blobs(blobs).unwrap(); // Assert commitment equality assert_eq!( @@ -300,7 +271,7 @@ mod tests { } // Generate a BlobTransactionSidecar from the blobs - let sidecar = generate_blob_sidecar(blobs.clone()); + let sidecar = BlobTransactionSidecar::try_from_blobs(blobs).unwrap(); // Assert sidecar size assert_eq!(sidecar.size(), 524672); @@ -325,7 +296,7 @@ mod tests { .unwrap()]; // Generate a BlobTransactionSidecar from the blobs - let sidecar = generate_blob_sidecar(blobs); + let sidecar = BlobTransactionSidecar::try_from_blobs(blobs).unwrap(); // Create a vector to store the encoded RLP let mut encoded_rlp = Vec::new(); @@ -356,7 +327,7 @@ mod tests { .unwrap()]; // Generate a BlobTransactionSidecar from the blobs - let sidecar = generate_blob_sidecar(blobs); + let sidecar = BlobTransactionSidecar::try_from_blobs(blobs).unwrap(); // Create a vector to store the encoded RLP let mut encoded_rlp = Vec::new(); diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 69f17504f79f..fef0fd0eea03 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -1236,7 +1236,8 @@ mod tests { validate::ValidTransaction, BlockInfo, PoolConfig, SubPoolLimit, TransactionOrigin, TransactionValidationOutcome, U256, }; - use reth_primitives::{kzg::Blob, transaction::generate_blob_sidecar}; + use alloy_eips::eip4844::BlobTransactionSidecar; + use reth_primitives::kzg::Blob; use std::{fs, path::PathBuf}; #[test] @@ -1271,7 +1272,7 @@ mod tests { .unwrap()]; // Generate a BlobTransactionSidecar from the blobs. - let sidecar = generate_blob_sidecar(blobs); + let sidecar = BlobTransactionSidecar::try_from_blobs(blobs).unwrap(); // Create an in-memory blob store. let blob_store = InMemoryBlobStore::default(); From 1006ce78c530088fd103bf9ffe14cc3ae6123ce9 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 29 Oct 2024 23:37:41 +0900 Subject: [PATCH 173/239] feat(provider): use `NodeTypes` on `DatabaseProvider` instead (#12166) --- crates/stages/stages/benches/criterion.rs | 6 +- crates/stages/stages/benches/setup/mod.rs | 11 +- .../stages/src/stages/hashing_account.rs | 7 +- crates/stages/stages/src/test_utils/runner.rs | 5 +- .../provider/src/providers/consistent.rs | 8 +- .../provider/src/providers/database/mod.rs | 8 +- .../src/providers/database/provider.rs | 151 +++++++++--------- .../storage/provider/src/test_utils/blocks.rs | 5 +- .../storage/provider/src/test_utils/mock.rs | 14 +- crates/trie/db/tests/trie.rs | 11 +- 10 files changed, 118 insertions(+), 108 deletions(-) diff --git a/crates/stages/stages/benches/criterion.rs b/crates/stages/stages/benches/criterion.rs index 7519d81a3622..0f876dd7011a 100644 --- a/crates/stages/stages/benches/criterion.rs +++ b/crates/stages/stages/benches/criterion.rs @@ -2,12 +2,11 @@ use criterion::{criterion_main, measurement::WallTime, BenchmarkGroup, Criterion}; #[cfg(not(target_os = "windows"))] use pprof::criterion::{Output, PProfProfiler}; -use reth_chainspec::ChainSpec; use reth_config::config::{EtlConfig, TransactionLookupConfig}; use reth_db::{test_utils::TempDatabase, Database, DatabaseEnv}; use alloy_primitives::BlockNumber; -use reth_provider::{DatabaseProvider, DatabaseProviderFactory}; +use reth_provider::{test_utils::MockNodeTypesWithDB, DatabaseProvider, DatabaseProviderFactory}; use reth_stages::{ stages::{MerkleStage, SenderRecoveryStage, TransactionLookupStage}, test_utils::TestStageDB, @@ -148,7 +147,8 @@ fn measure_stage( block_interval: RangeInclusive, label: String, ) where - S: Clone + Stage as Database>::TXMut, ChainSpec>>, + S: Clone + + Stage as Database>::TXMut, MockNodeTypesWithDB>>, F: Fn(S, &TestStageDB, StageRange), { let stage_range = ( diff --git a/crates/stages/stages/benches/setup/mod.rs b/crates/stages/stages/benches/setup/mod.rs index 4812fb13c39a..e6ae33f9c299 100644 --- a/crates/stages/stages/benches/setup/mod.rs +++ b/crates/stages/stages/benches/setup/mod.rs @@ -1,14 +1,15 @@ #![allow(unreachable_pub)] use alloy_primitives::{Address, Sealable, B256, U256}; use itertools::concat; -use reth_chainspec::ChainSpec; use reth_db::{tables, test_utils::TempDatabase, Database, DatabaseEnv}; use reth_db_api::{ cursor::DbCursorRO, transaction::{DbTx, DbTxMut}, }; use reth_primitives::{Account, SealedBlock, SealedHeader}; -use reth_provider::{DatabaseProvider, DatabaseProviderFactory, TrieWriter}; +use reth_provider::{ + test_utils::MockNodeTypesWithDB, DatabaseProvider, DatabaseProviderFactory, TrieWriter, +}; use reth_stages::{ stages::{AccountHashingStage, StorageHashingStage}, test_utils::{StorageKind, TestStageDB}, @@ -31,7 +32,8 @@ use reth_trie_db::DatabaseStateRoot; pub(crate) type StageRange = (ExecInput, UnwindInput); pub(crate) fn stage_unwind< - S: Clone + Stage as Database>::TXMut, ChainSpec>>, + S: Clone + + Stage as Database>::TXMut, MockNodeTypesWithDB>>, >( stage: S, db: &TestStageDB, @@ -63,7 +65,8 @@ pub(crate) fn stage_unwind< pub(crate) fn unwind_hashes(stage: S, db: &TestStageDB, range: StageRange) where - S: Clone + Stage as Database>::TXMut, ChainSpec>>, + S: Clone + + Stage as Database>::TXMut, MockNodeTypesWithDB>>, { let (input, unwind) = range; diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 5b4f720972f1..1ca0e1aa1325 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -58,11 +58,8 @@ impl AccountHashingStage { /// /// Proceeds to go to the `BlockTransitionIndex` end, go back `transitions` and change the /// account state in the `AccountChangeSets` table. - pub fn seed< - Tx: DbTx + DbTxMut + 'static, - Spec: Send + Sync + 'static + reth_chainspec::EthereumHardforks, - >( - provider: &reth_provider::DatabaseProvider, + pub fn seed( + provider: &reth_provider::DatabaseProvider, opts: SeedOpts, ) -> Result, StageError> { use alloy_primitives::U256; diff --git a/crates/stages/stages/src/test_utils/runner.rs b/crates/stages/stages/src/test_utils/runner.rs index 26f245c1304d..c3d25b99536a 100644 --- a/crates/stages/stages/src/test_utils/runner.rs +++ b/crates/stages/stages/src/test_utils/runner.rs @@ -1,7 +1,6 @@ use super::TestStageDB; -use reth_chainspec::ChainSpec; use reth_db::{test_utils::TempDatabase, Database, DatabaseEnv}; -use reth_provider::{DatabaseProvider, ProviderError}; +use reth_provider::{test_utils::MockNodeTypesWithDB, DatabaseProvider, ProviderError}; use reth_stages_api::{ ExecInput, ExecOutput, Stage, StageError, StageExt, UnwindInput, UnwindOutput, }; @@ -20,7 +19,7 @@ pub(crate) enum TestRunnerError { /// A generic test runner for stages. pub(crate) trait StageTestRunner { - type S: Stage as Database>::TXMut, ChainSpec>> + type S: Stage as Database>::TXMut, MockNodeTypesWithDB>> + 'static; /// Return a reference to the database. diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index d6847fa1b8f6..e6ca1a919324 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -296,7 +296,7 @@ impl ConsistentProvider { ) -> ProviderResult> where F: FnOnce( - &DatabaseProviderRO, + &DatabaseProviderRO, RangeInclusive, &mut P, ) -> ProviderResult>, @@ -413,7 +413,7 @@ impl ConsistentProvider { ) -> ProviderResult> where S: FnOnce( - &DatabaseProviderRO, + &DatabaseProviderRO, RangeInclusive, ) -> ProviderResult>, M: Fn(RangeInclusive, &BlockState) -> ProviderResult>, @@ -511,7 +511,7 @@ impl ConsistentProvider { fetch_from_block_state: M, ) -> ProviderResult> where - S: FnOnce(&DatabaseProviderRO) -> ProviderResult>, + S: FnOnce(&DatabaseProviderRO) -> ProviderResult>, M: Fn(usize, TxNumber, &BlockState) -> ProviderResult>, { let in_mem_chain = self.head_block.iter().flat_map(|b| b.chain()).collect::>(); @@ -578,7 +578,7 @@ impl ConsistentProvider { fetch_from_block_state: M, ) -> ProviderResult where - S: FnOnce(&DatabaseProviderRO) -> ProviderResult, + S: FnOnce(&DatabaseProviderRO) -> ProviderResult, M: Fn(&BlockState) -> ProviderResult, { if let Some(Some(block_state)) = self.head_block.as_ref().map(|b| b.block_on_chain(id)) { diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 54186dca6f69..04a30ce90aaa 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -130,7 +130,7 @@ impl ProviderFactory { /// This sets the [`PruneModes`] to [`None`], because they should only be relevant for writing /// data. #[track_caller] - pub fn provider(&self) -> ProviderResult> { + pub fn provider(&self) -> ProviderResult> { Ok(DatabaseProvider::new( self.db.tx()?, self.chain_spec.clone(), @@ -144,7 +144,7 @@ impl ProviderFactory { /// [`BlockHashReader`]. This may fail if the inner read/write database transaction fails to /// open. #[track_caller] - pub fn provider_rw(&self) -> ProviderResult> { + pub fn provider_rw(&self) -> ProviderResult> { Ok(DatabaseProviderRW(DatabaseProvider::new_rw( self.db.tx_mut()?, self.chain_spec.clone(), @@ -186,8 +186,8 @@ impl ProviderFactory { impl DatabaseProviderFactory for ProviderFactory { type DB = N::DB; - type Provider = DatabaseProvider<::TX, N::ChainSpec>; - type ProviderRW = DatabaseProvider<::TXMut, N::ChainSpec>; + type Provider = DatabaseProvider<::TX, N>; + type ProviderRW = DatabaseProvider<::TXMut, N>; fn database_provider_ro(&self) -> ProviderResult { self.provider() diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 2dcc3f92d705..81affa0d804f 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -38,6 +38,7 @@ use reth_db_api::{ use reth_evm::ConfigureEvmEnv; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_network_p2p::headers::downloader::SyncTarget; +use reth_node_types::NodeTypes; use reth_primitives::{ Account, Block, BlockBody, BlockWithSenders, Bytecode, GotExpected, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment, StorageEntry, @@ -70,40 +71,40 @@ use tokio::sync::watch; use tracing::{debug, error, trace, warn}; /// A [`DatabaseProvider`] that holds a read-only database transaction. -pub type DatabaseProviderRO = DatabaseProvider<::TX, Spec>; +pub type DatabaseProviderRO = DatabaseProvider<::TX, N>; /// A [`DatabaseProvider`] that holds a read-write database transaction. /// /// Ideally this would be an alias type. However, there's some weird compiler error (), that forces us to wrap this in a struct instead. /// Once that issue is solved, we can probably revert back to being an alias type. #[derive(Debug)] -pub struct DatabaseProviderRW( - pub DatabaseProvider<::TXMut, Spec>, +pub struct DatabaseProviderRW( + pub DatabaseProvider<::TXMut, N>, ); -impl Deref for DatabaseProviderRW { - type Target = DatabaseProvider<::TXMut, Spec>; +impl Deref for DatabaseProviderRW { + type Target = DatabaseProvider<::TXMut, N>; fn deref(&self) -> &Self::Target { &self.0 } } -impl DerefMut for DatabaseProviderRW { +impl DerefMut for DatabaseProviderRW { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } } -impl AsRef::TXMut, Spec>> - for DatabaseProviderRW +impl AsRef::TXMut, N>> + for DatabaseProviderRW { - fn as_ref(&self) -> &DatabaseProvider<::TXMut, Spec> { + fn as_ref(&self) -> &DatabaseProvider<::TXMut, N> { &self.0 } } -impl DatabaseProviderRW { +impl DatabaseProviderRW { /// Commit database transaction and static file if it exists. pub fn commit(self) -> ProviderResult { self.0.commit() @@ -115,10 +116,10 @@ impl DatabaseProviderRW { } } -impl From> - for DatabaseProvider<::TXMut, Spec> +impl From> + for DatabaseProvider<::TXMut, N> { - fn from(provider: DatabaseProviderRW) -> Self { + fn from(provider: DatabaseProviderRW) -> Self { provider.0 } } @@ -126,25 +127,25 @@ impl From> /// A provider struct that fetches data from the database. /// Wrapper around [`DbTx`] and [`DbTxMut`]. Example: [`HeaderProvider`] [`BlockHashReader`] #[derive(Debug)] -pub struct DatabaseProvider { +pub struct DatabaseProvider { /// Database transaction. tx: TX, /// Chain spec - chain_spec: Arc, + chain_spec: Arc, /// Static File provider static_file_provider: StaticFileProvider, /// Pruning configuration prune_modes: PruneModes, } -impl DatabaseProvider { +impl DatabaseProvider { /// Returns reference to prune modes. pub const fn prune_modes_ref(&self) -> &PruneModes { &self.prune_modes } } -impl DatabaseProvider { +impl DatabaseProvider { /// State provider for latest block pub fn latest<'a>(&'a self) -> ProviderResult> { trace!(target: "providers::db", "Returning latest state provider"); @@ -202,28 +203,28 @@ impl DatabaseProvider { } } -impl StaticFileProviderFactory for DatabaseProvider { +impl StaticFileProviderFactory for DatabaseProvider { /// Returns a static file provider fn static_file_provider(&self) -> StaticFileProvider { self.static_file_provider.clone() } } -impl ChainSpecProvider - for DatabaseProvider +impl> ChainSpecProvider + for DatabaseProvider { - type ChainSpec = Spec; + type ChainSpec = N::ChainSpec; fn chain_spec(&self) -> Arc { self.chain_spec.clone() } } -impl DatabaseProvider { +impl DatabaseProvider { /// Creates a provider with an inner read-write transaction. pub const fn new_rw( tx: TX, - chain_spec: Arc, + chain_spec: Arc, static_file_provider: StaticFileProvider, prune_modes: PruneModes, ) -> Self { @@ -231,15 +232,13 @@ impl DatabaseProvider { } } -impl AsRef for DatabaseProvider { +impl AsRef for DatabaseProvider { fn as_ref(&self) -> &Self { self } } -impl TryIntoHistoricalStateProvider - for DatabaseProvider -{ +impl TryIntoHistoricalStateProvider for DatabaseProvider { fn try_into_history_at_block( self, mut block_number: BlockNumber, @@ -282,8 +281,8 @@ impl TryIntoHistoricalStateProvider } } -impl - DatabaseProvider +impl + 'static> + DatabaseProvider { // TODO: uncomment below, once `reth debug_cmd` has been feature gated with dev. // #[cfg(any(test, feature = "test-utils"))] @@ -365,11 +364,11 @@ where Ok(Vec::new()) } -impl DatabaseProvider { +impl DatabaseProvider { /// Creates a provider with an inner read-only transaction. pub const fn new( tx: TX, - chain_spec: Arc, + chain_spec: Arc, static_file_provider: StaticFileProvider, prune_modes: PruneModes, ) -> Self { @@ -392,7 +391,7 @@ impl DatabaseProvider { } /// Returns a reference to the chain specification. - pub fn chain_spec(&self) -> &Spec { + pub fn chain_spec(&self) -> &N::ChainSpec { &self.chain_spec } @@ -490,7 +489,7 @@ impl DatabaseProvider { construct_block: BF, ) -> ProviderResult> where - Spec: EthereumHardforks, + N::ChainSpec: EthereumHardforks, H: AsRef
, HF: FnOnce(BlockNumber) -> ProviderResult>, BF: FnOnce( @@ -556,7 +555,7 @@ impl DatabaseProvider { mut assemble_block: F, ) -> ProviderResult> where - Spec: EthereumHardforks, + N::ChainSpec: EthereumHardforks, H: AsRef
, HF: FnOnce(RangeInclusive) -> ProviderResult>, F: FnMut(H, Range, Vec
, Option) -> ProviderResult, @@ -633,7 +632,7 @@ impl DatabaseProvider { assemble_block: BF, ) -> ProviderResult> where - Spec: EthereumHardforks, + N::ChainSpec: EthereumHardforks, H: AsRef
, HF: Fn(RangeInclusive) -> ProviderResult>, BF: Fn( @@ -853,7 +852,7 @@ impl DatabaseProvider { } } -impl DatabaseProvider { +impl DatabaseProvider { /// Commit database transaction. pub fn commit(self) -> ProviderResult { Ok(self.tx.commit()?) @@ -1079,7 +1078,7 @@ impl DatabaseProvider { range: impl RangeBounds + Clone, ) -> ProviderResult> where - Spec: EthereumHardforks, + N::ChainSpec: EthereumHardforks, { // For blocks we need: // @@ -1218,13 +1217,13 @@ impl DatabaseProvider { } } -impl AccountReader for DatabaseProvider { +impl AccountReader for DatabaseProvider { fn basic_account(&self, address: Address) -> ProviderResult> { Ok(self.tx.get::(address)?) } } -impl AccountExtReader for DatabaseProvider { +impl AccountExtReader for DatabaseProvider { fn changed_accounts_with_range( &self, range: impl RangeBounds, @@ -1268,7 +1267,7 @@ impl AccountExtReader for DatabaseProvider StorageChangeSetReader for DatabaseProvider { +impl StorageChangeSetReader for DatabaseProvider { fn storage_changeset( &self, block_number: BlockNumber, @@ -1283,7 +1282,7 @@ impl StorageChangeSetReader for DatabaseProvider ChangeSetReader for DatabaseProvider { +impl ChangeSetReader for DatabaseProvider { fn account_block_changeset( &self, block_number: BlockNumber, @@ -1300,7 +1299,7 @@ impl ChangeSetReader for DatabaseProvider } } -impl HeaderSyncGapProvider for DatabaseProvider { +impl HeaderSyncGapProvider for DatabaseProvider { fn sync_gap( &self, tip: watch::Receiver, @@ -1344,8 +1343,8 @@ impl HeaderSyncGapProvider for DatabaseProvider HeaderProvider - for DatabaseProvider +impl> HeaderProvider + for DatabaseProvider { fn header(&self, block_hash: &BlockHash) -> ProviderResult> { if let Some(num) = self.block_number(*block_hash)? { @@ -1444,7 +1443,7 @@ impl HeaderProvider } } -impl BlockHashReader for DatabaseProvider { +impl BlockHashReader for DatabaseProvider { fn block_hash(&self, number: u64) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( StaticFileSegment::Headers, @@ -1471,7 +1470,7 @@ impl BlockHashReader for DatabaseProvider } } -impl BlockNumReader for DatabaseProvider { +impl BlockNumReader for DatabaseProvider { fn chain_info(&self) -> ProviderResult { let best_number = self.best_block_number()?; let best_hash = self.block_hash(best_number)?.unwrap_or_default(); @@ -1502,7 +1501,7 @@ impl BlockNumReader for DatabaseProvider } } -impl BlockReader for DatabaseProvider { +impl> BlockReader for DatabaseProvider { fn find_block_by_hash(&self, hash: B256, source: BlockSource) -> ProviderResult> { if source.is_canonical() { self.block(hash.into()) @@ -1677,8 +1676,8 @@ impl BlockReader for DatabasePr } } -impl TransactionsProviderExt - for DatabaseProvider +impl> TransactionsProviderExt + for DatabaseProvider { /// Recovers transaction hashes by walking through `Transactions` table and /// calculating them in a parallel manner. Returned unsorted. @@ -1747,8 +1746,8 @@ impl TransactionsProviderExt } // Calculates the hash of the given transaction -impl TransactionsProvider - for DatabaseProvider +impl> TransactionsProvider + for DatabaseProvider { fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult> { Ok(self.tx.get::(tx_hash)?) @@ -1907,8 +1906,8 @@ impl TransactionsProvider } } -impl ReceiptProvider - for DatabaseProvider +impl> ReceiptProvider + for DatabaseProvider { fn receipt(&self, id: TxNumber) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( @@ -1955,8 +1954,8 @@ impl ReceiptProvider } } -impl WithdrawalsProvider - for DatabaseProvider +impl> WithdrawalsProvider + for DatabaseProvider { fn withdrawals_by_block( &self, @@ -1985,8 +1984,8 @@ impl WithdrawalsProvider } } -impl EvmEnvProvider - for DatabaseProvider +impl> EvmEnvProvider + for DatabaseProvider { fn fill_env_at( &self, @@ -2051,7 +2050,7 @@ impl EvmEnvProvider } } -impl StageCheckpointReader for DatabaseProvider { +impl StageCheckpointReader for DatabaseProvider { fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult> { Ok(self.tx.get::(id.to_string())?) } @@ -2070,7 +2069,7 @@ impl StageCheckpointReader for DatabaseProvider StageCheckpointWriter for DatabaseProvider { +impl StageCheckpointWriter for DatabaseProvider { /// Save stage checkpoint. fn save_stage_checkpoint( &self, @@ -2111,7 +2110,7 @@ impl StageCheckpointWriter for DatabaseProvider< } } -impl StorageReader for DatabaseProvider { +impl StorageReader for DatabaseProvider { fn plain_state_storages( &self, addresses_with_keys: impl IntoIterator)>, @@ -2174,7 +2173,7 @@ impl StorageReader for DatabaseProvider { } } -impl StateChangeWriter for DatabaseProvider { +impl StateChangeWriter for DatabaseProvider { fn write_state_reverts( &self, reverts: PlainStateReverts, @@ -2551,7 +2550,7 @@ impl StateChangeWriter for DatabaseProvid } } -impl TrieWriter for DatabaseProvider { +impl TrieWriter for DatabaseProvider { /// Writes trie updates. Returns the number of entries modified. fn write_trie_updates(&self, trie_updates: &TrieUpdates) -> ProviderResult { if trie_updates.is_empty() { @@ -2601,7 +2600,7 @@ impl TrieWriter for DatabaseProvider StorageTrieWriter for DatabaseProvider { +impl StorageTrieWriter for DatabaseProvider { /// Writes storage trie updates from the given storage trie map. First sorts the storage trie /// updates by the hashed address, writing in sorted order. fn write_storage_trie_updates( @@ -2638,7 +2637,7 @@ impl StorageTrieWriter for DatabaseProvid } } -impl HashingWriter for DatabaseProvider { +impl HashingWriter for DatabaseProvider { fn unwind_account_hashing<'a>( &self, changesets: impl Iterator, @@ -2863,7 +2862,7 @@ impl HashingWriter for DatabaseProvider HistoryWriter for DatabaseProvider { +impl HistoryWriter for DatabaseProvider { fn unwind_account_history_indices<'a>( &self, changesets: impl Iterator, @@ -2997,14 +2996,14 @@ impl HistoryWriter for DatabaseProvider StateReader for DatabaseProvider { +impl StateReader for DatabaseProvider { fn get_state(&self, block: BlockNumber) -> ProviderResult> { self.get_state(block..=block) } } -impl - BlockExecutionWriter for DatabaseProvider +impl + 'static> + BlockExecutionWriter for DatabaseProvider { fn take_block_and_execution_range( &self, @@ -3205,8 +3204,8 @@ impl BlockWriter - for DatabaseProvider +impl + 'static> BlockWriter + for DatabaseProvider { /// Inserts the block into the database, always modifying the following tables: /// * [`CanonicalHeaders`](tables::CanonicalHeaders) @@ -3418,7 +3417,7 @@ impl PruneCheckpointReader for DatabaseProvider { +impl PruneCheckpointReader for DatabaseProvider { fn get_prune_checkpoint( &self, segment: PruneSegment, @@ -3435,7 +3434,7 @@ impl PruneCheckpointReader for DatabaseProvider PruneCheckpointWriter for DatabaseProvider { +impl PruneCheckpointWriter for DatabaseProvider { fn save_prune_checkpoint( &self, segment: PruneSegment, @@ -3445,7 +3444,7 @@ impl PruneCheckpointWriter for DatabaseProvider< } } -impl StatsReader for DatabaseProvider { +impl StatsReader for DatabaseProvider { fn count_entries(&self) -> ProviderResult { let db_entries = self.tx.entries::()?; let static_file_entries = match self.static_file_provider.count_entries::() { @@ -3458,7 +3457,7 @@ impl StatsReader for DatabaseProvider { } } -impl ChainStateBlockReader for DatabaseProvider { +impl ChainStateBlockReader for DatabaseProvider { fn last_finalized_block_number(&self) -> ProviderResult> { let mut finalized_blocks = self .tx @@ -3484,7 +3483,7 @@ impl ChainStateBlockReader for DatabaseProvider ChainStateBlockWriter for DatabaseProvider { +impl ChainStateBlockWriter for DatabaseProvider { fn save_finalized_block_number(&self, block_number: BlockNumber) -> ProviderResult<()> { Ok(self .tx @@ -3498,7 +3497,7 @@ impl ChainStateBlockWriter for DatabaseProvider< } } -impl DBProvider for DatabaseProvider { +impl DBProvider for DatabaseProvider { type Tx = TX; fn tx_ref(&self) -> &Self::Tx { diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index 07486f5557cb..cacb71b351de 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -8,6 +8,7 @@ use alloy_primitives::{ use reth_db::tables; use reth_db_api::{database::Database, models::StoredBlockBodyIndices}; +use reth_node_types::NodeTypes; use reth_primitives::{ Account, BlockBody, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, Signature, Transaction, TransactionSigned, TxType, Withdrawal, Withdrawals, @@ -17,8 +18,8 @@ use revm::{db::BundleState, primitives::AccountInfo}; use std::{str::FromStr, sync::LazyLock}; /// Assert genesis block -pub fn assert_genesis_block( - provider: &DatabaseProviderRW, +pub fn assert_genesis_block( + provider: &DatabaseProviderRW, g: SealedBlock, ) { let n = g.number; diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index ed861f5f1820..1053b4778fdd 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -19,6 +19,7 @@ use reth_db::mock::{DatabaseMock, TxMock}; use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_evm::ConfigureEvmEnv; use reth_execution_types::ExecutionOutcome; +use reth_node_types::NodeTypes; use reth_primitives::{ Account, Block, BlockWithSenders, Bytecode, GotExpected, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, TransactionMeta, TransactionSigned, @@ -149,10 +150,19 @@ impl MockEthProvider { } } +/// Mock node. +#[derive(Debug)] +pub struct MockNode; + +impl NodeTypes for MockNode { + type Primitives = (); + type ChainSpec = ChainSpec; +} + impl DatabaseProviderFactory for MockEthProvider { type DB = DatabaseMock; - type Provider = DatabaseProvider; - type ProviderRW = DatabaseProvider; + type Provider = DatabaseProvider; + type ProviderRW = DatabaseProvider; fn database_provider_ro(&self) -> ProviderResult { Err(ConsistentViewError::Syncing { best_block: GotExpected::new(0, 0) }.into()) diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index f5823404c899..aee264364798 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -11,7 +11,8 @@ use reth_db_api::{ }; use reth_primitives::{Account, StorageEntry}; use reth_provider::{ - test_utils::create_test_provider_factory, DatabaseProviderRW, StorageTrieWriter, TrieWriter, + providers::ProviderNodeTypes, test_utils::create_test_provider_factory, DatabaseProviderRW, + StorageTrieWriter, TrieWriter, }; use reth_trie::{ prefix_set::PrefixSetMut, @@ -693,8 +694,8 @@ fn storage_trie_around_extension_node() { assert_trie_updates(updates.storage_nodes_ref()); } -fn extension_node_storage_trie( - tx: &DatabaseProviderRW>, Spec>, +fn extension_node_storage_trie( + tx: &DatabaseProviderRW>, N>, hashed_address: B256, ) -> (B256, StorageTrieUpdates) { let value = U256::from(1); @@ -721,8 +722,8 @@ fn extension_node_storage_trie( (root, trie_updates) } -fn extension_node_trie( - tx: &DatabaseProviderRW>, Spec>, +fn extension_node_trie( + tx: &DatabaseProviderRW>, N>, ) -> B256 { let a = Account { nonce: 0, balance: U256::from(1u64), bytecode_hash: Some(B256::random()) }; let val = encode_account(a, None); From 2e750f0ca03043cd2370036cc4ed7c9e1c12f173 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:57:35 +0100 Subject: [PATCH 174/239] test(prune): add unit tests for `ReceiptsLogPruneConfig` (#11916) --- crates/prune/types/src/lib.rs | 230 ++++++++++++++++++++++++++++++++- crates/prune/types/src/mode.rs | 5 + 2 files changed, 231 insertions(+), 4 deletions(-) diff --git a/crates/prune/types/src/lib.rs b/crates/prune/types/src/lib.rs index 6e06d6fc5dc9..8483b7b73705 100644 --- a/crates/prune/types/src/lib.rs +++ b/crates/prune/types/src/lib.rs @@ -27,6 +27,7 @@ use std::collections::BTreeMap; pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE}; use alloy_primitives::{Address, BlockNumber}; +use std::ops::Deref; /// Configuration for pruning receipts not associated with logs emitted by the specified contracts. #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] @@ -59,7 +60,7 @@ impl ReceiptsLogPruneConfig { pruned_block: Option, ) -> Result>, PruneSegmentError> { let mut map = BTreeMap::new(); - let pruned_block = pruned_block.unwrap_or_default(); + let base_block = pruned_block.unwrap_or_default() + 1; for (address, mode) in &self.0 { // Getting `None`, means that there is nothing to prune yet, so we need it to include in @@ -69,7 +70,7 @@ impl ReceiptsLogPruneConfig { // // Reminder, that we increment because the [`BlockNumber`] key of the new map should be // viewed as `PruneMode::Before(block)` - let block = (pruned_block + 1).max( + let block = base_block.max( mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)? .map(|(block, _)| block) .unwrap_or_default() + @@ -90,8 +91,8 @@ impl ReceiptsLogPruneConfig { let pruned_block = pruned_block.unwrap_or_default(); let mut lowest = None; - for mode in self.0.values() { - if let PruneMode::Distance(_) = mode { + for mode in self.values() { + if mode.is_distance() { if let Some((block, _)) = mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)? { @@ -103,3 +104,224 @@ impl ReceiptsLogPruneConfig { Ok(lowest.map(|lowest| lowest.max(pruned_block))) } } + +impl Deref for ReceiptsLogPruneConfig { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_group_by_block_empty_config() { + let config = ReceiptsLogPruneConfig(BTreeMap::new()); + let tip = 1000; + let pruned_block = None; + + let result = config.group_by_block(tip, pruned_block).unwrap(); + assert!(result.is_empty(), "The result should be empty when the config is empty"); + } + + #[test] + fn test_group_by_block_single_entry() { + let mut config_map = BTreeMap::new(); + let address = Address::new([1; 20]); + let prune_mode = PruneMode::Before(500); + config_map.insert(address, prune_mode); + + let config = ReceiptsLogPruneConfig(config_map); + // Big tip to have something to prune for the target block + let tip = 3000000; + let pruned_block = Some(400); + + let result = config.group_by_block(tip, pruned_block).unwrap(); + + // Expect one entry with block 500 and the corresponding address + assert_eq!(result.len(), 1); + assert_eq!(result[&500], vec![&address], "Address should be grouped under block 500"); + + // Tip smaller than the target block, so that we have nothing to prune for the block + let tip = 300; + let pruned_block = Some(400); + + let result = config.group_by_block(tip, pruned_block).unwrap(); + + // Expect one entry with block 400 and the corresponding address + assert_eq!(result.len(), 1); + assert_eq!(result[&401], vec![&address], "Address should be grouped under block 400"); + } + + #[test] + fn test_group_by_block_multiple_entries() { + let mut config_map = BTreeMap::new(); + let address1 = Address::new([1; 20]); + let address2 = Address::new([2; 20]); + let prune_mode1 = PruneMode::Before(600); + let prune_mode2 = PruneMode::Before(800); + config_map.insert(address1, prune_mode1); + config_map.insert(address2, prune_mode2); + + let config = ReceiptsLogPruneConfig(config_map); + let tip = 900000; + let pruned_block = Some(400); + + let result = config.group_by_block(tip, pruned_block).unwrap(); + + // Expect two entries: one for block 600 and another for block 800 + assert_eq!(result.len(), 2); + assert_eq!(result[&600], vec![&address1], "Address1 should be grouped under block 600"); + assert_eq!(result[&800], vec![&address2], "Address2 should be grouped under block 800"); + } + + #[test] + fn test_group_by_block_with_distance_prune_mode() { + let mut config_map = BTreeMap::new(); + let address = Address::new([1; 20]); + let prune_mode = PruneMode::Distance(100000); + config_map.insert(address, prune_mode); + + let config = ReceiptsLogPruneConfig(config_map); + let tip = 100100; + // Pruned block is smaller than the target block + let pruned_block = Some(50); + + let result = config.group_by_block(tip, pruned_block).unwrap(); + + // Expect the entry to be grouped under block 100 (tip - distance) + assert_eq!(result.len(), 1); + assert_eq!(result[&101], vec![&address], "Address should be grouped under block 100"); + + let tip = 100100; + // Pruned block is larger than the target block + let pruned_block = Some(800); + + let result = config.group_by_block(tip, pruned_block).unwrap(); + + // Expect the entry to be grouped under block 800 which is larger than tip - distance + assert_eq!(result.len(), 1); + assert_eq!(result[&801], vec![&address], "Address should be grouped under block 800"); + } + + #[test] + fn test_lowest_block_with_distance_empty_config() { + let config = ReceiptsLogPruneConfig(BTreeMap::new()); + let tip = 1000; + let pruned_block = None; + + let result = config.lowest_block_with_distance(tip, pruned_block).unwrap(); + assert_eq!(result, None, "The result should be None when the config is empty"); + } + + #[test] + fn test_lowest_block_with_distance_no_distance_mode() { + let mut config_map = BTreeMap::new(); + let address = Address::new([1; 20]); + let prune_mode = PruneMode::Before(500); + config_map.insert(address, prune_mode); + + let config = ReceiptsLogPruneConfig(config_map); + let tip = 1000; + let pruned_block = None; + + let result = config.lowest_block_with_distance(tip, pruned_block).unwrap(); + assert_eq!(result, None, "The result should be None when there are no Distance modes"); + } + + #[test] + fn test_lowest_block_with_distance_single_entry() { + let mut config_map = BTreeMap::new(); + let address = Address::new([1; 20]); + let prune_mode = PruneMode::Distance(100000); + config_map.insert(address, prune_mode); + + let config = ReceiptsLogPruneConfig(config_map); + + let tip = 100100; + let pruned_block = Some(400); + + // Expect the lowest block to be 400 as 400 > 100100 - 100000 (tip - distance) + assert_eq!( + config.lowest_block_with_distance(tip, pruned_block).unwrap(), + Some(400), + "The lowest block should be 400" + ); + + let tip = 100100; + let pruned_block = Some(50); + + // Expect the lowest block to be 100 as 100 > 50 (pruned block) + assert_eq!( + config.lowest_block_with_distance(tip, pruned_block).unwrap(), + Some(100), + "The lowest block should be 100" + ); + } + + #[test] + fn test_lowest_block_with_distance_multiple_entries_last() { + let mut config_map = BTreeMap::new(); + let address1 = Address::new([1; 20]); + let address2 = Address::new([2; 20]); + let prune_mode1 = PruneMode::Distance(100100); + let prune_mode2 = PruneMode::Distance(100300); + config_map.insert(address1, prune_mode1); + config_map.insert(address2, prune_mode2); + + let config = ReceiptsLogPruneConfig(config_map); + let tip = 200300; + let pruned_block = Some(100); + + // The lowest block should be 200300 - 100300 = 100000: + // - First iteration will return 100200 => 200300 - 100100 = 100200 + // - Second iteration will return 100000 => 200300 - 100300 = 100000 < 100200 + // - Final result is 100000 + assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(100000)); + } + + #[test] + fn test_lowest_block_with_distance_multiple_entries_first() { + let mut config_map = BTreeMap::new(); + let address1 = Address::new([1; 20]); + let address2 = Address::new([2; 20]); + let prune_mode1 = PruneMode::Distance(100400); + let prune_mode2 = PruneMode::Distance(100300); + config_map.insert(address1, prune_mode1); + config_map.insert(address2, prune_mode2); + + let config = ReceiptsLogPruneConfig(config_map); + let tip = 200300; + let pruned_block = Some(100); + + // The lowest block should be 200300 - 100400 = 99900: + // - First iteration, lowest block is 200300 - 100400 = 99900 + // - Second iteration, lowest block is still 99900 < 200300 - 100300 = 100000 + // - Final result is 99900 + assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(99900)); + } + + #[test] + fn test_lowest_block_with_distance_multiple_entries_pruned_block() { + let mut config_map = BTreeMap::new(); + let address1 = Address::new([1; 20]); + let address2 = Address::new([2; 20]); + let prune_mode1 = PruneMode::Distance(100400); + let prune_mode2 = PruneMode::Distance(100300); + config_map.insert(address1, prune_mode1); + config_map.insert(address2, prune_mode2); + + let config = ReceiptsLogPruneConfig(config_map); + let tip = 200300; + let pruned_block = Some(100000); + + // The lowest block should be 100000 because: + // - Lowest is 200300 - 100400 = 99900 < 200300 - 100300 = 100000 + // - Lowest is compared to the pruned block 100000: 100000 > 99900 + // - Finally the lowest block is 100000 + assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(100000)); + } +} diff --git a/crates/prune/types/src/mode.rs b/crates/prune/types/src/mode.rs index 9a8e55bb383a..de9b9e6dc081 100644 --- a/crates/prune/types/src/mode.rs +++ b/crates/prune/types/src/mode.rs @@ -74,6 +74,11 @@ impl PruneMode { pub const fn is_full(&self) -> bool { matches!(self, Self::Full) } + + /// Returns true if the prune mode is [`PruneMode::Distance`]. + pub const fn is_distance(&self) -> bool { + matches!(self, Self::Distance(_)) + } } #[cfg(test)] From 82784183e74deb78556eb01da5b015904eb8d194 Mon Sep 17 00:00:00 2001 From: tedison <76473430+edisontim@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:47:12 -0400 Subject: [PATCH 175/239] feat: transaction trait (#11877) --- crates/primitives-traits/Cargo.toml | 1 - crates/primitives-traits/src/lib.rs | 3 +- .../primitives-traits/src/transaction/mod.rs | 55 +++++++++++++++---- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/crates/primitives-traits/Cargo.toml b/crates/primitives-traits/Cargo.toml index 4319232f8248..6cafe8b8b1e5 100644 --- a/crates/primitives-traits/Cargo.toml +++ b/crates/primitives-traits/Cargo.toml @@ -44,7 +44,6 @@ reth-testing-utils.workspace = true alloy-primitives = { workspace = true, features = ["arbitrary"] } alloy-consensus = { workspace = true, features = ["arbitrary"] } -arbitrary = { workspace = true, features = ["derive"] } bincode.workspace = true proptest-arbitrary-interop.workspace = true proptest.workspace = true diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index 57d1119b0351..0489a250bbd1 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -14,6 +14,7 @@ extern crate alloc; /// Common constants. pub mod constants; + pub use constants::gas_units::{format_gas, format_gas_throughput}; /// Minimal account @@ -24,7 +25,7 @@ pub mod receipt; pub use receipt::Receipt; pub mod transaction; -pub use transaction::{signed::SignedTransaction, Transaction}; +pub use transaction::{signed::SignedTransaction, FullTransaction, Transaction}; mod integer_list; pub use integer_list::{IntegerList, IntegerListError}; diff --git a/crates/primitives-traits/src/transaction/mod.rs b/crates/primitives-traits/src/transaction/mod.rs index a306c5f76ed5..a1ad81ab3270 100644 --- a/crates/primitives-traits/src/transaction/mod.rs +++ b/crates/primitives-traits/src/transaction/mod.rs @@ -1,28 +1,61 @@ //! Transaction abstraction -pub mod signed; +use core::{fmt::Debug, hash::Hash}; -use alloc::fmt; +use alloy_primitives::{TxKind, B256}; use reth_codecs::Compact; use serde::{Deserialize, Serialize}; -/// Helper trait that unifies all behaviour required by transaction to support full node operations. -pub trait FullTransaction: Transaction + Compact {} - -impl FullTransaction for T where T: Transaction + Compact {} +pub mod signed; +#[allow(dead_code)] /// Abstraction of a transaction. pub trait Transaction: - alloy_consensus::Transaction + Debug + + Default + Clone - + fmt::Debug - + PartialEq + Eq - + Default + + PartialEq + + Hash + + Serialize + alloy_rlp::Encodable + alloy_rlp::Decodable - + Serialize + for<'de> Deserialize<'de> + + alloy_consensus::Transaction + + MaybeArbitrary { + /// Heavy operation that return signature hash over rlp encoded transaction. + /// It is only for signature signing or signer recovery. + fn signature_hash(&self) -> B256; + + /// Gets the transaction's [`TxKind`], which is the address of the recipient or + /// [`TxKind::Create`] if the transaction is a contract creation. + fn kind(&self) -> TxKind; + + /// Returns true if the tx supports dynamic fees + fn is_dynamic_fee(&self) -> bool; + + /// Returns the effective gas price for the given base fee. + fn effective_gas_price(&self, base_fee: Option) -> u128; + + /// This encodes the transaction _without_ the signature, and is only suitable for creating a + /// hash intended for signing. + fn encode_without_signature(&self, out: &mut dyn bytes::BufMut); + + /// Calculates a heuristic for the in-memory size of the [Transaction]. + fn size(&self) -> usize; } + +#[cfg(not(feature = "arbitrary"))] +/// Helper trait that requires arbitrary implementation if the feature is enabled. +pub trait MaybeArbitrary {} + +#[cfg(feature = "arbitrary")] +/// Helper trait that requires arbitrary implementation if the feature is enabled. +pub trait MaybeArbitrary: for<'a> arbitrary::Arbitrary<'a> {} + +/// Helper trait that unifies all behaviour required by transaction to support full node operations. +pub trait FullTransaction: Transaction + Compact {} + +impl FullTransaction for T where T: Transaction + Compact {} From 734c78fdfb46cc5a97971450ed74c6cbdf62d5af Mon Sep 17 00:00:00 2001 From: Kufre Samuel Date: Tue, 29 Oct 2024 22:24:35 +0100 Subject: [PATCH 176/239] feat(discv4): neighbors packet logging (#12042) Co-authored-by: Oliver Nordbjerg --- Cargo.lock | 1 + crates/net/discv4/Cargo.toml | 1 + crates/net/discv4/src/lib.rs | 13 ++++++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5b62244576ef..6545563d95c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6922,6 +6922,7 @@ dependencies = [ "discv5", "enr", "generic-array", + "itertools 0.13.0", "parking_lot", "rand 0.8.5", "reth-ethereum-forks", diff --git a/crates/net/discv4/Cargo.toml b/crates/net/discv4/Cargo.toml index 6fb669388851..f1c8410eeba9 100644 --- a/crates/net/discv4/Cargo.toml +++ b/crates/net/discv4/Cargo.toml @@ -42,6 +42,7 @@ parking_lot.workspace = true rand = { workspace = true, optional = true } generic-array.workspace = true serde = { workspace = true, optional = true } +itertools.workspace = true [dev-dependencies] assert_matches.workspace = true diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index a99906bdf09b..788e93048f10 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -38,6 +38,7 @@ use discv5::{ ConnectionDirection, ConnectionState, }; use enr::Enr; +use itertools::Itertools; use parking_lot::Mutex; use proto::{EnrRequest, EnrResponse}; use reth_ethereum_forks::ForkId; @@ -861,7 +862,7 @@ impl Discv4Service { let Some(bucket) = self.kbuckets.get_bucket(&key) else { return false }; if bucket.num_entries() < MAX_NODES_PER_BUCKET / 2 { // skip half empty bucket - return false; + return false } self.remove_key(node_id, key) } @@ -1406,6 +1407,16 @@ impl Discv4Service { } }; + // log the peers we discovered + trace!(target: "discv4", + target=format!("{:#?}", node_id), + peers_count=msg.nodes.len(), + peers=format!("[{:#}]", msg.nodes.iter() + .map(|node_rec| node_rec.id + ).format(", ")), + "Received peers from Neighbours packet" + ); + // This is the recursive lookup step where we initiate new FindNode requests for new nodes // that were discovered. for node in msg.nodes.into_iter().map(NodeRecord::into_ipv4_mapped) { From 129f3ba9116da0fcbfafb78e66b7a0163e5464d3 Mon Sep 17 00:00:00 2001 From: frisitano <35734660+frisitano@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:48:43 +0800 Subject: [PATCH 177/239] feat: introduce StateCommitment type (#11842) --- Cargo.lock | 5 ++ crates/ethereum/node/Cargo.toml | 2 + crates/ethereum/node/src/node.rs | 2 + crates/exex/test-utils/Cargo.toml | 1 + crates/exex/test-utils/src/lib.rs | 1 + crates/node/builder/src/node.rs | 2 + crates/node/types/Cargo.toml | 1 + crates/node/types/src/lib.rs | 49 +++++++++++++------ crates/optimism/node/Cargo.toml | 2 + crates/optimism/node/src/node.rs | 2 + .../storage/provider/src/test_utils/mock.rs | 2 + crates/storage/provider/src/test_utils/mod.rs | 1 + crates/trie/common/src/key.rs | 18 +++++++ crates/trie/common/src/lib.rs | 3 ++ crates/trie/db/src/commitment.rs | 39 +++++++++++++++ crates/trie/db/src/lib.rs | 2 + examples/custom-engine-types/Cargo.toml | 1 + examples/custom-engine-types/src/main.rs | 2 + 18 files changed, 120 insertions(+), 15 deletions(-) create mode 100644 crates/trie/common/src/key.rs create mode 100644 crates/trie/db/src/commitment.rs diff --git a/Cargo.lock b/Cargo.lock index 6545563d95c4..27581ba3a800 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2856,6 +2856,7 @@ dependencies = [ "reth-payload-builder", "reth-primitives", "reth-tracing", + "reth-trie-db", "serde", "thiserror", "tokio", @@ -7564,6 +7565,7 @@ dependencies = [ "reth-provider", "reth-tasks", "reth-transaction-pool", + "reth-trie-db", "tempfile", "thiserror", "tokio", @@ -8024,6 +8026,7 @@ dependencies = [ "reth-tasks", "reth-tracing", "reth-transaction-pool", + "reth-trie-db", "revm", "serde_json", "tokio", @@ -8086,6 +8089,7 @@ dependencies = [ "reth-engine-primitives", "reth-primitives", "reth-primitives-traits", + "reth-trie-db", ] [[package]] @@ -8246,6 +8250,7 @@ dependencies = [ "reth-revm", "reth-tracing", "reth-transaction-pool", + "reth-trie-db", "revm", "serde", "serde_json", diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index 98224b99e26b..83b62034740b 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -31,6 +31,7 @@ reth-node-api.workspace = true reth-chainspec.workspace = true reth-primitives.workspace = true reth-revm = { workspace = true, features = ["std"] } +reth-trie-db.workspace = true # revm with required ethereum features revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] } @@ -72,6 +73,7 @@ test-utils = [ "reth-db/test-utils", "reth-provider/test-utils", "reth-transaction-pool/test-utils", + "reth-trie-db/test-utils", "revm/test-utils", "reth-evm/test-utils" ] diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index dbd6ce0a134a..1942f8a9e56b 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -34,6 +34,7 @@ use reth_transaction_pool::{ blobstore::DiskFileBlobStore, EthTransactionPool, TransactionPool, TransactionValidationTaskExecutor, }; +use reth_trie_db::MerklePatriciaTrie; use crate::{EthEngineTypes, EthEvmConfig}; @@ -81,6 +82,7 @@ impl EthereumNode { impl NodeTypes for EthereumNode { type Primitives = EthPrimitives; type ChainSpec = ChainSpec; + type StateCommitment = MerklePatriciaTrie; } impl NodeTypesWithEngine for EthereumNode { diff --git a/crates/exex/test-utils/Cargo.toml b/crates/exex/test-utils/Cargo.toml index b850295a332b..cd0e0831b496 100644 --- a/crates/exex/test-utils/Cargo.toml +++ b/crates/exex/test-utils/Cargo.toml @@ -31,6 +31,7 @@ reth-primitives.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } reth-tasks.workspace = true reth-transaction-pool = { workspace = true, features = ["test-utils"] } +reth-trie-db.workspace = true ## async futures-util.workspace = true diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index 1f6ea75ce6d9..06aa8c81c7cd 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -119,6 +119,7 @@ pub struct TestNode; impl NodeTypes for TestNode { type Primitives = (); type ChainSpec = ChainSpec; + type StateCommitment = reth_trie_db::MerklePatriciaTrie; } impl NodeTypesWithEngine for TestNode { diff --git a/crates/node/builder/src/node.rs b/crates/node/builder/src/node.rs index 3b2f467d61c0..62c710ea8022 100644 --- a/crates/node/builder/src/node.rs +++ b/crates/node/builder/src/node.rs @@ -69,6 +69,8 @@ where type Primitives = ::Primitives; type ChainSpec = ::ChainSpec; + + type StateCommitment = ::StateCommitment; } impl NodeTypesWithEngine for AnyNode diff --git a/crates/node/types/Cargo.toml b/crates/node/types/Cargo.toml index 5747abe9c34b..21facae5460a 100644 --- a/crates/node/types/Cargo.toml +++ b/crates/node/types/Cargo.toml @@ -17,3 +17,4 @@ reth-db-api.workspace = true reth-engine-primitives.workspace = true reth-primitives.workspace = true reth-primitives-traits.workspace = true +reth-trie-db.workspace = true diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index 5ba03e6795ab..38e194bd4fbf 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -18,6 +18,7 @@ use reth_db_api::{ Database, }; use reth_engine_primitives::EngineTypes; +use reth_trie_db::StateCommitment; /// Configures all the primitive types of the node. pub trait NodePrimitives { @@ -39,6 +40,8 @@ pub trait NodeTypes: Send + Sync + Unpin + 'static { type Primitives: NodePrimitives; /// The type used for configuration of the EVM. type ChainSpec: EthChainSpec; + /// The type used to perform state commitment operations. + type StateCommitment: StateCommitment; } /// The type that configures an Ethereum-like node with an engine for consensus. @@ -89,6 +92,7 @@ where { type Primitives = Types::Primitives; type ChainSpec = Types::ChainSpec; + type StateCommitment = Types::StateCommitment; } impl NodeTypesWithEngine for NodeTypesWithDBAdapter @@ -109,70 +113,85 @@ where /// A [`NodeTypes`] type builder. #[derive(Default, Debug)] -pub struct AnyNodeTypes

(PhantomData

, PhantomData); +pub struct AnyNodeTypes

(PhantomData

, PhantomData, PhantomData); -impl AnyNodeTypes { +impl AnyNodeTypes { /// Sets the `Primitives` associated type. - pub const fn primitives(self) -> AnyNodeTypes { - AnyNodeTypes::(PhantomData::, PhantomData::) + pub const fn primitives(self) -> AnyNodeTypes { + AnyNodeTypes::(PhantomData::, PhantomData::, PhantomData::) } /// Sets the `ChainSpec` associated type. - pub const fn chain_spec(self) -> AnyNodeTypes { - AnyNodeTypes::(PhantomData::

, PhantomData::) + pub const fn chain_spec(self) -> AnyNodeTypes { + AnyNodeTypes::(PhantomData::

, PhantomData::, PhantomData::) + } + + /// Sets the `StateCommitment` associated type. + pub const fn state_commitment(self) -> AnyNodeTypes { + AnyNodeTypes::(PhantomData::

, PhantomData::, PhantomData::) } } -impl NodeTypes for AnyNodeTypes +impl NodeTypes for AnyNodeTypes where P: NodePrimitives + Send + Sync + Unpin + 'static, C: EthChainSpec + 'static, + S: StateCommitment, { type Primitives = P; type ChainSpec = C; + type StateCommitment = S; } /// A [`NodeTypesWithEngine`] type builder. #[derive(Default, Debug)] -pub struct AnyNodeTypesWithEngine

{ +pub struct AnyNodeTypesWithEngine

{ /// Embedding the basic node types. - base: AnyNodeTypes, + base: AnyNodeTypes, /// Phantom data for the engine. _engine: PhantomData, } -impl AnyNodeTypesWithEngine { +impl AnyNodeTypesWithEngine { /// Sets the `Primitives` associated type. - pub const fn primitives(self) -> AnyNodeTypesWithEngine { + pub const fn primitives(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine { base: self.base.primitives::(), _engine: PhantomData } } /// Sets the `Engine` associated type. - pub const fn engine(self) -> AnyNodeTypesWithEngine { + pub const fn engine(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine { base: self.base, _engine: PhantomData:: } } /// Sets the `ChainSpec` associated type. - pub const fn chain_spec(self) -> AnyNodeTypesWithEngine { + pub const fn chain_spec(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine { base: self.base.chain_spec::(), _engine: PhantomData } } + + /// Sets the `StateCommitment` associated type. + pub const fn state_commitment(self) -> AnyNodeTypesWithEngine { + AnyNodeTypesWithEngine { base: self.base.state_commitment::(), _engine: PhantomData } + } } -impl NodeTypes for AnyNodeTypesWithEngine +impl NodeTypes for AnyNodeTypesWithEngine where P: NodePrimitives + Send + Sync + Unpin + 'static, E: EngineTypes + Send + Sync + Unpin, C: EthChainSpec + 'static, + S: StateCommitment, { type Primitives = P; type ChainSpec = C; + type StateCommitment = S; } -impl NodeTypesWithEngine for AnyNodeTypesWithEngine +impl NodeTypesWithEngine for AnyNodeTypesWithEngine where P: NodePrimitives + Send + Sync + Unpin + 'static, E: EngineTypes + Send + Sync + Unpin, C: EthChainSpec + 'static, + S: StateCommitment, { type Engine = E; } diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 37cf4a328ea0..deabbac52499 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -28,6 +28,7 @@ reth-network.workspace = true reth-evm.workspace = true reth-revm = { workspace = true, features = ["std"] } reth-beacon-consensus.workspace = true +reth-trie-db.workspace = true # op-reth reth-optimism-payload-builder.workspace = true @@ -99,5 +100,6 @@ test-utils = [ "reth-db/test-utils", "reth-provider/test-utils", "reth-transaction-pool/test-utils", + "reth-trie-db/test-utils", "revm/test-utils" ] diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 9492bb8c429a..4328a55fb15d 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -30,6 +30,7 @@ use reth_transaction_pool::{ blobstore::DiskFileBlobStore, CoinbaseTipOrdering, TransactionPool, TransactionValidationTaskExecutor, }; +use reth_trie_db::MerklePatriciaTrie; use crate::{ args::RollupArgs, @@ -122,6 +123,7 @@ where impl NodeTypes for OptimismNode { type Primitives = OpPrimitives; type ChainSpec = OpChainSpec; + type StateCommitment = MerklePatriciaTrie; } impl NodeTypesWithEngine for OptimismNode { diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 1053b4778fdd..e2e08e61a86f 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -34,6 +34,7 @@ use reth_trie::{ updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, StorageProof, TrieInput, }; +use reth_trie_db::MerklePatriciaTrie; use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; use std::{ collections::BTreeMap, @@ -157,6 +158,7 @@ pub struct MockNode; impl NodeTypes for MockNode { type Primitives = (); type ChainSpec = ChainSpec; + type StateCommitment = MerklePatriciaTrie; } impl DatabaseProviderFactory for MockEthProvider { diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index 2200781096d0..c0e80930b318 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -25,6 +25,7 @@ pub type MockNodeTypes = reth_node_types::AnyNodeTypesWithEngine< (), reth_ethereum_engine_primitives::EthEngineTypes, reth_chainspec::ChainSpec, + reth_trie_db::MerklePatriciaTrie, >; /// Mock [`reth_node_types::NodeTypesWithDB`] for testing. diff --git a/crates/trie/common/src/key.rs b/crates/trie/common/src/key.rs new file mode 100644 index 000000000000..9e440d199fa1 --- /dev/null +++ b/crates/trie/common/src/key.rs @@ -0,0 +1,18 @@ +use alloy_primitives::B256; +use revm_primitives::keccak256; + +/// Trait for hashing keys in state. +pub trait KeyHasher: Default + Clone + Send + Sync + 'static { + /// Hashes the given bytes into a 256-bit hash. + fn hash_key>(bytes: T) -> B256; +} + +/// A key hasher that uses the Keccak-256 hash function. +#[derive(Clone, Debug, Default)] +pub struct KeccakKeyHasher; + +impl KeyHasher for KeccakKeyHasher { + fn hash_key>(bytes: T) -> B256 { + keccak256(bytes) + } +} diff --git a/crates/trie/common/src/lib.rs b/crates/trie/common/src/lib.rs index bdec36028b94..7645ebd3a1cb 100644 --- a/crates/trie/common/src/lib.rs +++ b/crates/trie/common/src/lib.rs @@ -14,6 +14,9 @@ pub mod hash_builder; mod account; pub use account::TrieAccount; +mod key; +pub use key::{KeccakKeyHasher, KeyHasher}; + mod nibbles; pub use nibbles::{Nibbles, StoredNibbles, StoredNibblesSubKey}; diff --git a/crates/trie/db/src/commitment.rs b/crates/trie/db/src/commitment.rs new file mode 100644 index 000000000000..c608aefff8ae --- /dev/null +++ b/crates/trie/db/src/commitment.rs @@ -0,0 +1,39 @@ +use crate::{ + DatabaseHashedCursorFactory, DatabaseProof, DatabaseStateRoot, DatabaseStorageRoot, + DatabaseTrieCursorFactory, DatabaseTrieWitness, +}; +use reth_db::transaction::DbTx; +use reth_trie::{ + proof::Proof, witness::TrieWitness, KeccakKeyHasher, KeyHasher, StateRoot, StorageRoot, +}; + +/// The `StateCommitment` trait provides associated types for state commitment operations. +pub trait StateCommitment: std::fmt::Debug + Send + Sync + Unpin + 'static { + /// The state root type. + type StateRoot<'a, TX: DbTx + 'a>: DatabaseStateRoot<'a, TX>; + /// The storage root type. + type StorageRoot<'a, TX: DbTx + 'a>: DatabaseStorageRoot<'a, TX>; + /// The state proof type. + type StateProof<'a, TX: DbTx + 'a>: DatabaseProof<'a, TX>; + /// The state witness type. + type StateWitness<'a, TX: DbTx + 'a>: DatabaseTrieWitness<'a, TX>; + /// The key hasher type. + type KeyHasher: KeyHasher; +} + +/// The state commitment type for Ethereum's Merkle Patricia Trie. +#[derive(Debug)] +#[non_exhaustive] +pub struct MerklePatriciaTrie; + +impl StateCommitment for MerklePatriciaTrie { + type StateRoot<'a, TX: DbTx + 'a> = + StateRoot, DatabaseHashedCursorFactory<'a, TX>>; + type StorageRoot<'a, TX: DbTx + 'a> = + StorageRoot, DatabaseHashedCursorFactory<'a, TX>>; + type StateProof<'a, TX: DbTx + 'a> = + Proof, DatabaseHashedCursorFactory<'a, TX>>; + type StateWitness<'a, TX: DbTx + 'a> = + TrieWitness, DatabaseHashedCursorFactory<'a, TX>>; + type KeyHasher = KeccakKeyHasher; +} diff --git a/crates/trie/db/src/lib.rs b/crates/trie/db/src/lib.rs index 3a9b1e328239..27c18af6cbfd 100644 --- a/crates/trie/db/src/lib.rs +++ b/crates/trie/db/src/lib.rs @@ -1,5 +1,6 @@ //! An integration of [`reth-trie`] with [`reth-db`]. +mod commitment; mod hashed_cursor; mod prefix_set; mod proof; @@ -8,6 +9,7 @@ mod storage; mod trie_cursor; mod witness; +pub use commitment::{MerklePatriciaTrie, StateCommitment}; pub use hashed_cursor::{ DatabaseHashedAccountCursor, DatabaseHashedCursorFactory, DatabaseHashedStorageCursor, }; diff --git a/examples/custom-engine-types/Cargo.toml b/examples/custom-engine-types/Cargo.toml index f826451d2038..1fbb3c4947ac 100644 --- a/examples/custom-engine-types/Cargo.toml +++ b/examples/custom-engine-types/Cargo.toml @@ -16,6 +16,7 @@ reth-basic-payload-builder.workspace = true reth-ethereum-payload-builder.workspace = true reth-node-ethereum = { workspace = true, features = ["test-utils"] } reth-tracing.workspace = true +reth-trie-db.workspace = true alloy-genesis.workspace = true alloy-rpc-types = { workspace = true, features = ["engine"] } alloy-primitives.workspace = true diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index 30f89a0b9d78..a48c44b200b9 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -70,6 +70,7 @@ use reth_payload_builder::{ }; use reth_primitives::Withdrawals; use reth_tracing::{RethTracer, Tracer}; +use reth_trie_db::MerklePatriciaTrie; /// A custom payload attributes type. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -228,6 +229,7 @@ struct MyCustomNode; impl NodeTypes for MyCustomNode { type Primitives = (); type ChainSpec = ChainSpec; + type StateCommitment = MerklePatriciaTrie; } /// Configure the node types with the custom engine types From 367b4ed18af583305c18adb118be0b303e55aa8f Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:10:14 +0100 Subject: [PATCH 178/239] chore(meta): update SECURITY.md (#12190) --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 5260d529f5a4..bea27ad1140c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,4 +2,4 @@ ## Reporting a Vulnerability -Contact georgios at paradigm.xyz. +Contact [security@ithaca.xyz](mailto:security@ithaca.xyz). From 6e794ee6738c4f6c610010330d76e5a845b091b2 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 30 Oct 2024 07:29:48 -0400 Subject: [PATCH 179/239] fix(ecies): bound initial header body size (#12172) --- crates/net/ecies/src/algorithm.rs | 3 ++- crates/net/ecies/src/codec.rs | 36 +++++++++++++++++++++++++++---- crates/net/ecies/src/error.rs | 8 +++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/crates/net/ecies/src/algorithm.rs b/crates/net/ecies/src/algorithm.rs index 6bf9717fe52c..e4266d9a06f7 100644 --- a/crates/net/ecies/src/algorithm.rs +++ b/crates/net/ecies/src/algorithm.rs @@ -650,7 +650,8 @@ impl ECIES { out.extend_from_slice(tag.as_slice()); } - /// Extracts the header from slice and returns the body size. + /// Reads the `RLPx` header from the slice, setting up the MAC and AES, returning the body + /// size contained in the header. pub fn read_header(&mut self, data: &mut [u8]) -> Result { // If the data is not large enough to fit the header and mac bytes, return an error // diff --git a/crates/net/ecies/src/codec.rs b/crates/net/ecies/src/codec.rs index c3e9b8d58cc9..b5a10284cf20 100644 --- a/crates/net/ecies/src/codec.rs +++ b/crates/net/ecies/src/codec.rs @@ -1,12 +1,15 @@ //! This contains the main codec for `RLPx` ECIES messages -use crate::{algorithm::ECIES, ECIESError, EgressECIESValue, IngressECIESValue}; +use crate::{algorithm::ECIES, ECIESError, ECIESErrorImpl, EgressECIESValue, IngressECIESValue}; use alloy_primitives::{bytes::BytesMut, B512 as PeerId}; use secp256k1::SecretKey; use std::{fmt::Debug, io}; use tokio_util::codec::{Decoder, Encoder}; use tracing::{instrument, trace}; +/// The max size that the initial handshake packet can be. Currently 2KiB. +const MAX_INITIAL_HANDSHAKE_SIZE: usize = 2048; + /// Tokio codec for ECIES #[derive(Debug)] pub struct ECIESCodec { @@ -26,6 +29,11 @@ pub enum ECIESState { /// message containing the nonce and other metadata. Ack, + /// This is the same as the [`ECIESState::Header`] stage, but occurs only after the first + /// [`ECIESState::Ack`] message. This is so that the initial handshake message can be properly + /// validated. + InitialHeader, + /// The third stage of the ECIES handshake, where header is parsed, message integrity checks /// performed, and message is decrypted. Header, @@ -70,7 +78,7 @@ impl Decoder for ECIESCodec { self.ecies.read_auth(&mut buf.split_to(total_size))?; - self.state = ECIESState::Header; + self.state = ECIESState::InitialHeader; return Ok(Some(IngressECIESValue::AuthReceive(self.ecies.remote_id()))) } ECIESState::Ack => { @@ -89,9 +97,29 @@ impl Decoder for ECIESCodec { self.ecies.read_ack(&mut buf.split_to(total_size))?; - self.state = ECIESState::Header; + self.state = ECIESState::InitialHeader; return Ok(Some(IngressECIESValue::Ack)) } + ECIESState::InitialHeader => { + if buf.len() < ECIES::header_len() { + trace!("current len {}, need {}", buf.len(), ECIES::header_len()); + return Ok(None) + } + + let body_size = + self.ecies.read_header(&mut buf.split_to(ECIES::header_len()))?; + + if body_size > MAX_INITIAL_HANDSHAKE_SIZE { + trace!(?body_size, max=?MAX_INITIAL_HANDSHAKE_SIZE, "Header exceeds max initial handshake size"); + return Err(ECIESErrorImpl::InitialHeaderBodyTooLarge { + body_size, + max_body_size: MAX_INITIAL_HANDSHAKE_SIZE, + } + .into()) + } + + self.state = ECIESState::Body; + } ECIESState::Header => { if buf.len() < ECIES::header_len() { trace!("current len {}, need {}", buf.len(), ECIES::header_len()); @@ -131,7 +159,7 @@ impl Encoder for ECIESCodec { Ok(()) } EgressECIESValue::Ack => { - self.state = ECIESState::Header; + self.state = ECIESState::InitialHeader; self.ecies.write_ack(buf); Ok(()) } diff --git a/crates/net/ecies/src/error.rs b/crates/net/ecies/src/error.rs index 79965f73303f..9dabfc16183d 100644 --- a/crates/net/ecies/src/error.rs +++ b/crates/net/ecies/src/error.rs @@ -62,6 +62,14 @@ pub enum ECIESErrorImpl { /// The encrypted data is not large enough for all fields #[error("encrypted data is not large enough for all fields")] EncryptedDataTooSmall, + /// The initial header body is too large. + #[error("initial header body is {body_size} but the max is {max_body_size}")] + InitialHeaderBodyTooLarge { + /// The body size from the header + body_size: usize, + /// The max body size + max_body_size: usize, + }, /// Error when trying to split an array beyond its length #[error("requested {idx} but array len is {len}")] OutOfBounds { From 2778ba3d52605e3837ef1abd29f9ef956e0e8140 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Wed, 30 Oct 2024 12:30:49 +0100 Subject: [PATCH 180/239] tx-pool: fix `ExceedsGasLimit` error message order (#12191) --- crates/transaction-pool/src/pool/txpool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index b11815fc4b54..bd0aacccfee9 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -628,8 +628,8 @@ impl TxPool { *transaction.hash(), PoolErrorKind::InvalidTransaction( InvalidPoolTransactionError::ExceedsGasLimit( - block_gas_limit, tx_gas_limit, + block_gas_limit, ), ), )), From 93a9b8a218309a296d96d5841df4f4b7b60701cd Mon Sep 17 00:00:00 2001 From: cody-wang-cb Date: Wed, 30 Oct 2024 08:59:19 -0400 Subject: [PATCH 181/239] feat: Eip1559 params in extradata (#11887) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> Co-authored-by: Matthias Seitz --- .../ethereum/engine-primitives/src/payload.rs | 2 +- crates/ethereum/evm/src/lib.rs | 7 +- crates/ethereum/payload/src/lib.rs | 10 +- crates/evm/Cargo.toml | 16 +- crates/evm/src/lib.rs | 5 +- crates/optimism/chainspec/src/lib.rs | 174 +++++++++++++++++- crates/optimism/evm/src/config.rs | 8 +- crates/optimism/evm/src/lib.rs | 15 +- crates/optimism/hardforks/src/hardfork.rs | 4 + crates/optimism/node/src/engine.rs | 137 +++++++++++++- crates/optimism/node/tests/e2e/utils.rs | 1 + crates/optimism/payload/src/builder.rs | 37 +++- crates/optimism/payload/src/error.rs | 14 ++ crates/optimism/payload/src/payload.rs | 64 ++++++- examples/custom-evm/src/main.rs | 5 +- examples/stateful-precompile/src/main.rs | 5 +- 16 files changed, 461 insertions(+), 43 deletions(-) diff --git a/crates/ethereum/engine-primitives/src/payload.rs b/crates/ethereum/engine-primitives/src/payload.rs index 2d162ef15050..ed377d003dd4 100644 --- a/crates/ethereum/engine-primitives/src/payload.rs +++ b/crates/ethereum/engine-primitives/src/payload.rs @@ -184,7 +184,7 @@ impl From for ExecutionPayloadEnvelopeV4 { } /// Container type for all components required to build a payload. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct EthPayloadBuilderAttributes { /// Id of the payload pub id: PayloadId, diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index 91da90d99f0e..1c340c0927ba 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -17,6 +17,8 @@ extern crate alloc; +use core::convert::Infallible; + use alloc::{sync::Arc, vec::Vec}; use alloy_primitives::{Address, Bytes, TxKind, U256}; use reth_chainspec::{ChainSpec, Head}; @@ -59,6 +61,7 @@ impl EthEvmConfig { impl ConfigureEvmEnv for EthEvmConfig { type Header = Header; + type Error = Infallible; fn fill_tx_env(&self, tx_env: &mut TxEnv, transaction: &TransactionSigned, sender: Address) { transaction.fill_tx_env(tx_env, sender); @@ -131,7 +134,7 @@ impl ConfigureEvmEnv for EthEvmConfig { &self, parent: &Self::Header, attributes: NextBlockEnvAttributes, - ) -> (CfgEnvWithHandlerCfg, BlockEnv) { + ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), Self::Error> { // configure evm env based on parent block let cfg = CfgEnv::default().with_chain_id(self.chain_spec.chain().id()); @@ -179,7 +182,7 @@ impl ConfigureEvmEnv for EthEvmConfig { blob_excess_gas_and_price, }; - (CfgEnvWithHandlerCfg::new_with_spec_id(cfg, spec_id), block_env) + Ok((CfgEnvWithHandlerCfg::new_with_spec_id(cfg, spec_id), block_env)) } } diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 8e188f890fd9..27d9d98bcca2 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -73,7 +73,7 @@ where &self, config: &PayloadConfig, parent: &Header, - ) -> (CfgEnvWithHandlerCfg, BlockEnv) { + ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), EvmConfig::Error> { let next_attributes = NextBlockEnvAttributes { timestamp: config.attributes.timestamp(), suggested_fee_recipient: config.attributes.suggested_fee_recipient(), @@ -97,7 +97,9 @@ where &self, args: BuildArguments, ) -> Result, PayloadBuilderError> { - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); + let (cfg_env, block_env) = self + .cfg_and_block_env(&args.config, &args.config.parent_header) + .map_err(PayloadBuilderError::other)?; let pool = args.pool.clone(); default_ethereum_payload(self.evm_config.clone(), args, cfg_env, block_env, |attributes| { @@ -120,7 +122,9 @@ where None, ); - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); + let (cfg_env, block_env) = self + .cfg_and_block_env(&args.config, &args.config.parent_header) + .map_err(PayloadBuilderError::other)?; let pool = args.pool.clone(); diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index 90fd532828e1..a4ce3c3893ba 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -57,12 +57,12 @@ std = [ "revm/std", ] test-utils = [ - "dep:parking_lot", - "reth-chainspec/test-utils", - "reth-consensus/test-utils", - "reth-primitives/test-utils", - "reth-primitives-traits/test-utils", - "reth-revm/test-utils", - "revm/test-utils", - "reth-prune-types/test-utils" + "dep:parking_lot", + "reth-chainspec/test-utils", + "reth-consensus/test-utils", + "reth-primitives/test-utils", + "reth-primitives-traits/test-utils", + "reth-revm/test-utils", + "revm/test-utils", + "reth-prune-types/test-utils" ] diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index b75feea83a15..e30ff9b1a7ad 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -116,6 +116,9 @@ pub trait ConfigureEvmEnv: Send + Sync + Unpin + Clone + 'static { /// The header type used by the EVM. type Header: BlockHeader; + /// The error type that is returned by [`Self::next_cfg_and_block_env`]. + type Error: core::error::Error + Send + Sync; + /// Returns a [`TxEnv`] from a [`TransactionSigned`] and [`Address`]. fn tx_env(&self, transaction: &TransactionSigned, signer: Address) -> TxEnv { let mut tx_env = TxEnv::default(); @@ -192,7 +195,7 @@ pub trait ConfigureEvmEnv: Send + Sync + Unpin + Clone + 'static { &self, parent: &Self::Header, attributes: NextBlockEnvAttributes, - ) -> (CfgEnvWithHandlerCfg, BlockEnv); + ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), Self::Error>; } /// Represents additional attributes required to configure the next block. diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 03ce75aec040..70adf2272cfb 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -20,10 +20,10 @@ mod op_sepolia; use alloc::{vec, vec::Vec}; use alloy_chains::Chain; use alloy_genesis::Genesis; -use alloy_primitives::{B256, U256}; +use alloy_primitives::{Bytes, Parity, Signature, B256, U256}; pub use base::BASE_MAINNET; pub use base_sepolia::BASE_SEPOLIA; -use derive_more::{Constructor, Deref, From, Into}; +use derive_more::{Constructor, Deref, Display, From, Into}; pub use dev::OP_DEV; #[cfg(not(feature = "std"))] pub(crate) use once_cell::sync::Lazy as LazyLock; @@ -159,6 +159,16 @@ impl OpChainSpecBuilder { self } + /// Enable Holocene at genesis + pub fn holocene_activated(mut self) -> Self { + self = self.granite_activated(); + self.inner = self.inner.with_fork( + reth_optimism_forks::OptimismHardfork::Holocene, + ForkCondition::Timestamp(0), + ); + self + } + /// Build the resulting [`OpChainSpec`]. /// /// # Panics @@ -177,6 +187,81 @@ pub struct OpChainSpec { pub inner: ChainSpec, } +impl OpChainSpec { + /// Read from parent to determine the base fee for the next block + pub fn next_block_base_fee( + &self, + parent: &Header, + timestamp: u64, + ) -> Result { + let is_holocene_activated = self.inner.is_fork_active_at_timestamp( + reth_optimism_forks::OptimismHardfork::Holocene, + timestamp, + ); + // If we are in the Holocene, we need to use the base fee params + // from the parent block's extra data. + // Else, use the base fee params (default values) from chainspec + if is_holocene_activated { + let (denominator, elasticity) = decode_holocene_1559_params(parent.extra_data.clone())?; + if elasticity == 0 && denominator == 0 { + return Ok(U256::from( + parent + .next_block_base_fee(self.base_fee_params_at_timestamp(timestamp)) + .unwrap_or_default(), + )); + } + let base_fee_params = BaseFeeParams::new(denominator as u128, elasticity as u128); + Ok(U256::from(parent.next_block_base_fee(base_fee_params).unwrap_or_default())) + } else { + Ok(U256::from( + parent + .next_block_base_fee(self.base_fee_params_at_timestamp(timestamp)) + .unwrap_or_default(), + )) + } + } +} + +#[derive(Clone, Debug, Display, Eq, PartialEq)] +/// Error type for decoding Holocene 1559 parameters +pub enum DecodeError { + #[display("Insufficient data to decode")] + /// Insufficient data to decode + InsufficientData, + #[display("Invalid denominator parameter")] + /// Invalid denominator parameter + InvalidDenominator, + #[display("Invalid elasticity parameter")] + /// Invalid elasticity parameter + InvalidElasticity, +} + +impl core::error::Error for DecodeError { + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + // None of the errors have sub-errors + None + } +} + +/// Extracts the Holcene 1599 parameters from the encoded form: +/// +pub fn decode_holocene_1559_params(extra_data: Bytes) -> Result<(u32, u32), DecodeError> { + if extra_data.len() < 9 { + return Err(DecodeError::InsufficientData); + } + let denominator: [u8; 4] = + extra_data[1..5].try_into().map_err(|_| DecodeError::InvalidDenominator)?; + let elasticity: [u8; 4] = + extra_data[5..9].try_into().map_err(|_| DecodeError::InvalidElasticity)?; + Ok((u32::from_be_bytes(denominator), u32::from_be_bytes(elasticity))) +} + +/// Returns the signature for the optimism deposit transactions, which don't include a +/// signature. +pub fn optimism_deposit_tx_signature() -> Signature { + Signature::new(U256::ZERO, U256::ZERO, Parity::Parity(false)) +} + impl EthChainSpec for OpChainSpec { fn chain(&self) -> alloy_chains::Chain { self.inner.chain() @@ -405,6 +490,8 @@ impl OptimismGenesisInfo { #[cfg(test)] mod tests { + use std::sync::Arc; + use alloy_genesis::{ChainConfig, Genesis}; use alloy_primitives::b256; use reth_chainspec::{test_fork_ids, BaseFeeParams, BaseFeeParamsKind}; @@ -919,4 +1006,87 @@ mod tests { .all(|(expected, actual)| &**expected == *actual)); assert_eq!(expected_hardforks.len(), hardforks.len()); } + + #[test] + fn test_get_base_fee_pre_holocene() { + let op_chain_spec = &BASE_SEPOLIA; + let parent = Header { + base_fee_per_gas: Some(1), + gas_used: 15763614, + gas_limit: 144000000, + ..Default::default() + }; + let base_fee = op_chain_spec.next_block_base_fee(&parent, 0); + assert_eq!( + base_fee.unwrap(), + U256::from( + parent + .next_block_base_fee(op_chain_spec.base_fee_params_at_timestamp(0)) + .unwrap_or_default() + ) + ); + } + + fn holocene_chainspec() -> Arc { + let mut hardforks = OptimismHardfork::base_sepolia(); + hardforks.insert(OptimismHardfork::Holocene.boxed(), ForkCondition::Timestamp(1800000000)); + Arc::new(OpChainSpec { + inner: ChainSpec { + chain: BASE_SEPOLIA.inner.chain, + genesis: BASE_SEPOLIA.inner.genesis.clone(), + genesis_hash: BASE_SEPOLIA.inner.genesis_hash.clone(), + paris_block_and_final_difficulty: Some((0, U256::from(0))), + hardforks, + base_fee_params: BASE_SEPOLIA.inner.base_fee_params.clone(), + max_gas_limit: crate::constants::BASE_SEPOLIA_MAX_GAS_LIMIT, + prune_delete_limit: 10000, + ..Default::default() + }, + }) + } + + #[test] + fn test_get_base_fee_holocene_nonce_not_set() { + let op_chain_spec = holocene_chainspec(); + let parent = Header { + base_fee_per_gas: Some(1), + gas_used: 15763614, + gas_limit: 144000000, + timestamp: 1800000003, + extra_data: Bytes::from_static(&[0, 0, 0, 0, 0, 0, 0, 0, 0]), + ..Default::default() + }; + let base_fee = op_chain_spec.next_block_base_fee(&parent, 1800000005); + assert_eq!( + base_fee.unwrap(), + U256::from( + parent + .next_block_base_fee(op_chain_spec.base_fee_params_at_timestamp(0)) + .unwrap_or_default() + ) + ); + } + + #[test] + fn test_get_base_fee_holocene_nonce_set() { + let op_chain_spec = holocene_chainspec(); + let parent = Header { + base_fee_per_gas: Some(1), + gas_used: 15763614, + gas_limit: 144000000, + extra_data: Bytes::from_static(&[0, 0, 0, 0, 8, 0, 0, 0, 8]), + timestamp: 1800000003, + ..Default::default() + }; + + let base_fee = op_chain_spec.next_block_base_fee(&parent, 1800000005); + assert_eq!( + base_fee.unwrap(), + U256::from( + parent + .next_block_base_fee(BaseFeeParams::new(0x00000008, 0x00000008)) + .unwrap_or_default() + ) + ); + } } diff --git a/crates/optimism/evm/src/config.rs b/crates/optimism/evm/src/config.rs index 668fcba4ddc4..b00341ff6776 100644 --- a/crates/optimism/evm/src/config.rs +++ b/crates/optimism/evm/src/config.rs @@ -12,7 +12,9 @@ pub fn revm_spec_by_timestamp_after_bedrock( chain_spec: &OpChainSpec, timestamp: u64, ) -> revm_primitives::SpecId { - if chain_spec.fork(OptimismHardfork::Granite).active_at_timestamp(timestamp) { + if chain_spec.fork(OptimismHardfork::Holocene).active_at_timestamp(timestamp) { + revm_primitives::HOLOCENE + } else if chain_spec.fork(OptimismHardfork::Granite).active_at_timestamp(timestamp) { revm_primitives::GRANITE } else if chain_spec.fork(OptimismHardfork::Fjord).active_at_timestamp(timestamp) { revm_primitives::FJORD @@ -29,7 +31,9 @@ pub fn revm_spec_by_timestamp_after_bedrock( /// Map the latest active hardfork at the given block to a revm [`SpecId`](revm_primitives::SpecId). pub fn revm_spec(chain_spec: &OpChainSpec, block: &Head) -> revm_primitives::SpecId { - if chain_spec.fork(OptimismHardfork::Granite).active_at_head(block) { + if chain_spec.fork(OptimismHardfork::Holocene).active_at_head(block) { + revm_primitives::HOLOCENE + } else if chain_spec.fork(OptimismHardfork::Granite).active_at_head(block) { revm_primitives::GRANITE } else if chain_spec.fork(OptimismHardfork::Fjord).active_at_head(block) { revm_primitives::FJORD diff --git a/crates/optimism/evm/src/lib.rs b/crates/optimism/evm/src/lib.rs index bc46f3ea9c20..03aecf2c83e8 100644 --- a/crates/optimism/evm/src/lib.rs +++ b/crates/optimism/evm/src/lib.rs @@ -15,7 +15,7 @@ extern crate alloc; use alloc::{sync::Arc, vec::Vec}; use alloy_primitives::{Address, U256}; use reth_evm::{ConfigureEvm, ConfigureEvmEnv, NextBlockEnvAttributes}; -use reth_optimism_chainspec::OpChainSpec; +use reth_optimism_chainspec::{DecodeError, OpChainSpec}; use reth_primitives::{ revm_primitives::{AnalysisKind, CfgEnvWithHandlerCfg, TxEnv}, transaction::FillTxEnv, @@ -56,6 +56,7 @@ impl OptimismEvmConfig { impl ConfigureEvmEnv for OptimismEvmConfig { type Header = Header; + type Error = DecodeError; fn fill_tx_env(&self, tx_env: &mut TxEnv, transaction: &TransactionSigned, sender: Address) { transaction.fill_tx_env(tx_env, sender); @@ -134,7 +135,7 @@ impl ConfigureEvmEnv for OptimismEvmConfig { &self, parent: &Self::Header, attributes: NextBlockEnvAttributes, - ) -> (CfgEnvWithHandlerCfg, BlockEnv) { + ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), Self::Error> { // configure evm env based on parent block let cfg = CfgEnv::default().with_chain_id(self.chain_spec.chain().id()); @@ -156,13 +157,7 @@ impl ConfigureEvmEnv for OptimismEvmConfig { prevrandao: Some(attributes.prev_randao), gas_limit: U256::from(parent.gas_limit), // calculate basefee based on parent block's gas usage - basefee: U256::from( - parent - .next_block_base_fee( - self.chain_spec.base_fee_params_at_timestamp(attributes.timestamp), - ) - .unwrap_or_default(), - ), + basefee: self.chain_spec.next_block_base_fee(parent, attributes.timestamp)?, // calculate excess gas based on parent block's blob gas usage blob_excess_gas_and_price, }; @@ -175,7 +170,7 @@ impl ConfigureEvmEnv for OptimismEvmConfig { }; } - (cfg_with_handler_cfg, block_env) + Ok((cfg_with_handler_cfg, block_env)) } } diff --git a/crates/optimism/hardforks/src/hardfork.rs b/crates/optimism/hardforks/src/hardfork.rs index 011c4ae72fdd..440314e3711f 100644 --- a/crates/optimism/hardforks/src/hardfork.rs +++ b/crates/optimism/hardforks/src/hardfork.rs @@ -31,6 +31,8 @@ hardfork!( Fjord, /// Granite: Granite, + /// Holocene: + Holocene, } ); @@ -156,6 +158,7 @@ impl OptimismHardfork { Self::Ecotone => Some(1708534800), Self::Fjord => Some(1716998400), Self::Granite => Some(1723478400), + Self::Holocene => None, }, ) } @@ -190,6 +193,7 @@ impl OptimismHardfork { Self::Ecotone => Some(1710374401), Self::Fjord => Some(1720627201), Self::Granite => Some(1726070401), + Self::Holocene => None, }, ) } diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index cec609671a38..966d87279c5a 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -15,7 +15,9 @@ use reth_node_api::{ }; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_forks::OptimismHardfork; -use reth_optimism_payload_builder::{OptimismBuiltPayload, OptimismPayloadBuilderAttributes}; +use reth_optimism_payload_builder::{ + builder::decode_eip_1559_params, OptimismBuiltPayload, OptimismPayloadBuilderAttributes, +}; /// The types used in the optimism beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] @@ -147,6 +149,139 @@ where )) } + if self.chain_spec.is_fork_active_at_timestamp( + OptimismHardfork::Holocene, + attributes.payload_attributes.timestamp, + ) { + let Some(eip_1559_params) = attributes.eip_1559_params else { + return Err(EngineObjectValidationError::InvalidParams( + "MissingEip1559ParamsInPayloadAttributes".to_string().into(), + )) + }; + let (elasticity, denominator) = decode_eip_1559_params(eip_1559_params); + if elasticity != 0 && denominator == 0 { + return Err(EngineObjectValidationError::InvalidParams( + "Eip1559ParamsDenominatorZero".to_string().into(), + )) + } + } + Ok(()) } } + +#[cfg(test)] +mod test { + + use crate::engine; + use alloy_primitives::{b64, Address, B256, B64}; + use alloy_rpc_types_engine::PayloadAttributes; + use reth_chainspec::ForkCondition; + use reth_optimism_chainspec::BASE_SEPOLIA; + + use super::*; + + fn get_chainspec(is_holocene: bool) -> Arc { + let mut hardforks = OptimismHardfork::base_sepolia(); + if is_holocene { + hardforks + .insert(OptimismHardfork::Holocene.boxed(), ForkCondition::Timestamp(1800000000)); + } + Arc::new(OpChainSpec { + inner: ChainSpec { + chain: BASE_SEPOLIA.inner.chain, + genesis: BASE_SEPOLIA.inner.genesis.clone(), + genesis_hash: BASE_SEPOLIA.inner.genesis_hash.clone(), + paris_block_and_final_difficulty: BASE_SEPOLIA + .inner + .paris_block_and_final_difficulty, + hardforks, + base_fee_params: BASE_SEPOLIA.inner.base_fee_params.clone(), + max_gas_limit: BASE_SEPOLIA.inner.max_gas_limit, + prune_delete_limit: 10000, + ..Default::default() + }, + }) + } + + const fn get_attributes(eip_1559_params: Option, timestamp: u64) -> OpPayloadAttributes { + OpPayloadAttributes { + gas_limit: Some(1000), + eip_1559_params, + transactions: None, + no_tx_pool: None, + payload_attributes: PayloadAttributes { + timestamp, + prev_randao: B256::ZERO, + suggested_fee_recipient: Address::ZERO, + withdrawals: Some(vec![]), + parent_beacon_block_root: Some(B256::ZERO), + }, + } + } + + #[test] + fn test_well_formed_attributes_pre_holocene() { + let validator = OptimismEngineValidator::new(get_chainspec(false)); + let attributes = get_attributes(None, 1799999999); + + let result = >::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes + ); + assert!(result.is_ok()); + } + + #[test] + fn test_well_formed_attributes_holocene_no_eip1559_params() { + let validator = OptimismEngineValidator::new(get_chainspec(true)); + let attributes = get_attributes(None, 1800000000); + + let result = >::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes + ); + assert!(matches!(result, Err(EngineObjectValidationError::InvalidParams(_)))); + } + + #[test] + fn test_well_formed_attributes_holocene_eip1559_params_zero_denominator() { + let validator = OptimismEngineValidator::new(get_chainspec(true)); + let attributes = get_attributes(Some(b64!("0000000000000008")), 1800000000); + + let result = >::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes + ); + assert!(matches!(result, Err(EngineObjectValidationError::InvalidParams(_)))); + } + + #[test] + fn test_well_formed_attributes_holocene_valid() { + let validator = OptimismEngineValidator::new(get_chainspec(true)); + let attributes = get_attributes(Some(b64!("0000000800000008")), 1800000000); + + let result = >::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes + ); + assert!(result.is_ok()); + } + + #[test] + fn test_well_formed_attributes_holocene_valid_all_zero() { + let validator = OptimismEngineValidator::new(get_chainspec(true)); + let attributes = get_attributes(Some(b64!("0000000000000000")), 1800000000); + + let result = >::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes + ); + assert!(result.is_ok()); + } +} diff --git a/crates/optimism/node/tests/e2e/utils.rs b/crates/optimism/node/tests/e2e/utils.rs index 48175e5b21ac..d4219b0fea1f 100644 --- a/crates/optimism/node/tests/e2e/utils.rs +++ b/crates/optimism/node/tests/e2e/utils.rs @@ -63,5 +63,6 @@ pub(crate) fn optimism_payload_attributes(timestamp: u64) -> OptimismPayloadBuil transactions: vec![], no_tx_pool: false, gas_limit: Some(30_000_000), + eip_1559_params: None, } } diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 0550adeaa8d5..e9b7e2c76f80 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -1,10 +1,9 @@ //! Optimism payload builder implementation. - use std::sync::Arc; use alloy_consensus::EMPTY_OMMER_ROOT_HASH; use alloy_eips::merge::BEACON_NONCE; -use alloy_primitives::U256; +use alloy_primitives::{B64, U256}; use reth_basic_payload_builder::*; use reth_chain_state::ExecutedBlock; use reth_chainspec::ChainSpecProvider; @@ -80,7 +79,7 @@ where &self, config: &PayloadConfig, parent: &Header, - ) -> (CfgEnvWithHandlerCfg, BlockEnv) { + ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), EvmConfig::Error> { let next_attributes = NextBlockEnvAttributes { timestamp: config.attributes.timestamp(), suggested_fee_recipient: config.attributes.suggested_fee_recipient(), @@ -104,7 +103,9 @@ where &self, args: BuildArguments, ) -> Result, PayloadBuilderError> { - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); + let (cfg_env, block_env) = self + .cfg_and_block_env(&args.config, &args.config.parent_header) + .map_err(PayloadBuilderError::other)?; optimism_payload(&self.evm_config, args, cfg_env, block_env, self.compute_pending_block) } @@ -133,7 +134,9 @@ where cancel: Default::default(), best_payload: None, }; - let (cfg_env, block_env) = self.cfg_and_block_env(&args.config, &args.config.parent_header); + let (cfg_env, block_env) = self + .cfg_and_block_env(&args.config, &args.config.parent_header) + .map_err(PayloadBuilderError::other)?; optimism_payload(&self.evm_config, args, cfg_env, block_env, false)? .into_payload() .ok_or_else(|| PayloadBuilderError::MissingPayload) @@ -168,7 +171,7 @@ where let state = StateProviderDatabase::new(state_provider); let mut db = State::builder().with_database_ref(cached_reads.as_db(state)).with_bundle_update().build(); - let PayloadConfig { parent_header, attributes, extra_data } = config; + let PayloadConfig { parent_header, attributes, mut extra_data } = config; debug!(target: "payload_builder", id=%attributes.payload_attributes.payload_id(), parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload"); @@ -470,6 +473,19 @@ where (None, None) }; + let is_holocene = chain_spec.is_fork_active_at_timestamp( + OptimismHardfork::Holocene, + attributes.payload_attributes.timestamp, + ); + + if is_holocene { + extra_data = attributes + .get_holocene_extra_data( + chain_spec.base_fee_params_at_timestamp(attributes.payload_attributes.timestamp), + ) + .map_err(PayloadBuilderError::other)?; + } + let header = Header { parent_hash: parent_header.hash(), ommers_hash: EMPTY_OMMER_ROOT_HASH, @@ -532,3 +548,12 @@ where Ok(BuildOutcome::Better { payload, cached_reads }) } } + +/// Extracts the Holocene 1599 parameters from the encoded form: +/// +pub fn decode_eip_1559_params(eip_1559_params: B64) -> (u32, u32) { + let denominator: [u8; 4] = eip_1559_params.0[..4].try_into().expect("sufficient length"); + let elasticity: [u8; 4] = eip_1559_params.0[4..8].try_into().expect("sufficient length"); + + (u32::from_be_bytes(elasticity), u32::from_be_bytes(denominator)) +} diff --git a/crates/optimism/payload/src/error.rs b/crates/optimism/payload/src/error.rs index 2016fdc6dd93..ce5f584a1cec 100644 --- a/crates/optimism/payload/src/error.rs +++ b/crates/optimism/payload/src/error.rs @@ -21,3 +21,17 @@ pub enum OptimismPayloadBuilderError { #[error("blob transaction included in sequencer block")] BlobTransactionRejected, } + +/// Error type for EIP-1559 parameters +#[derive(Debug, thiserror::Error)] +pub enum EIP1559ParamError { + /// No EIP-1559 parameters provided + #[error("No EIP-1559 parameters provided")] + NoEIP1559Params, + /// Denominator overflow + #[error("Denominator overflow")] + DenominatorOverflow, + /// Elasticity overflow + #[error("Elasticity overflow")] + ElasticityOverflow, +} diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index 7f95d04ad9fa..056edfe7b638 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -2,8 +2,9 @@ //! Optimism builder support -use alloy_eips::{eip2718::Decodable2718, eip7685::Requests}; -use alloy_primitives::{keccak256, Address, B256, U256}; +use crate::{builder::decode_eip_1559_params, error::EIP1559ParamError}; +use alloy_eips::{eip1559::BaseFeeParams, eip2718::Decodable2718, eip7685::Requests}; +use alloy_primitives::{keccak256, Address, Bytes, B256, B64, U256}; use alloy_rlp::Encodable; use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV2, ExecutionPayloadV1, PayloadId}; /// Re-export for use in downstream arguments. @@ -23,7 +24,7 @@ use reth_rpc_types_compat::engine::payload::{ use std::sync::Arc; /// Optimism Payload Builder Attributes -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct OptimismPayloadBuilderAttributes { /// Inner ethereum payload builder attributes pub payload_attributes: EthPayloadBuilderAttributes, @@ -34,6 +35,42 @@ pub struct OptimismPayloadBuilderAttributes { pub transactions: Vec>, /// The gas limit for the generated payload pub gas_limit: Option, + /// EIP-1559 parameters for the generated payload + pub eip_1559_params: Option, +} + +impl OptimismPayloadBuilderAttributes { + /// Extracts the `eip1559` parameters for the payload. + pub fn get_holocene_extra_data( + &self, + default_base_fee_params: BaseFeeParams, + ) -> Result { + let eip_1559_params = self.eip_1559_params.ok_or(EIP1559ParamError::NoEIP1559Params)?; + + let mut extra_data = [0u8; 9]; + // If eip 1559 params aren't set, use the canyon base fee param constants + // otherwise use them + if eip_1559_params.is_zero() { + // Try casting max_change_denominator to u32 + let max_change_denominator: u32 = (default_base_fee_params.max_change_denominator) + .try_into() + .map_err(|_| EIP1559ParamError::DenominatorOverflow)?; + + // Try casting elasticity_multiplier to u32 + let elasticity_multiplier: u32 = (default_base_fee_params.elasticity_multiplier) + .try_into() + .map_err(|_| EIP1559ParamError::ElasticityOverflow)?; + + // Copy the values safely + extra_data[1..5].copy_from_slice(&max_change_denominator.to_be_bytes()); + extra_data[5..9].copy_from_slice(&elasticity_multiplier.to_be_bytes()); + } else { + let (elasticity, denominator) = decode_eip_1559_params(eip_1559_params); + extra_data[1..5].copy_from_slice(&denominator.to_be_bytes()); + extra_data[5..9].copy_from_slice(&elasticity.to_be_bytes()); + } + Ok(Bytes::copy_from_slice(&extra_data)) + } } impl PayloadBuilderAttributes for OptimismPayloadBuilderAttributes { @@ -82,6 +119,7 @@ impl PayloadBuilderAttributes for OptimismPayloadBuilderAttributes { no_tx_pool: attributes.no_tx_pool.unwrap_or_default(), transactions, gas_limit: attributes.gas_limit, + eip_1559_params: attributes.eip_1559_params, }) } @@ -370,4 +408,24 @@ mod tests { ) ); } + + #[test] + fn test_get_extra_data_post_holocene() { + let attributes = OptimismPayloadBuilderAttributes { + eip_1559_params: Some(B64::from_str("0x0000000800000008").unwrap()), + ..Default::default() + }; + let extra_data = attributes.get_holocene_extra_data(BaseFeeParams::new(80, 60)); + assert_eq!(extra_data.unwrap(), Bytes::copy_from_slice(&[0, 0, 0, 0, 8, 0, 0, 0, 8])); + } + + #[test] + fn test_get_extra_data_post_holocene_default() { + let attributes = OptimismPayloadBuilderAttributes { + eip_1559_params: Some(B64::ZERO), + ..Default::default() + }; + let extra_data = attributes.get_holocene_extra_data(BaseFeeParams::new(80, 60)); + assert_eq!(extra_data.unwrap(), Bytes::copy_from_slice(&[0, 0, 0, 0, 80, 0, 0, 0, 60])); + } } diff --git a/examples/custom-evm/src/main.rs b/examples/custom-evm/src/main.rs index 55063fc9bbcb..16aad63c0932 100644 --- a/examples/custom-evm/src/main.rs +++ b/examples/custom-evm/src/main.rs @@ -38,7 +38,7 @@ use reth_primitives::{ Header, TransactionSigned, }; use reth_tracing::{RethTracer, Tracer}; -use std::sync::Arc; +use std::{convert::Infallible, sync::Arc}; /// Custom EVM configuration #[derive(Debug, Clone)] @@ -87,6 +87,7 @@ impl MyEvmConfig { impl ConfigureEvmEnv for MyEvmConfig { type Header = Header; + type Error = Infallible; fn fill_tx_env(&self, tx_env: &mut TxEnv, transaction: &TransactionSigned, sender: Address) { self.inner.fill_tx_env(tx_env, transaction, sender); @@ -115,7 +116,7 @@ impl ConfigureEvmEnv for MyEvmConfig { &self, parent: &Self::Header, attributes: NextBlockEnvAttributes, - ) -> (CfgEnvWithHandlerCfg, BlockEnv) { + ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), Self::Error> { self.inner.next_cfg_and_block_env(parent, attributes) } } diff --git a/examples/stateful-precompile/src/main.rs b/examples/stateful-precompile/src/main.rs index b0165e4de26c..371fbf4f78bc 100644 --- a/examples/stateful-precompile/src/main.rs +++ b/examples/stateful-precompile/src/main.rs @@ -30,7 +30,7 @@ use reth_primitives::{ }; use reth_tracing::{RethTracer, Tracer}; use schnellru::{ByLength, LruMap}; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, convert::Infallible, sync::Arc}; /// Type alias for the LRU cache used within the [`PrecompileCache`]. type PrecompileLRUCache = LruMap<(Bytes, u64), PrecompileResult>; @@ -147,6 +147,7 @@ impl StatefulPrecompileMut for WrappedPrecompile { impl ConfigureEvmEnv for MyEvmConfig { type Header = Header; + type Error = Infallible; fn fill_tx_env(&self, tx_env: &mut TxEnv, transaction: &TransactionSigned, sender: Address) { self.inner.fill_tx_env(tx_env, transaction, sender) @@ -175,7 +176,7 @@ impl ConfigureEvmEnv for MyEvmConfig { &self, parent: &Self::Header, attributes: NextBlockEnvAttributes, - ) -> (CfgEnvWithHandlerCfg, BlockEnv) { + ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), Self::Error> { self.inner.next_cfg_and_block_env(parent, attributes) } } From ff9a42ae8fbf441f4fb9ca8dcea84c765b284721 Mon Sep 17 00:00:00 2001 From: Abhishek kochar Date: Wed, 30 Oct 2024 22:13:03 +0800 Subject: [PATCH 182/239] feat(eth69): support for ETH69 (#12158) Signed-off-by: Abhishekkochar Co-authored-by: Matthias Seitz --- crates/net/eth-wire-types/src/message.rs | 10 +++++++++- crates/net/eth-wire-types/src/version.rs | 20 ++++++++++++++++---- crates/net/network/src/transactions/mod.rs | 2 +- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/net/eth-wire-types/src/message.rs b/crates/net/eth-wire-types/src/message.rs index 4afcb34e13bf..8546bfe14c8f 100644 --- a/crates/net/eth-wire-types/src/message.rs +++ b/crates/net/eth-wire-types/src/message.rs @@ -50,9 +50,17 @@ impl ProtocolMessage { let message = match message_type { EthMessageID::Status => EthMessage::Status(Status::decode(buf)?), EthMessageID::NewBlockHashes => { + if version.is_eth69() { + return Err(MessageError::Invalid(version, EthMessageID::NewBlockHashes)); + } EthMessage::NewBlockHashes(NewBlockHashes::decode(buf)?) } - EthMessageID::NewBlock => EthMessage::NewBlock(Box::new(NewBlock::decode(buf)?)), + EthMessageID::NewBlock => { + if version.is_eth69() { + return Err(MessageError::Invalid(version, EthMessageID::NewBlock)); + } + EthMessage::NewBlock(Box::new(NewBlock::decode(buf)?)) + } EthMessageID::Transactions => EthMessage::Transactions(Transactions::decode(buf)?), EthMessageID::NewPooledTransactionHashes => { if version >= EthVersion::Eth68 { diff --git a/crates/net/eth-wire-types/src/version.rs b/crates/net/eth-wire-types/src/version.rs index 4fd3e792dcc9..5a2e0ff9651d 100644 --- a/crates/net/eth-wire-types/src/version.rs +++ b/crates/net/eth-wire-types/src/version.rs @@ -18,12 +18,12 @@ pub struct ParseVersionError(String); pub enum EthVersion { /// The `eth` protocol version 66. Eth66 = 66, - /// The `eth` protocol version 67. Eth67 = 67, - /// The `eth` protocol version 68. Eth68 = 68, + /// The `eth` protocol version 69. + Eth69 = 69, } impl EthVersion { @@ -38,6 +38,8 @@ impl EthVersion { // eth/67,68 are eth/66 minus GetNodeData and NodeData messages 13 } + // eth69 is both eth67 and eth68 minus NewBlockHashes and NewBlock + Self::Eth69 => 11, } } @@ -55,6 +57,11 @@ impl EthVersion { pub const fn is_eth68(&self) -> bool { matches!(self, Self::Eth68) } + + /// Returns true if the version is eth/69 + pub const fn is_eth69(&self) -> bool { + matches!(self, Self::Eth69) + } } /// Allow for converting from a `&str` to an `EthVersion`. @@ -75,6 +82,7 @@ impl TryFrom<&str> for EthVersion { "66" => Ok(Self::Eth66), "67" => Ok(Self::Eth67), "68" => Ok(Self::Eth68), + "69" => Ok(Self::Eth69), _ => Err(ParseVersionError(s.to_string())), } } @@ -98,6 +106,7 @@ impl TryFrom for EthVersion { 66 => Ok(Self::Eth66), 67 => Ok(Self::Eth67), 68 => Ok(Self::Eth68), + 69 => Ok(Self::Eth69), _ => Err(ParseVersionError(u.to_string())), } } @@ -126,6 +135,7 @@ impl From for &'static str { EthVersion::Eth66 => "66", EthVersion::Eth67 => "67", EthVersion::Eth68 => "68", + EthVersion::Eth69 => "69", } } } @@ -179,7 +189,8 @@ mod tests { assert_eq!(EthVersion::Eth66, EthVersion::try_from("66").unwrap()); assert_eq!(EthVersion::Eth67, EthVersion::try_from("67").unwrap()); assert_eq!(EthVersion::Eth68, EthVersion::try_from("68").unwrap()); - assert_eq!(Err(ParseVersionError("69".to_string())), EthVersion::try_from("69")); + assert_eq!(EthVersion::Eth69, EthVersion::try_from("69").unwrap()); + assert_eq!(Err(ParseVersionError("70".to_string())), EthVersion::try_from("70")); } #[test] @@ -187,6 +198,7 @@ mod tests { assert_eq!(EthVersion::Eth66, "66".parse().unwrap()); assert_eq!(EthVersion::Eth67, "67".parse().unwrap()); assert_eq!(EthVersion::Eth68, "68".parse().unwrap()); - assert_eq!(Err(ParseVersionError("69".to_string())), "69".parse::()); + assert_eq!(EthVersion::Eth69, "69".parse().unwrap()); + assert_eq!(Err(ParseVersionError("70".to_string())), "70".parse::()); } } diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index b6e589c6ff79..4e23c8527b47 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1668,7 +1668,7 @@ impl PooledTransactionsHashesBuilder { fn new(version: EthVersion) -> Self { match version { EthVersion::Eth66 | EthVersion::Eth67 => Self::Eth66(Default::default()), - EthVersion::Eth68 => Self::Eth68(Default::default()), + EthVersion::Eth68 | EthVersion::Eth69 => Self::Eth68(Default::default()), } } From bb8da983b0260c8aa2912b20516a8518f2f8cd20 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Oct 2024 16:39:12 +0100 Subject: [PATCH 183/239] feat: add is_ethereum trait fn (#12197) --- crates/chainspec/src/api.rs | 5 +++++ crates/net/discv5/src/network_stack_id.rs | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index 3751789cac8c..36640f34b709 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -57,6 +57,11 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { fn is_optimism(&self) -> bool { self.chain().is_optimism() } + + /// Returns `true` if this chain contains Ethereum configuration. + fn is_ethereum(&self) -> bool { + self.chain().is_ethereum() + } } impl EthChainSpec for ChainSpec { diff --git a/crates/net/discv5/src/network_stack_id.rs b/crates/net/discv5/src/network_stack_id.rs index f707c7de7b7f..a7b6944f355a 100644 --- a/crates/net/discv5/src/network_stack_id.rs +++ b/crates/net/discv5/src/network_stack_id.rs @@ -20,12 +20,11 @@ impl NetworkStackId { /// ENR fork ID kv-pair key, for an Optimism CL node. pub const OPSTACK: &'static [u8] = b"opstack"; - #[allow(clippy::missing_const_for_fn)] /// Returns the [`NetworkStackId`] that matches the given chain spec. pub fn id(chain: impl EthChainSpec) -> Option<&'static [u8]> { - if chain.chain().is_optimism() { + if chain.is_optimism() { return Some(Self::OPEL) - } else if chain.chain().is_ethereum() { + } else if chain.is_ethereum() { return Some(Self::ETH) } From 755fac08ddeec0f4260fa5565bac72d6cb392348 Mon Sep 17 00:00:00 2001 From: Kaushik Donthi Date: Wed, 30 Oct 2024 09:06:37 -0700 Subject: [PATCH 184/239] Wrap sidecar in arcs (#11554) Co-authored-by: Matthias Seitz --- crates/ethereum/payload/src/lib.rs | 2 +- crates/transaction-pool/src/blobstore/disk.rs | 44 ++++++++++++------- crates/transaction-pool/src/blobstore/mem.rs | 21 ++++----- crates/transaction-pool/src/blobstore/mod.rs | 12 +++-- crates/transaction-pool/src/blobstore/noop.rs | 10 +++-- crates/transaction-pool/src/lib.rs | 9 ++-- crates/transaction-pool/src/maintain.rs | 2 + crates/transaction-pool/src/noop.rs | 9 ++-- crates/transaction-pool/src/pool/mod.rs | 4 +- crates/transaction-pool/src/traits.rs | 9 ++-- .../src/mined_sidecar.rs | 8 ++-- 11 files changed, 84 insertions(+), 46 deletions(-) diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 27d9d98bcca2..8e92d0aa8706 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -456,7 +456,7 @@ where EthBuiltPayload::new(attributes.id, sealed_block, total_fees, Some(executed), requests); // extend the payload with the blob sidecars from the executed txs - payload.extend_sidecars(blob_sidecars); + payload.extend_sidecars(blob_sidecars.into_iter().map(Arc::unwrap_or_clone)); Ok(BuildOutcome::Better { payload, cached_reads }) } diff --git a/crates/transaction-pool/src/blobstore/disk.rs b/crates/transaction-pool/src/blobstore/disk.rs index 787d4985ff1b..987264853db8 100644 --- a/crates/transaction-pool/src/blobstore/disk.rs +++ b/crates/transaction-pool/src/blobstore/disk.rs @@ -103,7 +103,7 @@ impl BlobStore for DiskFileBlobStore { stat } - fn get(&self, tx: B256) -> Result, BlobStoreError> { + fn get(&self, tx: B256) -> Result>, BlobStoreError> { self.inner.get_one(tx) } @@ -114,14 +114,17 @@ impl BlobStore for DiskFileBlobStore { fn get_all( &self, txs: Vec, - ) -> Result, BlobStoreError> { + ) -> Result)>, BlobStoreError> { if txs.is_empty() { return Ok(Vec::new()) } self.inner.get_all(txs) } - fn get_exact(&self, txs: Vec) -> Result, BlobStoreError> { + fn get_exact( + &self, + txs: Vec, + ) -> Result>, BlobStoreError> { if txs.is_empty() { return Ok(Vec::new()) } @@ -164,7 +167,7 @@ impl BlobStore for DiskFileBlobStore { struct DiskFileBlobStoreInner { blob_dir: PathBuf, - blob_cache: Mutex>, + blob_cache: Mutex, ByLength>>, size_tracker: BlobStoreSize, file_lock: RwLock<()>, txs_to_delete: RwLock>, @@ -205,7 +208,7 @@ impl DiskFileBlobStoreInner { fn insert_one(&self, tx: B256, data: BlobTransactionSidecar) -> Result<(), BlobStoreError> { let mut buf = Vec::with_capacity(data.rlp_encoded_fields_length()); data.rlp_encode_fields(&mut buf); - self.blob_cache.lock().insert(tx, data); + self.blob_cache.lock().insert(tx, Arc::new(data)); let size = self.write_one_encoded(tx, &buf)?; self.size_tracker.add_size(size); @@ -227,7 +230,7 @@ impl DiskFileBlobStoreInner { { let mut cache = self.blob_cache.lock(); for (tx, data) in txs { - cache.insert(tx, data); + cache.insert(tx, Arc::new(data)); } } let mut add = 0; @@ -278,15 +281,19 @@ impl DiskFileBlobStoreInner { } /// Retrieves the blob for the given transaction hash from the blob cache or disk. - fn get_one(&self, tx: B256) -> Result, BlobStoreError> { + fn get_one(&self, tx: B256) -> Result>, BlobStoreError> { if let Some(blob) = self.blob_cache.lock().get(&tx) { return Ok(Some(blob.clone())) } let blob = self.read_one(tx)?; + if let Some(blob) = &blob { - self.blob_cache.lock().insert(tx, blob.clone()); + let blob_arc = Arc::new(blob.clone()); + self.blob_cache.lock().insert(tx, blob_arc.clone()); + return Ok(Some(blob_arc)) } - Ok(blob) + + Ok(None) } /// Returns the path to the blob file for the given transaction hash. @@ -374,7 +381,7 @@ impl DiskFileBlobStoreInner { fn get_all( &self, txs: Vec, - ) -> Result, BlobStoreError> { + ) -> Result)>, BlobStoreError> { let mut res = Vec::with_capacity(txs.len()); let mut cache_miss = Vec::new(); { @@ -396,8 +403,9 @@ impl DiskFileBlobStoreInner { } let mut cache = self.blob_cache.lock(); for (tx, data) in from_disk { - cache.insert(tx, data.clone()); - res.push((tx, data)); + let arc = Arc::new(data.clone()); + cache.insert(tx, arc.clone()); + res.push((tx, arc.clone())); } Ok(res) @@ -407,7 +415,10 @@ impl DiskFileBlobStoreInner { /// /// Returns an error if there are any missing blobs. #[inline] - fn get_exact(&self, txs: Vec) -> Result, BlobStoreError> { + fn get_exact( + &self, + txs: Vec, + ) -> Result>, BlobStoreError> { txs.into_iter() .map(|tx| self.get_one(tx)?.ok_or(BlobStoreError::MissingSidecar(tx))) .collect() @@ -514,14 +525,17 @@ mod tests { let blobs = rng_blobs(10); let all_hashes = blobs.iter().map(|(tx, _)| *tx).collect::>(); store.insert_all(blobs.clone()).unwrap(); + // all cached for (tx, blob) in &blobs { assert!(store.is_cached(tx)); - assert_eq!(store.get(*tx).unwrap().unwrap(), *blob); + let b = store.get(*tx).unwrap().map(Arc::unwrap_or_clone).unwrap(); + assert_eq!(b, *blob); } + let all = store.get_all(all_hashes.clone()).unwrap(); for (tx, blob) in all { - assert!(blobs.contains(&(tx, blob)), "missing blob {tx:?}"); + assert!(blobs.contains(&(tx, Arc::unwrap_or_clone(blob))), "missing blob {tx:?}"); } assert!(store.contains(all_hashes[0]).unwrap()); diff --git a/crates/transaction-pool/src/blobstore/mem.rs b/crates/transaction-pool/src/blobstore/mem.rs index c98a01b88c18..cea1837bdcd3 100644 --- a/crates/transaction-pool/src/blobstore/mem.rs +++ b/crates/transaction-pool/src/blobstore/mem.rs @@ -15,7 +15,7 @@ pub struct InMemoryBlobStore { #[derive(Debug, Default)] struct InMemoryBlobStoreInner { /// Storage for all blob data. - store: RwLock>, + store: RwLock>>, size_tracker: BlobStoreSize, } @@ -75,7 +75,7 @@ impl BlobStore for InMemoryBlobStore { } // Retrieves the decoded blob data for the given transaction hash. - fn get(&self, tx: B256) -> Result, BlobStoreError> { + fn get(&self, tx: B256) -> Result>, BlobStoreError> { Ok(self.inner.store.read().get(&tx).cloned()) } @@ -86,16 +86,17 @@ impl BlobStore for InMemoryBlobStore { fn get_all( &self, txs: Vec, - ) -> Result, BlobStoreError> { + ) -> Result)>, BlobStoreError> { let store = self.inner.store.read(); Ok(txs.into_iter().filter_map(|tx| store.get(&tx).map(|item| (tx, item.clone()))).collect()) } - fn get_exact(&self, txs: Vec) -> Result, BlobStoreError> { + fn get_exact( + &self, + txs: Vec, + ) -> Result>, BlobStoreError> { let store = self.inner.store.read(); - txs.into_iter() - .map(|tx| store.get(&tx).cloned().ok_or_else(|| BlobStoreError::MissingSidecar(tx))) - .collect() + Ok(txs.into_iter().filter_map(|tx| store.get(&tx).cloned()).collect()) } fn get_by_versioned_hashes( @@ -134,7 +135,7 @@ impl BlobStore for InMemoryBlobStore { /// Removes the given blob from the store and returns the size of the blob that was removed. #[inline] -fn remove_size(store: &mut HashMap, tx: &B256) -> usize { +fn remove_size(store: &mut HashMap>, tx: &B256) -> usize { store.remove(tx).map(|rem| rem.size()).unwrap_or_default() } @@ -143,11 +144,11 @@ fn remove_size(store: &mut HashMap, tx: &B256) -> /// We don't need to handle the size updates for replacements because transactions are unique. #[inline] fn insert_size( - store: &mut HashMap, + store: &mut HashMap>, tx: B256, blob: BlobTransactionSidecar, ) -> usize { let add = blob.size(); - store.insert(tx, blob); + store.insert(tx, Arc::new(blob)); add } diff --git a/crates/transaction-pool/src/blobstore/mod.rs b/crates/transaction-pool/src/blobstore/mod.rs index ee98e3eed85e..f8d37bfcc0f2 100644 --- a/crates/transaction-pool/src/blobstore/mod.rs +++ b/crates/transaction-pool/src/blobstore/mod.rs @@ -8,7 +8,10 @@ pub use noop::NoopBlobStore; use reth_primitives::BlobTransactionSidecar; use std::{ fmt, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; pub use tracker::{BlobStoreCanonTracker, BlobStoreUpdates}; @@ -44,7 +47,7 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static { fn cleanup(&self) -> BlobStoreCleanupStat; /// Retrieves the decoded blob data for the given transaction hash. - fn get(&self, tx: B256) -> Result, BlobStoreError>; + fn get(&self, tx: B256) -> Result>, BlobStoreError>; /// Checks if the given transaction hash is in the blob store. fn contains(&self, tx: B256) -> Result; @@ -58,13 +61,14 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static { fn get_all( &self, txs: Vec, - ) -> Result, BlobStoreError>; + ) -> Result)>, BlobStoreError>; /// Returns the exact [`BlobTransactionSidecar`] for the given transaction hashes in the exact /// order they were requested. /// /// Returns an error if any of the blobs are not found in the blob store. - fn get_exact(&self, txs: Vec) -> Result, BlobStoreError>; + fn get_exact(&self, txs: Vec) + -> Result>, BlobStoreError>; /// Return the [`BlobTransactionSidecar`]s for a list of blob versioned hashes. fn get_by_versioned_hashes( diff --git a/crates/transaction-pool/src/blobstore/noop.rs b/crates/transaction-pool/src/blobstore/noop.rs index 0e99858bd627..0f2935735562 100644 --- a/crates/transaction-pool/src/blobstore/noop.rs +++ b/crates/transaction-pool/src/blobstore/noop.rs @@ -1,6 +1,7 @@ use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobTransactionSidecar}; use alloy_eips::eip4844::BlobAndProofV1; use alloy_primitives::B256; +use std::sync::Arc; /// A blobstore implementation that does nothing #[derive(Clone, Copy, Debug, PartialOrd, PartialEq, Eq, Default)] @@ -28,7 +29,7 @@ impl BlobStore for NoopBlobStore { BlobStoreCleanupStat::default() } - fn get(&self, _tx: B256) -> Result, BlobStoreError> { + fn get(&self, _tx: B256) -> Result>, BlobStoreError> { Ok(None) } @@ -39,11 +40,14 @@ impl BlobStore for NoopBlobStore { fn get_all( &self, _txs: Vec, - ) -> Result, BlobStoreError> { + ) -> Result)>, BlobStoreError> { Ok(vec![]) } - fn get_exact(&self, txs: Vec) -> Result, BlobStoreError> { + fn get_exact( + &self, + txs: Vec, + ) -> Result>, BlobStoreError> { if txs.is_empty() { return Ok(vec![]) } diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 609ab987f502..3a5e547ba4e7 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -561,21 +561,24 @@ where self.pool.unique_senders() } - fn get_blob(&self, tx_hash: TxHash) -> Result, BlobStoreError> { + fn get_blob( + &self, + tx_hash: TxHash, + ) -> Result>, BlobStoreError> { self.pool.blob_store().get(tx_hash) } fn get_all_blobs( &self, tx_hashes: Vec, - ) -> Result, BlobStoreError> { + ) -> Result)>, BlobStoreError> { self.pool.blob_store().get_all(tx_hashes) } fn get_all_blobs_exact( &self, tx_hashes: Vec, - ) -> Result, BlobStoreError> { + ) -> Result>, BlobStoreError> { self.pool.blob_store().get_exact(tx_hashes) } diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 09c042ae66dc..b62a6c18c0bc 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -27,6 +27,7 @@ use std::{ collections::HashSet, hash::{Hash, Hasher}, path::{Path, PathBuf}, + sync::Arc, }; use tokio::sync::oneshot; use tracing::{debug, error, info, trace, warn}; @@ -328,6 +329,7 @@ pub async fn maintain_transaction_pool( pool.get_blob(tx.hash) .ok() .flatten() + .map(Arc::unwrap_or_clone) .and_then(|sidecar| { PooledTransactionsElementEcRecovered::try_from_blob_transaction( tx, sidecar, diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 817ea7bad7a0..4f4e5a3813ae 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -275,21 +275,24 @@ impl TransactionPool for NoopTransactionPool { Default::default() } - fn get_blob(&self, _tx_hash: TxHash) -> Result, BlobStoreError> { + fn get_blob( + &self, + _tx_hash: TxHash, + ) -> Result>, BlobStoreError> { Ok(None) } fn get_all_blobs( &self, _tx_hashes: Vec, - ) -> Result, BlobStoreError> { + ) -> Result)>, BlobStoreError> { Ok(vec![]) } fn get_all_blobs_exact( &self, tx_hashes: Vec, - ) -> Result, BlobStoreError> { + ) -> Result>, BlobStoreError> { if tx_hashes.is_empty() { return Ok(vec![]) } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index fef0fd0eea03..2e73409546d1 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -307,7 +307,9 @@ where /// Caution: this assumes the given transaction is eip-4844 fn get_blob_transaction(&self, transaction: TransactionSigned) -> Option { if let Ok(Some(sidecar)) = self.blob_store.get(transaction.hash()) { - if let Ok(blob) = BlobTransaction::try_from_signed(transaction, sidecar) { + if let Ok(blob) = + BlobTransaction::try_from_signed(transaction, Arc::unwrap_or_clone(sidecar)) + { return Some(blob) } } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index fbbddb98f43c..1b3004154921 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -443,7 +443,10 @@ pub trait TransactionPool: Send + Sync + Clone { /// Returns the [BlobTransactionSidecar] for the given transaction hash if it exists in the blob /// store. - fn get_blob(&self, tx_hash: TxHash) -> Result, BlobStoreError>; + fn get_blob( + &self, + tx_hash: TxHash, + ) -> Result>, BlobStoreError>; /// Returns all [BlobTransactionSidecar] for the given transaction hashes if they exists in the /// blob store. @@ -453,7 +456,7 @@ pub trait TransactionPool: Send + Sync + Clone { fn get_all_blobs( &self, tx_hashes: Vec, - ) -> Result, BlobStoreError>; + ) -> Result)>, BlobStoreError>; /// Returns the exact [BlobTransactionSidecar] for the given transaction hashes in the order /// they were requested. @@ -462,7 +465,7 @@ pub trait TransactionPool: Send + Sync + Clone { fn get_all_blobs_exact( &self, tx_hashes: Vec, - ) -> Result, BlobStoreError>; + ) -> Result>, BlobStoreError>; /// Return the [`BlobTransactionSidecar`]s for a list of blob versioned hashes. fn get_blobs_for_versioned_hashes( diff --git a/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs b/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs index 1c53e4f41051..2436ee0210e4 100644 --- a/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs +++ b/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::VecDeque, pin::Pin, + sync::Arc, task::{Context, Poll}, }; use thiserror::Error; @@ -110,9 +111,10 @@ where match self.pool.get_all_blobs_exact(txs.iter().map(|(tx, _)| tx.hash()).collect()) { Ok(blobs) => { actions_to_queue.reserve_exact(txs.len()); - for ((tx, _), sidecar) in txs.iter().zip(blobs.iter()) { - let transaction = BlobTransaction::try_from_signed(tx.clone(), sidecar.clone()) - .expect("should not fail to convert blob tx if it is already eip4844"); + for ((tx, _), sidecar) in txs.iter().zip(blobs.into_iter()) { + let transaction = + BlobTransaction::try_from_signed(tx.clone(), Arc::unwrap_or_clone(sidecar)) + .expect("should not fail to convert blob tx if it is already eip4844"); let block_metadata = BlockMetadata { block_hash: block.hash(), From b3e8327065ccb2570913ec9f99aa142f7bb8a962 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Oct 2024 18:56:36 +0100 Subject: [PATCH 185/239] chore: rename v1 type (#12205) --- crates/engine/primitives/src/lib.rs | 18 ++++++++++++------ crates/ethereum/engine-primitives/src/lib.rs | 2 +- crates/optimism/node/src/engine.rs | 2 +- crates/rpc/rpc-api/src/engine.rs | 5 ++++- crates/rpc/rpc-engine-api/src/engine_api.rs | 4 ++-- examples/custom-engine-types/src/main.rs | 2 +- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/crates/engine/primitives/src/lib.rs b/crates/engine/primitives/src/lib.rs index fab96b0d17e9..949ebf0155c4 100644 --- a/crates/engine/primitives/src/lib.rs +++ b/crates/engine/primitives/src/lib.rs @@ -23,7 +23,7 @@ use serde::{de::DeserializeOwned, ser::Serialize}; /// payload job. Hence this trait is also [`PayloadTypes`]. pub trait EngineTypes: PayloadTypes< - BuiltPayload: TryInto + BuiltPayload: TryInto + TryInto + TryInto + TryInto, @@ -31,9 +31,15 @@ pub trait EngineTypes: + Serialize + 'static { - /// Execution Payload V1 type. - type ExecutionPayloadV1: DeserializeOwned + Serialize + Clone + Unpin + Send + Sync + 'static; - /// Execution Payload V2 type. + /// Execution Payload V1 envelope type. + type ExecutionPayloadEnvelopeV1: DeserializeOwned + + Serialize + + Clone + + Unpin + + Send + + Sync + + 'static; + /// Execution Payload V2 envelope type. type ExecutionPayloadEnvelopeV2: DeserializeOwned + Serialize + Clone @@ -41,7 +47,7 @@ pub trait EngineTypes: + Send + Sync + 'static; - /// Execution Payload V3 type. + /// Execution Payload V3 envelope type. type ExecutionPayloadEnvelopeV3: DeserializeOwned + Serialize + Clone @@ -49,7 +55,7 @@ pub trait EngineTypes: + Send + Sync + 'static; - /// Execution Payload V4 type. + /// Execution Payload V4 envelope type. type ExecutionPayloadEnvelopeV4: DeserializeOwned + Serialize + Clone diff --git a/crates/ethereum/engine-primitives/src/lib.rs b/crates/ethereum/engine-primitives/src/lib.rs index 20a55883680b..5addf2a18c51 100644 --- a/crates/ethereum/engine-primitives/src/lib.rs +++ b/crates/ethereum/engine-primitives/src/lib.rs @@ -43,7 +43,7 @@ where + TryInto + TryInto, { - type ExecutionPayloadV1 = ExecutionPayloadV1; + type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1; type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3; type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4; diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index 966d87279c5a..7e19b2f93f3c 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -39,7 +39,7 @@ where + TryInto + TryInto, { - type ExecutionPayloadV1 = ExecutionPayloadV1; + type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1; type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadEnvelopeV3 = OpExecutionPayloadEnvelopeV3; type ExecutionPayloadEnvelopeV4 = OpExecutionPayloadEnvelopeV4; diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index ddf6d846119d..d92173112ebb 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -110,7 +110,10 @@ pub trait EngineApi { /// Note: /// > Provider software MAY stop the corresponding build process after serving this call. #[method(name = "getPayloadV1")] - async fn get_payload_v1(&self, payload_id: PayloadId) -> RpcResult; + async fn get_payload_v1( + &self, + payload_id: PayloadId, + ) -> RpcResult; /// See also /// diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index cca9f5d6b64c..383da2d21ffd 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -281,7 +281,7 @@ where pub async fn get_payload_v1( &self, payload_id: PayloadId, - ) -> EngineApiResult { + ) -> EngineApiResult { self.inner .payload_store .resolve(payload_id) @@ -775,7 +775,7 @@ where async fn get_payload_v1( &self, payload_id: PayloadId, - ) -> RpcResult { + ) -> RpcResult { trace!(target: "rpc::engine", "Serving engine_getPayloadV1"); let start = Instant::now(); let res = Self::get_payload_v1(self, payload_id).await; diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index a48c44b200b9..896a4b55f6bb 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -161,7 +161,7 @@ impl PayloadTypes for CustomEngineTypes { } impl EngineTypes for CustomEngineTypes { - type ExecutionPayloadV1 = ExecutionPayloadV1; + type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1; type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3; type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4; From 0c39704950e370c3c97acd4253f52eca5a57a94c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Oct 2024 19:04:44 +0100 Subject: [PATCH 186/239] feat: add missing is active at timestamp fns (#12206) --- crates/optimism/hardforks/src/lib.rs | 21 +++++++++++++++++++-- crates/optimism/node/src/engine.rs | 8 +++----- crates/optimism/payload/src/builder.rs | 19 ++++++------------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/crates/optimism/hardforks/src/lib.rs b/crates/optimism/hardforks/src/lib.rs index 91c11d3fd23c..bac0d0e04ed6 100644 --- a/crates/optimism/hardforks/src/lib.rs +++ b/crates/optimism/hardforks/src/lib.rs @@ -26,18 +26,35 @@ pub trait OptimismHardforks: EthereumHardforks { self.fork(OptimismHardfork::Bedrock).active_at_block(block_number) } + /// Returns `true` if [`Canyon`](OptimismHardfork::Canyon) is active at given block timestamp. + fn is_canyon_active_at_timestamp(&self, timestamp: u64) -> bool { + self.fork(OptimismHardfork::Canyon).active_at_timestamp(timestamp) + } + /// Returns `true` if [`Ecotone`](OptimismHardfork::Ecotone) is active at given block timestamp. fn is_ecotone_active_at_timestamp(&self, timestamp: u64) -> bool { self.fork(OptimismHardfork::Ecotone).active_at_timestamp(timestamp) } - /// Returns `true` if [`Ecotone`](OptimismHardfork::Ecotone) is active at given block timestamp. + /// Returns `true` if [`Fjord`](OptimismHardfork::Fjord) is active at given block timestamp. fn is_fjord_active_at_timestamp(&self, timestamp: u64) -> bool { - self.fork(OptimismHardfork::Ecotone).active_at_timestamp(timestamp) + self.fork(OptimismHardfork::Fjord).active_at_timestamp(timestamp) } /// Returns `true` if [`Granite`](OptimismHardfork::Granite) is active at given block timestamp. fn is_granite_active_at_timestamp(&self, timestamp: u64) -> bool { self.fork(OptimismHardfork::Granite).active_at_timestamp(timestamp) } + + /// Returns `true` if [`Holocene`](OptimismHardfork::Holocene) is active at given block + /// timestamp. + fn is_holocene_active_at_timestamp(&self, timestamp: u64) -> bool { + self.fork(OptimismHardfork::Holocene).active_at_timestamp(timestamp) + } + + /// Returns `true` if [`Regolith`](OptimismHardfork::Regolith) is active at given block + /// timestamp. + fn is_regolith_active_at_timestamp(&self, timestamp: u64) -> bool { + self.fork(OptimismHardfork::Regolith).active_at_timestamp(timestamp) + } } diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index 7e19b2f93f3c..da8fde2b4d36 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -14,7 +14,7 @@ use reth_node_api::{ validate_version_specific_fields, EngineTypes, EngineValidator, }; use reth_optimism_chainspec::OpChainSpec; -use reth_optimism_forks::OptimismHardfork; +use reth_optimism_forks::{OptimismHardfork, OptimismHardforks}; use reth_optimism_payload_builder::{ builder::decode_eip_1559_params, OptimismBuiltPayload, OptimismPayloadBuilderAttributes, }; @@ -149,10 +149,8 @@ where )) } - if self.chain_spec.is_fork_active_at_timestamp( - OptimismHardfork::Holocene, - attributes.payload_attributes.timestamp, - ) { + if self.chain_spec.is_holocene_active_at_timestamp(attributes.payload_attributes.timestamp) + { let Some(eip_1559_params) = attributes.eip_1559_params else { return Err(EngineObjectValidationError::InvalidParams( "MissingEip1559ParamsInPayloadAttributes".to_string().into(), diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index e9b7e2c76f80..96ac28c5d1a4 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -11,7 +11,7 @@ use reth_evm::{system_calls::SystemCaller, ConfigureEvm, ConfigureEvmEnv, NextBl use reth_execution_types::ExecutionOutcome; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_consensus::calculate_receipt_root_no_memo_optimism; -use reth_optimism_forks::{OptimismHardfork, OptimismHardforks}; +use reth_optimism_forks::OptimismHardforks; use reth_payload_primitives::{PayloadBuilderAttributes, PayloadBuilderError}; use reth_primitives::{ proofs, @@ -193,10 +193,8 @@ where let block_number = initialized_block_env.number.to::(); - let is_regolith = chain_spec.is_fork_active_at_timestamp( - OptimismHardfork::Regolith, - attributes.payload_attributes.timestamp, - ); + let is_regolith = + chain_spec.is_regolith_active_at_timestamp(attributes.payload_attributes.timestamp); // apply eip-4788 pre block contract call let mut system_caller = SystemCaller::new(evm_config.clone(), &chain_spec); @@ -315,10 +313,7 @@ where // receipt hashes should be computed when set. The state transition process // ensures this is only set for post-Canyon deposit transactions. deposit_receipt_version: chain_spec - .is_fork_active_at_timestamp( - OptimismHardfork::Canyon, - attributes.payload_attributes.timestamp, - ) + .is_canyon_active_at_timestamp(attributes.payload_attributes.timestamp) .then_some(1), })); @@ -473,10 +468,8 @@ where (None, None) }; - let is_holocene = chain_spec.is_fork_active_at_timestamp( - OptimismHardfork::Holocene, - attributes.payload_attributes.timestamp, - ); + let is_holocene = + chain_spec.is_holocene_active_at_timestamp(attributes.payload_attributes.timestamp); if is_holocene { extra_data = attributes From 09c666d676dc857692e883a6b93b87a06524ed36 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Oct 2024 19:36:38 +0100 Subject: [PATCH 187/239] test: add test case for BestTransactions (#12209) --- crates/transaction-pool/src/pool/best.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 6ade15be7d8a..36a14edaa232 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -401,6 +401,29 @@ mod tests { assert!(best.next().is_none()); } + #[test] + fn test_best_transactions_iter_invalid() { + let mut pool = PendingPool::new(MockOrdering::default()); + let mut f = MockTransactionFactory::default(); + + let num_tx = 10; + // insert 10 gapless tx + let tx = MockTransaction::eip1559(); + for nonce in 0..num_tx { + let tx = tx.clone().rng_hash().with_nonce(nonce); + let valid_tx = f.validated(tx); + pool.add_transaction(Arc::new(valid_tx), 0); + } + + let mut best: Box< + dyn crate::traits::BestTransactions>>, + > = Box::new(pool.best()); + + let tx = best.next().unwrap(); + best.mark_invalid(&tx); + assert!(best.next().is_none()); + } + #[test] fn test_best_with_fees_iter_base_fee_satisfied() { let mut pool = PendingPool::new(MockOrdering::default()); From b42b189210a6a4802dbf97521fee6f4fea8d8e11 Mon Sep 17 00:00:00 2001 From: "0xriazaka.eth" <168359025+0xriazaka@users.noreply.github.com> Date: Wed, 30 Oct 2024 22:45:16 +0100 Subject: [PATCH 188/239] Reth primitives traits tx type (#11720) Co-authored-by: Emilia Hane --- crates/primitives-traits/src/lib.rs | 4 ++++ crates/primitives-traits/src/tx_type.rs | 28 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 crates/primitives-traits/src/tx_type.rs diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index 0489a250bbd1..9f41bbd47fb2 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -45,6 +45,10 @@ pub use alloy_primitives::{logs_bloom, Log, LogData}; mod storage; pub use storage::StorageEntry; +/// Transaction types +pub mod tx_type; +pub use tx_type::TxType; + /// Common header types pub mod header; #[cfg(any(test, feature = "arbitrary", feature = "test-utils"))] diff --git a/crates/primitives-traits/src/tx_type.rs b/crates/primitives-traits/src/tx_type.rs new file mode 100644 index 000000000000..aebf7584fe9c --- /dev/null +++ b/crates/primitives-traits/src/tx_type.rs @@ -0,0 +1,28 @@ +use alloy_eips::eip2718::Eip2718Error; +use alloy_primitives::{U64, U8}; +use alloy_rlp::{Decodable, Encodable}; +use core::fmt::{Debug, Display}; + +/// Trait representing the behavior of a transaction type. +pub trait TxType: + Into + + Into + + PartialEq + + Eq + + PartialEq + + TryFrom + + TryFrom + + TryFrom + + From + + Debug + + Display + + Clone + + Copy + + Default + + Encodable + + Decodable + + Send + + Sync + + 'static +{ +} From e5fc048139ad61f96ed581cb89c5aa4f0da12ac2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Oct 2024 23:43:26 +0100 Subject: [PATCH 189/239] docs: add context truncated input (#12207) --- crates/rpc/rpc/src/otterscan.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/otterscan.rs b/crates/rpc/rpc/src/otterscan.rs index 024bdd172fb9..a772dd501d42 100644 --- a/crates/rpc/rpc/src/otterscan.rs +++ b/crates/rpc/rpc/src/otterscan.rs @@ -227,7 +227,8 @@ where *transactions = transactions.drain(page_start..page_end).collect::>(); // The input field returns only the 4 bytes method selector instead of the entire - // calldata byte blob. + // calldata byte blob + // See also: for tx in transactions.iter_mut() { if tx.input().len() > 4 { Eth::TransactionCompat::otterscan_api_truncate_input(tx); From c19af293a63fdfee4e92d7dd9a915e40c0a98afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:33:33 +0700 Subject: [PATCH 190/239] feat: add rlp support for `EthVersion` (#12221) --- crates/net/eth-wire-types/src/status.rs | 22 ++++---- crates/net/eth-wire-types/src/version.rs | 65 ++++++++++++++++++++++++ crates/net/eth-wire/src/errors/eth.rs | 3 +- crates/net/eth-wire/src/ethstream.rs | 12 ++--- crates/net/eth-wire/src/test_utils.rs | 2 +- crates/net/network/tests/it/session.rs | 4 +- examples/manual-p2p/src/main.rs | 3 +- 7 files changed, 89 insertions(+), 22 deletions(-) diff --git a/crates/net/eth-wire-types/src/status.rs b/crates/net/eth-wire-types/src/status.rs index 90e1731c90ab..d9e8d4319b5b 100644 --- a/crates/net/eth-wire-types/src/status.rs +++ b/crates/net/eth-wire-types/src/status.rs @@ -19,7 +19,7 @@ use std::fmt::{Debug, Display}; pub struct Status { /// The current protocol version. For example, peers running `eth/66` would have a version of /// 66. - pub version: u8, + pub version: EthVersion, /// The chain id, as introduced in /// [EIP155](https://eips.ethereum.org/EIPS/eip-155#list-of-chain-ids). @@ -50,7 +50,7 @@ impl Status { /// Sets the [`EthVersion`] for the status. pub fn set_eth_version(&mut self, version: EthVersion) { - self.version = version as u8; + self.version = version; } /// Create a [`StatusBuilder`] from the given [`EthChainSpec`] and head block. @@ -122,7 +122,7 @@ impl Default for Status { fn default() -> Self { let mainnet_genesis = MAINNET.genesis_hash(); Self { - version: EthVersion::Eth68 as u8, + version: EthVersion::Eth68, chain: Chain::from_named(NamedChain::Mainnet), total_difficulty: U256::from(17_179_869_184u64), blockhash: mainnet_genesis, @@ -145,7 +145,7 @@ impl Default for Status { /// /// // this is just an example status message! /// let status = Status::builder() -/// .version(EthVersion::Eth66.into()) +/// .version(EthVersion::Eth66) /// .chain(Chain::mainnet()) /// .total_difficulty(U256::from(100)) /// .blockhash(B256::from(MAINNET_GENESIS_HASH)) @@ -156,7 +156,7 @@ impl Default for Status { /// assert_eq!( /// status, /// Status { -/// version: EthVersion::Eth66.into(), +/// version: EthVersion::Eth66, /// chain: Chain::mainnet(), /// total_difficulty: U256::from(100), /// blockhash: B256::from(MAINNET_GENESIS_HASH), @@ -177,7 +177,7 @@ impl StatusBuilder { } /// Sets the protocol version. - pub const fn version(mut self, version: u8) -> Self { + pub const fn version(mut self, version: EthVersion) -> Self { self.status.version = version; self } @@ -229,7 +229,7 @@ mod tests { fn encode_eth_status_message() { let expected = hex!("f85643018a07aac59dabcdd74bc567a0feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13da0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3c684b715077d80"); let status = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: Chain::from_named(NamedChain::Mainnet), total_difficulty: U256::from(36206751599115524359527u128), blockhash: B256::from_str( @@ -249,7 +249,7 @@ mod tests { fn decode_eth_status_message() { let data = hex!("f85643018a07aac59dabcdd74bc567a0feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13da0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3c684b715077d80"); let expected = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: Chain::from_named(NamedChain::Mainnet), total_difficulty: U256::from(36206751599115524359527u128), blockhash: B256::from_str( @@ -267,7 +267,7 @@ mod tests { fn encode_network_status_message() { let expected = hex!("f850423884024190faa0f8514c4680ef27700751b08f37645309ce65a449616a3ea966bf39dd935bb27ba00d21840abff46b96c84b2ac9e10e4f5cdaeb5693cb665db62a2f3b02d2d57b5bc6845d43d2fd80"); let status = Status { - version: EthVersion::Eth66 as u8, + version: EthVersion::Eth66, chain: Chain::from_named(NamedChain::BinanceSmartChain), total_difficulty: U256::from(37851386u64), blockhash: B256::from_str( @@ -290,7 +290,7 @@ mod tests { fn decode_network_status_message() { let data = hex!("f850423884024190faa0f8514c4680ef27700751b08f37645309ce65a449616a3ea966bf39dd935bb27ba00d21840abff46b96c84b2ac9e10e4f5cdaeb5693cb665db62a2f3b02d2d57b5bc6845d43d2fd80"); let expected = Status { - version: EthVersion::Eth66 as u8, + version: EthVersion::Eth66, chain: Chain::from_named(NamedChain::BinanceSmartChain), total_difficulty: U256::from(37851386u64), blockhash: B256::from_str( @@ -311,7 +311,7 @@ mod tests { fn decode_another_network_status_message() { let data = hex!("f86142820834936d68fcffffffffffffffffffffffffdeab81b8a0523e8163a6d620a4cc152c547a05f28a03fec91a2a615194cb86df9731372c0ca06499dccdc7c7def3ebb1ce4c6ee27ec6bd02aee570625ca391919faf77ef27bdc6841a67ccd880"); let expected = Status { - version: EthVersion::Eth66 as u8, + version: EthVersion::Eth66, chain: Chain::from_id(2100), total_difficulty: U256::from_str( "0x000000000000000000000000006d68fcffffffffffffffffffffffffdeab81b8", diff --git a/crates/net/eth-wire-types/src/version.rs b/crates/net/eth-wire-types/src/version.rs index 5a2e0ff9651d..40d51cb55180 100644 --- a/crates/net/eth-wire-types/src/version.rs +++ b/crates/net/eth-wire-types/src/version.rs @@ -15,6 +15,8 @@ pub struct ParseVersionError(String); /// The `eth` protocol version. #[repr(u8)] #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Display)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] pub enum EthVersion { /// The `eth` protocol version 66. Eth66 = 66, @@ -64,6 +66,26 @@ impl EthVersion { } } +/// RLP encodes `EthVersion` as a single byte (66-69). +impl Encodable for EthVersion { + fn encode(&self, out: &mut dyn BufMut) { + (*self as u8).encode(out) + } + + fn length(&self) -> usize { + (*self as u8).length() + } +} + +/// RLP decodes a single byte into `EthVersion`. +/// Returns error if byte is not a valid version (66-69). +impl Decodable for EthVersion { + fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { + let version = u8::decode(buf)?; + Self::try_from(version).map_err(|_| RlpError::Custom("invalid eth version")) + } +} + /// Allow for converting from a `&str` to an `EthVersion`. /// /// # Example @@ -183,6 +205,8 @@ impl Decodable for ProtocolVersion { #[cfg(test)] mod tests { use super::{EthVersion, ParseVersionError}; + use alloy_rlp::{Decodable, Encodable, Error as RlpError}; + use bytes::BytesMut; #[test] fn test_eth_version_try_from_str() { @@ -201,4 +225,45 @@ mod tests { assert_eq!(EthVersion::Eth69, "69".parse().unwrap()); assert_eq!(Err(ParseVersionError("70".to_string())), "70".parse::()); } + + #[test] + fn test_eth_version_rlp_encode() { + let versions = [EthVersion::Eth66, EthVersion::Eth67, EthVersion::Eth68, EthVersion::Eth69]; + + for version in versions { + let mut encoded = BytesMut::new(); + version.encode(&mut encoded); + + assert_eq!(encoded.len(), 1); + assert_eq!(encoded[0], version as u8); + } + } + #[test] + fn test_eth_version_rlp_decode() { + let test_cases = [ + (66_u8, Ok(EthVersion::Eth66)), + (67_u8, Ok(EthVersion::Eth67)), + (68_u8, Ok(EthVersion::Eth68)), + (69_u8, Ok(EthVersion::Eth69)), + (70_u8, Err(RlpError::Custom("invalid eth version"))), + (65_u8, Err(RlpError::Custom("invalid eth version"))), + ]; + + for (input, expected) in test_cases { + let mut encoded = BytesMut::new(); + input.encode(&mut encoded); + + let mut slice = encoded.as_ref(); + let result = EthVersion::decode(&mut slice); + assert_eq!(result, expected); + } + } + + #[test] + fn test_eth_version_total_messages() { + assert_eq!(EthVersion::Eth66.total_messages(), 15); + assert_eq!(EthVersion::Eth67.total_messages(), 13); + assert_eq!(EthVersion::Eth68.total_messages(), 13); + assert_eq!(EthVersion::Eth69.total_messages(), 11); + } } diff --git a/crates/net/eth-wire/src/errors/eth.rs b/crates/net/eth-wire/src/errors/eth.rs index 557fbd66a00f..1f8b995afda4 100644 --- a/crates/net/eth-wire/src/errors/eth.rs +++ b/crates/net/eth-wire/src/errors/eth.rs @@ -5,6 +5,7 @@ use crate::{ }; use alloy_primitives::B256; use reth_chainspec::Chain; +use reth_eth_wire_types::EthVersion; use reth_primitives::{GotExpected, GotExpectedBoxed, ValidationError}; use std::io; @@ -88,7 +89,7 @@ pub enum EthHandshakeError { MismatchedGenesis(GotExpectedBoxed), #[error("mismatched protocol version in status message: {0}")] /// Mismatched protocol versions in status messages. - MismatchedProtocolVersion(GotExpected), + MismatchedProtocolVersion(GotExpected), #[error("mismatched chain in status message: {0}")] /// Mismatch in chain details in status messages. MismatchedChain(GotExpected), diff --git a/crates/net/eth-wire/src/ethstream.rs b/crates/net/eth-wire/src/ethstream.rs index 9deca99fb587..74f3fab2be64 100644 --- a/crates/net/eth-wire/src/ethstream.rs +++ b/crates/net/eth-wire/src/ethstream.rs @@ -102,7 +102,7 @@ where return Err(EthStreamError::MessageTooBig(their_msg.len())) } - let version = EthVersion::try_from(status.version)?; + let version = status.version; let msg = match ProtocolMessage::decode_message(version, &mut their_msg.as_ref()) { Ok(m) => m, Err(err) => { @@ -368,7 +368,7 @@ mod tests { let fork_filter = ForkFilter::new(Head::default(), genesis, 0, Vec::new()); let status = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: NamedChain::Mainnet.into(), total_difficulty: U256::ZERO, blockhash: B256::random(), @@ -415,7 +415,7 @@ mod tests { let fork_filter = ForkFilter::new(Head::default(), genesis, 0, Vec::new()); let status = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: NamedChain::Mainnet.into(), total_difficulty: U256::from(2).pow(U256::from(100)) - U256::from(1), blockhash: B256::random(), @@ -462,7 +462,7 @@ mod tests { let fork_filter = ForkFilter::new(Head::default(), genesis, 0, Vec::new()); let status = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: NamedChain::Mainnet.into(), total_difficulty: U256::from(2).pow(U256::from(100)), blockhash: B256::random(), @@ -603,7 +603,7 @@ mod tests { let fork_filter = ForkFilter::new(Head::default(), genesis, 0, Vec::new()); let status = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: NamedChain::Mainnet.into(), total_difficulty: U256::ZERO, blockhash: B256::random(), @@ -674,7 +674,7 @@ mod tests { let fork_filter = ForkFilter::new(Head::default(), genesis, 0, Vec::new()); let status = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: NamedChain::Mainnet.into(), total_difficulty: U256::ZERO, blockhash: B256::random(), diff --git a/crates/net/eth-wire/src/test_utils.rs b/crates/net/eth-wire/src/test_utils.rs index e516c0aee7df..d7a3aa582b72 100644 --- a/crates/net/eth-wire/src/test_utils.rs +++ b/crates/net/eth-wire/src/test_utils.rs @@ -37,7 +37,7 @@ pub fn eth_handshake() -> (Status, ForkFilter) { let fork_filter = ForkFilter::new(Head::default(), genesis, 0, Vec::new()); let status = Status { - version: EthVersion::Eth67 as u8, + version: EthVersion::Eth67, chain: Chain::mainnet(), total_difficulty: U256::ZERO, blockhash: B256::random(), diff --git a/crates/net/network/tests/it/session.rs b/crates/net/network/tests/it/session.rs index 6bc029d8a7b3..3f74db3d37f1 100644 --- a/crates/net/network/tests/it/session.rs +++ b/crates/net/network/tests/it/session.rs @@ -33,7 +33,7 @@ async fn test_session_established_with_highest_version() { } NetworkEvent::SessionEstablished { peer_id, status, .. } => { assert_eq!(handle1.peer_id(), &peer_id); - assert_eq!(status.version, EthVersion::Eth68 as u8); + assert_eq!(status.version, EthVersion::Eth68); } ev => { panic!("unexpected event {ev:?}") @@ -71,7 +71,7 @@ async fn test_session_established_with_different_capability() { } NetworkEvent::SessionEstablished { peer_id, status, .. } => { assert_eq!(handle1.peer_id(), &peer_id); - assert_eq!(status.version, EthVersion::Eth66 as u8); + assert_eq!(status.version, EthVersion::Eth66); } ev => { panic!("unexpected event: {ev:?}") diff --git a/examples/manual-p2p/src/main.rs b/examples/manual-p2p/src/main.rs index 857a8a1c1269..79a2ff26a274 100644 --- a/examples/manual-p2p/src/main.rs +++ b/examples/manual-p2p/src/main.rs @@ -106,7 +106,8 @@ async fn handshake_eth(p2p_stream: AuthedP2PStream) -> eyre::Result<(AuthedEthSt .forkid(MAINNET.hardfork_fork_id(EthereumHardfork::Shanghai).unwrap()) .build(); - let status = Status { version: p2p_stream.shared_capabilities().eth()?.version(), ..status }; + let status = + Status { version: p2p_stream.shared_capabilities().eth()?.version().try_into()?, ..status }; let eth_unauthed = UnauthedEthStream::new(p2p_stream); Ok(eth_unauthed.handshake(status, fork_filter).await?) } From 9659717e83b58d2655f5f35a714c8ff8d9366f6c Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 31 Oct 2024 02:34:11 -0600 Subject: [PATCH 191/239] renamed OptimismPayloadBuilderAttributes to OpPayloadBuilderAttributes (#12213) --- crates/optimism/node/src/engine.rs | 4 ++-- crates/optimism/node/src/lib.rs | 2 +- crates/optimism/node/tests/e2e/utils.rs | 8 ++++---- crates/optimism/payload/src/builder.rs | 12 ++++++------ crates/optimism/payload/src/lib.rs | 2 +- crates/optimism/payload/src/payload.rs | 18 ++++++++---------- 6 files changed, 22 insertions(+), 24 deletions(-) diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index da8fde2b4d36..27a609d953ae 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -16,7 +16,7 @@ use reth_node_api::{ use reth_optimism_chainspec::OpChainSpec; use reth_optimism_forks::{OptimismHardfork, OptimismHardforks}; use reth_optimism_payload_builder::{ - builder::decode_eip_1559_params, OptimismBuiltPayload, OptimismPayloadBuilderAttributes, + builder::decode_eip_1559_params, OpPayloadBuilderAttributes, OptimismBuiltPayload, }; /// The types used in the optimism beacon consensus engine. @@ -53,7 +53,7 @@ pub struct OptimismPayloadTypes; impl PayloadTypes for OptimismPayloadTypes { type BuiltPayload = OptimismBuiltPayload; type PayloadAttributes = OpPayloadAttributes; - type PayloadBuilderAttributes = OptimismPayloadBuilderAttributes; + type PayloadBuilderAttributes = OpPayloadBuilderAttributes; } /// Validator for Optimism engine API. diff --git a/crates/optimism/node/src/lib.rs b/crates/optimism/node/src/lib.rs index 768f4d94efd5..a8ef472da769 100644 --- a/crates/optimism/node/src/lib.rs +++ b/crates/optimism/node/src/lib.rs @@ -23,7 +23,7 @@ pub use node::OptimismNode; pub mod txpool; pub use reth_optimism_payload_builder::{ - OptimismBuiltPayload, OptimismPayloadBuilder, OptimismPayloadBuilderAttributes, + OpPayloadBuilderAttributes, OptimismBuiltPayload, OptimismPayloadBuilder, }; pub use reth_optimism_evm::*; diff --git a/crates/optimism/node/tests/e2e/utils.rs b/crates/optimism/node/tests/e2e/utils.rs index d4219b0fea1f..b445af33b4e4 100644 --- a/crates/optimism/node/tests/e2e/utils.rs +++ b/crates/optimism/node/tests/e2e/utils.rs @@ -6,7 +6,7 @@ use reth_e2e_test_utils::{ }; use reth_optimism_chainspec::OpChainSpecBuilder; use reth_optimism_node::{ - node::OptimismAddOns, OptimismBuiltPayload, OptimismNode, OptimismPayloadBuilderAttributes, + node::OptimismAddOns, OpPayloadBuilderAttributes, OptimismBuiltPayload, OptimismNode, }; use reth_payload_builder::EthPayloadBuilderAttributes; use std::sync::Arc; @@ -31,7 +31,7 @@ pub(crate) async fn advance_chain( length: usize, node: &mut OpNode, wallet: Arc>, -) -> eyre::Result> { +) -> eyre::Result> { node.advance(length as u64, |_| { let wallet = wallet.clone(); Box::pin(async move { @@ -49,7 +49,7 @@ pub(crate) async fn advance_chain( } /// Helper function to create a new eth payload attributes -pub(crate) fn optimism_payload_attributes(timestamp: u64) -> OptimismPayloadBuilderAttributes { +pub(crate) fn optimism_payload_attributes(timestamp: u64) -> OpPayloadBuilderAttributes { let attributes = PayloadAttributes { timestamp, prev_randao: B256::ZERO, @@ -58,7 +58,7 @@ pub(crate) fn optimism_payload_attributes(timestamp: u64) -> OptimismPayloadBuil parent_beacon_block_root: Some(B256::ZERO), }; - OptimismPayloadBuilderAttributes { + OpPayloadBuilderAttributes { payload_attributes: EthPayloadBuilderAttributes::new(B256::ZERO, attributes), transactions: vec![], no_tx_pool: false, diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 96ac28c5d1a4..3095ce3512eb 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -33,7 +33,7 @@ use tracing::{debug, trace, warn}; use crate::{ error::OptimismPayloadBuilderError, - payload::{OptimismBuiltPayload, OptimismPayloadBuilderAttributes}, + payload::{OpPayloadBuilderAttributes, OptimismBuiltPayload}, }; use op_alloy_consensus::DepositTransaction; @@ -77,7 +77,7 @@ where /// (that has the `parent` as its parent). pub fn cfg_and_block_env( &self, - config: &PayloadConfig, + config: &PayloadConfig, parent: &Header, ) -> Result<(CfgEnvWithHandlerCfg, BlockEnv), EvmConfig::Error> { let next_attributes = NextBlockEnvAttributes { @@ -96,12 +96,12 @@ where Pool: TransactionPool, EvmConfig: ConfigureEvm

, { - type Attributes = OptimismPayloadBuilderAttributes; + type Attributes = OpPayloadBuilderAttributes; type BuiltPayload = OptimismBuiltPayload; fn try_build( &self, - args: BuildArguments, + args: BuildArguments, ) -> Result, PayloadBuilderError> { let (cfg_env, block_env) = self .cfg_and_block_env(&args.config, &args.config.parent_header) @@ -111,7 +111,7 @@ where fn on_missing_payload( &self, - _args: BuildArguments, + _args: BuildArguments, ) -> MissingPayloadBehaviour { // we want to await the job that's already in progress because that should be returned as // is, there's no benefit in racing another job @@ -154,7 +154,7 @@ where #[inline] pub(crate) fn optimism_payload( evm_config: &EvmConfig, - args: BuildArguments, + args: BuildArguments, initialized_cfg: CfgEnvWithHandlerCfg, initialized_block_env: BlockEnv, _compute_pending_block: bool, diff --git a/crates/optimism/payload/src/lib.rs b/crates/optimism/payload/src/lib.rs index c06b49c53762..1c7bcaf70765 100644 --- a/crates/optimism/payload/src/lib.rs +++ b/crates/optimism/payload/src/lib.rs @@ -15,4 +15,4 @@ pub mod builder; pub use builder::OptimismPayloadBuilder; pub mod error; pub mod payload; -pub use payload::{OpPayloadAttributes, OptimismBuiltPayload, OptimismPayloadBuilderAttributes}; +pub use payload::{OpPayloadAttributes, OpPayloadBuilderAttributes, OptimismBuiltPayload}; diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index 056edfe7b638..ecfebdf00b9a 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -25,7 +25,7 @@ use std::sync::Arc; /// Optimism Payload Builder Attributes #[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct OptimismPayloadBuilderAttributes { +pub struct OpPayloadBuilderAttributes { /// Inner ethereum payload builder attributes pub payload_attributes: EthPayloadBuilderAttributes, /// `NoTxPool` option for the generated payload @@ -39,7 +39,7 @@ pub struct OptimismPayloadBuilderAttributes { pub eip_1559_params: Option, } -impl OptimismPayloadBuilderAttributes { +impl OpPayloadBuilderAttributes { /// Extracts the `eip1559` parameters for the payload. pub fn get_holocene_extra_data( &self, @@ -73,7 +73,7 @@ impl OptimismPayloadBuilderAttributes { } } -impl PayloadBuilderAttributes for OptimismPayloadBuilderAttributes { +impl PayloadBuilderAttributes for OpPayloadBuilderAttributes { type RpcPayloadAttributes = OpPayloadAttributes; type Error = alloy_rlp::Error; @@ -169,7 +169,7 @@ pub struct OptimismBuiltPayload { /// The rollup's chainspec. pub(crate) chain_spec: Arc, /// The payload attributes. - pub(crate) attributes: OptimismPayloadBuilderAttributes, + pub(crate) attributes: OpPayloadBuilderAttributes, } // === impl BuiltPayload === @@ -181,7 +181,7 @@ impl OptimismBuiltPayload { block: SealedBlock, fees: U256, chain_spec: Arc, - attributes: OptimismPayloadBuilderAttributes, + attributes: OpPayloadBuilderAttributes, executed_block: Option, ) -> Self { Self { id, block, executed_block, fees, sidecars: Vec::new(), chain_spec, attributes } @@ -411,7 +411,7 @@ mod tests { #[test] fn test_get_extra_data_post_holocene() { - let attributes = OptimismPayloadBuilderAttributes { + let attributes = OpPayloadBuilderAttributes { eip_1559_params: Some(B64::from_str("0x0000000800000008").unwrap()), ..Default::default() }; @@ -421,10 +421,8 @@ mod tests { #[test] fn test_get_extra_data_post_holocene_default() { - let attributes = OptimismPayloadBuilderAttributes { - eip_1559_params: Some(B64::ZERO), - ..Default::default() - }; + let attributes = + OpPayloadBuilderAttributes { eip_1559_params: Some(B64::ZERO), ..Default::default() }; let extra_data = attributes.get_holocene_extra_data(BaseFeeParams::new(80, 60)); assert_eq!(extra_data.unwrap(), Bytes::copy_from_slice(&[0, 0, 0, 0, 80, 0, 0, 0, 60])); } From 66cc619128450a0368d6814a177f3b812758618a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 31 Oct 2024 09:34:31 +0100 Subject: [PATCH 192/239] chore: rm deprecated txpool fn (#12198) --- crates/transaction-pool/src/lib.rs | 7 ------- crates/transaction-pool/src/noop.rs | 7 ------- crates/transaction-pool/src/traits.rs | 10 ---------- 3 files changed, 24 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 3a5e547ba4e7..f8f06b805f93 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -430,13 +430,6 @@ where Box::new(self.pool.best_transactions()) } - fn best_transactions_with_base_fee( - &self, - base_fee: u64, - ) -> Box>>> { - self.pool.best_transactions_with_attributes(BestTransactionsAttributes::base_fee(base_fee)) - } - fn best_transactions_with_attributes( &self, best_transactions_attributes: BestTransactionsAttributes, diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 4f4e5a3813ae..0f87f06f7dd3 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -150,13 +150,6 @@ impl TransactionPool for NoopTransactionPool { Box::new(std::iter::empty()) } - fn best_transactions_with_base_fee( - &self, - _: u64, - ) -> Box>>> { - Box::new(std::iter::empty()) - } - fn best_transactions_with_attributes( &self, _: BestTransactionsAttributes, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 1b3004154921..0d8f6dbb54ad 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -248,16 +248,6 @@ pub trait TransactionPool: Send + Sync + Clone { &self, ) -> Box>>>; - /// Returns an iterator that yields transactions that are ready for block production with the - /// given base fee. - /// - /// Consumer: Block production - #[deprecated(note = "Use best_transactions_with_attributes instead.")] - fn best_transactions_with_base_fee( - &self, - base_fee: u64, - ) -> Box>>>; - /// Returns an iterator that yields transactions that are ready for block production with the /// given base fee and optional blob fee attributes. /// From 41044a2601aab03a76838921ff92009debeb27e3 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 31 Oct 2024 03:07:20 -0600 Subject: [PATCH 193/239] Apply beacon system call to trace_block (#12030) Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/debug.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 6da03b046758..8b9e16023217 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -98,6 +98,7 @@ where cfg: CfgEnvWithHandlerCfg, block_env: BlockEnv, opts: GethDebugTracingOptions, + parent_beacon_block_root: Option, ) -> Result, Eth::Error> { if transactions.is_empty() { // nothing to trace @@ -111,6 +112,26 @@ where let block_hash = at.as_block_hash(); let mut results = Vec::with_capacity(transactions.len()); let mut db = CacheDB::new(StateProviderDatabase::new(state)); + + let mut system_caller = SystemCaller::new( + RpcNodeCore::evm_config(this.eth_api()).clone(), + RpcNodeCore::provider(this.eth_api()).chain_spec(), + ); + + // apply relevant system calls + system_caller + .pre_block_beacon_root_contract_call( + &mut db, + &cfg, + &block_env, + parent_beacon_block_root, + ) + .map_err(|_| { + EthApiError::EvmCustom( + "failed to apply 4788 beacon root system call".to_string(), + ) + })?; + let mut transactions = transactions.into_iter().enumerate().peekable(); let mut inspector = None; while let Some((index, tx)) = transactions.next() { @@ -170,6 +191,9 @@ where // we trace on top the block's parent block let parent = block.parent_hash; + // we need the beacon block root for a system call + let parent_beacon_block_root = block.parent_beacon_block_root; + // Depending on EIP-2 we need to recover the transactions differently let transactions = if self.inner.provider.chain_spec().is_homestead_active_at_block(block.number) { @@ -196,7 +220,15 @@ where .collect::, Eth::Error>>()? }; - self.trace_block(parent.into(), transactions, cfg, block_env, opts).await + self.trace_block( + parent.into(), + transactions, + cfg, + block_env, + opts, + parent_beacon_block_root, + ) + .await } /// Replays a block and returns the trace of each transaction. @@ -228,6 +260,7 @@ where cfg, block_env, opts, + block.parent_beacon_block_root, ) .await } From 76c5aef911d909161be47113cd88b7ea36fafffd Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 31 Oct 2024 11:53:59 +0100 Subject: [PATCH 194/239] fix(trie): move to sibling on invalid tree mask (#12193) Co-authored-by: Federico Gimenez --- crates/trie/trie/src/metrics.rs | 22 +++++++++++++++++++++- crates/trie/trie/src/walker.rs | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/crates/trie/trie/src/metrics.rs b/crates/trie/trie/src/metrics.rs index 7582f37418d9..006dc7e36555 100644 --- a/crates/trie/trie/src/metrics.rs +++ b/crates/trie/trie/src/metrics.rs @@ -1,5 +1,5 @@ use crate::stats::TrieStats; -use metrics::Histogram; +use metrics::{Counter, Histogram}; use reth_metrics::Metrics; /// Wrapper for state root metrics. @@ -63,3 +63,23 @@ impl TrieType { } } } + +/// Metrics for trie walker +#[derive(Clone, Metrics)] +#[metrics(scope = "trie.walker")] +pub struct WalkerMetrics { + /// The number of subnodes out of order due to wrong tree mask. + out_of_order_subnode: Counter, +} + +impl WalkerMetrics { + /// Create new metrics for the given trie type. + pub fn new(ty: TrieType) -> Self { + Self::new_with_labels(&[("type", ty.as_str())]) + } + + /// Increment `out_of_order_subnode`. + pub fn inc_out_of_order_subnode(&self, amount: u64) { + self.out_of_order_subnode.increment(amount); + } +} diff --git a/crates/trie/trie/src/walker.rs b/crates/trie/trie/src/walker.rs index e75a96d0f1f4..aaff293b379d 100644 --- a/crates/trie/trie/src/walker.rs +++ b/crates/trie/trie/src/walker.rs @@ -7,6 +7,9 @@ use alloy_primitives::B256; use reth_storage_errors::db::DatabaseError; use std::collections::HashSet; +#[cfg(feature = "metrics")] +use crate::metrics::WalkerMetrics; + /// `TrieWalker` is a structure that enables traversal of a Merkle trie. /// It allows moving through the trie in a depth-first manner, skipping certain branches /// if they have not changed. @@ -24,13 +27,23 @@ pub struct TrieWalker { pub changes: PrefixSet, /// The retained trie node keys that need to be removed. removed_keys: Option>, + #[cfg(feature = "metrics")] + /// Walker metrics. + metrics: WalkerMetrics, } impl TrieWalker { /// Constructs a new `TrieWalker` from existing stack and a cursor. pub fn from_stack(cursor: C, stack: Vec, changes: PrefixSet) -> Self { - let mut this = - Self { cursor, changes, stack, can_skip_current_node: false, removed_keys: None }; + let mut this = Self { + cursor, + changes, + stack, + can_skip_current_node: false, + removed_keys: None, + #[cfg(feature = "metrics")] + metrics: WalkerMetrics::default(), + }; this.update_skip_node(); this } @@ -113,6 +126,8 @@ impl TrieWalker { stack: vec![CursorSubNode::default()], can_skip_current_node: false, removed_keys: None, + #[cfg(feature = "metrics")] + metrics: WalkerMetrics::default(), }; // Set up the root node of the trie in the stack, if it exists. @@ -179,6 +194,19 @@ impl TrieWalker { self.stack[0].set_nibble(key[0] as i8); } + // The current tree mask might have been set incorrectly. + // Sanity check that the newly retrieved trie node key is the child of the last item + // on the stack. If not, advance to the next sibling instead of adding the node to the + // stack. + if let Some(subnode) = self.stack.last() { + if !key.starts_with(subnode.full_key()) { + #[cfg(feature = "metrics")] + self.metrics.inc_out_of_order_subnode(1); + self.move_to_next_sibling(false)?; + return Ok(()) + } + } + // Create a new CursorSubNode and push it to the stack. let subnode = CursorSubNode::new(key, Some(node)); let nibble = subnode.nibble(); From 1f1dcc950dc2f1b3bee5a562a1c2c7cd9db04fff Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 31 Oct 2024 14:45:55 +0100 Subject: [PATCH 195/239] chore: simplify SystemCaller setup (#12223) --- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 12 +---- crates/rpc/rpc/src/debug.rs | 60 ++++++++++----------- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index da1d1cdb919b..fa70b2df2efa 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -195,11 +195,7 @@ pub trait Trace: LoadState> { let block_txs = block.transactions_with_sender(); // apply relevant system calls - let mut system_caller = SystemCaller::new( - this.evm_config().clone(), - RpcNodeCore::provider(&this).chain_spec(), - ); - system_caller + SystemCaller::new(this.evm_config().clone(), this.provider().chain_spec()) .pre_block_beacon_root_contract_call( &mut db, &cfg, @@ -338,11 +334,7 @@ pub trait Trace: LoadState> { CacheDB::new(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))); // apply relevant system calls - let mut system_caller = SystemCaller::new( - this.evm_config().clone(), - RpcNodeCore::provider(&this).chain_spec(), - ); - system_caller + SystemCaller::new(this.evm_config().clone(), this.provider().chain_spec()) .pre_block_beacon_root_contract_call( &mut db, &cfg, diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 8b9e16023217..e2746a53cd04 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -113,24 +113,22 @@ where let mut results = Vec::with_capacity(transactions.len()); let mut db = CacheDB::new(StateProviderDatabase::new(state)); - let mut system_caller = SystemCaller::new( - RpcNodeCore::evm_config(this.eth_api()).clone(), - RpcNodeCore::provider(this.eth_api()).chain_spec(), - ); - // apply relevant system calls - system_caller - .pre_block_beacon_root_contract_call( - &mut db, - &cfg, - &block_env, - parent_beacon_block_root, + SystemCaller::new( + this.eth_api().evm_config().clone(), + this.eth_api().provider().chain_spec(), + ) + .pre_block_beacon_root_contract_call( + &mut db, + &cfg, + &block_env, + parent_beacon_block_root, + ) + .map_err(|_| { + EthApiError::EvmCustom( + "failed to apply 4788 beacon root system call".to_string(), ) - .map_err(|_| { - EthApiError::EvmCustom( - "failed to apply 4788 beacon root system call".to_string(), - ) - })?; + })?; let mut transactions = transactions.into_iter().enumerate().peekable(); let mut inspector = None; @@ -296,23 +294,21 @@ where let mut db = CacheDB::new(StateProviderDatabase::new(state)); // apply relevant system calls - let mut system_caller = SystemCaller::new( - RpcNodeCore::evm_config(this.eth_api()).clone(), - RpcNodeCore::provider(this.eth_api()).chain_spec(), - ); - - system_caller - .pre_block_beacon_root_contract_call( - &mut db, - &cfg, - &block_env, - parent_beacon_block_root, + SystemCaller::new( + this.eth_api().evm_config().clone(), + this.eth_api().provider().chain_spec(), + ) + .pre_block_beacon_root_contract_call( + &mut db, + &cfg, + &block_env, + parent_beacon_block_root, + ) + .map_err(|_| { + EthApiError::EvmCustom( + "failed to apply 4788 beacon root system call".to_string(), ) - .map_err(|_| { - EthApiError::EvmCustom( - "failed to apply 4788 beacon root system call".to_string(), - ) - })?; + })?; // replay all transactions prior to the targeted transaction let index = this.eth_api().replay_transactions_until( From 460e26cc51c1902cac88b7287c7491976478ca18 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 31 Oct 2024 09:51:33 -0600 Subject: [PATCH 196/239] renamed OptimismBuiltPayload to OpBuiltPayload (#12227) --- crates/optimism/node/src/engine.rs | 4 ++-- crates/optimism/node/src/lib.rs | 2 +- crates/optimism/node/tests/e2e/utils.rs | 4 ++-- crates/optimism/payload/src/builder.rs | 18 +++++++-------- crates/optimism/payload/src/lib.rs | 2 +- crates/optimism/payload/src/payload.rs | 30 ++++++++++++------------- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index 27a609d953ae..0f48dc347062 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -16,7 +16,7 @@ use reth_node_api::{ use reth_optimism_chainspec::OpChainSpec; use reth_optimism_forks::{OptimismHardfork, OptimismHardforks}; use reth_optimism_payload_builder::{ - builder::decode_eip_1559_params, OpPayloadBuilderAttributes, OptimismBuiltPayload, + builder::decode_eip_1559_params, OpBuiltPayload, OpPayloadBuilderAttributes, }; /// The types used in the optimism beacon consensus engine. @@ -51,7 +51,7 @@ where pub struct OptimismPayloadTypes; impl PayloadTypes for OptimismPayloadTypes { - type BuiltPayload = OptimismBuiltPayload; + type BuiltPayload = OpBuiltPayload; type PayloadAttributes = OpPayloadAttributes; type PayloadBuilderAttributes = OpPayloadBuilderAttributes; } diff --git a/crates/optimism/node/src/lib.rs b/crates/optimism/node/src/lib.rs index a8ef472da769..9bc15ef26685 100644 --- a/crates/optimism/node/src/lib.rs +++ b/crates/optimism/node/src/lib.rs @@ -23,7 +23,7 @@ pub use node::OptimismNode; pub mod txpool; pub use reth_optimism_payload_builder::{ - OpPayloadBuilderAttributes, OptimismBuiltPayload, OptimismPayloadBuilder, + OpBuiltPayload, OpPayloadBuilderAttributes, OptimismPayloadBuilder, }; pub use reth_optimism_evm::*; diff --git a/crates/optimism/node/tests/e2e/utils.rs b/crates/optimism/node/tests/e2e/utils.rs index b445af33b4e4..16eb974914d5 100644 --- a/crates/optimism/node/tests/e2e/utils.rs +++ b/crates/optimism/node/tests/e2e/utils.rs @@ -6,7 +6,7 @@ use reth_e2e_test_utils::{ }; use reth_optimism_chainspec::OpChainSpecBuilder; use reth_optimism_node::{ - node::OptimismAddOns, OpPayloadBuilderAttributes, OptimismBuiltPayload, OptimismNode, + node::OptimismAddOns, OpBuiltPayload, OpPayloadBuilderAttributes, OptimismNode, }; use reth_payload_builder::EthPayloadBuilderAttributes; use std::sync::Arc; @@ -31,7 +31,7 @@ pub(crate) async fn advance_chain( length: usize, node: &mut OpNode, wallet: Arc>, -) -> eyre::Result> { +) -> eyre::Result> { node.advance(length as u64, |_| { let wallet = wallet.clone(); Box::pin(async move { diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 3095ce3512eb..a1fa3d47bebd 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -33,7 +33,7 @@ use tracing::{debug, trace, warn}; use crate::{ error::OptimismPayloadBuilderError, - payload::{OpPayloadBuilderAttributes, OptimismBuiltPayload}, + payload::{OpBuiltPayload, OpPayloadBuilderAttributes}, }; use op_alloy_consensus::DepositTransaction; @@ -97,12 +97,12 @@ where EvmConfig: ConfigureEvm
, { type Attributes = OpPayloadBuilderAttributes; - type BuiltPayload = OptimismBuiltPayload; + type BuiltPayload = OpBuiltPayload; fn try_build( &self, - args: BuildArguments, - ) -> Result, PayloadBuilderError> { + args: BuildArguments, + ) -> Result, PayloadBuilderError> { let (cfg_env, block_env) = self .cfg_and_block_env(&args.config, &args.config.parent_header) .map_err(PayloadBuilderError::other)?; @@ -111,7 +111,7 @@ where fn on_missing_payload( &self, - _args: BuildArguments, + _args: BuildArguments, ) -> MissingPayloadBehaviour { // we want to await the job that's already in progress because that should be returned as // is, there's no benefit in racing another job @@ -124,7 +124,7 @@ where &self, client: &Client, config: PayloadConfig, - ) -> Result { + ) -> Result { let args = BuildArguments { client, config, @@ -154,11 +154,11 @@ where #[inline] pub(crate) fn optimism_payload( evm_config: &EvmConfig, - args: BuildArguments, + args: BuildArguments, initialized_cfg: CfgEnvWithHandlerCfg, initialized_block_env: BlockEnv, _compute_pending_block: bool, -) -> Result, PayloadBuilderError> +) -> Result, PayloadBuilderError> where EvmConfig: ConfigureEvm
, Client: StateProviderFactory + ChainSpecProvider, @@ -523,7 +523,7 @@ where let no_tx_pool = attributes.no_tx_pool; - let payload = OptimismBuiltPayload::new( + let payload = OpBuiltPayload::new( attributes.payload_attributes.id, sealed_block, total_fees, diff --git a/crates/optimism/payload/src/lib.rs b/crates/optimism/payload/src/lib.rs index 1c7bcaf70765..e1f55e51badc 100644 --- a/crates/optimism/payload/src/lib.rs +++ b/crates/optimism/payload/src/lib.rs @@ -15,4 +15,4 @@ pub mod builder; pub use builder::OptimismPayloadBuilder; pub mod error; pub mod payload; -pub use payload::{OpPayloadAttributes, OpPayloadBuilderAttributes, OptimismBuiltPayload}; +pub use payload::{OpBuiltPayload, OpPayloadAttributes, OpPayloadBuilderAttributes}; diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index ecfebdf00b9a..3a7d87acc4c3 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -154,7 +154,7 @@ impl PayloadBuilderAttributes for OpPayloadBuilderAttributes { /// Contains the built payload. #[derive(Debug, Clone)] -pub struct OptimismBuiltPayload { +pub struct OpBuiltPayload { /// Identifier of the payload pub(crate) id: PayloadId, /// The built block @@ -174,7 +174,7 @@ pub struct OptimismBuiltPayload { // === impl BuiltPayload === -impl OptimismBuiltPayload { +impl OpBuiltPayload { /// Initializes the payload with the given initial block. pub const fn new( id: PayloadId, @@ -208,7 +208,7 @@ impl OptimismBuiltPayload { } } -impl BuiltPayload for OptimismBuiltPayload { +impl BuiltPayload for OpBuiltPayload { fn block(&self) -> &SealedBlock { &self.block } @@ -226,7 +226,7 @@ impl BuiltPayload for OptimismBuiltPayload { } } -impl BuiltPayload for &OptimismBuiltPayload { +impl BuiltPayload for &OpBuiltPayload { fn block(&self) -> &SealedBlock { (**self).block() } @@ -245,24 +245,24 @@ impl BuiltPayload for &OptimismBuiltPayload { } // V1 engine_getPayloadV1 response -impl From for ExecutionPayloadV1 { - fn from(value: OptimismBuiltPayload) -> Self { +impl From for ExecutionPayloadV1 { + fn from(value: OpBuiltPayload) -> Self { block_to_payload_v1(value.block) } } // V2 engine_getPayloadV2 response -impl From for ExecutionPayloadEnvelopeV2 { - fn from(value: OptimismBuiltPayload) -> Self { - let OptimismBuiltPayload { block, fees, .. } = value; +impl From for ExecutionPayloadEnvelopeV2 { + fn from(value: OpBuiltPayload) -> Self { + let OpBuiltPayload { block, fees, .. } = value; Self { block_value: fees, execution_payload: convert_block_to_payload_field_v2(block) } } } -impl From for OpExecutionPayloadEnvelopeV3 { - fn from(value: OptimismBuiltPayload) -> Self { - let OptimismBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; +impl From for OpExecutionPayloadEnvelopeV3 { + fn from(value: OpBuiltPayload) -> Self { + let OpBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; let parent_beacon_block_root = if chain_spec.is_cancun_active_at_timestamp(attributes.timestamp()) { @@ -287,9 +287,9 @@ impl From for OpExecutionPayloadEnvelopeV3 { } } } -impl From for OpExecutionPayloadEnvelopeV4 { - fn from(value: OptimismBuiltPayload) -> Self { - let OptimismBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; +impl From for OpExecutionPayloadEnvelopeV4 { + fn from(value: OpBuiltPayload) -> Self { + let OpBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; let parent_beacon_block_root = if chain_spec.is_cancun_active_at_timestamp(attributes.timestamp()) { From 219def95821b6ba631e8e1457aca86acdc3589eb Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Thu, 31 Oct 2024 17:21:06 +0100 Subject: [PATCH 197/239] chore(ci): pin kurtosis to working version (#12225) --- .github/workflows/kurtosis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index 74d26dbd3eea..43f5c3605abe 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -79,6 +79,7 @@ jobs: - name: Run kurtosis uses: ethpandaops/kurtosis-assertoor-github-action@v1 with: + kurtosis_version: 1.3.1 ethereum_package_args: '.github/assets/kurtosis_network_params.yaml' notify-on-error: From d555f9ef3a057c9143c7cf7b4ea69eb30859295d Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:21:55 -0400 Subject: [PATCH 198/239] chore(book): fix engine api typo (#12231) --- book/run/mainnet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book/run/mainnet.md b/book/run/mainnet.md index 4412f51c7bf0..6f1ec144df81 100644 --- a/book/run/mainnet.md +++ b/book/run/mainnet.md @@ -83,4 +83,4 @@ In the meantime, consider setting up [observability](./observability.md) to moni ## Running without a Consensus Layer -We provide a method for running Reth without a Consensus Layer via the `--debug.tip ` parameter. If you provide that to your node, it will simulate sending a `engine_forkChoiceUpdated` message _once_ and will trigger syncing to the provided block hash. This is useful for testing and debugging purposes, but in order to have a node that can keep up with the tip you'll need to run a CL alongside it. At the moment we have no plans of including a Consensus Layer implementation in Reth, and we are open to including light clients other methods of syncing like importing Lighthouse as a library. +We provide a method for running Reth without a Consensus Layer via the `--debug.tip ` parameter. If you provide that to your node, it will simulate sending an `engine_forkchoiceUpdated` message _once_ and will trigger syncing to the provided block hash. This is useful for testing and debugging purposes, but in order to have a node that can keep up with the tip you'll need to run a CL alongside it. At the moment we have no plans of including a Consensus Layer implementation in Reth, and we are open to including light clients other methods of syncing like importing Lighthouse as a library. From 998b3b3d3a57b3a567166dda57c4f2d856539ec3 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:21:38 -0600 Subject: [PATCH 199/239] renamed OptimismPayloadBuilder to OpPayloadBuilder (#12234) --- crates/optimism/node/src/lib.rs | 2 +- crates/optimism/node/src/node.rs | 17 ++++++++--------- crates/optimism/payload/src/builder.rs | 12 ++++++------ crates/optimism/payload/src/lib.rs | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/optimism/node/src/lib.rs b/crates/optimism/node/src/lib.rs index 9bc15ef26685..ff25e7173a66 100644 --- a/crates/optimism/node/src/lib.rs +++ b/crates/optimism/node/src/lib.rs @@ -23,7 +23,7 @@ pub use node::OptimismNode; pub mod txpool; pub use reth_optimism_payload_builder::{ - OpBuiltPayload, OpPayloadBuilderAttributes, OptimismPayloadBuilder, + OpBuiltPayload, OpPayloadBuilder, OpPayloadBuilderAttributes, }; pub use reth_optimism_evm::*; diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 4328a55fb15d..8e06dbcc5000 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -67,7 +67,7 @@ impl OptimismNode { ) -> ComponentsBuilder< Node, OptimismPoolBuilder, - OptimismPayloadBuilder, + OpPayloadBuilder, OptimismNetworkBuilder, OptimismExecutorBuilder, OptimismConsensusBuilder, @@ -81,7 +81,7 @@ impl OptimismNode { ComponentsBuilder::default() .node_types::() .pool(OptimismPoolBuilder::default()) - .payload(OptimismPayloadBuilder::new(compute_pending_block)) + .payload(OpPayloadBuilder::new(compute_pending_block)) .network(OptimismNetworkBuilder { disable_txpool_gossip, disable_discovery_v4: !discovery_v4, @@ -100,7 +100,7 @@ where type ComponentsBuilder = ComponentsBuilder< N, OptimismPoolBuilder, - OptimismPayloadBuilder, + OpPayloadBuilder, OptimismNetworkBuilder, OptimismExecutorBuilder, OptimismConsensusBuilder, @@ -288,7 +288,7 @@ where /// A basic optimism payload service builder #[derive(Debug, Default, Clone)] -pub struct OptimismPayloadBuilder { +pub struct OpPayloadBuilder { /// By default the pending block equals the latest block /// to save resources and not leak txs from the tx-pool, /// this flag enables computing of the pending block @@ -300,7 +300,7 @@ pub struct OptimismPayloadBuilder { pub compute_pending_block: bool, } -impl OptimismPayloadBuilder { +impl OpPayloadBuilder { /// Create a new instance with the given `compute_pending_block` flag. pub const fn new(compute_pending_block: bool) -> Self { Self { compute_pending_block } @@ -320,9 +320,8 @@ impl OptimismPayloadBuilder { Pool: TransactionPool + Unpin + 'static, Evm: ConfigureEvm
, { - let payload_builder = - reth_optimism_payload_builder::OptimismPayloadBuilder::new(evm_config) - .set_compute_pending_block(self.compute_pending_block); + let payload_builder = reth_optimism_payload_builder::OpPayloadBuilder::new(evm_config) + .set_compute_pending_block(self.compute_pending_block); let conf = ctx.payload_builder_config(); let payload_job_config = BasicPayloadJobGeneratorConfig::default() @@ -348,7 +347,7 @@ impl OptimismPayloadBuilder { } } -impl PayloadServiceBuilder for OptimismPayloadBuilder +impl PayloadServiceBuilder for OpPayloadBuilder where Node: FullNodeTypes< Types: NodeTypesWithEngine, diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index a1fa3d47bebd..09a443d0f1df 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -39,7 +39,7 @@ use op_alloy_consensus::DepositTransaction; /// Optimism's payload builder #[derive(Debug, Clone, PartialEq, Eq)] -pub struct OptimismPayloadBuilder { +pub struct OpPayloadBuilder { /// The rollup's compute pending block configuration option. // TODO(clabby): Implement this feature. pub compute_pending_block: bool, @@ -47,8 +47,8 @@ pub struct OptimismPayloadBuilder { pub evm_config: EvmConfig, } -impl OptimismPayloadBuilder { - /// `OptimismPayloadBuilder` constructor. +impl OpPayloadBuilder { + /// `OpPayloadBuilder` constructor. pub const fn new(evm_config: EvmConfig) -> Self { Self { compute_pending_block: true, evm_config } } @@ -69,7 +69,7 @@ impl OptimismPayloadBuilder { self.compute_pending_block } } -impl OptimismPayloadBuilder +impl OpPayloadBuilder where EvmConfig: ConfigureEvmEnv
, { @@ -89,8 +89,8 @@ where } } -/// Implementation of the [`PayloadBuilder`] trait for [`OptimismPayloadBuilder`]. -impl PayloadBuilder for OptimismPayloadBuilder +/// Implementation of the [`PayloadBuilder`] trait for [`OpPayloadBuilder`]. +impl PayloadBuilder for OpPayloadBuilder where Client: StateProviderFactory + ChainSpecProvider, Pool: TransactionPool, diff --git a/crates/optimism/payload/src/lib.rs b/crates/optimism/payload/src/lib.rs index e1f55e51badc..8447026d783a 100644 --- a/crates/optimism/payload/src/lib.rs +++ b/crates/optimism/payload/src/lib.rs @@ -12,7 +12,7 @@ #![cfg(feature = "optimism")] pub mod builder; -pub use builder::OptimismPayloadBuilder; +pub use builder::OpPayloadBuilder; pub mod error; pub mod payload; pub use payload::{OpBuiltPayload, OpPayloadAttributes, OpPayloadBuilderAttributes}; From d020b41f6aa053dd6a13d3647f7ad708ee35dc66 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Thu, 31 Oct 2024 23:22:42 +0400 Subject: [PATCH 200/239] feat: `flashbots_validateBuilderSubmissionV3` (#12168) --- Cargo.lock | 8 + book/cli/reth/node.md | 3 + crates/cli/util/Cargo.toml | 1 + crates/cli/util/src/parsers.rs | 7 + crates/e2e-test-utils/src/lib.rs | 8 +- crates/ethereum/consensus/src/lib.rs | 4 +- crates/ethereum/node/Cargo.toml | 1 + crates/ethereum/node/tests/e2e/rpc.rs | 86 +++- crates/node/builder/src/rpc.rs | 3 + crates/node/core/src/args/rpc_server.rs | 14 + crates/payload/validator/src/lib.rs | 4 +- crates/rpc/rpc-api/Cargo.toml | 2 + crates/rpc/rpc-api/src/lib.rs | 2 +- crates/rpc/rpc-api/src/validation.rs | 21 +- crates/rpc/rpc-builder/Cargo.toml | 1 + crates/rpc/rpc-builder/src/config.rs | 10 +- crates/rpc/rpc-builder/src/lib.rs | 322 +++++++++++--- crates/rpc/rpc-builder/tests/it/utils.rs | 3 + .../rpc/rpc-eth-types/src/builder/config.rs | 2 +- crates/rpc/rpc-server-types/src/result.rs | 2 + crates/rpc/rpc/Cargo.toml | 5 +- crates/rpc/rpc/src/lib.rs | 2 +- crates/rpc/rpc/src/validation.rs | 400 ++++++++++++++++-- examples/rpc-db/src/main.rs | 6 +- 24 files changed, 801 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27581ba3a800..5fc3574ebc43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6697,6 +6697,7 @@ dependencies = [ "rand 0.8.5", "reth-fs-util", "secp256k1", + "serde", "thiserror", "tikv-jemallocator", "tracy-client", @@ -7997,6 +7998,7 @@ dependencies = [ "alloy-genesis", "alloy-primitives", "alloy-provider", + "alloy-rpc-types-beacon", "alloy-signer", "alloy-sol-types", "eyre", @@ -8626,13 +8628,16 @@ dependencies = [ "pin-project", "rand 0.8.5", "reth-chainspec", + "reth-consensus", "reth-consensus-common", "reth-errors", + "reth-ethereum-consensus", "reth-evm", "reth-evm-ethereum", "reth-network-api", "reth-network-peers", "reth-network-types", + "reth-payload-validator", "reth-primitives", "reth-provider", "reth-revm", @@ -8682,6 +8687,8 @@ dependencies = [ "reth-network-peers", "reth-primitives", "reth-rpc-eth-api", + "serde", + "serde_with", ] [[package]] @@ -8721,6 +8728,7 @@ dependencies = [ "pin-project", "reth-beacon-consensus", "reth-chainspec", + "reth-consensus", "reth-engine-primitives", "reth-ethereum-engine-primitives", "reth-evm", diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 52f597279f06..5f0090ef8961 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -367,6 +367,9 @@ RPC: [default: 25] + --builder.disallow + Path to file containing disallowed addresses, json-encoded list of strings. Block validation API will reject blocks containing transactions from these addresses + RPC State Cache: --rpc-cache.max-blocks Max number of blocks in cache diff --git a/crates/cli/util/Cargo.toml b/crates/cli/util/Cargo.toml index d96a882a672a..70515f83b4b7 100644 --- a/crates/cli/util/Cargo.toml +++ b/crates/cli/util/Cargo.toml @@ -24,6 +24,7 @@ eyre.workspace = true rand.workspace = true secp256k1 = { workspace = true, features = ["rand"] } thiserror.workspace = true +serde.workspace = true tracy-client = { workspace = true, optional = true, features = ["demangle"] } diff --git a/crates/cli/util/src/parsers.rs b/crates/cli/util/src/parsers.rs index 202744a4bb79..9bb803bcca89 100644 --- a/crates/cli/util/src/parsers.rs +++ b/crates/cli/util/src/parsers.rs @@ -1,7 +1,9 @@ use alloy_eips::BlockHashOrNumber; use alloy_primitives::B256; +use reth_fs_util::FsPathError; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, + path::Path, str::FromStr, time::Duration, }; @@ -82,6 +84,11 @@ pub fn parse_socket_address(value: &str) -> eyre::Result(path: &str) -> Result { + reth_fs_util::read_json_file(Path::new(path)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 1e9717d082cd..1e9b39058e63 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -7,6 +7,7 @@ use reth::{ args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}, builder::{NodeBuilder, NodeConfig, NodeHandle}, network::PeersHandleProvider, + rpc::server_types::RpcModuleSelection, tasks::TaskManager, }; use reth_chainspec::{EthChainSpec, EthereumHardforks}; @@ -147,7 +148,12 @@ where let node_config = NodeConfig::new(chain_spec.clone()) .with_network(network_config.clone()) .with_unused_ports() - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()) + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ) .set_dev(is_dev); let span = span!(Level::INFO, "node", idx); diff --git a/crates/ethereum/consensus/src/lib.rs b/crates/ethereum/consensus/src/lib.rs index dd286584a598..07c2a71e8cf2 100644 --- a/crates/ethereum/consensus/src/lib.rs +++ b/crates/ethereum/consensus/src/lib.rs @@ -24,7 +24,7 @@ use reth_primitives::{ use std::{fmt::Debug, sync::Arc, time::SystemTime}; /// The bound divisor of the gas limit, used in update calculations. -const GAS_LIMIT_BOUND_DIVISOR: u64 = 1024; +pub const GAS_LIMIT_BOUND_DIVISOR: u64 = 1024; mod validation; pub use validation::validate_block_post_execution; @@ -32,7 +32,7 @@ pub use validation::validate_block_post_execution; /// Ethereum beacon consensus /// /// This consensus engine does basic checks as outlined in the execution specs. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct EthBeaconConsensus { /// Configuration chain_spec: Arc, diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index 83b62034740b..49663ffc2cc2 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -59,6 +59,7 @@ alloy-signer.workspace = true alloy-eips.workspace = true alloy-sol-types.workspace = true alloy-contract.workspace = true +alloy-rpc-types-beacon.workspace = true [features] default = [] diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index ddf3d5cba2a6..c8b127b9b7f5 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -1,8 +1,14 @@ use crate::utils::eth_payload_attributes; -use alloy_eips::calc_next_block_base_fee; -use alloy_primitives::U256; -use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder}; +use alloy_eips::{calc_next_block_base_fee, eip2718::Encodable2718}; +use alloy_primitives::{Address, B256, U256}; +use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder, SendableTx}; +use alloy_rpc_types_beacon::relay::{BidTrace, SignedBidSubmissionV3}; use rand::{rngs::StdRng, Rng, SeedableRng}; +use reth::rpc::{ + api::BuilderBlockValidationRequestV3, + compat::engine::payload::block_to_payload_v3, + types::{engine::BlobsBundleV1, TransactionRequest}, +}; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::setup_engine; use reth_node_ethereum::EthereumNode; @@ -107,3 +113,77 @@ async fn test_fee_history() -> eyre::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_flashbots_validate() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + + let (mut nodes, _tasks, wallet) = + setup_engine::(1, chain_spec.clone(), false, eth_payload_attributes).await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .with_recommended_fillers() + .wallet(EthereumWallet::new(wallet.gen().swap_remove(0))) + .on_http(node.rpc_url()); + + node.advance(100, |_| { + let provider = provider.clone(); + Box::pin(async move { + let SendableTx::Envelope(tx) = + provider.fill(TransactionRequest::default().to(Address::ZERO)).await.unwrap() + else { + unreachable!() + }; + + tx.encoded_2718().into() + }) + }) + .await?; + + let _ = provider.send_transaction(TransactionRequest::default().to(Address::ZERO)).await?; + let (payload, attrs) = node.new_payload().await?; + + let mut request = BuilderBlockValidationRequestV3 { + request: SignedBidSubmissionV3 { + message: BidTrace { + parent_hash: payload.block().parent_hash, + block_hash: payload.block().hash(), + gas_used: payload.block().gas_used, + gas_limit: payload.block().gas_limit, + ..Default::default() + }, + execution_payload: block_to_payload_v3(payload.block().clone()), + blobs_bundle: BlobsBundleV1::new([]), + signature: Default::default(), + }, + parent_beacon_block_root: attrs.parent_beacon_block_root.unwrap(), + registered_gas_limit: payload.block().gas_limit, + }; + + assert!(provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV3".into(), (&request,)) + .await + .is_ok()); + + request.registered_gas_limit -= 1; + assert!(provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV3".into(), (&request,)) + .await + .is_err()); + request.registered_gas_limit += 1; + + request.request.execution_payload.payload_inner.payload_inner.state_root = B256::ZERO; + assert!(provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV3".into(), (&request,)) + .await + .is_err()); + Ok(()) +} diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 4c1ea32d045d..8819aa4ac4f7 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -198,6 +198,7 @@ pub struct RpcRegistry { Node::Provider, EthApi, Node::Executor, + Node::Consensus, >, } @@ -214,6 +215,7 @@ where Node::Provider, EthApi, Node::Executor, + Node::Consensus, >; fn deref(&self) -> &Self::Target { @@ -442,6 +444,7 @@ where .with_executor(node.task_executor().clone()) .with_evm_config(node.evm_config().clone()) .with_block_executor(node.block_executor().clone()) + .with_consensus(node.consensus().clone()) .build_with_auth_server(module_config, engine_api, eth_api_builder); // in dev mode we generate 20 random dev-signer accounts diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 382f22d37764..fe9b80cec472 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -1,11 +1,13 @@ //! clap [Args](clap::Args) for RPC related arguments. use std::{ + collections::HashSet, ffi::OsStr, net::{IpAddr, Ipv4Addr}, path::PathBuf, }; +use alloy_primitives::Address; use alloy_rpc_types_engine::JwtSecret; use clap::{ builder::{PossibleValue, RangedU64ValueParser, TypedValueParser}, @@ -183,6 +185,11 @@ pub struct RpcServerArgs { #[arg(long = "rpc.proof-permits", alias = "rpc-proof-permits", value_name = "COUNT", default_value_t = constants::DEFAULT_PROOF_PERMITS)] pub rpc_proof_permits: usize, + /// Path to file containing disallowed addresses, json-encoded list of strings. Block + /// validation API will reject blocks containing transactions from these addresses. + #[arg(long = "builder.disallow", value_name = "PATH", value_parser = reth_cli_util::parsers::read_json_from_file::>)] + pub builder_disallow: Option>, + /// State cache configuration. #[command(flatten)] pub rpc_state_cache: RpcStateCacheArgs, @@ -199,6 +206,12 @@ impl RpcServerArgs { self } + /// Configures modules for the HTTP-RPC server. + pub fn with_http_api(mut self, http_api: RpcModuleSelection) -> Self { + self.http_api = Some(http_api); + self + } + /// Enables the WS-RPC server. pub const fn with_ws(mut self) -> Self { self.ws = true; @@ -318,6 +331,7 @@ impl Default for RpcServerArgs { gas_price_oracle: GasPriceOracleArgs::default(), rpc_state_cache: RpcStateCacheArgs::default(), rpc_proof_permits: constants::DEFAULT_PROOF_PERMITS, + builder_disallow: Default::default(), } } } diff --git a/crates/payload/validator/src/lib.rs b/crates/payload/validator/src/lib.rs index 9952815fd982..38e53bac42a5 100644 --- a/crates/payload/validator/src/lib.rs +++ b/crates/payload/validator/src/lib.rs @@ -23,7 +23,7 @@ pub struct ExecutionPayloadValidator { chain_spec: Arc, } -impl ExecutionPayloadValidator { +impl ExecutionPayloadValidator { /// Create a new validator. pub const fn new(chain_spec: Arc) -> Self { Self { chain_spec } @@ -34,7 +34,9 @@ impl ExecutionPayloadValidator { pub fn chain_spec(&self) -> &ChainSpec { &self.chain_spec } +} +impl ExecutionPayloadValidator { /// Returns true if the Cancun hardfork is active at the given timestamp. #[inline] fn is_cancun_active_at_timestamp(&self, timestamp: u64) -> bool { diff --git a/crates/rpc/rpc-api/Cargo.toml b/crates/rpc/rpc-api/Cargo.toml index 363e22955301..60146e8b2c2d 100644 --- a/crates/rpc/rpc-api/Cargo.toml +++ b/crates/rpc/rpc-api/Cargo.toml @@ -36,6 +36,8 @@ alloy-rpc-types-engine.workspace = true # misc jsonrpsee = { workspace = true, features = ["server", "macros"] } +serde.workspace = true +serde_with.workspace = true [features] client = [ diff --git a/crates/rpc/rpc-api/src/lib.rs b/crates/rpc/rpc-api/src/lib.rs index 73775112dcf5..63e6e54466dd 100644 --- a/crates/rpc/rpc-api/src/lib.rs +++ b/crates/rpc/rpc-api/src/lib.rs @@ -46,7 +46,7 @@ pub mod servers { rpc::RpcApiServer, trace::TraceApiServer, txpool::TxPoolApiServer, - validation::BlockSubmissionValidationApiServer, + validation::{BlockSubmissionValidationApiServer, BuilderBlockValidationRequestV3}, web3::Web3ApiServer, }; pub use reth_rpc_eth_api::{ diff --git a/crates/rpc/rpc-api/src/validation.rs b/crates/rpc/rpc-api/src/validation.rs index e1819dde440c..d8f55b668c9e 100644 --- a/crates/rpc/rpc-api/src/validation.rs +++ b/crates/rpc/rpc-api/src/validation.rs @@ -1,9 +1,28 @@ //! API for block submission validation. +use alloy_primitives::B256; use alloy_rpc_types_beacon::relay::{ - BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, BuilderBlockValidationRequestV3, + BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, SignedBidSubmissionV3, }; use jsonrpsee::proc_macros::rpc; +use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; + +/// A Request to validate a [`SignedBidSubmissionV3`] +/// +/// +#[serde_as] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BuilderBlockValidationRequestV3 { + /// The request to be validated. + #[serde(flatten)] + pub request: SignedBidSubmissionV3, + /// The registered gas limit for the validation request. + #[serde_as(as = "DisplayFromStr")] + pub registered_gas_limit: u64, + /// The parent beacon block root for the validation request. + pub parent_beacon_block_root: B256, +} /// Block validation rpc interface. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "flashbots"))] diff --git a/crates/rpc/rpc-builder/Cargo.toml b/crates/rpc/rpc-builder/Cargo.toml index b9b511a078bc..711e44381330 100644 --- a/crates/rpc/rpc-builder/Cargo.toml +++ b/crates/rpc/rpc-builder/Cargo.toml @@ -15,6 +15,7 @@ workspace = true # reth reth-ipc.workspace = true reth-chainspec.workspace = true +reth-consensus.workspace = true reth-network-api.workspace = true reth-node-core.workspace = true reth-provider.workspace = true diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index 4ff98ae8d501..daff81fa2ae1 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -2,6 +2,7 @@ use std::{net::SocketAddr, path::PathBuf}; use jsonrpsee::server::ServerBuilder; use reth_node_core::{args::RpcServerArgs, utils::get_or_create_jwt_secret_from_path}; +use reth_rpc::ValidationApiConfig; use reth_rpc_eth_types::{EthConfig, EthStateCacheConfig, GasPriceOracleConfig}; use reth_rpc_layer::{JwtError, JwtSecret}; use reth_rpc_server_types::RpcModuleSelection; @@ -27,6 +28,9 @@ pub trait RethRpcServerConfig { /// The configured ethereum RPC settings. fn eth_config(&self) -> EthConfig; + /// The configured ethereum RPC settings. + fn flashbots_config(&self) -> ValidationApiConfig; + /// Returns state cache configuration. fn state_cache_config(&self) -> EthStateCacheConfig; @@ -101,6 +105,10 @@ impl RethRpcServerConfig for RpcServerArgs { .proof_permits(self.rpc_proof_permits) } + fn flashbots_config(&self) -> ValidationApiConfig { + ValidationApiConfig { disallow: self.builder_disallow.clone().unwrap_or_default() } + } + fn state_cache_config(&self) -> EthStateCacheConfig { EthStateCacheConfig { max_blocks: self.rpc_state_cache.max_blocks, @@ -124,7 +132,7 @@ impl RethRpcServerConfig for RpcServerArgs { fn transport_rpc_module_config(&self) -> TransportRpcModuleConfig { let mut config = TransportRpcModuleConfig::default() - .with_config(RpcModuleConfig::new(self.eth_config())); + .with_config(RpcModuleConfig::new(self.eth_config(), self.flashbots_config())); if self.http { config = config.with_http( diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 787dce08b8d0..5d5fd31aab8c 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -27,13 +27,14 @@ //! use reth_tasks::TokioTaskExecutor; //! use reth_transaction_pool::TransactionPool; //! -//! pub async fn launch( +//! pub async fn launch( //! provider: Provider, //! pool: Pool, //! network: Network, //! events: Events, //! evm_config: EvmConfig, //! block_executor: BlockExecutor, +//! consensus: Consensus, //! ) where //! Provider: FullRpcProvider + AccountReader + ChangeSetReader, //! Pool: TransactionPool + Unpin + 'static, @@ -41,6 +42,7 @@ //! Events: CanonStateSubscriptions + Clone + 'static, //! EvmConfig: ConfigureEvm
, //! BlockExecutor: BlockExecutorProvider, +//! Consensus: reth_consensus::Consensus + Clone + 'static, //! { //! // configure the rpc module per transport //! let transports = TransportRpcModuleConfig::default().with_http(vec![ @@ -57,6 +59,7 @@ //! events, //! evm_config, //! block_executor, +//! consensus, //! ) //! .build(transports, Box::new(EthApi::with_spawner)); //! let handle = RpcServerConfig::default() @@ -95,6 +98,7 @@ //! EngineT, //! EvmConfig, //! BlockExecutor, +//! Consensus, //! >( //! provider: Provider, //! pool: Pool, @@ -103,6 +107,7 @@ //! engine_api: EngineApi, //! evm_config: EvmConfig, //! block_executor: BlockExecutor, +//! consensus: Consensus, //! ) where //! Provider: FullRpcProvider + AccountReader + ChangeSetReader, //! Pool: TransactionPool + Unpin + 'static, @@ -112,6 +117,7 @@ //! EngineT: EngineTypes, //! EvmConfig: ConfigureEvm
, //! BlockExecutor: BlockExecutorProvider, +//! Consensus: reth_consensus::Consensus + Clone + 'static, //! { //! // configure the rpc module per transport //! let transports = TransportRpcModuleConfig::default().with_http(vec![ @@ -128,6 +134,7 @@ //! events, //! evm_config, //! block_executor, +//! consensus, //! ); //! //! // configure the server modules @@ -155,6 +162,7 @@ use std::{ collections::HashMap, fmt::Debug, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; @@ -170,6 +178,7 @@ use jsonrpsee::{ Methods, RpcModule, }; use reth_chainspec::EthereumHardforks; +use reth_consensus::Consensus; use reth_engine_primitives::EngineTypes; use reth_evm::{execute::BlockExecutorProvider, ConfigureEvm}; use reth_network_api::{noop::NoopNetwork, NetworkInfo, Peers}; @@ -180,7 +189,7 @@ use reth_provider::{ }; use reth_rpc::{ AdminApi, DebugApi, EngineEthApi, EthBundle, NetApi, OtterscanApi, RPCApi, RethApi, TraceApi, - TxPoolApi, ValidationApi, Web3Api, + TxPoolApi, ValidationApi, ValidationApiConfig, Web3Api, }; use reth_rpc_api::servers::*; use reth_rpc_eth_api::{ @@ -243,6 +252,7 @@ pub async fn launch, block_executor: BlockExecutor, + consensus: Arc, ) -> Result where Provider: FullRpcProvider + AccountReader + ChangeSetReader, @@ -266,6 +276,7 @@ where events, evm_config, block_executor, + consensus, ) .build(module_config, eth), ) @@ -276,7 +287,16 @@ where /// /// This is the main entrypoint and the easiest way to configure an RPC server. #[derive(Debug, Clone)] -pub struct RpcModuleBuilder { +pub struct RpcModuleBuilder< + Provider, + Pool, + Network, + Tasks, + Events, + EvmConfig, + BlockExecutor, + Consensus, +> { /// The Provider type to when creating all rpc handlers provider: Provider, /// The Pool type to when creating all rpc handlers @@ -291,14 +311,17 @@ pub struct RpcModuleBuilder - RpcModuleBuilder +impl + RpcModuleBuilder { /// Create a new instance of the builder + #[allow(clippy::too_many_arguments)] pub const fn new( provider: Provider, pool: Pool, @@ -307,32 +330,54 @@ impl events: Events, evm_config: EvmConfig, block_executor: BlockExecutor, + consensus: Consensus, ) -> Self { - Self { provider, pool, network, executor, events, evm_config, block_executor } + Self { provider, pool, network, executor, events, evm_config, block_executor, consensus } } /// Configure the provider instance. pub fn with_provider

( self, provider: P, - ) -> RpcModuleBuilder + ) -> RpcModuleBuilder where P: BlockReader + StateProviderFactory + EvmEnvProvider + 'static, { - let Self { pool, network, executor, events, evm_config, block_executor, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, events, evm_config, block_executor } + let Self { pool, network, executor, events, evm_config, block_executor, consensus, .. } = + self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } } /// Configure the transaction pool instance. pub fn with_pool

( self, pool: P, - ) -> RpcModuleBuilder + ) -> RpcModuleBuilder where P: TransactionPool + 'static, { - let Self { provider, network, executor, events, evm_config, block_executor, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, events, evm_config, block_executor } + let Self { + provider, network, executor, events, evm_config, block_executor, consensus, .. + } = self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } } /// Configure a [`NoopTransactionPool`] instance. @@ -350,8 +395,11 @@ impl Events, EvmConfig, BlockExecutor, + Consensus, > { - let Self { provider, executor, events, network, evm_config, block_executor, .. } = self; + let Self { + provider, executor, events, network, evm_config, block_executor, consensus, .. + } = self; RpcModuleBuilder { provider, executor, @@ -360,6 +408,7 @@ impl evm_config, block_executor, pool: NoopTransactionPool::default(), + consensus, } } @@ -367,12 +416,23 @@ impl pub fn with_network( self, network: N, - ) -> RpcModuleBuilder + ) -> RpcModuleBuilder where N: NetworkInfo + Peers + 'static, { - let Self { provider, pool, executor, events, evm_config, block_executor, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, events, evm_config, block_executor } + let Self { + provider, pool, executor, events, evm_config, block_executor, consensus, .. + } = self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } } /// Configure a [`NoopNetwork`] instance. @@ -382,9 +442,19 @@ impl /// [`EthApi`](reth_rpc::eth::EthApi) which requires a [`NetworkInfo`] implementation. pub fn with_noop_network( self, - ) -> RpcModuleBuilder - { - let Self { provider, pool, executor, events, evm_config, block_executor, .. } = self; + ) -> RpcModuleBuilder< + Provider, + Pool, + NoopNetwork, + Tasks, + Events, + EvmConfig, + BlockExecutor, + Consensus, + > { + let Self { + provider, pool, executor, events, evm_config, block_executor, consensus, .. + } = self; RpcModuleBuilder { provider, pool, @@ -393,6 +463,7 @@ impl network: NoopNetwork::default(), evm_config, block_executor, + consensus, } } @@ -400,12 +471,22 @@ impl pub fn with_executor( self, executor: T, - ) -> RpcModuleBuilder + ) -> RpcModuleBuilder where T: TaskSpawner + 'static, { - let Self { pool, network, provider, events, evm_config, block_executor, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, events, evm_config, block_executor } + let Self { pool, network, provider, events, evm_config, block_executor, consensus, .. } = + self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } } /// Configure [`TokioTaskExecutor`] as the task executor to use for additional tasks. @@ -422,8 +503,10 @@ impl Events, EvmConfig, BlockExecutor, + Consensus, > { - let Self { pool, network, provider, events, evm_config, block_executor, .. } = self; + let Self { pool, network, provider, events, evm_config, block_executor, consensus, .. } = + self; RpcModuleBuilder { provider, network, @@ -432,6 +515,7 @@ impl executor: TokioTaskExecutor::default(), evm_config, block_executor, + consensus, } } @@ -439,41 +523,90 @@ impl pub fn with_events( self, events: E, - ) -> RpcModuleBuilder + ) -> RpcModuleBuilder where E: CanonStateSubscriptions + 'static, { - let Self { provider, pool, executor, network, evm_config, block_executor, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, events, evm_config, block_executor } + let Self { + provider, pool, executor, network, evm_config, block_executor, consensus, .. + } = self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } } /// Configure the evm configuration type pub fn with_evm_config( self, evm_config: E, - ) -> RpcModuleBuilder + ) -> RpcModuleBuilder where E: ConfigureEvm + 'static, { - let Self { provider, pool, executor, network, events, block_executor, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, events, evm_config, block_executor } + let Self { provider, pool, executor, network, events, block_executor, consensus, .. } = + self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } } /// Configure the block executor provider pub fn with_block_executor( self, block_executor: BE, - ) -> RpcModuleBuilder + ) -> RpcModuleBuilder where BE: BlockExecutorProvider, { - let Self { provider, network, pool, executor, events, evm_config, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, events, evm_config, block_executor } + let Self { provider, network, pool, executor, events, evm_config, consensus, .. } = self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } + } + + /// Configure the consensus implementation. + pub fn with_consensus( + self, + consensus: C, + ) -> RpcModuleBuilder { + let Self { provider, network, pool, executor, events, evm_config, block_executor, .. } = + self; + RpcModuleBuilder { + provider, + network, + pool, + executor, + events, + evm_config, + block_executor, + consensus, + } } } -impl - RpcModuleBuilder +impl + RpcModuleBuilder where Provider: FullRpcProvider + AccountReader + ChangeSetReader, Pool: TransactionPool + 'static, @@ -482,6 +615,7 @@ where Events: CanonStateSubscriptions + Clone + 'static, EvmConfig: ConfigureEvm

, BlockExecutor: BlockExecutorProvider, + Consensus: reth_consensus::Consensus + Clone + 'static, { /// Configures all [`RpcModule`]s specific to the given [`TransportRpcModuleConfig`] which can /// be used to start the transport server(s). @@ -498,14 +632,23 @@ where ) -> ( TransportRpcModules, AuthRpcModule, - RpcRegistryInner, + RpcRegistryInner, ) where EngineT: EngineTypes, EngineApi: EngineApiServer, EthApi: FullEthApiServer, { - let Self { provider, pool, network, executor, events, evm_config, block_executor } = self; + let Self { + provider, + pool, + network, + executor, + events, + evm_config, + block_executor, + consensus, + } = self; let config = module_config.config.clone().unwrap_or_default(); @@ -515,6 +658,7 @@ where network, executor, events, + consensus, config, evm_config, eth, @@ -536,6 +680,7 @@ where /// # Example /// /// ```no_run + /// use reth_consensus::noop::NoopConsensus; /// use reth_evm::ConfigureEvm; /// use reth_evm_ethereum::execute::EthExecutorProvider; /// use reth_network_api::noop::NoopNetwork; @@ -555,6 +700,7 @@ where /// .with_events(TestCanonStateSubscriptions::default()) /// .with_evm_config(evm) /// .with_block_executor(EthExecutorProvider::mainnet()) + /// .with_consensus(NoopConsensus::default()) /// .into_registry(Default::default(), Box::new(EthApi::with_spawner)); /// /// let eth_api = registry.eth_api(); @@ -564,17 +710,27 @@ where self, config: RpcModuleConfig, eth: DynEthApiBuilder, - ) -> RpcRegistryInner + ) -> RpcRegistryInner where EthApi: EthApiTypes + 'static, { - let Self { provider, pool, network, executor, events, evm_config, block_executor } = self; + let Self { + provider, + pool, + network, + executor, + events, + evm_config, + block_executor, + consensus, + } = self; RpcRegistryInner::new( provider, pool, network, executor, events, + consensus, config, evm_config, eth, @@ -594,7 +750,16 @@ where { let mut modules = TransportRpcModules::default(); - let Self { provider, pool, network, executor, events, evm_config, block_executor } = self; + let Self { + provider, + pool, + network, + executor, + events, + evm_config, + block_executor, + consensus, + } = self; if !module_config.is_empty() { let TransportRpcModuleConfig { http, ws, ipc, config } = module_config.clone(); @@ -605,6 +770,7 @@ where network, executor, events, + consensus, config.unwrap_or_default(), evm_config, eth, @@ -621,9 +787,9 @@ where } } -impl Default for RpcModuleBuilder<(), (), (), (), (), (), ()> { +impl Default for RpcModuleBuilder<(), (), (), (), (), (), (), ()> { fn default() -> Self { - Self::new((), (), (), (), (), (), ()) + Self::new((), (), (), (), (), (), (), ()) } } @@ -632,6 +798,8 @@ impl Default for RpcModuleBuilder<(), (), (), (), (), (), ()> { pub struct RpcModuleConfig { /// `eth` namespace settings eth: EthConfig, + /// `flashbots` namespace settings + flashbots: ValidationApiConfig, } // === impl RpcModuleConfig === @@ -643,8 +811,8 @@ impl RpcModuleConfig { } /// Returns a new RPC module config given the eth namespace config - pub const fn new(eth: EthConfig) -> Self { - Self { eth } + pub const fn new(eth: EthConfig, flashbots: ValidationApiConfig) -> Self { + Self { eth, flashbots } } /// Get a reference to the eth namespace config @@ -662,6 +830,7 @@ impl RpcModuleConfig { #[derive(Clone, Debug, Default)] pub struct RpcModuleConfigBuilder { eth: Option, + flashbots: Option, } // === impl RpcModuleConfigBuilder === @@ -673,10 +842,16 @@ impl RpcModuleConfigBuilder { self } + /// Configures a custom flashbots namespace config + pub fn flashbots(mut self, flashbots: ValidationApiConfig) -> Self { + self.flashbots = Some(flashbots); + self + } + /// Consumes the type and creates the [`RpcModuleConfig`] pub fn build(self) -> RpcModuleConfig { - let Self { eth } = self; - RpcModuleConfig { eth: eth.unwrap_or_default() } + let Self { eth, flashbots } = self; + RpcModuleConfig { eth: eth.unwrap_or_default(), flashbots: flashbots.unwrap_or_default() } } /// Get a reference to the eth namespace config, if any @@ -705,6 +880,7 @@ pub struct RpcRegistryInner< Events, EthApi: EthApiTypes, BlockExecutor, + Consensus, > { provider: Provider, pool: Pool, @@ -712,6 +888,9 @@ pub struct RpcRegistryInner< executor: Tasks, events: Events, block_executor: BlockExecutor, + consensus: Consensus, + /// Holds the configuration for the RPC modules + config: RpcModuleConfig, /// Holds a all `eth_` namespace handlers eth: EthHandlers, /// to put trace calls behind semaphore @@ -722,8 +901,8 @@ pub struct RpcRegistryInner< // === impl RpcRegistryInner === -impl - RpcRegistryInner +impl + RpcRegistryInner where Provider: StateProviderFactory + BlockReader + EvmEnvProvider + Clone + Unpin + 'static, Pool: Send + Sync + Clone + 'static, @@ -741,6 +920,7 @@ where network: Network, executor: Tasks, events: Events, + consensus: Consensus, config: RpcModuleConfig, evm_config: EvmConfig, eth_api_builder: DynEthApiBuilder< @@ -776,6 +956,8 @@ where network, eth, executor, + consensus, + config, modules: Default::default(), blocking_pool_guard, events, @@ -784,8 +966,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where EthApi: EthApiTypes, { @@ -842,8 +1024,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where Network: NetworkInfo + Clone + 'static, EthApi: EthApiTypes, @@ -881,8 +1063,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where Provider: FullRpcProvider + AccountReader + ChangeSetReader, Network: NetworkInfo + Peers + Clone + 'static, @@ -994,8 +1176,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where Provider: FullRpcProvider + AccountReader + ChangeSetReader, Network: NetworkInfo + Peers + Clone + 'static, @@ -1069,13 +1251,21 @@ where } /// Instantiates `ValidationApi` - pub fn validation_api(&self) -> ValidationApi { - ValidationApi::new(self.provider.clone()) + pub fn validation_api(&self) -> ValidationApi + where + Consensus: reth_consensus::Consensus + Clone + 'static, + { + ValidationApi::new( + self.provider.clone(), + Arc::new(self.consensus.clone()), + self.block_executor.clone(), + self.config.flashbots.clone(), + ) } } -impl - RpcRegistryInner +impl + RpcRegistryInner where Provider: FullRpcProvider + AccountReader + ChangeSetReader, Pool: TransactionPool + 'static, @@ -1084,6 +1274,7 @@ where Events: CanonStateSubscriptions + Clone + 'static, EthApi: FullEthApiServer, BlockExecutor: BlockExecutorProvider, + Consensus: reth_consensus::Consensus + Clone + 'static, { /// Configures the auth module that includes the /// * `engine_` namespace @@ -1228,9 +1419,14 @@ where .into_rpc() .into() } - RethRpcModule::Flashbots => { - ValidationApi::new(self.provider.clone()).into_rpc().into() - } + RethRpcModule::Flashbots => ValidationApi::new( + self.provider.clone(), + Arc::new(self.consensus.clone()), + self.block_executor.clone(), + self.config.flashbots.clone(), + ) + .into_rpc() + .into(), }) .clone() }) @@ -1688,7 +1884,7 @@ impl TransportRpcModuleConfig { } /// Sets a custom [`RpcModuleConfig`] for the configured modules. - pub const fn with_config(mut self, config: RpcModuleConfig) -> Self { + pub fn with_config(mut self, config: RpcModuleConfig) -> Self { self.config = Some(config); self } diff --git a/crates/rpc/rpc-builder/tests/it/utils.rs b/crates/rpc/rpc-builder/tests/it/utils.rs index 44614ea49a85..175992c0f14b 100644 --- a/crates/rpc/rpc-builder/tests/it/utils.rs +++ b/crates/rpc/rpc-builder/tests/it/utils.rs @@ -3,6 +3,7 @@ use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use alloy_rpc_types_engine::{ClientCode, ClientVersionV1}; use reth_beacon_consensus::BeaconConsensusEngineHandle; use reth_chainspec::MAINNET; +use reth_consensus::noop::NoopConsensus; use reth_ethereum_engine_primitives::{EthEngineTypes, EthereumEngineValidator}; use reth_evm::execute::BasicBlockExecutorProvider; use reth_evm_ethereum::{execute::EthExecutionStrategyFactory, EthEvmConfig}; @@ -126,6 +127,7 @@ pub fn test_rpc_builder() -> RpcModuleBuilder< TestCanonStateSubscriptions, EthEvmConfig, BasicBlockExecutorProvider, + NoopConsensus, > { RpcModuleBuilder::default() .with_provider(NoopProvider::default()) @@ -137,4 +139,5 @@ pub fn test_rpc_builder() -> RpcModuleBuilder< .with_block_executor( BasicBlockExecutorProvider::new(EthExecutionStrategyFactory::mainnet()), ) + .with_consensus(NoopConsensus::default()) } diff --git a/crates/rpc/rpc-eth-types/src/builder/config.rs b/crates/rpc/rpc-eth-types/src/builder/config.rs index a016d021586b..532c10772035 100644 --- a/crates/rpc/rpc-eth-types/src/builder/config.rs +++ b/crates/rpc/rpc-eth-types/src/builder/config.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; pub const DEFAULT_STALE_FILTER_TTL: Duration = Duration::from_secs(5 * 60); /// Additional config values for the eth namespace. -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] pub struct EthConfig { /// Settings for the caching layer pub cache: EthStateCacheConfig, diff --git a/crates/rpc/rpc-server-types/src/result.rs b/crates/rpc/rpc-server-types/src/result.rs index 78e6436643a7..10ce1650ad15 100644 --- a/crates/rpc/rpc-server-types/src/result.rs +++ b/crates/rpc/rpc-server-types/src/result.rs @@ -4,6 +4,7 @@ use std::fmt; use alloy_rpc_types_engine::PayloadError; use jsonrpsee_core::RpcResult; +use reth_errors::ConsensusError; use reth_primitives::BlockId; /// Helper trait to easily convert various `Result` types into [`RpcResult`] @@ -102,6 +103,7 @@ macro_rules! impl_to_rpc_result { } impl_to_rpc_result!(PayloadError); +impl_to_rpc_result!(ConsensusError); impl_to_rpc_result!(reth_errors::RethError); impl_to_rpc_result!(reth_errors::ProviderError); impl_to_rpc_result!(reth_network_api::NetworkError); diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 00799d761d39..876467d1f4b6 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -18,6 +18,7 @@ reth-primitives = { workspace = true, features = ["secp256k1"] } reth-rpc-api.workspace = true reth-rpc-eth-api.workspace = true reth-errors.workspace = true +reth-ethereum-consensus.workspace = true reth-provider.workspace = true reth-transaction-pool.workspace = true reth-network-api.workspace = true @@ -33,12 +34,14 @@ reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true reth-network-types.workspace = true reth-trie.workspace = true +reth-consensus.workspace = true +reth-payload-validator.workspace = true # ethereum alloy-consensus.workspace = true alloy-signer.workspace = true alloy-signer-local.workspace = true -alloy-eips.workspace = true +alloy-eips = { workspace = true, features = ["kzg"] } alloy-dyn-abi.workspace = true alloy-genesis.workspace = true alloy-network.workspace = true diff --git a/crates/rpc/rpc/src/lib.rs b/crates/rpc/rpc/src/lib.rs index 027edea3cc1e..76fb96f91629 100644 --- a/crates/rpc/rpc/src/lib.rs +++ b/crates/rpc/rpc/src/lib.rs @@ -55,5 +55,5 @@ pub use reth::RethApi; pub use rpc::RPCApi; pub use trace::TraceApi; pub use txpool::TxPoolApi; -pub use validation::ValidationApi; +pub use validation::{ValidationApi, ValidationApiConfig}; pub use web3::Web3Api; diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index c6419dc12c01..fe9d0eb4475d 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -1,56 +1,376 @@ +use alloy_consensus::{BlobTransactionValidationError, EnvKzgSettings, Transaction}; +use alloy_eips::eip4844::kzg_to_versioned_hash; +use alloy_rpc_types::engine::{ + BlobsBundleV1, CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, +}; use alloy_rpc_types_beacon::relay::{ - BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, BuilderBlockValidationRequestV3, + BidTrace, BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, }; use async_trait::async_trait; use jsonrpsee::core::RpcResult; -use reth_chainspec::ChainSpecProvider; +use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; +use reth_consensus::{Consensus, PostExecutionInput}; +use reth_errors::{BlockExecutionError, ConsensusError, ProviderError, RethError}; +use reth_ethereum_consensus::GAS_LIMIT_BOUND_DIVISOR; +use reth_evm::execute::{BlockExecutorProvider, Executor}; +use reth_payload_validator::ExecutionPayloadValidator; +use reth_primitives::{Block, GotExpected, Receipt, SealedBlockWithSenders, SealedHeader}; use reth_provider::{ - AccountReader, BlockReaderIdExt, HeaderProvider, StateProviderFactory, WithdrawalsProvider, + AccountReader, BlockExecutionInput, BlockExecutionOutput, BlockReaderIdExt, HeaderProvider, + StateProviderFactory, WithdrawalsProvider, }; -use reth_rpc_api::BlockSubmissionValidationApiServer; -use reth_rpc_server_types::result::internal_rpc_err; -use std::sync::Arc; -use tracing::warn; +use reth_revm::database::StateProviderDatabase; +use reth_rpc_api::{BlockSubmissionValidationApiServer, BuilderBlockValidationRequestV3}; +use reth_rpc_eth_types::EthApiError; +use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; +use reth_trie::HashedPostState; +use revm_primitives::{Address, B256, U256}; +use serde::{Deserialize, Serialize}; +use std::{collections::HashSet, sync::Arc}; + +/// Configuration for validation API. +#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct ValidationApiConfig { + /// Disallowed addresses. + pub disallow: HashSet
, +} + +#[derive(Debug, thiserror::Error)] +pub enum ValidationApiError { + #[error("block gas limit mismatch: {_0}")] + GasLimitMismatch(GotExpected), + #[error("block gas used mismatch: {_0}")] + GasUsedMismatch(GotExpected), + #[error("block parent hash mismatch: {_0}")] + ParentHashMismatch(GotExpected), + #[error("block hash mismatch: {_0}")] + BlockHashMismatch(GotExpected), + #[error("missing latest block in database")] + MissingLatestBlock, + #[error("could not verify proposer payment")] + ProposerPayment, + #[error("invalid blobs bundle")] + InvalidBlobsBundle, + #[error("block accesses blacklisted address: {_0}")] + Blacklist(Address), + #[error(transparent)] + Blob(#[from] BlobTransactionValidationError), + #[error(transparent)] + Consensus(#[from] ConsensusError), + #[error(transparent)] + Provider(#[from] ProviderError), + #[error(transparent)] + Execution(#[from] BlockExecutionError), +} + +#[derive(Debug, Clone)] +pub struct ValidationApiInner { + /// The provider that can interact with the chain. + provider: Provider, + /// Consensus implementation. + consensus: Arc, + /// Execution payload validator. + payload_validator: ExecutionPayloadValidator, + /// Block executor factory. + executor_provider: E, + /// Set of disallowed addresses + disallow: HashSet
, +} /// The type that implements the `validation` rpc namespace trait -pub struct ValidationApi { - inner: Arc>, +#[derive(Clone, Debug, derive_more::Deref)] +pub struct ValidationApi { + #[deref] + inner: Arc>, } -impl ValidationApi +impl ValidationApi +where + Provider: ChainSpecProvider, +{ + /// Create a new instance of the [`ValidationApi`] + pub fn new( + provider: Provider, + consensus: Arc, + executor_provider: E, + config: ValidationApiConfig, + ) -> Self { + let ValidationApiConfig { disallow } = config; + + let payload_validator = ExecutionPayloadValidator::new(provider.chain_spec()); + let inner = Arc::new(ValidationApiInner { + provider, + consensus, + payload_validator, + executor_provider, + disallow, + }); + + Self { inner } + } +} + +impl ValidationApi where Provider: BlockReaderIdExt - + ChainSpecProvider + + ChainSpecProvider + StateProviderFactory + HeaderProvider + AccountReader + WithdrawalsProvider + Clone + 'static, + E: BlockExecutorProvider, { - /// The provider that can interact with the chain. - pub fn provider(&self) -> Provider { - self.inner.provider.clone() + /// Validates the given block and a [`BidTrace`] against it. + pub fn validate_message_against_block( + &self, + block: SealedBlockWithSenders, + message: BidTrace, + registered_gas_limit: u64, + ) -> Result<(), ValidationApiError> { + self.validate_message_against_header(&block.header, &message)?; + + self.consensus.validate_header_with_total_difficulty(&block.header, U256::MAX)?; + self.consensus.validate_header(&block.header)?; + self.consensus.validate_block_pre_execution(&block)?; + + if !self.disallow.is_empty() { + if self.disallow.contains(&block.beneficiary) { + return Err(ValidationApiError::Blacklist(block.beneficiary)) + } + if self.disallow.contains(&message.proposer_fee_recipient) { + return Err(ValidationApiError::Blacklist(message.proposer_fee_recipient)) + } + for (sender, tx) in block.senders.iter().zip(block.transactions()) { + if self.disallow.contains(sender) { + return Err(ValidationApiError::Blacklist(*sender)) + } + if let Some(to) = tx.to() { + if self.disallow.contains(&to) { + return Err(ValidationApiError::Blacklist(to)) + } + } + } + } + + let latest_header = + self.provider.latest_header()?.ok_or_else(|| ValidationApiError::MissingLatestBlock)?; + + if latest_header.hash() != block.header.parent_hash { + return Err(ConsensusError::ParentHashMismatch( + GotExpected { got: block.header.parent_hash, expected: latest_header.hash() } + .into(), + ) + .into()) + } + self.consensus.validate_header_against_parent(&block.header, &latest_header)?; + self.validate_gas_limit(registered_gas_limit, &latest_header, &block.header)?; + + let state_provider = self.provider.state_by_block_hash(latest_header.hash())?; + let executor = self.executor_provider.executor(StateProviderDatabase::new(&state_provider)); + + let block = block.unseal(); + let mut accessed_blacklisted = None; + let output = executor.execute_with_state_closure( + BlockExecutionInput::new(&block, U256::MAX), + |state| { + if !self.disallow.is_empty() { + for account in state.cache.accounts.keys() { + if self.disallow.contains(account) { + accessed_blacklisted = Some(*account); + } + } + } + }, + )?; + + if let Some(account) = accessed_blacklisted { + return Err(ValidationApiError::Blacklist(account)) + } + + self.consensus.validate_block_post_execution( + &block, + PostExecutionInput::new(&output.receipts, &output.requests), + )?; + + self.ensure_payment(&block, &output, &message)?; + + let state_root = + state_provider.state_root(HashedPostState::from_bundle_state(&output.state.state))?; + + if state_root != block.state_root { + return Err(ConsensusError::BodyStateRootDiff( + GotExpected { got: state_root, expected: block.state_root }.into(), + ) + .into()) + } + + Ok(()) } - /// Create a new instance of the [`ValidationApi`] - pub fn new(provider: Provider) -> Self { - let inner = Arc::new(ValidationApiInner { provider }); - Self { inner } + /// Ensures that fields of [`BidTrace`] match the fields of the [`SealedHeader`]. + fn validate_message_against_header( + &self, + header: &SealedHeader, + message: &BidTrace, + ) -> Result<(), ValidationApiError> { + if header.hash() != message.block_hash { + Err(ValidationApiError::BlockHashMismatch(GotExpected { + got: message.block_hash, + expected: header.hash(), + })) + } else if header.parent_hash != message.parent_hash { + Err(ValidationApiError::ParentHashMismatch(GotExpected { + got: message.parent_hash, + expected: header.parent_hash, + })) + } else if header.gas_limit != message.gas_limit { + Err(ValidationApiError::GasLimitMismatch(GotExpected { + got: message.gas_limit, + expected: header.gas_limit, + })) + } else if header.gas_used != message.gas_used { + return Err(ValidationApiError::GasUsedMismatch(GotExpected { + got: message.gas_used, + expected: header.gas_used, + })) + } else { + Ok(()) + } + } + + /// Ensures that the chosen gas limit is the closest possible value for the validator's + /// registered gas limit. + /// + /// Ref: + fn validate_gas_limit( + &self, + registered_gas_limit: u64, + parent_header: &SealedHeader, + header: &SealedHeader, + ) -> Result<(), ValidationApiError> { + let max_gas_limit = + parent_header.gas_limit + parent_header.gas_limit / GAS_LIMIT_BOUND_DIVISOR - 1; + let min_gas_limit = + parent_header.gas_limit - parent_header.gas_limit / GAS_LIMIT_BOUND_DIVISOR + 1; + + let best_gas_limit = + std::cmp::max(min_gas_limit, std::cmp::min(max_gas_limit, registered_gas_limit)); + + if best_gas_limit != header.gas_limit { + return Err(ValidationApiError::GasLimitMismatch(GotExpected { + got: header.gas_limit, + expected: best_gas_limit, + })) + } + + Ok(()) + } + + /// Ensures that the proposer has received [`BidTrace::value`] for this block. + /// + /// Firstly attempts to verify the payment by checking the state changes, otherwise falls back + /// to checking the latest block transaction. + fn ensure_payment( + &self, + block: &Block, + output: &BlockExecutionOutput, + message: &BidTrace, + ) -> Result<(), ValidationApiError> { + let (mut balance_before, balance_after) = if let Some(acc) = + output.state.state.get(&message.proposer_fee_recipient) + { + let balance_before = acc.original_info.as_ref().map(|i| i.balance).unwrap_or_default(); + let balance_after = acc.info.as_ref().map(|i| i.balance).unwrap_or_default(); + + (balance_before, balance_after) + } else { + // account might have balance but considering it zero is fine as long as we know + // that balance have not changed + (U256::ZERO, U256::ZERO) + }; + + if let Some(withdrawals) = &block.body.withdrawals { + for withdrawal in withdrawals { + if withdrawal.address == message.proposer_fee_recipient { + balance_before += withdrawal.amount_wei(); + } + } + } + + if balance_after >= balance_before + message.value { + return Ok(()) + } + + let (receipt, tx) = output + .receipts + .last() + .zip(block.body.transactions.last()) + .ok_or(ValidationApiError::ProposerPayment)?; + + if !receipt.success { + return Err(ValidationApiError::ProposerPayment) + } + + if tx.to() != Some(message.proposer_fee_recipient) { + return Err(ValidationApiError::ProposerPayment) + } + + if tx.value() != message.value { + return Err(ValidationApiError::ProposerPayment) + } + + if !tx.input().is_empty() { + return Err(ValidationApiError::ProposerPayment) + } + + if let Some(block_base_fee) = block.base_fee_per_gas { + if tx.effective_tip_per_gas(block_base_fee).unwrap_or_default() != 0 { + return Err(ValidationApiError::ProposerPayment) + } + } + + Ok(()) + } + + /// Validates the given [`BlobsBundleV1`] and returns versioned hashes for blobs. + pub fn validate_blobs_bundle( + &self, + mut blobs_bundle: BlobsBundleV1, + ) -> Result, ValidationApiError> { + if blobs_bundle.commitments.len() != blobs_bundle.proofs.len() || + blobs_bundle.commitments.len() != blobs_bundle.blobs.len() + { + return Err(ValidationApiError::InvalidBlobsBundle) + } + + let versioned_hashes = blobs_bundle + .commitments + .iter() + .map(|c| kzg_to_versioned_hash(c.as_slice())) + .collect::>(); + + let sidecar = blobs_bundle.pop_sidecar(blobs_bundle.blobs.len()); + + sidecar.validate(&versioned_hashes, EnvKzgSettings::default().get())?; + + Ok(versioned_hashes) } } #[async_trait] -impl BlockSubmissionValidationApiServer for ValidationApi +impl BlockSubmissionValidationApiServer for ValidationApi where Provider: BlockReaderIdExt - + ChainSpecProvider + + ChainSpecProvider + StateProviderFactory + HeaderProvider + AccountReader + WithdrawalsProvider + Clone + 'static, + E: BlockExecutorProvider, { async fn validate_builder_submission_v1( &self, @@ -71,24 +391,28 @@ where &self, request: BuilderBlockValidationRequestV3, ) -> RpcResult<()> { - warn!("flashbots_validateBuilderSubmissionV3: blindly accepting request without validation {:?}", request); - Ok(()) - } -} - -impl std::fmt::Debug for ValidationApi { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ValidationApi").finish_non_exhaustive() - } -} + let block = self + .payload_validator + .ensure_well_formed_payload( + ExecutionPayload::V3(request.request.execution_payload), + ExecutionPayloadSidecar::v3(CancunPayloadFields { + parent_beacon_block_root: request.parent_beacon_block_root, + versioned_hashes: self + .validate_blobs_bundle(request.request.blobs_bundle) + .map_err(|e| RethError::Other(e.into())) + .to_rpc_result()?, + }), + ) + .to_rpc_result()? + .try_seal_with_senders() + .map_err(|_| EthApiError::InvalidTransactionSignature)?; -impl Clone for ValidationApi { - fn clone(&self) -> Self { - Self { inner: Arc::clone(&self.inner) } + self.validate_message_against_block( + block, + request.request.message, + request.registered_gas_limit, + ) + .map_err(|e| RethError::Other(e.into())) + .to_rpc_result() } } - -struct ValidationApiInner { - /// The provider that can interact with the chain. - provider: Provider, -} diff --git a/examples/rpc-db/src/main.rs b/examples/rpc-db/src/main.rs index 1b2899a6485b..92ae86f00bb9 100644 --- a/examples/rpc-db/src/main.rs +++ b/examples/rpc-db/src/main.rs @@ -16,6 +16,7 @@ use std::{path::Path, sync::Arc}; use reth::{ api::NodeTypesWithDBAdapter, + beacon_consensus::EthBeaconConsensus, providers::{ providers::{BlockchainProvider, StaticFileProvider}, ProviderFactory, @@ -66,9 +67,10 @@ async fn main() -> eyre::Result<()> { .with_noop_pool() .with_noop_network() .with_executor(TokioTaskExecutor::default()) - .with_evm_config(EthEvmConfig::new(spec)) + .with_evm_config(EthEvmConfig::new(spec.clone())) .with_events(TestCanonStateSubscriptions::default()) - .with_block_executor(EthExecutorProvider::ethereum(provider.chain_spec())); + .with_block_executor(EthExecutorProvider::ethereum(provider.chain_spec())) + .with_consensus(EthBeaconConsensus::new(spec)); // Pick which namespaces to expose. let config = TransportRpcModuleConfig::default().with_http([RethRpcModule::Eth]); From bd8c4eceb20c39c6e501d06cf906469329340bb9 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:20:41 -0600 Subject: [PATCH 201/239] replace DisplayHardforks with Box (#12219) Co-authored-by: Matthias Seitz --- crates/chainspec/src/api.rs | 11 +++++------ crates/optimism/chainspec/src/lib.rs | 10 +++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index 36640f34b709..ee25f72bae8d 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -1,11 +1,10 @@ use crate::{ChainSpec, DepositContract}; -use alloc::vec::Vec; +use alloc::{boxed::Box, vec::Vec}; use alloy_chains::Chain; use alloy_eips::eip1559::BaseFeeParams; use alloy_genesis::Genesis; use alloy_primitives::B256; -use core::fmt::Debug; -use reth_ethereum_forks::DisplayHardforks; +use core::fmt::{Debug, Display}; use reth_network_peers::NodeRecord; use reth_primitives_traits::Header; @@ -39,7 +38,7 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { fn prune_delete_limit(&self) -> usize; /// Returns a string representation of the hardforks. - fn display_hardforks(&self) -> DisplayHardforks; + fn display_hardforks(&self) -> Box; /// The genesis header. fn genesis_header(&self) -> &Header; @@ -89,8 +88,8 @@ impl EthChainSpec for ChainSpec { self.prune_delete_limit } - fn display_hardforks(&self) -> DisplayHardforks { - self.display_hardforks() + fn display_hardforks(&self) -> Box { + Box::new(Self::display_hardforks(self)) } fn genesis_header(&self) -> &Header { diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 70adf2272cfb..8ad36c66afb0 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -17,7 +17,7 @@ mod dev; mod op; mod op_sepolia; -use alloc::{vec, vec::Vec}; +use alloc::{boxed::Box, vec, vec::Vec}; use alloy_chains::Chain; use alloy_genesis::Genesis; use alloy_primitives::{Bytes, Parity, Signature, B256, U256}; @@ -30,8 +30,8 @@ pub(crate) use once_cell::sync::Lazy as LazyLock; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; use reth_chainspec::{ - BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, DepositContract, - DisplayHardforks, EthChainSpec, EthereumHardforks, ForkFilter, ForkId, Hardforks, Head, + BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, DepositContract, EthChainSpec, + EthereumHardforks, ForkFilter, ForkId, Hardforks, Head, }; use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition, Hardfork}; use reth_network_peers::NodeRecord; @@ -287,8 +287,8 @@ impl EthChainSpec for OpChainSpec { self.inner.prune_delete_limit() } - fn display_hardforks(&self) -> DisplayHardforks { - self.inner.display_hardforks() + fn display_hardforks(&self) -> Box { + Box::new(ChainSpec::display_hardforks(self)) } fn genesis_header(&self) -> &Header { From 921d1cc4b5d37195188f2afcede39fe789cf9582 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Fri, 1 Nov 2024 03:44:16 -0600 Subject: [PATCH 202/239] renamed OptimismPoolBuilder to OpPoolBuilder (#12246) --- crates/optimism/node/src/node.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 8e06dbcc5000..925e7204c9e2 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -66,7 +66,7 @@ impl OptimismNode { args: RollupArgs, ) -> ComponentsBuilder< Node, - OptimismPoolBuilder, + OpPoolBuilder, OpPayloadBuilder, OptimismNetworkBuilder, OptimismExecutorBuilder, @@ -80,7 +80,7 @@ impl OptimismNode { let RollupArgs { disable_txpool_gossip, compute_pending_block, discovery_v4, .. } = args; ComponentsBuilder::default() .node_types::() - .pool(OptimismPoolBuilder::default()) + .pool(OpPoolBuilder::default()) .payload(OpPayloadBuilder::new(compute_pending_block)) .network(OptimismNetworkBuilder { disable_txpool_gossip, @@ -99,7 +99,7 @@ where { type ComponentsBuilder = ComponentsBuilder< N, - OptimismPoolBuilder, + OpPoolBuilder, OpPayloadBuilder, OptimismNetworkBuilder, OptimismExecutorBuilder, @@ -206,12 +206,12 @@ where /// This contains various settings that can be configured and take precedence over the node's /// config. #[derive(Debug, Default, Clone)] -pub struct OptimismPoolBuilder { +pub struct OpPoolBuilder { /// Enforced overrides that are applied to the pool config. pub pool_config_overrides: PoolBuilderConfigOverrides, } -impl PoolBuilder for OptimismPoolBuilder +impl PoolBuilder for OpPoolBuilder where Node: FullNodeTypes>, { From 8d31b652425d0e67be1577d41be77331028b93d6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 1 Nov 2024 11:17:43 +0100 Subject: [PATCH 203/239] chore: clippy happy (#12248) --- crates/rpc/rpc-testing-util/src/debug.rs | 4 ++-- crates/rpc/rpc-testing-util/src/trace.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-testing-util/src/debug.rs b/crates/rpc/rpc-testing-util/src/debug.rs index f50064e80ce9..97fe008fa97c 100644 --- a/crates/rpc/rpc-testing-util/src/debug.rs +++ b/crates/rpc/rpc-testing-util/src/debug.rs @@ -292,7 +292,7 @@ pub struct DebugTraceTransactionsStream<'a> { stream: Pin + 'a>>, } -impl<'a> DebugTraceTransactionsStream<'a> { +impl DebugTraceTransactionsStream<'_> { /// Returns the next error result of the stream. pub async fn next_err(&mut self) -> Option<(RpcError, TxHash)> { loop { @@ -324,7 +324,7 @@ pub struct DebugTraceBlockStream<'a> { stream: Pin + 'a>>, } -impl<'a> DebugTraceBlockStream<'a> { +impl DebugTraceBlockStream<'_> { /// Returns the next error result of the stream. pub async fn next_err(&mut self) -> Option<(RpcError, BlockId)> { loop { diff --git a/crates/rpc/rpc-testing-util/src/trace.rs b/crates/rpc/rpc-testing-util/src/trace.rs index c6dc16cf1063..0fefef7c997f 100644 --- a/crates/rpc/rpc-testing-util/src/trace.rs +++ b/crates/rpc/rpc-testing-util/src/trace.rs @@ -381,7 +381,7 @@ pub struct TraceBlockStream<'a> { stream: Pin + 'a>>, } -impl<'a> TraceBlockStream<'a> { +impl TraceBlockStream<'_> { /// Returns the next error result of the stream. pub async fn next_err(&mut self) -> Option<(RpcError, BlockId)> { loop { From 39bc8ce81a237fd5452ae31d0362a1777a7d3c5f Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:35:06 +0100 Subject: [PATCH 204/239] refactor(revm): simplify `Database` impl for `StateProviderDatabase` (#12241) --- crates/revm/src/database.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/revm/src/database.rs b/crates/revm/src/database.rs index 8f40d2be8d91..5f662fea7cfe 100644 --- a/crates/revm/src/database.rs +++ b/crates/revm/src/database.rs @@ -101,21 +101,21 @@ impl Database for StateProviderDatabase { /// Returns `Ok` with `Some(AccountInfo)` if the account exists, /// `None` if it doesn't, or an error if encountered. fn basic(&mut self, address: Address) -> Result, Self::Error> { - DatabaseRef::basic_ref(self, address) + self.basic_ref(address) } /// Retrieves the bytecode associated with a given code hash. /// /// Returns `Ok` with the bytecode if found, or the default bytecode otherwise. fn code_by_hash(&mut self, code_hash: B256) -> Result { - DatabaseRef::code_by_hash_ref(self, code_hash) + self.code_by_hash_ref(code_hash) } /// Retrieves the storage value at a specific index for a given address. /// /// Returns `Ok` with the storage value, or the default value if not found. fn storage(&mut self, address: Address, index: U256) -> Result { - DatabaseRef::storage_ref(self, address, index) + self.storage_ref(address, index) } /// Retrieves the block hash for a given block number. @@ -123,7 +123,7 @@ impl Database for StateProviderDatabase { /// Returns `Ok` with the block hash if found, or the default hash otherwise. /// Note: It safely casts the `number` to `u64`. fn block_hash(&mut self, number: u64) -> Result { - DatabaseRef::block_hash_ref(self, number) + self.block_hash_ref(number) } } From f0cef9dc51fa738490d09f9d64bf03ae27a01fd6 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:33:58 +0100 Subject: [PATCH 205/239] revm: add `Database` `Either` helper type (#12240) --- crates/revm/src/either.rs | 52 +++++++++++++++++++++++++++++++++++++++ crates/revm/src/lib.rs | 3 +++ 2 files changed, 55 insertions(+) create mode 100644 crates/revm/src/either.rs diff --git a/crates/revm/src/either.rs b/crates/revm/src/either.rs new file mode 100644 index 000000000000..e93ba3a8d010 --- /dev/null +++ b/crates/revm/src/either.rs @@ -0,0 +1,52 @@ +use alloy_primitives::{Address, B256, U256}; +use revm::{ + primitives::{AccountInfo, Bytecode}, + Database, +}; + +/// An enum type that can hold either of two different [`Database`] implementations. +/// +/// This allows flexible usage of different [`Database`] types in the same context. +#[derive(Debug, Clone)] +pub enum Either { + /// A value of type `L`. + Left(L), + /// A value of type `R`. + Right(R), +} + +impl Database for Either +where + L: Database, + R: Database, +{ + type Error = L::Error; + + fn basic(&mut self, address: Address) -> Result, Self::Error> { + match self { + Self::Left(db) => db.basic(address), + Self::Right(db) => db.basic(address), + } + } + + fn code_by_hash(&mut self, code_hash: B256) -> Result { + match self { + Self::Left(db) => db.code_by_hash(code_hash), + Self::Right(db) => db.code_by_hash(code_hash), + } + } + + fn storage(&mut self, address: Address, index: U256) -> Result { + match self { + Self::Left(db) => db.storage(address, index), + Self::Right(db) => db.storage(address, index), + } + } + + fn block_hash(&mut self, number: u64) -> Result { + match self { + Self::Left(db) => db.block_hash(number), + Self::Right(db) => db.block_hash(number), + } + } +} diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index 02eb182ee119..8b544a537289 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -22,3 +22,6 @@ pub mod test_utils; // Convenience re-exports. pub use revm::{self, *}; + +/// Either type for flexible usage of different database types in the same context. +pub mod either; From 249c600dd9c8a7ae8556ae004f366b66644ac569 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 1 Nov 2024 11:34:48 +0100 Subject: [PATCH 206/239] rpc: add `rename` method in `TransportRpcModules` (#12239) --- crates/rpc/rpc-builder/src/lib.rs | 56 +++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 5d5fd31aab8c..696d35014306 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -2015,7 +2015,7 @@ impl TransportRpcModules { Ok(false) } - /// Merge the given [Methods] in all configured methods. + /// Merge the given [`Methods`] in all configured methods. /// /// Fails if any of the methods in other is present already. pub fn merge_configured( @@ -2106,7 +2106,22 @@ impl TransportRpcModules { http_removed || ws_removed || ipc_removed } - /// Replace the given [Methods] in the configured http methods. + /// Renames a method in all configured transports by: + /// 1. Removing the old method name. + /// 2. Adding the new method. + pub fn rename( + &mut self, + old_name: &'static str, + new_method: impl Into, + ) -> Result<(), RegisterMethodError> { + // Remove the old method from all configured transports + self.remove_method_from_configured(old_name); + + // Merge the new method into the configured transports + self.merge_configured(new_method) + } + + /// Replace the given [`Methods`] in the configured http methods. /// /// Fails if any of the methods in other is present already or if the method being removed is /// not present @@ -2478,6 +2493,43 @@ mod tests { assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); } + #[test] + fn test_transport_rpc_module_rename() { + let mut modules = TransportRpcModules { + http: Some(create_test_module()), + ws: Some(create_test_module()), + ipc: Some(create_test_module()), + ..Default::default() + }; + + // Verify that the old we want to rename exists at the start + assert!(modules.http.as_ref().unwrap().method("anything").is_some()); + assert!(modules.ws.as_ref().unwrap().method("anything").is_some()); + assert!(modules.ipc.as_ref().unwrap().method("anything").is_some()); + + // Verify that the new method does not exist at the start + assert!(modules.http.as_ref().unwrap().method("something").is_none()); + assert!(modules.ws.as_ref().unwrap().method("something").is_none()); + assert!(modules.ipc.as_ref().unwrap().method("something").is_none()); + + // Create another module + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + // Rename the method + modules.rename("anything", other_module).expect("rename failed"); + + // Verify that the old method was removed from all transports + assert!(modules.http.as_ref().unwrap().method("anything").is_none()); + assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); + assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); + + // Verify that the new method was added to all transports + assert!(modules.http.as_ref().unwrap().method("something").is_some()); + assert!(modules.ws.as_ref().unwrap().method("something").is_some()); + assert!(modules.ipc.as_ref().unwrap().method("something").is_some()); + } + #[test] fn test_replace_http_method() { let mut modules = From 2758a560c0ed1afb9b395123eef788e2426b951b Mon Sep 17 00:00:00 2001 From: Darshan Kathiriya <8559992+lakshya-sky@users.noreply.github.com> Date: Fri, 1 Nov 2024 07:10:55 -0400 Subject: [PATCH 207/239] txpool: added a helper to filter pending txns by predicate (#12204) --- crates/transaction-pool/src/lib.rs | 7 +++++++ crates/transaction-pool/src/noop.rs | 7 +++++++ crates/transaction-pool/src/pool/mod.rs | 8 ++++++++ crates/transaction-pool/src/pool/txpool.rs | 8 ++++++++ crates/transaction-pool/src/traits.rs | 6 ++++++ 5 files changed, 36 insertions(+) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index f8f06b805f93..020375994320 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -496,6 +496,13 @@ where self.pool.get_transactions_by_sender(sender) } + fn get_pending_transactions_with_predicate( + &self, + predicate: impl FnMut(&ValidPoolTransaction) -> bool, + ) -> Vec>> { + self.pool.pending_transactions_with_predicate(predicate) + } + fn get_pending_transactions_by_sender( &self, sender: Address, diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 0f87f06f7dd3..47a26ee29a3b 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -213,6 +213,13 @@ impl TransactionPool for NoopTransactionPool { vec![] } + fn get_pending_transactions_with_predicate( + &self, + _predicate: impl FnMut(&ValidPoolTransaction) -> bool, + ) -> Vec>> { + vec![] + } + fn get_pending_transactions_by_sender( &self, _sender: Address, diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 2e73409546d1..77446a523757 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -787,6 +787,14 @@ where self.get_pool_data().pending_txs_by_sender(sender_id) } + /// Returns all pending transactions filtered by predicate + pub(crate) fn pending_transactions_with_predicate( + &self, + predicate: impl FnMut(&ValidPoolTransaction) -> bool, + ) -> Vec>> { + self.get_pool_data().pending_transactions_with_predicate(predicate) + } + /// Returns all pending transactions of the address by sender pub(crate) fn get_pending_transactions_by_sender( &self, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index bd0aacccfee9..8679a4318bed 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -364,6 +364,14 @@ impl TxPool { self.pending_pool.all() } + /// Returns all pending transactions filtered by predicate + pub(crate) fn pending_transactions_with_predicate( + &self, + mut predicate: impl FnMut(&ValidPoolTransaction) -> bool, + ) -> Vec>> { + self.pending_transactions_iter().filter(|tx| predicate(tx)).collect() + } + /// Returns all pending transactions for the specified sender pub(crate) fn pending_txs_by_sender( &self, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 0d8f6dbb54ad..2667143b7c8c 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -342,6 +342,12 @@ pub trait TransactionPool: Send + Sync + Clone { sender: Address, ) -> Vec>>; + /// Returns all pending transactions filtered by predicate + fn get_pending_transactions_with_predicate( + &self, + predicate: impl FnMut(&ValidPoolTransaction) -> bool, + ) -> Vec>>; + /// Returns all pending transactions sent by a given user fn get_pending_transactions_by_sender( &self, From c6b740801fda7963baeb33ac395227032825bb7e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 1 Nov 2024 12:11:54 +0100 Subject: [PATCH 208/239] chore: apply same member order (#12253) --- crates/blockchain-tree/src/noop.rs | 12 ++-- crates/exex/exex/src/notifications.rs | 10 ++-- crates/node/builder/src/builder/states.rs | 14 ++--- crates/optimism/rpc/src/eth/mod.rs | 2 +- crates/primitives/src/block.rs | 14 ++--- crates/rpc/rpc-eth-api/src/node.rs | 2 +- crates/rpc/rpc/src/eth/core.rs | 2 +- .../src/providers/database/provider.rs | 10 ++-- crates/storage/provider/src/providers/mod.rs | 58 +++++++++---------- .../storage/provider/src/test_utils/mock.rs | 24 ++++---- .../storage/provider/src/test_utils/noop.rs | 32 +++++----- crates/trie/trie/src/hashed_cursor/noop.rs | 8 +-- crates/trie/trie/src/prefix_set.rs | 2 +- 13 files changed, 95 insertions(+), 95 deletions(-) diff --git a/crates/blockchain-tree/src/noop.rs b/crates/blockchain-tree/src/noop.rs index 925b8f03add7..862b02e76070 100644 --- a/crates/blockchain-tree/src/noop.rs +++ b/crates/blockchain-tree/src/noop.rs @@ -60,6 +60,12 @@ impl BlockchainTreeEngine for NoopBlockchainTree { Ok(()) } + fn update_block_hashes_and_clear_buffered( + &self, + ) -> Result, CanonicalError> { + Ok(BTreeMap::new()) + } + fn connect_buffered_blocks_to_canonical_hashes(&self) -> Result<(), CanonicalError> { Ok(()) } @@ -67,12 +73,6 @@ impl BlockchainTreeEngine for NoopBlockchainTree { fn make_canonical(&self, block_hash: BlockHash) -> Result { Err(BlockchainTreeError::BlockHashNotFoundInChain { block_hash }.into()) } - - fn update_block_hashes_and_clear_buffered( - &self, - ) -> Result, CanonicalError> { - Ok(BTreeMap::new()) - } } impl BlockchainTreeViewer for NoopBlockchainTree { diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index 90a0ee230a4d..14cfe9be4d92 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -108,11 +108,6 @@ where }); } - fn without_head(mut self) -> Self { - self.set_without_head(); - self - } - fn set_with_head(&mut self, exex_head: ExExHead) { let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); self.inner = ExExNotificationsInner::WithHead(match current { @@ -131,6 +126,11 @@ where }); } + fn without_head(mut self) -> Self { + self.set_without_head(); + self + } + fn with_head(mut self, exex_head: ExExHead) -> Self { self.set_with_head(exex_head); self diff --git a/crates/node/builder/src/builder/states.rs b/crates/node/builder/src/builder/states.rs index e75a07802a6e..ca5a57d0db60 100644 --- a/crates/node/builder/src/builder/states.rs +++ b/crates/node/builder/src/builder/states.rs @@ -95,8 +95,8 @@ impl> FullNodeComponents for NodeAdapter< type Pool = C::Pool; type Evm = C::Evm; type Executor = C::Executor; - type Network = C::Network; type Consensus = C::Consensus; + type Network = C::Network; fn pool(&self) -> &Self::Pool { self.components.pool() @@ -110,8 +110,8 @@ impl> FullNodeComponents for NodeAdapter< self.components.block_executor() } - fn provider(&self) -> &Self::Provider { - &self.provider + fn consensus(&self) -> &Self::Consensus { + self.components.consensus() } fn network(&self) -> &Self::Network { @@ -122,12 +122,12 @@ impl> FullNodeComponents for NodeAdapter< self.components.payload_builder() } - fn task_executor(&self) -> &TaskExecutor { - &self.task_executor + fn provider(&self) -> &Self::Provider { + &self.provider } - fn consensus(&self) -> &Self::Consensus { - self.components.consensus() + fn task_executor(&self) -> &TaskExecutor { + &self.task_executor } } diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index 7b427467b2c5..dc6e8e59fa6b 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -119,8 +119,8 @@ where { type Provider = N::Provider; type Pool = N::Pool; - type Network = ::Network; type Evm = ::Evm; + type Network = ::Network; #[inline] fn pool(&self) -> &Self::Pool { diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index a06979300acd..7e5e76f1b06c 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -150,27 +150,27 @@ mod block_rlp { } impl Encodable for Block { - fn length(&self) -> usize { - let helper: HelperRef<'_, _> = self.into(); - helper.length() - } - fn encode(&self, out: &mut dyn bytes::BufMut) { let helper: HelperRef<'_, _> = self.into(); helper.encode(out) } - } - impl Encodable for SealedBlock { fn length(&self) -> usize { let helper: HelperRef<'_, _> = self.into(); helper.length() } + } + impl Encodable for SealedBlock { fn encode(&self, out: &mut dyn bytes::BufMut) { let helper: HelperRef<'_, _> = self.into(); helper.encode(out) } + + fn length(&self) -> usize { + let helper: HelperRef<'_, _> = self.into(); + helper.length() + } } } diff --git a/crates/rpc/rpc-eth-api/src/node.rs b/crates/rpc/rpc-eth-api/src/node.rs index 851b26b72b99..4ae79c083411 100644 --- a/crates/rpc/rpc-eth-api/src/node.rs +++ b/crates/rpc/rpc-eth-api/src/node.rs @@ -38,8 +38,8 @@ where { type Provider = T::Provider; type Pool = T::Pool; - type Network = ::Network; type Evm = ::Evm; + type Network = ::Network; #[inline] fn pool(&self) -> &Self::Pool { diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index 339f2200c67a..026c87153f2f 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -150,8 +150,8 @@ where { type Provider = Provider; type Pool = Pool; - type Network = Network; type Evm = EvmConfig; + type Network = Network; fn pool(&self) -> &Self::Pool { self.inner.pool() diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 81affa0d804f..2af22cec0b59 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -2055,6 +2055,11 @@ impl StageCheckpointReader for DatabaseProvider { Ok(self.tx.get::(id.to_string())?) } + /// Get stage checkpoint progress. + fn get_stage_checkpoint_progress(&self, id: StageId) -> ProviderResult>> { + Ok(self.tx.get::(id.to_string())?) + } + fn get_all_checkpoints(&self) -> ProviderResult> { self.tx .cursor_read::()? @@ -2062,11 +2067,6 @@ impl StageCheckpointReader for DatabaseProvider { .collect::, _>>() .map_err(ProviderError::Database) } - - /// Get stage checkpoint progress. - fn get_stage_checkpoint_progress(&self, id: StageId) -> ProviderResult>> { - Ok(self.tx.get::(id.to_string())?) - } } impl StageCheckpointWriter for DatabaseProvider { diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index c81ef05d2eaa..3b24617fd95b 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -601,35 +601,6 @@ impl StateProviderFactory for BlockchainProvider { self.database.latest() } - fn history_by_block_number( - &self, - block_number: BlockNumber, - ) -> ProviderResult { - trace!(target: "providers::blockchain", ?block_number, "Getting history by block number"); - self.ensure_canonical_block(block_number)?; - self.database.history_by_block_number(block_number) - } - - fn history_by_block_hash(&self, block_hash: BlockHash) -> ProviderResult { - trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); - self.database.history_by_block_hash(block_hash) - } - - fn state_by_block_hash(&self, block: BlockHash) -> ProviderResult { - trace!(target: "providers::blockchain", ?block, "Getting state by block hash"); - let mut state = self.history_by_block_hash(block); - - // we failed to get the state by hash, from disk, hash block be the pending block - if state.is_err() { - if let Ok(Some(pending)) = self.pending_state_by_hash(block) { - // we found pending block by hash - state = Ok(pending) - } - } - - state - } - /// Returns a [`StateProviderBox`] indexed by the given block number or tag. /// /// Note: if a number is provided this will only look at historical(canonical) state. @@ -662,6 +633,35 @@ impl StateProviderFactory for BlockchainProvider { } } + fn history_by_block_number( + &self, + block_number: BlockNumber, + ) -> ProviderResult { + trace!(target: "providers::blockchain", ?block_number, "Getting history by block number"); + self.ensure_canonical_block(block_number)?; + self.database.history_by_block_number(block_number) + } + + fn history_by_block_hash(&self, block_hash: BlockHash) -> ProviderResult { + trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); + self.database.history_by_block_hash(block_hash) + } + + fn state_by_block_hash(&self, block: BlockHash) -> ProviderResult { + trace!(target: "providers::blockchain", ?block, "Getting state by block hash"); + let mut state = self.history_by_block_hash(block); + + // we failed to get the state by hash, from disk, hash block be the pending block + if state.is_err() { + if let Ok(Some(pending)) = self.pending_state_by_hash(block) { + // we found pending block by hash + state = Ok(pending) + } + } + + state + } + /// Returns the state provider for pending state. /// /// If there's no pending block available then the latest state provider is returned: diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index e2e08e61a86f..07fa505b2b49 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -760,18 +760,6 @@ impl StateProviderFactory for MockEthProvider { Ok(Box::new(self.clone())) } - fn history_by_block_number(&self, _block: BlockNumber) -> ProviderResult { - Ok(Box::new(self.clone())) - } - - fn history_by_block_hash(&self, _block: BlockHash) -> ProviderResult { - Ok(Box::new(self.clone())) - } - - fn state_by_block_hash(&self, _block: BlockHash) -> ProviderResult { - Ok(Box::new(self.clone())) - } - fn state_by_block_number_or_tag( &self, number_or_tag: BlockNumberOrTag, @@ -798,6 +786,18 @@ impl StateProviderFactory for MockEthProvider { } } + fn history_by_block_number(&self, _block: BlockNumber) -> ProviderResult { + Ok(Box::new(self.clone())) + } + + fn history_by_block_hash(&self, _block: BlockHash) -> ProviderResult { + Ok(Box::new(self.clone())) + } + + fn state_by_block_hash(&self, _block: BlockHash) -> ProviderResult { + Ok(Box::new(self.clone())) + } + fn pending(&self) -> ProviderResult { Ok(Box::new(self.clone())) } diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index f6f7e185de63..e09437647722 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -465,22 +465,6 @@ impl StateProviderFactory for NoopProvider { Ok(Box::new(*self)) } - fn history_by_block_number(&self, _block: BlockNumber) -> ProviderResult { - Ok(Box::new(*self)) - } - - fn history_by_block_hash(&self, _block: BlockHash) -> ProviderResult { - Ok(Box::new(*self)) - } - - fn state_by_block_hash(&self, _block: BlockHash) -> ProviderResult { - Ok(Box::new(*self)) - } - - fn pending(&self) -> ProviderResult { - Ok(Box::new(*self)) - } - fn state_by_block_number_or_tag( &self, number_or_tag: BlockNumberOrTag, @@ -507,6 +491,22 @@ impl StateProviderFactory for NoopProvider { } } + fn history_by_block_number(&self, _block: BlockNumber) -> ProviderResult { + Ok(Box::new(*self)) + } + + fn history_by_block_hash(&self, _block: BlockHash) -> ProviderResult { + Ok(Box::new(*self)) + } + + fn state_by_block_hash(&self, _block: BlockHash) -> ProviderResult { + Ok(Box::new(*self)) + } + + fn pending(&self) -> ProviderResult { + Ok(Box::new(*self)) + } + fn pending_state_by_hash(&self, _block_hash: B256) -> ProviderResult> { Ok(Some(Box::new(*self))) } diff --git a/crates/trie/trie/src/hashed_cursor/noop.rs b/crates/trie/trie/src/hashed_cursor/noop.rs index 4783d5afd9d4..a21e1026b380 100644 --- a/crates/trie/trie/src/hashed_cursor/noop.rs +++ b/crates/trie/trie/src/hashed_cursor/noop.rs @@ -32,11 +32,11 @@ pub struct NoopHashedAccountCursor; impl HashedCursor for NoopHashedAccountCursor { type Value = Account; - fn next(&mut self) -> Result, DatabaseError> { + fn seek(&mut self, _key: B256) -> Result, DatabaseError> { Ok(None) } - fn seek(&mut self, _key: B256) -> Result, DatabaseError> { + fn next(&mut self) -> Result, DatabaseError> { Ok(None) } } @@ -49,11 +49,11 @@ pub struct NoopHashedStorageCursor; impl HashedCursor for NoopHashedStorageCursor { type Value = U256; - fn next(&mut self) -> Result, DatabaseError> { + fn seek(&mut self, _key: B256) -> Result, DatabaseError> { Ok(None) } - fn seek(&mut self, _key: B256) -> Result, DatabaseError> { + fn next(&mut self) -> Result, DatabaseError> { Ok(None) } } diff --git a/crates/trie/trie/src/prefix_set.rs b/crates/trie/trie/src/prefix_set.rs index 0cf16f939d7b..d904ef38fdd5 100644 --- a/crates/trie/trie/src/prefix_set.rs +++ b/crates/trie/trie/src/prefix_set.rs @@ -211,8 +211,8 @@ impl PrefixSet { } impl<'a> IntoIterator for &'a PrefixSet { - type IntoIter = std::slice::Iter<'a, reth_trie_common::Nibbles>; type Item = &'a reth_trie_common::Nibbles; + type IntoIter = std::slice::Iter<'a, reth_trie_common::Nibbles>; fn into_iter(self) -> Self::IntoIter { self.iter() } From f52186cc4dbc1598be8e6a7fa17cf0cedd56a1a3 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Fri, 1 Nov 2024 19:30:03 +0800 Subject: [PATCH 209/239] db-api: opt StorageShardedKey encode, decode (#12143) --- .../storage/db-api/src/models/storage_sharded_key.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/storage/db-api/src/models/storage_sharded_key.rs b/crates/storage/db-api/src/models/storage_sharded_key.rs index 5fd79ba655c1..a7a1ffb71be3 100644 --- a/crates/storage/db-api/src/models/storage_sharded_key.rs +++ b/crates/storage/db-api/src/models/storage_sharded_key.rs @@ -12,6 +12,10 @@ use super::ShardedKey; /// Number of indices in one shard. pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000; +/// The size of [`StorageShardedKey`] encode bytes. +/// The fields are: 20-byte address, 32-byte key, and 8-byte block number +const STORAGE_SHARD_KEY_BYTES_SIZE: usize = 20 + 32 + 8; + /// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data /// into different shards. Example: /// @@ -53,7 +57,8 @@ impl Encode for StorageShardedKey { type Encoded = Vec; fn encode(self) -> Self::Encoded { - let mut buf: Vec = Encode::encode(self.address).into(); + let mut buf: Vec = Vec::with_capacity(STORAGE_SHARD_KEY_BYTES_SIZE); + buf.extend_from_slice(&Encode::encode(self.address)); buf.extend_from_slice(&Encode::encode(self.sharded_key.key)); buf.extend_from_slice(&self.sharded_key.highest_block_number.to_be_bytes()); buf @@ -62,6 +67,9 @@ impl Encode for StorageShardedKey { impl Decode for StorageShardedKey { fn decode(value: &[u8]) -> Result { + if value.len() != STORAGE_SHARD_KEY_BYTES_SIZE { + return Err(DatabaseError::Decode) + } let tx_num_index = value.len() - 8; let highest_tx_number = u64::from_be_bytes( From 927be855ffbdf8186a69ae90168a784bdd0651cc Mon Sep 17 00:00:00 2001 From: caglarkaya Date: Fri, 1 Nov 2024 14:32:12 +0300 Subject: [PATCH 210/239] feat: track buffered outgoing messages (#12220) --- crates/net/network/src/metrics.rs | 2 ++ crates/net/network/src/session/active.rs | 31 ++++++++++++++++++++++-- crates/net/network/src/session/mod.rs | 5 +++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/crates/net/network/src/metrics.rs b/crates/net/network/src/metrics.rs index 4333cf1408b0..bda5f84c76b8 100644 --- a/crates/net/network/src/metrics.rs +++ b/crates/net/network/src/metrics.rs @@ -85,6 +85,8 @@ pub struct SessionManagerMetrics { pub(crate) total_dial_successes: Counter, /// Number of dropped outgoing peer messages. pub(crate) total_outgoing_peer_messages_dropped: Counter, + /// Number of queued outgoing messages + pub(crate) queued_outgoing_messages: Gauge, } /// Metrics for the [`TransactionsManager`](crate::transactions::TransactionsManager). diff --git a/crates/net/network/src/session/active.rs b/crates/net/network/src/session/active.rs index e83f5d9f125a..10048823c549 100644 --- a/crates/net/network/src/session/active.rs +++ b/crates/net/network/src/session/active.rs @@ -12,6 +12,7 @@ use std::{ }; use futures::{stream::Fuse, SinkExt, StreamExt}; +use metrics::Gauge; use reth_eth_wire::{ errors::{EthHandshakeError, EthStreamError, P2PStreamError}, message::{EthBroadcastMessage, RequestPair}, @@ -87,7 +88,7 @@ pub(crate) struct ActiveSession { /// All requests that were sent by the remote peer and we're waiting on an internal response pub(crate) received_requests_from_remote: Vec, /// Buffered messages that should be handled and sent to the peer. - pub(crate) queued_outgoing: VecDeque, + pub(crate) queued_outgoing: QueuedOutgoingMessages, /// The maximum time we wait for a response from a peer. pub(crate) internal_request_timeout: Arc, /// Interval when to check for timed out requests. @@ -757,6 +758,32 @@ fn calculate_new_timeout(current_timeout: Duration, estimated_rtt: Duration) -> smoothened_timeout.clamp(MINIMUM_TIMEOUT, MAXIMUM_TIMEOUT) } + +/// A helper struct that wraps the queue of outgoing messages and a metric to track their count +pub(crate) struct QueuedOutgoingMessages { + messages: VecDeque, + count: Gauge, +} + +impl QueuedOutgoingMessages { + pub(crate) const fn new(metric: Gauge) -> Self { + Self { messages: VecDeque::new(), count: metric } + } + + pub(crate) fn push_back(&mut self, message: OutgoingMessage) { + self.messages.push_back(message); + self.count.increment(1); + } + + pub(crate) fn pop_front(&mut self) -> Option { + self.messages.pop_front().inspect(|_| self.count.decrement(1)) + } + + pub(crate) fn shrink_to_fit(&mut self) { + self.messages.shrink_to_fit(); + } +} + #[cfg(test)] mod tests { use super::*; @@ -882,7 +909,7 @@ mod tests { internal_request_tx: ReceiverStream::new(messages_rx).fuse(), inflight_requests: Default::default(), conn, - queued_outgoing: Default::default(), + queued_outgoing: QueuedOutgoingMessages::new(Gauge::noop()), received_requests_from_remote: Default::default(), internal_request_timeout_interval: tokio::time::interval( INITIAL_REQUEST_TIMEOUT, diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index 3522aa6a75b2..712f076b47de 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -5,6 +5,7 @@ mod conn; mod counter; mod handle; +use active::QueuedOutgoingMessages; pub use conn::EthRlpxConnection; pub use handle::{ ActiveSessionHandle, ActiveSessionMessage, PendingSessionEvent, PendingSessionHandle, @@ -495,7 +496,9 @@ impl SessionManager { internal_request_tx: ReceiverStream::new(messages_rx).fuse(), inflight_requests: Default::default(), conn, - queued_outgoing: Default::default(), + queued_outgoing: QueuedOutgoingMessages::new( + self.metrics.queued_outgoing_messages.clone(), + ), received_requests_from_remote: Default::default(), internal_request_timeout_interval: tokio::time::interval( self.initial_internal_request_timeout, From f93dbf54c3b1181b166a0813cfeb95296ec0c6cd Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 1 Nov 2024 19:49:37 +0800 Subject: [PATCH 211/239] Remove redundant `SignedTransaction::Signature` (#12185) --- crates/primitives-traits/src/transaction/signed.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/primitives-traits/src/transaction/signed.rs b/crates/primitives-traits/src/transaction/signed.rs index 1bc8308b13f0..748ef39666dc 100644 --- a/crates/primitives-traits/src/transaction/signed.rs +++ b/crates/primitives-traits/src/transaction/signed.rs @@ -5,7 +5,7 @@ use core::hash::Hash; use alloy_consensus::Transaction; use alloy_eips::eip2718::{Decodable2718, Encodable2718}; -use alloy_primitives::{keccak256, Address, TxHash, B256}; +use alloy_primitives::{keccak256, Address, Signature, TxHash, B256}; /// A signed transaction. pub trait SignedTransaction: @@ -26,9 +26,6 @@ pub trait SignedTransaction: /// Transaction type that is signed. type Transaction: Transaction; - /// Signature type that results from signing transaction. - type Signature; - /// Returns reference to transaction hash. fn tx_hash(&self) -> &TxHash; @@ -36,7 +33,7 @@ pub trait SignedTransaction: fn transaction(&self) -> &Self::Transaction; /// Returns reference to signature. - fn signature(&self) -> &Self::Signature; + fn signature(&self) -> &Signature; /// Recover signer from signature and hash. /// @@ -59,10 +56,8 @@ pub trait SignedTransaction: /// Create a new signed transaction from a transaction and its signature. /// /// This will also calculate the transaction hash using its encoding. - fn from_transaction_and_signature( - transaction: Self::Transaction, - signature: Self::Signature, - ) -> Self; + fn from_transaction_and_signature(transaction: Self::Transaction, signature: Signature) + -> Self; /// Calculate transaction hash, eip2728 transaction does not contain rlp header and start with /// tx type. From f6bd11c7116dbf46a4450cb69121d015b4b705b6 Mon Sep 17 00:00:00 2001 From: Aliaksei Misiukevich Date: Fri, 1 Nov 2024 14:54:55 +0100 Subject: [PATCH 212/239] feature: transaction's input truncate function (#12236) Signed-off-by: nadtech-hub --- crates/rpc/rpc-types-compat/src/transaction/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-types-compat/src/transaction/mod.rs b/crates/rpc/rpc-types-compat/src/transaction/mod.rs index f8b46454dc23..2d92747d4015 100644 --- a/crates/rpc/rpc-types-compat/src/transaction/mod.rs +++ b/crates/rpc/rpc-types-compat/src/transaction/mod.rs @@ -102,7 +102,9 @@ impl TransactionCompat for () { WithOtherFields::default() } - fn otterscan_api_truncate_input(_tx: &mut Self::Transaction) {} + fn otterscan_api_truncate_input(tx: &mut Self::Transaction) { + tx.input = tx.input.slice(..4); + } fn tx_type(_tx: &Self::Transaction) -> u8 { 0 From c1a68f23cf2d655cd8137660fc82c7d83c094b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Fri, 1 Nov 2024 20:55:23 +0700 Subject: [PATCH 213/239] refactor: move `payload/builder/src/database.rs` to `revm/src/cached.rs` (#12252) Co-authored-by: Matthias Seitz --- Cargo.lock | 237 +++++++++--------- .../src/commands/debug_cmd/build_block.rs | 3 +- crates/payload/basic/Cargo.toml | 1 + crates/payload/basic/src/lib.rs | 5 +- crates/payload/builder/Cargo.toml | 10 +- crates/payload/builder/src/lib.rs | 1 - .../src/database.rs => revm/src/cached.rs} | 12 +- crates/revm/src/lib.rs | 8 +- 8 files changed, 147 insertions(+), 130 deletions(-) rename crates/{payload/builder/src/database.rs => revm/src/cached.rs} (96%) diff --git a/Cargo.lock b/Cargo.lock index 5fc3574ebc43..4c469cd21e2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.42" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca4a1469a3e572e9ba362920ff145f5d0a00a3e71a64ddcb4a3659cf64c76a7" +checksum = "4feb7c662fd0be3d0c926a456be4ac44e9cf8e05cbd91df6db7f7140b861016a" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -150,9 +150,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5647fce5a168f9630f935bf7821c4207b1755184edaeba783cb4e11d35058484" +checksum = "f5228b189b18b85761340dc9eaac0141148a8503657b36f9bc3a869413d987ca" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -229,9 +229,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b5671117c38b1c2306891f97ad3828d85487087f54ebe2c7591a055ea5bcea7" +checksum = "31a0f0d51db8a1a30a4d98a9f90e090a94c8f44cb4d9eafc7e03aa6d00aae984" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71738eb20c42c5fb149571e76536a0f309d142f3957c28791662b96baf77a3d" +checksum = "8edae627382349b56cd6a7a2106f4fd69b243a9233e560c55c2e03cabb7e1d3c" dependencies = [ "alloy-rlp", "arbitrary", @@ -415,7 +415,7 @@ checksum = "2b09cae092c27b6f1bde952653a22708691802e57bfef4a2973b80bea21efd3f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -630,23 +630,23 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0900b83f4ee1f45c640ceee596afbc118051921b9438fdb5a3175c1a7e05f8b" +checksum = "841eabaa4710f719fddbc24c95d386eae313f07e6da4babc25830ee37945be0c" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] name = "alloy-sol-macro-expander" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41b1e78dde06b5e12e6702fa8c1d30621bf07728ba75b801fb801c9c6a0ba10" +checksum = "6672337f19d837b9f7073c45853aeb528ed9f7dd6a4154ce683e9e5cb7794014" dependencies = [ "alloy-sol-macro-input", "const-hex", @@ -655,31 +655,31 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "syn-solidity", "tiny-keccak", ] [[package]] name = "alloy-sol-macro-input" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91dc311a561a306664393407b88d3e53ae58581624128afd8a15faa5de3627dc" +checksum = "0dff37dd20bfb118b777c96eda83b2067f4226d2644c5cfa00187b3bc01770ba" dependencies = [ "const-hex", "dunce", "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "syn-solidity", ] [[package]] name = "alloy-sol-type-parser" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d1fbee9e698f3ba176b6e7a145f4aefe6d2b746b611e8bb246fe11a0e9f6c4" +checksum = "5b853d42292dbb159671a3edae3b2750277ff130f32b726fe07dc2b17aa6f2b5" dependencies = [ "serde", "winnow", @@ -687,9 +687,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086f41bc6ebcd8cb15f38ba20e47be38dd03692149681ce8061c35d960dbf850" +checksum = "aa828bb1b9a6dc52208fbb18084fb9ce2c30facc2bfda6a5d922349b4990354f" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -862,9 +862,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" [[package]] name = "aquamarine" @@ -877,7 +877,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1100,7 +1100,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1111,7 +1111,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1149,7 +1149,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1255,7 +1255,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1437,7 +1437,7 @@ checksum = "240f4126219a83519bad05c9a40bfc0303921eeb571fc2d7e44c17ffac99d3f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "synstructure", ] @@ -1559,7 +1559,7 @@ checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -1677,6 +1677,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -1771,7 +1777,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2228,7 +2234,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2252,7 +2258,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2263,7 +2269,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2385,7 +2391,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2396,7 +2402,7 @@ checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2417,7 +2423,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "unicode-xid", ] @@ -2531,7 +2537,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2679,7 +2685,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2690,7 +2696,7 @@ checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -2747,7 +2753,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -3303,7 +3309,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -3788,9 +3794,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -3829,7 +3835,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -3979,7 +3985,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -4147,7 +4153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -4395,7 +4401,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -4571,9 +4577,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00419de735aac21d53b0de5ce2c03bd3627277cf471300f27ebc89f7d828047" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libp2p-identity" @@ -4813,7 +4819,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -4959,7 +4965,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -5207,7 +5213,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -5260,9 +5266,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "op-alloy-consensus" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7c98055fd048073738df0cc6d6537e992a0d8828f39d99a469e870db126dbd" +checksum = "f26c3b35b7b3e36d15e0563eebffe13c1d9ca16b7aaffcb6a64354633547e16b" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5278,9 +5284,9 @@ dependencies = [ [[package]] name = "op-alloy-genesis" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d631e8113cf88d30e621022677209caa148a9ca3ccb590fd34bbd1c731e3aff3" +checksum = "ccacc2efed3d60d98ea581bddb885df1c6c62a592e55de049cfefd94116112cd" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5292,9 +5298,9 @@ dependencies = [ [[package]] name = "op-alloy-network" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1eabe7683d7e19c7cc5171d664e49fc449176cf1334ffff82808e2a7eea5933a" +checksum = "5ff6fc0f94702ea0f4d8466bffdc990067ae6df9213465df9b7957f74f1e5461" dependencies = [ "alloy-consensus", "alloy-network", @@ -5306,26 +5312,29 @@ dependencies = [ [[package]] name = "op-alloy-protocol" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b39574acb1873315e6bd89df174f6223e897188fb87eeea2ad1eda04f7d28eb" +checksum = "f5f8e6ec6b91c6aaeb20860b455a52fd8e300acfe5d534e96e9073a24f853e74" dependencies = [ "alloy-consensus", "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-serde", + "async-trait", "derive_more 1.0.0", "op-alloy-consensus", "op-alloy-genesis", "serde", + "tracing", + "unsigned-varint", ] [[package]] name = "op-alloy-rpc-types" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919e9b69212d61f3c8932bfb717c7ad458ea3fc52072b3433d99994f8223d555" +checksum = "94bae9bf91b620e1e2c2291562e5998bc1247bd8ada011773e1997b31a95de99" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5341,9 +5350,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types-engine" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3a47ea24cee189b4351be247fd138c68571704ee57060cf5a722502f44412c" +checksum = "4b52ee59c86537cff83e8c7f2a6aa287a94f3608bb40c06d442aafd0c2e807a4" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -5561,7 +5570,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -5590,7 +5599,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -5762,7 +5771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -5813,7 +5822,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -5911,7 +5920,7 @@ checksum = "6ff7ff745a347b87471d859a377a9a404361e7efc2a971d73424a6d183c0fc77" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -5990,10 +5999,11 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", @@ -6229,9 +6239,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", @@ -6403,6 +6413,7 @@ dependencies = [ "reth-payload-primitives", "reth-primitives", "reth-provider", + "reth-revm", "reth-tasks", "reth-transaction-pool", "revm", @@ -6733,7 +6744,7 @@ dependencies = [ "proc-macro2", "quote", "similar-asserts", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -9606,9 +9617,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.15" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "log", "once_cell", @@ -9883,22 +9894,22 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.213" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.213" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -9933,7 +9944,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -9984,7 +9995,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10007,7 +10018,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10293,7 +10304,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10351,9 +10362,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" dependencies = [ "proc-macro2", "quote", @@ -10362,14 +10373,14 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5e0c2ea8db64b2898b62ea2fbd60204ca95e0b2c6bdf53ff768bbe916fbe4d" +checksum = "16320d4a2021ba1a32470b3759676114a918885e9800e68ad60f2c67969fba62" dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10395,7 +10406,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10472,7 +10483,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10496,22 +10507,22 @@ checksum = "a38c90d48152c236a3ab59271da4f4ae63d678c5d7ad6b7714d7cb9760be5e4b" [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10688,7 +10699,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -10889,7 +10900,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11280,7 +11291,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11351,7 +11362,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "wasm-bindgen-shared", ] @@ -11385,7 +11396,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -11541,7 +11552,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11552,7 +11563,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11563,7 +11574,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11574,7 +11585,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11849,7 +11860,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "synstructure", ] @@ -11871,7 +11882,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11891,7 +11902,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", "synstructure", ] @@ -11912,7 +11923,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] @@ -11934,7 +11945,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.86", ] [[package]] diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index a2dfb5ab3eaa..0559d473f621 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -26,7 +26,6 @@ use reth_node_api::{ EngineApiMessageVersion, NodeTypesWithDB, NodeTypesWithEngine, PayloadBuilderAttributes, }; use reth_node_ethereum::{EthEvmConfig, EthExecutorProvider}; -use reth_payload_builder::database::CachedReads; use reth_primitives::{ revm_primitives::KzgSettings, BlobTransaction, BlobTransactionSidecar, PooledTransactionsElement, SealedBlock, SealedBlockWithSenders, SealedHeader, Transaction, @@ -36,7 +35,7 @@ use reth_provider::{ providers::BlockchainProvider, BlockHashReader, BlockReader, BlockWriter, ChainSpecProvider, ProviderFactory, StageCheckpointReader, StateProviderFactory, }; -use reth_revm::{database::StateProviderDatabase, primitives::EnvKzgSettings}; +use reth_revm::{cached::CachedReads, database::StateProviderDatabase, primitives::EnvKzgSettings}; use reth_stages::StageId; use reth_transaction_pool::{ blobstore::InMemoryBlobStore, BlobStore, EthPooledTransaction, PoolConfig, TransactionOrigin, diff --git a/crates/payload/basic/Cargo.toml b/crates/payload/basic/Cargo.toml index 88ab99272dbd..74dea45d10d9 100644 --- a/crates/payload/basic/Cargo.toml +++ b/crates/payload/basic/Cargo.toml @@ -21,6 +21,7 @@ reth-payload-builder.workspace = true reth-payload-primitives.workspace = true reth-tasks.workspace = true reth-evm.workspace = true +reth-revm.workspace=true # ethereum alloy-rlp.workspace = true diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index bb8dc0ef66aa..7b1de980ce9c 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -16,9 +16,7 @@ use futures_core::ready; use futures_util::FutureExt; use reth_chainspec::{ChainSpec, EthereumHardforks}; use reth_evm::state_change::post_block_withdrawals_balance_increments; -use reth_payload_builder::{ - database::CachedReads, KeepPayloadJobAlive, PayloadId, PayloadJob, PayloadJobGenerator, -}; +use reth_payload_builder::{KeepPayloadJobAlive, PayloadId, PayloadJob, PayloadJobGenerator}; use reth_payload_primitives::{ BuiltPayload, PayloadBuilderAttributes, PayloadBuilderError, PayloadKind, }; @@ -28,6 +26,7 @@ use reth_primitives::{ use reth_provider::{ BlockReaderIdExt, BlockSource, CanonStateNotification, ProviderError, StateProviderFactory, }; +use reth_revm::cached::CachedReads; use reth_tasks::TaskSpawner; use reth_transaction_pool::TransactionPool; use revm::{Database, State}; diff --git a/crates/payload/builder/Cargo.toml b/crates/payload/builder/Cargo.toml index 3b71011e02e5..08399b6f9c6d 100644 --- a/crates/payload/builder/Cargo.toml +++ b/crates/payload/builder/Cargo.toml @@ -13,15 +13,15 @@ workspace = true [dependencies] # reth -reth-primitives.workspace = true +reth-primitives = { workspace = true, optional = true } reth-provider.workspace = true reth-payload-primitives.workspace = true reth-ethereum-engine-primitives.workspace = true reth-chain-state = { workspace = true, optional = true } # alloy +alloy-primitives = { workspace = true, optional = true } alloy-rpc-types = { workspace = true, features = ["engine"] } -alloy-primitives.workspace = true # async async-trait.workspace = true @@ -37,12 +37,16 @@ metrics.workspace = true tracing.workspace = true [dev-dependencies] +reth-primitives.workspace = true +reth-chain-state.workspace = true +alloy-primitives.workspace = true revm.workspace = true [features] test-utils = [ + "alloy-primitives", "reth-chain-state", - "reth-chain-state?/test-utils", + "reth-chain-state/test-utils", "reth-primitives/test-utils", "reth-provider/test-utils", "revm/test-utils" diff --git a/crates/payload/builder/src/lib.rs b/crates/payload/builder/src/lib.rs index 7af61ac4c682..2c46a4a9e165 100644 --- a/crates/payload/builder/src/lib.rs +++ b/crates/payload/builder/src/lib.rs @@ -101,7 +101,6 @@ #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] -pub mod database; mod metrics; mod service; mod traits; diff --git a/crates/payload/builder/src/database.rs b/crates/revm/src/cached.rs similarity index 96% rename from crates/payload/builder/src/database.rs rename to crates/revm/src/cached.rs index d63f7322dee2..b0eac39c44cb 100644 --- a/crates/payload/builder/src/database.rs +++ b/crates/revm/src/cached.rs @@ -1,13 +1,13 @@ //! Database adapters for payload building. -use alloy_primitives::{Address, B256, U256}; +use alloy_primitives::{ + map::{Entry, HashMap}, + Address, B256, U256, +}; +use core::cell::RefCell; use reth_primitives::revm_primitives::{ db::{Database, DatabaseRef}, AccountInfo, Bytecode, }; -use std::{ - cell::RefCell, - collections::{hash_map::Entry, HashMap}, -}; /// A container type that caches reads from an underlying [`DatabaseRef`]. /// @@ -17,7 +17,7 @@ use std::{ /// # Example /// /// ``` -/// use reth_payload_builder::database::CachedReads; +/// use reth_revm::cached::CachedReads; /// use revm::db::{DatabaseRef, State}; /// /// fn build_payload(db: DB) { diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index 8b544a537289..b06ee816f8d0 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -11,11 +11,15 @@ extern crate alloc; +pub mod batch; + +/// Cache database that reads from an underlying [`DatabaseRef`]. +/// Database adapters for payload building. +pub mod cached; + /// Contains glue code for integrating reth database into revm's [Database]. pub mod database; -pub mod batch; - /// Common test helpers #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; From d5a3a3a849265496af53b7f4fd460b572fb4fca3 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:57:34 +0100 Subject: [PATCH 214/239] test(chain-state): add unit tests for `CanonStateNotification` (#12110) --- crates/chain-state/src/notifications.rs | 226 ++++++++++++++++++++++++ crates/evm/execution-types/src/chain.rs | 2 +- 2 files changed, 227 insertions(+), 1 deletion(-) diff --git a/crates/chain-state/src/notifications.rs b/crates/chain-state/src/notifications.rs index fc717314b3fc..6e24bcbb4c8c 100644 --- a/crates/chain-state/src/notifications.rs +++ b/crates/chain-state/src/notifications.rs @@ -190,3 +190,229 @@ impl Stream for ForkChoiceStream { } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use reth_execution_types::ExecutionOutcome; + use reth_primitives::{Receipt, Receipts, TransactionSigned, TxType}; + + #[test] + fn test_commit_notification() { + let block = SealedBlockWithSenders::default(); + let block1_hash = B256::new([0x01; 32]); + let block2_hash = B256::new([0x02; 32]); + + let mut block1 = block.clone(); + block1.set_block_number(1); + block1.set_hash(block1_hash); + + let mut block2 = block; + block2.set_block_number(2); + block2.set_hash(block2_hash); + + let chain = Arc::new(Chain::new( + vec![block1.clone(), block2.clone()], + ExecutionOutcome::default(), + None, + )); + + // Create a commit notification + let notification = CanonStateNotification::Commit { new: chain.clone() }; + + // Test that `committed` returns the correct chain + assert_eq!(notification.committed(), chain); + + // Test that `reverted` returns None for `Commit` + assert!(notification.reverted().is_none()); + + // Test that `tip` returns the correct block + assert_eq!(*notification.tip(), block2); + } + + #[test] + fn test_reorg_notification() { + let block = SealedBlockWithSenders::default(); + let block1_hash = B256::new([0x01; 32]); + let block2_hash = B256::new([0x02; 32]); + let block3_hash = B256::new([0x03; 32]); + + let mut block1 = block.clone(); + block1.set_block_number(1); + block1.set_hash(block1_hash); + + let mut block2 = block.clone(); + block2.set_block_number(2); + block2.set_hash(block2_hash); + + let mut block3 = block; + block3.set_block_number(3); + block3.set_hash(block3_hash); + + let old_chain = + Arc::new(Chain::new(vec![block1.clone()], ExecutionOutcome::default(), None)); + let new_chain = Arc::new(Chain::new( + vec![block2.clone(), block3.clone()], + ExecutionOutcome::default(), + None, + )); + + // Create a reorg notification + let notification = + CanonStateNotification::Reorg { old: old_chain.clone(), new: new_chain.clone() }; + + // Test that `reverted` returns the old chain + assert_eq!(notification.reverted(), Some(old_chain)); + + // Test that `committed` returns the new chain + assert_eq!(notification.committed(), new_chain); + + // Test that `tip` returns the tip of the new chain (last block in the new chain) + assert_eq!(*notification.tip(), block3); + } + + #[test] + fn test_block_receipts_commit() { + // Create a default block instance for use in block definitions. + let block = SealedBlockWithSenders::default(); + + // Define unique hashes for two blocks to differentiate them in the chain. + let block1_hash = B256::new([0x01; 32]); + let block2_hash = B256::new([0x02; 32]); + + // Create a default transaction to include in block1's transactions. + let tx = TransactionSigned::default(); + + // Create a clone of the default block and customize it to act as block1. + let mut block1 = block.clone(); + block1.set_block_number(1); + block1.set_hash(block1_hash); + // Add the transaction to block1's transactions. + block1.block.body.transactions.push(tx); + + // Clone the default block and customize it to act as block2. + let mut block2 = block; + block2.set_block_number(2); + block2.set_hash(block2_hash); + + // Create a receipt for the transaction in block1. + let receipt1 = Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 12345, + logs: vec![], + success: true, + ..Default::default() + }; + + // Wrap the receipt in a `Receipts` structure, as expected in the `ExecutionOutcome`. + let receipts = Receipts { receipt_vec: vec![vec![Some(receipt1.clone())]] }; + + // Define an `ExecutionOutcome` with the created receipts. + let execution_outcome = ExecutionOutcome { receipts, ..Default::default() }; + + // Create a new chain segment with `block1` and `block2` and the execution outcome. + let new_chain = + Arc::new(Chain::new(vec![block1.clone(), block2.clone()], execution_outcome, None)); + + // Create a commit notification containing the new chain segment. + let notification = CanonStateNotification::Commit { new: new_chain }; + + // Call `block_receipts` on the commit notification to retrieve block receipts. + let block_receipts = notification.block_receipts(); + + // Assert that only one receipt entry exists in the `block_receipts` list. + assert_eq!(block_receipts.len(), 1); + + // Verify that the first entry matches block1's hash and transaction receipt. + assert_eq!( + block_receipts[0].0, + BlockReceipts { + block: block1.num_hash(), + tx_receipts: vec![(B256::default(), receipt1)] + } + ); + + // Assert that the receipt is from the committed segment (not reverted). + assert!(!block_receipts[0].1); + } + + #[test] + fn test_block_receipts_reorg() { + // Define block1 for the old chain segment, which will be reverted. + let mut old_block1 = SealedBlockWithSenders::default(); + old_block1.set_block_number(1); + old_block1.set_hash(B256::new([0x01; 32])); + old_block1.block.body.transactions.push(TransactionSigned::default()); + + // Create a receipt for a transaction in the reverted block. + let old_receipt = Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 54321, + logs: vec![], + success: false, + ..Default::default() + }; + let old_receipts = Receipts { receipt_vec: vec![vec![Some(old_receipt.clone())]] }; + + let old_execution_outcome = + ExecutionOutcome { receipts: old_receipts, ..Default::default() }; + + // Create an old chain segment to be reverted, containing `old_block1`. + let old_chain = Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, None)); + + // Define block2 for the new chain segment, which will be committed. + let mut new_block1 = SealedBlockWithSenders::default(); + new_block1.set_block_number(2); + new_block1.set_hash(B256::new([0x02; 32])); + new_block1.block.body.transactions.push(TransactionSigned::default()); + + // Create a receipt for a transaction in the new committed block. + let new_receipt = Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 12345, + logs: vec![], + success: true, + ..Default::default() + }; + let new_receipts = Receipts { receipt_vec: vec![vec![Some(new_receipt.clone())]] }; + + let new_execution_outcome = + ExecutionOutcome { receipts: new_receipts, ..Default::default() }; + + // Create a new chain segment to be committed, containing `new_block1`. + let new_chain = Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, None)); + + // Create a reorg notification with both reverted (old) and committed (new) chain segments. + let notification = CanonStateNotification::Reorg { old: old_chain, new: new_chain }; + + // Retrieve receipts from both old (reverted) and new (committed) segments. + let block_receipts = notification.block_receipts(); + + // Assert there are two receipt entries, one from each chain segment. + assert_eq!(block_receipts.len(), 2); + + // Verify that the first entry matches old_block1 and its receipt from the reverted segment. + assert_eq!( + block_receipts[0].0, + BlockReceipts { + block: old_block1.num_hash(), + tx_receipts: vec![(B256::default(), old_receipt)] + } + ); + // Confirm this is from the reverted segment. + assert!(block_receipts[0].1); + + // Verify that the second entry matches new_block1 and its receipt from the committed + // segment. + assert_eq!( + block_receipts[1].0, + BlockReceipts { + block: new_block1.num_hash(), + tx_receipts: vec![(B256::default(), new_receipt)] + } + ); + // Confirm this is from the committed segment. + assert!(!block_receipts[1].1); + } +} diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 65f96ff56387..dc633e2d7ab7 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -453,7 +453,7 @@ impl IntoIterator for ChainBlocks<'_> { } /// Used to hold receipts and their attachment. -#[derive(Default, Clone, Debug)] +#[derive(Default, Clone, Debug, PartialEq, Eq)] pub struct BlockReceipts { /// Block identifier pub block: BlockNumHash, From eaac2aa2cfb6ea5a61eea2d9a36a8b66b7854ce4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 1 Nov 2024 15:17:31 +0100 Subject: [PATCH 215/239] chore: simplify cached db usage (#12242) --- crates/ethereum/payload/src/lib.rs | 5 ++--- crates/optimism/payload/src/builder.rs | 5 ++--- crates/revm/src/cached.rs | 24 +++++++++++++++++++----- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 8e92d0aa8706..ed33292ef984 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -161,7 +161,7 @@ where let state_provider = client.state_by_block_hash(config.parent_header.hash())?; let state = StateProviderDatabase::new(state_provider); let mut db = - State::builder().with_database_ref(cached_reads.as_db(state)).with_bundle_update().build(); + State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build(); let PayloadConfig { parent_header, extra_data, attributes } = config; debug!(target: "payload_builder", id=%attributes.id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload"); @@ -372,8 +372,7 @@ where // calculate the state root let hashed_state = HashedPostState::from_bundle_state(&execution_outcome.state().state); let (state_root, trie_output) = { - let state_provider = db.database.0.inner.borrow_mut(); - state_provider.db.state_root_with_updates(hashed_state.clone()).inspect_err(|err| { + db.database.inner().state_root_with_updates(hashed_state.clone()).inspect_err(|err| { warn!(target: "payload_builder", parent_hash=%parent_header.hash(), %err, diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 09a443d0f1df..cae2d34bd499 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -170,7 +170,7 @@ where let state_provider = client.state_by_block_hash(config.parent_header.hash())?; let state = StateProviderDatabase::new(state_provider); let mut db = - State::builder().with_database_ref(cached_reads.as_db(state)).with_bundle_update().build(); + State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build(); let PayloadConfig { parent_header, attributes, mut extra_data } = config; debug!(target: "payload_builder", id=%attributes.payload_attributes.payload_id(), parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload"); @@ -445,8 +445,7 @@ where // calculate the state root let hashed_state = HashedPostState::from_bundle_state(&execution_outcome.state().state); let (state_root, trie_output) = { - let state_provider = db.database.0.inner.borrow_mut(); - state_provider.db.state_root_with_updates(hashed_state.clone()).inspect_err(|err| { + db.database.inner().state_root_with_updates(hashed_state.clone()).inspect_err(|err| { warn!(target: "payload_builder", parent_header=%parent_header.hash(), %err, diff --git a/crates/revm/src/cached.rs b/crates/revm/src/cached.rs index b0eac39c44cb..2152ca5bd733 100644 --- a/crates/revm/src/cached.rs +++ b/crates/revm/src/cached.rs @@ -22,10 +22,10 @@ use reth_primitives::revm_primitives::{ /// /// fn build_payload(db: DB) { /// let mut cached_reads = CachedReads::default(); -/// let db_ref = cached_reads.as_db(db); -/// // this is `Database` and can be used to build a payload, it never writes to `CachedReads` or the underlying database, but all reads from the underlying database are cached in `CachedReads`. +/// let db = cached_reads.as_db_mut(db); +/// // this is `Database` and can be used to build a payload, it never commits to `CachedReads` or the underlying database, but all reads from the underlying database are cached in `CachedReads`. /// // Subsequent payload build attempts can use cached reads and avoid hitting the underlying database. -/// let db = State::builder().with_database_ref(db_ref).build(); +/// let state = State::builder().with_database(db).build(); /// } /// ``` #[derive(Debug, Clone, Default)] @@ -40,10 +40,11 @@ pub struct CachedReads { impl CachedReads { /// Gets a [`DatabaseRef`] that will cache reads from the given database. pub fn as_db(&mut self, db: DB) -> CachedReadsDBRef<'_, DB> { - CachedReadsDBRef { inner: RefCell::new(self.as_db_mut(db)) } + self.as_db_mut(db).into_db() } - fn as_db_mut(&mut self, db: DB) -> CachedReadsDbMut<'_, DB> { + /// Gets a mutable [`Database`] that will cache reads from the underlying database. + pub fn as_db_mut(&mut self, db: DB) -> CachedReadsDbMut<'_, DB> { CachedReadsDbMut { cached: self, db } } @@ -67,6 +68,19 @@ pub struct CachedReadsDbMut<'a, DB> { pub db: DB, } +impl<'a, DB> CachedReadsDbMut<'a, DB> { + /// Converts this [`Database`] implementation into a [`DatabaseRef`] that will still cache + /// reads. + pub const fn into_db(self) -> CachedReadsDBRef<'a, DB> { + CachedReadsDBRef { inner: RefCell::new(self) } + } + + /// Returns access to wrapped [`DatabaseRef`]. + pub const fn inner(&self) -> &DB { + &self.db + } +} + impl Database for CachedReadsDbMut<'_, DB> { type Error = ::Error; From d8100012798c540f71266e757bc5dbcddd508b63 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Fri, 1 Nov 2024 09:34:48 -0600 Subject: [PATCH 216/239] renamed OptimismNetworkBuilder to OpNetworkBuilder (#12255) --- crates/optimism/node/src/node.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 925e7204c9e2..56487fb24351 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -68,7 +68,7 @@ impl OptimismNode { Node, OpPoolBuilder, OpPayloadBuilder, - OptimismNetworkBuilder, + OpNetworkBuilder, OptimismExecutorBuilder, OptimismConsensusBuilder, > @@ -82,7 +82,7 @@ impl OptimismNode { .node_types::() .pool(OpPoolBuilder::default()) .payload(OpPayloadBuilder::new(compute_pending_block)) - .network(OptimismNetworkBuilder { + .network(OpNetworkBuilder { disable_txpool_gossip, disable_discovery_v4: !discovery_v4, }) @@ -101,7 +101,7 @@ where N, OpPoolBuilder, OpPayloadBuilder, - OptimismNetworkBuilder, + OpNetworkBuilder, OptimismExecutorBuilder, OptimismConsensusBuilder, >; @@ -365,17 +365,17 @@ where /// A basic optimism network builder. #[derive(Debug, Default, Clone)] -pub struct OptimismNetworkBuilder { +pub struct OpNetworkBuilder { /// Disable transaction pool gossip pub disable_txpool_gossip: bool, /// Disable discovery v4 pub disable_discovery_v4: bool, } -impl OptimismNetworkBuilder { +impl OpNetworkBuilder { /// Returns the [`NetworkConfig`] that contains the settings to launch the p2p network. /// - /// This applies the configured [`OptimismNetworkBuilder`] settings. + /// This applies the configured [`OpNetworkBuilder`] settings. pub fn network_config( &self, ctx: &BuilderContext, @@ -420,7 +420,7 @@ impl OptimismNetworkBuilder { } } -impl NetworkBuilder for OptimismNetworkBuilder +impl NetworkBuilder for OpNetworkBuilder where Node: FullNodeTypes>, Pool: TransactionPool + Unpin + 'static, From 41c4bab0f781f7b67fbc430797ff96839149c603 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 1 Nov 2024 17:02:49 +0100 Subject: [PATCH 217/239] chore: use deref directly (#12256) --- crates/rpc/rpc-eth-api/src/helpers/pending_block.rs | 4 +--- crates/rpc/rpc-eth-types/src/simulate.rs | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index f2d1416139b8..a0065d79342f 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -399,9 +399,7 @@ pub trait LoadPendingBlock: execution_outcome.block_logs_bloom(block_number).expect("Block is present"); // calculate the state root - let state_provider = &db.database; - let state_root = - state_provider.state_root(hashed_state).map_err(Self::Error::from_eth_err)?; + let state_root = db.database.state_root(hashed_state).map_err(Self::Error::from_eth_err)?; // create the block header let transactions_root = calculate_transaction_root(&executed_txs); diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index 4249c78fe6ac..62f0e24b1c6d 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -274,7 +274,7 @@ pub fn build_block( } } - let state_root = db.db.0.state_root(hashed_state)?; + let state_root = db.db.state_root(hashed_state)?; let header = reth_primitives::Header { beneficiary: block_env.coinbase, From bc69f6348f553802c94a774e0305ca57b018aaf9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 1 Nov 2024 18:08:41 +0100 Subject: [PATCH 218/239] feat: add asref impls (#12257) --- crates/revm/src/cached.rs | 9 +++++++++ crates/revm/src/database.rs | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/crates/revm/src/cached.rs b/crates/revm/src/cached.rs index 2152ca5bd733..807b163c4cbc 100644 --- a/crates/revm/src/cached.rs +++ b/crates/revm/src/cached.rs @@ -81,6 +81,15 @@ impl<'a, DB> CachedReadsDbMut<'a, DB> { } } +impl AsRef for CachedReadsDbMut<'_, DB> +where + DB: AsRef, +{ + fn as_ref(&self) -> &T { + self.inner().as_ref() + } +} + impl Database for CachedReadsDbMut<'_, DB> { type Error = ::Error; diff --git a/crates/revm/src/database.rs b/crates/revm/src/database.rs index 5f662fea7cfe..682aca6cf379 100644 --- a/crates/revm/src/database.rs +++ b/crates/revm/src/database.rs @@ -79,6 +79,12 @@ impl StateProviderDatabase { } } +impl AsRef for StateProviderDatabase { + fn as_ref(&self) -> &DB { + self + } +} + impl Deref for StateProviderDatabase { type Target = DB; From 969ca3e63b2203abf7537808bf3513971ebf210b Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Sat, 2 Nov 2024 02:14:22 +0900 Subject: [PATCH 219/239] fix: check hashed state for loading `TriePrefixSets::destroyed_accounts` (#12235) --- crates/trie/db/src/prefix_set.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/trie/db/src/prefix_set.rs b/crates/trie/db/src/prefix_set.rs index 079fe393764d..cd50503bc703 100644 --- a/crates/trie/db/src/prefix_set.rs +++ b/crates/trie/db/src/prefix_set.rs @@ -36,13 +36,13 @@ impl PrefixSetLoader<'_, TX> { // Walk account changeset and insert account prefixes. let mut account_changeset_cursor = self.cursor_read::()?; - let mut account_plain_state_cursor = self.cursor_read::()?; + let mut account_hashed_state_cursor = self.cursor_read::()?; for account_entry in account_changeset_cursor.walk_range(range.clone())? { let (_, AccountBeforeTx { address, .. }) = account_entry?; let hashed_address = keccak256(address); account_prefix_set.insert(Nibbles::unpack(hashed_address)); - if account_plain_state_cursor.seek_exact(address)?.is_none() { + if account_hashed_state_cursor.seek_exact(hashed_address)?.is_none() { destroyed_accounts.insert(hashed_address); } } From 166a2346dcd2a9c8d5fea6995a9fc5e6426c62bb Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:19:47 -0600 Subject: [PATCH 220/239] renamed OptimismExecutorBuilder to OpExecutorBuilder (#12258) --- crates/optimism/node/src/node.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 56487fb24351..b4a4d2730ab5 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -69,7 +69,7 @@ impl OptimismNode { OpPoolBuilder, OpPayloadBuilder, OpNetworkBuilder, - OptimismExecutorBuilder, + OpExecutorBuilder, OptimismConsensusBuilder, > where @@ -86,7 +86,7 @@ impl OptimismNode { disable_txpool_gossip, disable_discovery_v4: !discovery_v4, }) - .executor(OptimismExecutorBuilder::default()) + .executor(OpExecutorBuilder::default()) .consensus(OptimismConsensusBuilder::default()) } } @@ -102,7 +102,7 @@ where OpPoolBuilder, OpPayloadBuilder, OpNetworkBuilder, - OptimismExecutorBuilder, + OpExecutorBuilder, OptimismConsensusBuilder, >; @@ -179,9 +179,9 @@ where /// A regular optimism evm and executor builder. #[derive(Debug, Default, Clone, Copy)] #[non_exhaustive] -pub struct OptimismExecutorBuilder; +pub struct OpExecutorBuilder; -impl ExecutorBuilder for OptimismExecutorBuilder +impl ExecutorBuilder for OpExecutorBuilder where Node: FullNodeTypes>, { From fdf10a7dc26bdaf669341eb095f3a801b595b9a4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 1 Nov 2024 21:58:59 +0100 Subject: [PATCH 221/239] test: make cargo t compile in codecs (#12261) --- crates/storage/codecs/src/alloy/mod.rs | 2 +- crates/storage/codecs/src/alloy/transaction/eip4844.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/storage/codecs/src/alloy/mod.rs b/crates/storage/codecs/src/alloy/mod.rs index f1bf6a00e694..697bac901e46 100644 --- a/crates/storage/codecs/src/alloy/mod.rs +++ b/crates/storage/codecs/src/alloy/mod.rs @@ -7,7 +7,7 @@ macro_rules! cond_mod { #[cfg(feature = "test-utils")] pub mod $mod_name; #[cfg(not(feature = "test-utils"))] - mod $mod_name; + pub(crate) mod $mod_name; )* }; } diff --git a/crates/storage/codecs/src/alloy/transaction/eip4844.rs b/crates/storage/codecs/src/alloy/transaction/eip4844.rs index 5ec36e06bf5c..c89e2b0785ba 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip4844.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip4844.rs @@ -106,7 +106,7 @@ impl<'a> arbitrary::Arbitrary<'a> for TxEip4844 { } } -#[cfg(any(test, feature = "test-utils"))] +#[cfg(feature = "test-utils")] fn serialize_placeholder(value: &Option<()>, serializer: S) -> Result where S: serde::Serializer, @@ -119,7 +119,7 @@ where } } -#[cfg(any(test, feature = "test-utils"))] +#[cfg(feature = "test-utils")] fn deserialize_placeholder<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, From a911104fe9ccb04c0946e0b11ac14337eec6b7e0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 1 Nov 2024 22:03:17 +0100 Subject: [PATCH 222/239] test: make cargo t compile in db-models (#12263) --- Cargo.lock | 2 +- crates/storage/db-models/Cargo.toml | 10 ++++++---- crates/storage/db-models/src/accounts.rs | 2 +- crates/storage/db-models/src/blocks.rs | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c469cd21e2f..6491a47eb4f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6920,7 +6920,7 @@ dependencies = [ "proptest", "proptest-arbitrary-interop", "reth-codecs", - "reth-primitives", + "reth-primitives-traits", "serde", "test-fuzz", ] diff --git a/crates/storage/db-models/Cargo.toml b/crates/storage/db-models/Cargo.toml index d5f773347b09..44b291959bae 100644 --- a/crates/storage/db-models/Cargo.toml +++ b/crates/storage/db-models/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] # reth reth-codecs.workspace = true -reth-primitives = { workspace = true, features = ["reth-codec"] } +reth-primitives-traits.workspace = true # ethereum alloy-primitives.workspace = true @@ -32,20 +32,22 @@ proptest = { workspace = true, optional = true } [dev-dependencies] # reth -reth-primitives = { workspace = true, features = ["arbitrary"] } +reth-primitives-traits = { workspace = true, features = ["arbitrary"] } reth-codecs.workspace = true +arbitrary = { workspace = true, features = ["derive"] } +proptest.workspace = true proptest-arbitrary-interop.workspace = true test-fuzz.workspace = true [features] test-utils = [ + "reth-primitives-traits/test-utils", "arbitrary", - "reth-primitives/test-utils", "reth-codecs/test-utils" ] arbitrary = [ - "reth-primitives/arbitrary", + "reth-primitives-traits/arbitrary", "dep:arbitrary", "dep:proptest", "alloy-primitives/arbitrary", diff --git a/crates/storage/db-models/src/accounts.rs b/crates/storage/db-models/src/accounts.rs index acfd45fe34e8..29a5cf305920 100644 --- a/crates/storage/db-models/src/accounts.rs +++ b/crates/storage/db-models/src/accounts.rs @@ -2,7 +2,7 @@ use reth_codecs::{add_arbitrary_tests, Compact}; use serde::Serialize; use alloy_primitives::{bytes::Buf, Address}; -use reth_primitives::Account; +use reth_primitives_traits::Account; /// Account as it is saved in the database. /// diff --git a/crates/storage/db-models/src/blocks.rs b/crates/storage/db-models/src/blocks.rs index 3e740a2e1aa8..b4399dc1e27e 100644 --- a/crates/storage/db-models/src/blocks.rs +++ b/crates/storage/db-models/src/blocks.rs @@ -2,7 +2,7 @@ use std::ops::Range; use alloy_primitives::TxNumber; use reth_codecs::{add_arbitrary_tests, Compact}; -use reth_primitives::Withdrawals; +use reth_primitives_traits::Withdrawals; use serde::{Deserialize, Serialize}; /// Total number of transactions. From c72f11cc851073518d670960b2930b74aa36b9b5 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Fri, 1 Nov 2024 15:34:14 -0600 Subject: [PATCH 223/239] renamed OptimismConsensusBuilder to OpConsensusBuilder (#12265) --- crates/optimism/node/src/node.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index b4a4d2730ab5..6a1f94cb7882 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -70,7 +70,7 @@ impl OptimismNode { OpPayloadBuilder, OpNetworkBuilder, OpExecutorBuilder, - OptimismConsensusBuilder, + OpConsensusBuilder, > where Node: FullNodeTypes< @@ -87,7 +87,7 @@ impl OptimismNode { disable_discovery_v4: !discovery_v4, }) .executor(OpExecutorBuilder::default()) - .consensus(OptimismConsensusBuilder::default()) + .consensus(OpConsensusBuilder::default()) } } @@ -103,7 +103,7 @@ where OpPayloadBuilder, OpNetworkBuilder, OpExecutorBuilder, - OptimismConsensusBuilder, + OpConsensusBuilder, >; type AddOns = OptimismAddOns< @@ -442,9 +442,9 @@ where /// A basic optimism consensus builder. #[derive(Debug, Default, Clone)] #[non_exhaustive] -pub struct OptimismConsensusBuilder; +pub struct OpConsensusBuilder; -impl ConsensusBuilder for OptimismConsensusBuilder +impl ConsensusBuilder for OpConsensusBuilder where Node: FullNodeTypes>, { From d8bbd36b2f081b3e7d4eb5c7cb90c4e8ab9343e0 Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Fri, 1 Nov 2024 14:35:47 -0700 Subject: [PATCH 224/239] feat: flashbots_validateBuilderSubmissionV4 (#12243) --- crates/ethereum/node/tests/e2e/rpc.rs | 90 +++++++++++++++++++++++++-- crates/rpc/rpc-api/src/lib.rs | 5 +- crates/rpc/rpc-api/src/validation.rs | 24 +++++++ crates/rpc/rpc/src/validation.rs | 38 ++++++++++- 4 files changed, 149 insertions(+), 8 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index c8b127b9b7f5..1f7ac32e048b 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -2,12 +2,15 @@ use crate::utils::eth_payload_attributes; use alloy_eips::{calc_next_block_base_fee, eip2718::Encodable2718}; use alloy_primitives::{Address, B256, U256}; use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder, SendableTx}; -use alloy_rpc_types_beacon::relay::{BidTrace, SignedBidSubmissionV3}; +use alloy_rpc_types_beacon::relay::{BidTrace, SignedBidSubmissionV3, SignedBidSubmissionV4}; use rand::{rngs::StdRng, Rng, SeedableRng}; -use reth::rpc::{ - api::BuilderBlockValidationRequestV3, - compat::engine::payload::block_to_payload_v3, - types::{engine::BlobsBundleV1, TransactionRequest}, +use reth::{ + payload::BuiltPayload, + rpc::{ + api::{BuilderBlockValidationRequestV3, BuilderBlockValidationRequestV4}, + compat::engine::payload::block_to_payload_v3, + types::{engine::BlobsBundleV1, TransactionRequest}, + }, }; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::setup_engine; @@ -115,7 +118,7 @@ async fn test_fee_history() -> eyre::Result<()> { } #[tokio::test] -async fn test_flashbots_validate() -> eyre::Result<()> { +async fn test_flashbots_validate_v3() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let chain_spec = Arc::new( @@ -187,3 +190,78 @@ async fn test_flashbots_validate() -> eyre::Result<()> { .is_err()); Ok(()) } + +#[tokio::test] +async fn test_flashbots_validate_v4() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .prague_activated() + .build(), + ); + + let (mut nodes, _tasks, wallet) = + setup_engine::(1, chain_spec.clone(), false, eth_payload_attributes).await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .with_recommended_fillers() + .wallet(EthereumWallet::new(wallet.gen().swap_remove(0))) + .on_http(node.rpc_url()); + + node.advance(100, |_| { + let provider = provider.clone(); + Box::pin(async move { + let SendableTx::Envelope(tx) = + provider.fill(TransactionRequest::default().to(Address::ZERO)).await.unwrap() + else { + unreachable!() + }; + + tx.encoded_2718().into() + }) + }) + .await?; + + let _ = provider.send_transaction(TransactionRequest::default().to(Address::ZERO)).await?; + let (payload, attrs) = node.new_payload().await?; + + let mut request = BuilderBlockValidationRequestV4 { + request: SignedBidSubmissionV4 { + message: BidTrace { + parent_hash: payload.block().parent_hash, + block_hash: payload.block().hash(), + gas_used: payload.block().gas_used, + gas_limit: payload.block().gas_limit, + ..Default::default() + }, + execution_payload: block_to_payload_v3(payload.block().clone()), + blobs_bundle: BlobsBundleV1::new([]), + execution_requests: payload.requests().unwrap_or_default().to_vec(), + signature: Default::default(), + }, + parent_beacon_block_root: attrs.parent_beacon_block_root.unwrap(), + registered_gas_limit: payload.block().gas_limit, + }; + + provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV4".into(), (&request,)) + .await + .expect("request should validate"); + + request.registered_gas_limit -= 1; + assert!(provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV4".into(), (&request,)) + .await + .is_err()); + request.registered_gas_limit += 1; + + request.request.execution_payload.payload_inner.payload_inner.state_root = B256::ZERO; + assert!(provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV4".into(), (&request,)) + .await + .is_err()); + Ok(()) +} diff --git a/crates/rpc/rpc-api/src/lib.rs b/crates/rpc/rpc-api/src/lib.rs index 63e6e54466dd..0a4fa9f660ed 100644 --- a/crates/rpc/rpc-api/src/lib.rs +++ b/crates/rpc/rpc-api/src/lib.rs @@ -46,7 +46,10 @@ pub mod servers { rpc::RpcApiServer, trace::TraceApiServer, txpool::TxPoolApiServer, - validation::{BlockSubmissionValidationApiServer, BuilderBlockValidationRequestV3}, + validation::{ + BlockSubmissionValidationApiServer, BuilderBlockValidationRequestV3, + BuilderBlockValidationRequestV4, + }, web3::Web3ApiServer, }; pub use reth_rpc_eth_api::{ diff --git a/crates/rpc/rpc-api/src/validation.rs b/crates/rpc/rpc-api/src/validation.rs index d8f55b668c9e..797eee7ae52c 100644 --- a/crates/rpc/rpc-api/src/validation.rs +++ b/crates/rpc/rpc-api/src/validation.rs @@ -3,6 +3,7 @@ use alloy_primitives::B256; use alloy_rpc_types_beacon::relay::{ BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, SignedBidSubmissionV3, + SignedBidSubmissionV4, }; use jsonrpsee::proc_macros::rpc; use serde::{Deserialize, Serialize}; @@ -24,6 +25,22 @@ pub struct BuilderBlockValidationRequestV3 { pub parent_beacon_block_root: B256, } +/// A Request to validate a [`SignedBidSubmissionV4`] +/// +/// +#[serde_as] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BuilderBlockValidationRequestV4 { + /// The request to be validated. + #[serde(flatten)] + pub request: SignedBidSubmissionV4, + /// The registered gas limit for the validation request. + #[serde_as(as = "DisplayFromStr")] + pub registered_gas_limit: u64, + /// The parent beacon block root for the validation request. + pub parent_beacon_block_root: B256, +} + /// Block validation rpc interface. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "flashbots"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "flashbots"))] @@ -48,4 +65,11 @@ pub trait BlockSubmissionValidationApi { &self, request: BuilderBlockValidationRequestV3, ) -> jsonrpsee::core::RpcResult<()>; + + /// A Request to validate a block submission. + #[method(name = "validateBuilderSubmissionV4")] + async fn validate_builder_submission_v4( + &self, + request: BuilderBlockValidationRequestV4, + ) -> jsonrpsee::core::RpcResult<()>; } diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index fe9d0eb4475d..1476180d431a 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -20,7 +20,10 @@ use reth_provider::{ StateProviderFactory, WithdrawalsProvider, }; use reth_revm::database::StateProviderDatabase; -use reth_rpc_api::{BlockSubmissionValidationApiServer, BuilderBlockValidationRequestV3}; +use reth_rpc_api::{ + BlockSubmissionValidationApiServer, BuilderBlockValidationRequestV3, + BuilderBlockValidationRequestV4, +}; use reth_rpc_eth_types::EthApiError; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use reth_trie::HashedPostState; @@ -415,4 +418,37 @@ where .map_err(|e| RethError::Other(e.into())) .to_rpc_result() } + + /// Validates a block submitted to the relay + async fn validate_builder_submission_v4( + &self, + request: BuilderBlockValidationRequestV4, + ) -> RpcResult<()> { + let block = self + .payload_validator + .ensure_well_formed_payload( + ExecutionPayload::V3(request.request.execution_payload), + ExecutionPayloadSidecar::v4( + CancunPayloadFields { + parent_beacon_block_root: request.parent_beacon_block_root, + versioned_hashes: self + .validate_blobs_bundle(request.request.blobs_bundle) + .map_err(|e| RethError::Other(e.into())) + .to_rpc_result()?, + }, + request.request.execution_requests.into(), + ), + ) + .to_rpc_result()? + .try_seal_with_senders() + .map_err(|_| EthApiError::InvalidTransactionSignature)?; + + self.validate_message_against_block( + block, + request.request.message, + request.registered_gas_limit, + ) + .map_err(|e| RethError::Other(e.into())) + .to_rpc_result() + } } From d7ead13bda6551958d5595cd20eac5a873debd48 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sat, 2 Nov 2024 11:17:31 +0400 Subject: [PATCH 225/239] fix: clippy lints (#12269) --- .github/workflows/lint.yml | 4 ++-- crates/chain-state/src/notifications.rs | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4723d8a4d57f..fa7b4f9f45c2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,10 +18,10 @@ jobs: matrix: include: - type: ethereum - args: --bin reth --workspace --locked + args: --bin reth --workspace --lib --examples --tests --benches --locked features: "ethereum asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" - type: optimism - args: --bin op-reth --workspace --locked + args: --bin op-reth --workspace --lib --examples --tests --benches --locked features: "optimism asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" - type: book args: --manifest-path book/sources/Cargo.toml --workspace --bins diff --git a/crates/chain-state/src/notifications.rs b/crates/chain-state/src/notifications.rs index 6e24bcbb4c8c..582e1d2a05d4 100644 --- a/crates/chain-state/src/notifications.rs +++ b/crates/chain-state/src/notifications.rs @@ -297,6 +297,7 @@ mod tests { block2.set_hash(block2_hash); // Create a receipt for the transaction in block1. + #[allow(clippy::needless_update)] let receipt1 = Receipt { tx_type: TxType::Legacy, cumulative_gas_used: 12345, @@ -346,6 +347,7 @@ mod tests { old_block1.block.body.transactions.push(TransactionSigned::default()); // Create a receipt for a transaction in the reverted block. + #[allow(clippy::needless_update)] let old_receipt = Receipt { tx_type: TxType::Legacy, cumulative_gas_used: 54321, @@ -368,6 +370,7 @@ mod tests { new_block1.block.body.transactions.push(TransactionSigned::default()); // Create a receipt for a transaction in the new committed block. + #[allow(clippy::needless_update)] let new_receipt = Receipt { tx_type: TxType::Legacy, cumulative_gas_used: 12345, From 962fa6685bd5763f8a162a1f9f9e67a92c9548a3 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Sat, 2 Nov 2024 01:41:11 -0600 Subject: [PATCH 226/239] Use Arc in SystemCaller (#12268) Co-authored-by: Matthias Seitz --- crates/engine/util/src/reorg.rs | 2 +- crates/ethereum/evm/src/execute.rs | 2 +- crates/evm/src/system_calls/mod.rs | 6 +++--- crates/optimism/evm/src/execute.rs | 2 +- crates/optimism/payload/src/builder.rs | 2 +- crates/payload/validator/src/lib.rs | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index 0d51d2dfab6f..69831389a658 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -303,7 +303,7 @@ where let mut evm = evm_config.evm_with_env(&mut state, env); // apply eip-4788 pre block contract call - let mut system_caller = SystemCaller::new(evm_config.clone(), chain_spec); + let mut system_caller = SystemCaller::new(evm_config.clone(), chain_spec.clone()); system_caller.apply_beacon_root_contract_call( reorg_target.timestamp, diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index f082a3a707ea..fa14e260d651 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -95,7 +95,7 @@ where { /// Creates a new [`EthExecutionStrategy`] pub fn new(state: State, chain_spec: Arc, evm_config: EvmConfig) -> Self { - let system_caller = SystemCaller::new(evm_config.clone(), (*chain_spec).clone()); + let system_caller = SystemCaller::new(evm_config.clone(), chain_spec.clone()); Self { state, chain_spec, evm_config, system_caller } } } diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index daaf1d1414f7..7fdb31d967dd 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -1,7 +1,7 @@ //! System contract call functions. use crate::ConfigureEvm; -use alloc::{boxed::Box, vec}; +use alloc::{boxed::Box, sync::Arc, vec}; use alloy_eips::eip7685::Requests; use alloy_primitives::Bytes; use core::fmt::Display; @@ -46,7 +46,7 @@ impl OnStateHook for NoopHook { #[allow(missing_debug_implementations)] pub struct SystemCaller { evm_config: EvmConfig, - chain_spec: Chainspec, + chain_spec: Arc, /// Optional hook to be called after each state change. hook: Option>, } @@ -54,7 +54,7 @@ pub struct SystemCaller { impl SystemCaller { /// Create a new system caller with the given EVM config, database, and chain spec, and creates /// the EVM with the given initialized config and block environment. - pub const fn new(evm_config: EvmConfig, chain_spec: Chainspec) -> Self { + pub const fn new(evm_config: EvmConfig, chain_spec: Arc) -> Self { Self { evm_config, chain_spec, hook: None } } diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 9c5db9d4b614..c2b006144369 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -90,7 +90,7 @@ where { /// Creates a new [`OpExecutionStrategy`] pub fn new(state: State, chain_spec: Arc, evm_config: EvmConfig) -> Self { - let system_caller = SystemCaller::new(evm_config.clone(), (*chain_spec).clone()); + let system_caller = SystemCaller::new(evm_config.clone(), chain_spec.clone()); Self { state, chain_spec, evm_config, system_caller } } } diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index cae2d34bd499..0cf45835a239 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -197,7 +197,7 @@ where chain_spec.is_regolith_active_at_timestamp(attributes.payload_attributes.timestamp); // apply eip-4788 pre block contract call - let mut system_caller = SystemCaller::new(evm_config.clone(), &chain_spec); + let mut system_caller = SystemCaller::new(evm_config.clone(), chain_spec.clone()); system_caller .pre_block_beacon_root_contract_call( diff --git a/crates/payload/validator/src/lib.rs b/crates/payload/validator/src/lib.rs index 38e53bac42a5..e74b5f48d40f 100644 --- a/crates/payload/validator/src/lib.rs +++ b/crates/payload/validator/src/lib.rs @@ -31,7 +31,7 @@ impl ExecutionPayloadValidator { /// Returns the chain spec used by the validator. #[inline] - pub fn chain_spec(&self) -> &ChainSpec { + pub const fn chain_spec(&self) -> &Arc { &self.chain_spec } } From 9593f4d08d25018eac0173fadff8768ce7d241b7 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Sat, 2 Nov 2024 02:42:42 -0600 Subject: [PATCH 227/239] renamed OptimismGenesisInfo to OpGenesisInfo (#12270) --- crates/optimism/chainspec/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 8ad36c66afb0..3248625d604c 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -351,7 +351,7 @@ impl OptimismHardforks for OpChainSpec {} impl From for OpChainSpec { fn from(genesis: Genesis) -> Self { use reth_optimism_forks::OptimismHardfork; - let optimism_genesis_info = OptimismGenesisInfo::extract_from(&genesis); + let optimism_genesis_info = OpGenesisInfo::extract_from(&genesis); let genesis_info = optimism_genesis_info.optimism_chain_info.genesis_info.unwrap_or_default(); @@ -441,12 +441,12 @@ impl From for OpChainSpec { } #[derive(Default, Debug)] -struct OptimismGenesisInfo { +struct OpGenesisInfo { optimism_chain_info: op_alloy_rpc_types::genesis::OpChainInfo, base_fee_params: BaseFeeParamsKind, } -impl OptimismGenesisInfo { +impl OpGenesisInfo { fn extract_from(genesis: &Genesis) -> Self { let mut info = Self { optimism_chain_info: op_alloy_rpc_types::genesis::OpChainInfo::extract_from( From dbdf60ba119c04df52dddba0e1e9a8f2843b063f Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Sat, 2 Nov 2024 03:22:58 -0600 Subject: [PATCH 228/239] renamed OptimismBeaconConsensus to OpBeaconConsensus (#12271) --- crates/optimism/consensus/src/lib.rs | 8 ++++---- crates/optimism/node/src/node.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/optimism/consensus/src/lib.rs b/crates/optimism/consensus/src/lib.rs index 16c1d5d37d7c..bf1428815d09 100644 --- a/crates/optimism/consensus/src/lib.rs +++ b/crates/optimism/consensus/src/lib.rs @@ -34,19 +34,19 @@ pub use validation::validate_block_post_execution; /// /// Provides basic checks as outlined in the execution specs. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct OptimismBeaconConsensus { +pub struct OpBeaconConsensus { /// Configuration chain_spec: Arc, } -impl OptimismBeaconConsensus { - /// Create a new instance of [`OptimismBeaconConsensus`] +impl OpBeaconConsensus { + /// Create a new instance of [`OpBeaconConsensus`] pub const fn new(chain_spec: Arc) -> Self { Self { chain_spec } } } -impl Consensus for OptimismBeaconConsensus { +impl Consensus for OpBeaconConsensus { fn validate_header(&self, header: &SealedHeader) -> Result<(), ConsensusError> { validate_header_gas(header)?; validate_header_base_fee(header, &self.chain_spec) diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 6a1f94cb7882..87266767da22 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -19,7 +19,7 @@ use reth_node_builder::{ BuilderContext, Node, NodeAdapter, NodeComponentsBuilder, PayloadBuilderConfig, }; use reth_optimism_chainspec::OpChainSpec; -use reth_optimism_consensus::OptimismBeaconConsensus; +use reth_optimism_consensus::OpBeaconConsensus; use reth_optimism_evm::{OpExecutionStrategyFactory, OptimismEvmConfig}; use reth_optimism_rpc::OpEthApi; use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; @@ -454,7 +454,7 @@ where if ctx.is_dev() { Ok(Arc::new(reth_auto_seal_consensus::AutoSealConsensus::new(ctx.chain_spec()))) } else { - Ok(Arc::new(OptimismBeaconConsensus::new(ctx.chain_spec()))) + Ok(Arc::new(OpBeaconConsensus::new(ctx.chain_spec()))) } } } From c74d2a06f2d84a96f779e62293ef90b3c984c56a Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Sat, 2 Nov 2024 11:30:17 +0100 Subject: [PATCH 229/239] chore(ci): unpin kurtosis (#12272) --- .github/workflows/kurtosis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index 43f5c3605abe..74d26dbd3eea 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -79,7 +79,6 @@ jobs: - name: Run kurtosis uses: ethpandaops/kurtosis-assertoor-github-action@v1 with: - kurtosis_version: 1.3.1 ethereum_package_args: '.github/assets/kurtosis_network_params.yaml' notify-on-error: From f2de55d8fe4988e8580c18a8302d642ce87a031d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 2 Nov 2024 13:16:08 +0100 Subject: [PATCH 230/239] chore: make some fields owned (#12274) --- crates/node/api/src/node.rs | 8 ++++---- crates/node/builder/src/launch/engine.rs | 6 +++--- crates/node/builder/src/launch/mod.rs | 6 +++--- crates/node/builder/src/rpc.rs | 10 ++++++---- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/crates/node/api/src/node.rs b/crates/node/api/src/node.rs index 3173fd2b3985..b016e01c295f 100644 --- a/crates/node/api/src/node.rs +++ b/crates/node/api/src/node.rs @@ -91,17 +91,17 @@ pub trait FullNodeComponents: FullNodeTypes + Clone + 'static { } /// Context passed to [`NodeAddOns::launch_add_ons`], -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct AddOnsContext<'a, N: FullNodeComponents> { /// Node with all configured components. - pub node: &'a N, + pub node: N, /// Node configuration. pub config: &'a NodeConfig<::ChainSpec>, /// Handle to the beacon consensus engine. pub beacon_engine_handle: - &'a BeaconConsensusEngineHandle<::Engine>, + BeaconConsensusEngineHandle<::Engine>, /// JWT secret for the node. - pub jwt_secret: &'a JwtSecret, + pub jwt_secret: JwtSecret, } /// Customizable node add-on types. diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 3de651cdcd0b..85401b8b9587 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -286,10 +286,10 @@ where let jwt_secret = ctx.auth_jwt_secret()?; let add_ons_ctx = AddOnsContext { - node: ctx.node_adapter(), + node: ctx.node_adapter().clone(), config: ctx.node_config(), - beacon_engine_handle: &beacon_engine_handle, - jwt_secret: &jwt_secret, + beacon_engine_handle, + jwt_secret, }; let RpcHandle { rpc_server_handles, rpc_registry } = diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index 50438e79d2b6..a623a7a9f23c 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -329,10 +329,10 @@ where let jwt_secret = ctx.auth_jwt_secret()?; let add_ons_ctx = AddOnsContext { - node: ctx.node_adapter(), + node: ctx.node_adapter().clone(), config: ctx.node_config(), - beacon_engine_handle: &beacon_engine_handle, - jwt_secret: &jwt_secret, + beacon_engine_handle, + jwt_secret, }; let RpcHandle { rpc_server_handles, rpc_registry } = diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 8819aa4ac4f7..8af1527cbcd8 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -409,9 +409,11 @@ where type Handle = RpcHandle; async fn launch_add_ons(self, ctx: AddOnsContext<'_, N>) -> eyre::Result { - let AddOnsContext { node, config, beacon_engine_handle, jwt_secret } = ctx; let Self { eth_api_builder, engine_validator_builder, hooks, _pd: _ } = self; + let engine_validator = engine_validator_builder.build(&ctx).await?; + let AddOnsContext { node, config, beacon_engine_handle, jwt_secret } = ctx; + let client = ClientVersionV1 { code: CLIENT_CODE, name: NAME_CLIENT.to_string(), @@ -422,17 +424,17 @@ where let engine_api = EngineApi::new( node.provider().clone(), config.chain.clone(), - beacon_engine_handle.clone(), + beacon_engine_handle, node.payload_builder().clone().into(), node.pool().clone(), Box::new(node.task_executor().clone()), client, EngineCapabilities::default(), - engine_validator_builder.build(&ctx).await?, + engine_validator, ); info!(target: "reth::cli", "Engine API handler initialized"); - let auth_config = config.rpc.auth_server_config(*jwt_secret)?; + let auth_config = config.rpc.auth_server_config(jwt_secret)?; let module_config = config.rpc.transport_rpc_module_config(); debug!(target: "reth::cli", http=?module_config.http(), ws=?module_config.ws(), "Using RPC module config"); From 2aba40b17c57c6a6f6bd68f1fa32279d8e7b0d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Sat, 2 Nov 2024 21:22:17 +0700 Subject: [PATCH 231/239] feat: add `CachedReads::extend` (#12277) Co-authored-by: Matthias Seitz --- crates/revm/src/cached.rs | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/crates/revm/src/cached.rs b/crates/revm/src/cached.rs index 807b163c4cbc..88a41e1d8957 100644 --- a/crates/revm/src/cached.rs +++ b/crates/revm/src/cached.rs @@ -57,6 +57,15 @@ impl CachedReads { ) { self.accounts.insert(address, CachedAccount { info: Some(info), storage }); } + + /// Extends current cache with entries from another [`CachedReads`] instance. + /// + /// Note: It is expected that both instances are based on the exact same state. + pub fn extend(&mut self, other: Self) { + self.accounts.extend(other.accounts); + self.contracts.extend(other.contracts); + self.block_hashes.extend(other.block_hashes); + } } /// A [Database] that caches reads inside [`CachedReads`]. @@ -184,3 +193,57 @@ impl CachedAccount { Self { info, storage: HashMap::default() } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extend_with_two_cached_reads() { + // Setup test data + let hash1 = B256::from_slice(&[1u8; 32]); + let hash2 = B256::from_slice(&[2u8; 32]); + let address1 = Address::from_slice(&[1u8; 20]); + let address2 = Address::from_slice(&[2u8; 20]); + + // Create primary cache + let mut primary = { + let mut cache = CachedReads::default(); + cache.accounts.insert(address1, CachedAccount::new(Some(AccountInfo::default()))); + cache.contracts.insert(hash1, Bytecode::default()); + cache.block_hashes.insert(1, hash1); + cache + }; + + // Create additional cache + let additional = { + let mut cache = CachedReads::default(); + cache.accounts.insert(address2, CachedAccount::new(Some(AccountInfo::default()))); + cache.contracts.insert(hash2, Bytecode::default()); + cache.block_hashes.insert(2, hash2); + cache + }; + + // Extending primary with additional cache + primary.extend(additional); + + // Verify the combined state + assert!( + primary.accounts.len() == 2 && + primary.contracts.len() == 2 && + primary.block_hashes.len() == 2, + "All maps should contain 2 entries" + ); + + // Verify specific entries + assert!( + primary.accounts.contains_key(&address1) && + primary.accounts.contains_key(&address2) && + primary.contracts.contains_key(&hash1) && + primary.contracts.contains_key(&hash2) && + primary.block_hashes.get(&1) == Some(&hash1) && + primary.block_hashes.get(&2) == Some(&hash2), + "All expected entries should be present" + ); + } +} From af38a1a2d0631d0f37c4da29d6aa8f86f7dec0d9 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Sat, 2 Nov 2024 12:04:25 -0600 Subject: [PATCH 232/239] renamed OptimismPayloadTypes to OpPayloadTypes (#12281) --- crates/optimism/node/src/engine.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index 0f48dc347062..af517b00ea0e 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -22,7 +22,7 @@ use reth_optimism_payload_builder::{ /// The types used in the optimism beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct OptimismEngineTypes { +pub struct OptimismEngineTypes { _marker: std::marker::PhantomData, } @@ -48,9 +48,9 @@ where /// A default payload type for [`OptimismEngineTypes`] #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct OptimismPayloadTypes; +pub struct OpPayloadTypes; -impl PayloadTypes for OptimismPayloadTypes { +impl PayloadTypes for OpPayloadTypes { type BuiltPayload = OpBuiltPayload; type PayloadAttributes = OpPayloadAttributes; type PayloadBuilderAttributes = OpPayloadBuilderAttributes; From adff5a9429e2f5f65e0b183526a554664161b4f4 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sat, 2 Nov 2024 22:14:10 +0400 Subject: [PATCH 233/239] feat: remove AnyNetwork usage (#12280) --- Cargo.lock | 5 - crates/optimism/rpc/src/eth/receipt.rs | 96 +++------ crates/optimism/rpc/src/eth/transaction.rs | 2 +- crates/rpc/rpc-builder/Cargo.toml | 5 - crates/rpc/rpc-builder/src/lib.rs | 9 +- crates/rpc/rpc-builder/tests/it/http.rs | 7 +- crates/rpc/rpc-builder/tests/it/middleware.rs | 2 +- crates/rpc/rpc-eth-api/src/types.rs | 15 +- crates/rpc/rpc-eth-types/Cargo.toml | 1 - crates/rpc/rpc-eth-types/src/lib.rs | 2 +- crates/rpc/rpc-eth-types/src/receipt.rs | 186 ++++++++++-------- crates/rpc/rpc-types-compat/Cargo.toml | 3 +- .../rpc-types-compat/src/transaction/mod.rs | 25 +-- crates/rpc/rpc/src/eth/core.rs | 5 +- crates/rpc/rpc/src/eth/helpers/block.rs | 10 +- crates/rpc/rpc/src/eth/helpers/receipt.rs | 5 +- crates/rpc/rpc/src/eth/helpers/types.rs | 60 +++--- crates/rpc/rpc/src/eth/pubsub.rs | 19 +- 18 files changed, 183 insertions(+), 274 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6491a47eb4f8..8aab2c0a8ebc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8725,13 +8725,10 @@ dependencies = [ name = "reth-rpc-builder" version = "1.1.0" dependencies = [ - "alloy-network", "alloy-primitives", - "alloy-rpc-types", "alloy-rpc-types-engine", "alloy-rpc-types-eth", "alloy-rpc-types-trace", - "alloy-serde", "clap", "http", "jsonrpsee", @@ -8860,7 +8857,6 @@ dependencies = [ "alloy-primitives", "alloy-rpc-types", "alloy-rpc-types-eth", - "alloy-serde", "alloy-sol-types", "derive_more 1.0.0", "futures", @@ -8936,7 +8932,6 @@ dependencies = [ "alloy-rpc-types", "alloy-rpc-types-engine", "alloy-rpc-types-eth", - "alloy-serde", "reth-primitives", "reth-trie-common", "serde", diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index 2734fb5458c8..f8e6b7fc21ec 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -1,7 +1,7 @@ //! Loads and formats OP receipt RPC response. use alloy_eips::eip2718::Encodable2718; -use alloy_rpc_types::{AnyReceiptEnvelope, Log, TransactionReceipt}; +use alloy_rpc_types::{Log, TransactionReceipt}; use op_alloy_consensus::{ DepositTransaction, OpDepositReceipt, OpDepositReceiptWithBloom, OpReceiptEnvelope, }; @@ -13,7 +13,7 @@ use reth_optimism_forks::OptimismHardforks; use reth_primitives::{Receipt, TransactionMeta, TransactionSigned, TxType}; use reth_provider::ChainSpecProvider; use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcReceipt}; -use reth_rpc_eth_types::{EthApiError, ReceiptBuilder}; +use reth_rpc_eth_types::{receipt::build_receipt, EthApiError}; use crate::{OpEthApi, OpEthApiError}; @@ -172,9 +172,7 @@ impl OpReceiptFieldsBuilder { #[derive(Debug)] pub struct OpReceiptBuilder { /// Core receipt, has all the fields of an L1 receipt and is the basis for the OP receipt. - pub core_receipt: TransactionReceipt>, - /// Transaction type. - pub tx_type: TxType, + pub core_receipt: TransactionReceipt>, /// Additional OP receipt fields. pub op_receipt_fields: OpTransactionReceiptFields, } @@ -189,11 +187,29 @@ impl OpReceiptBuilder { all_receipts: &[Receipt], l1_block_info: revm::L1BlockInfo, ) -> Result { - let ReceiptBuilder { base: core_receipt, .. } = - ReceiptBuilder::new(transaction, meta, receipt, all_receipts) - .map_err(OpEthApiError::Eth)?; - - let tx_type = transaction.tx_type(); + let core_receipt = + build_receipt(transaction, meta, receipt, all_receipts, |receipt_with_bloom| { + match receipt.tx_type { + TxType::Legacy => OpReceiptEnvelope::::Legacy(receipt_with_bloom), + TxType::Eip2930 => OpReceiptEnvelope::::Eip2930(receipt_with_bloom), + TxType::Eip1559 => OpReceiptEnvelope::::Eip1559(receipt_with_bloom), + TxType::Eip4844 => { + // TODO: unreachable + OpReceiptEnvelope::::Eip1559(receipt_with_bloom) + } + TxType::Eip7702 => OpReceiptEnvelope::::Eip7702(receipt_with_bloom), + TxType::Deposit => { + OpReceiptEnvelope::::Deposit(OpDepositReceiptWithBloom:: { + receipt: OpDepositReceipt:: { + inner: receipt_with_bloom.receipt, + deposit_nonce: receipt.deposit_nonce, + deposit_receipt_version: receipt.deposit_receipt_version, + }, + logs_bloom: receipt_with_bloom.logs_bloom, + }) + } + } + })?; let op_receipt_fields = OpReceiptFieldsBuilder::default() .l1_block_info(chain_spec, transaction, l1_block_info)? @@ -201,69 +217,15 @@ impl OpReceiptBuilder { .deposit_version(receipt.deposit_receipt_version) .build(); - Ok(Self { core_receipt, tx_type, op_receipt_fields }) + Ok(Self { core_receipt, op_receipt_fields }) } /// Builds [`OpTransactionReceipt`] by combing core (l1) receipt fields and additional OP /// receipt fields. pub fn build(self) -> OpTransactionReceipt { - let Self { core_receipt, tx_type, op_receipt_fields } = self; - - let OpTransactionReceiptFields { l1_block_info, deposit_nonce, deposit_receipt_version } = - op_receipt_fields; - - let TransactionReceipt { - inner: AnyReceiptEnvelope { inner: receipt_with_bloom, .. }, - transaction_hash, - transaction_index, - block_hash, - block_number, - gas_used, - effective_gas_price, - blob_gas_used, - blob_gas_price, - from, - to, - contract_address, - authorization_list, - } = core_receipt; - - let inner = match tx_type { - TxType::Legacy => OpReceiptEnvelope::::Legacy(receipt_with_bloom), - TxType::Eip2930 => OpReceiptEnvelope::::Eip2930(receipt_with_bloom), - TxType::Eip1559 => OpReceiptEnvelope::::Eip1559(receipt_with_bloom), - TxType::Eip4844 => { - // TODO: unreachable - OpReceiptEnvelope::::Eip1559(receipt_with_bloom) - } - TxType::Eip7702 => OpReceiptEnvelope::::Eip7702(receipt_with_bloom), - TxType::Deposit => { - OpReceiptEnvelope::::Deposit(OpDepositReceiptWithBloom:: { - receipt: OpDepositReceipt:: { - inner: receipt_with_bloom.receipt, - deposit_nonce, - deposit_receipt_version, - }, - logs_bloom: receipt_with_bloom.logs_bloom, - }) - } - }; + let Self { core_receipt: inner, op_receipt_fields } = self; - let inner = TransactionReceipt::> { - inner, - transaction_hash, - transaction_index, - block_hash, - block_number, - gas_used, - effective_gas_price, - blob_gas_used, - blob_gas_price, - from, - to, - contract_address, - authorization_list, - }; + let OpTransactionReceiptFields { l1_block_info, .. } = op_receipt_fields; OpTransactionReceipt { inner, l1_block_info } } diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 3994afe19841..6b5954391d9f 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -86,7 +86,7 @@ where let signed_tx = tx.clone().into_signed(); let hash = tx.hash; - let mut inner = EthTxBuilder.fill(tx, tx_info).inner; + let mut inner = EthTxBuilder.fill(tx, tx_info); if signed_tx.is_deposit() { inner.gas_price = Some(signed_tx.max_fee_per_gas()) diff --git a/crates/rpc/rpc-builder/Cargo.toml b/crates/rpc/rpc-builder/Cargo.toml index 711e44381330..2d10dabf8af8 100644 --- a/crates/rpc/rpc-builder/Cargo.toml +++ b/crates/rpc/rpc-builder/Cargo.toml @@ -31,11 +31,6 @@ reth-evm.workspace = true reth-engine-primitives.workspace = true reth-primitives.workspace = true -# ethereum -alloy-network.workspace = true -alloy-rpc-types.workspace = true -alloy-serde.workspace = true - # rpc/net jsonrpsee = { workspace = true, features = ["server"] } tower-http = { workspace = true, features = ["full"] } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 696d35014306..385b92af3d0b 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1094,14 +1094,7 @@ where /// If called outside of the tokio runtime. See also [`Self::eth_api`] pub fn register_ots(&mut self) -> &mut Self where - EthApi: TraceExt - + EthTransactions< - NetworkTypes: alloy_network::Network< - TransactionResponse = alloy_serde::WithOtherFields< - alloy_rpc_types::Transaction, - >, - >, - >, + EthApi: TraceExt + EthTransactions, { let otterscan_api = self.otterscan_api(); self.modules.insert(RethRpcModule::Ots, otterscan_api.into_rpc().into()); diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 7a8093c5062b..ed9ef56d62bf 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -3,11 +3,10 @@ use crate::utils::{launch_http, launch_http_ws, launch_ws}; use alloy_primitives::{hex_literal::hex, Address, Bytes, TxHash, B256, B64, U256, U64}; -use alloy_rpc_types::{ - Block, FeeHistory, Filter, Index, Log, PendingTransactionFilterKind, SyncStatus, Transaction, - TransactionReceipt, +use alloy_rpc_types_eth::{ + transaction::TransactionRequest, Block, FeeHistory, Filter, Index, Log, + PendingTransactionFilterKind, SyncStatus, Transaction, TransactionReceipt, }; -use alloy_rpc_types_eth::transaction::TransactionRequest; use alloy_rpc_types_trace::filter::TraceFilter; use jsonrpsee::{ core::{ diff --git a/crates/rpc/rpc-builder/tests/it/middleware.rs b/crates/rpc/rpc-builder/tests/it/middleware.rs index bcc26dcad895..0e2186e56eeb 100644 --- a/crates/rpc/rpc-builder/tests/it/middleware.rs +++ b/crates/rpc/rpc-builder/tests/it/middleware.rs @@ -1,5 +1,5 @@ use crate::utils::{test_address, test_rpc_builder}; -use alloy_rpc_types::{Block, Receipt, Transaction}; +use alloy_rpc_types_eth::{Block, Receipt, Transaction}; use jsonrpsee::{ server::{middleware::rpc::RpcServiceT, RpcServiceBuilder}, types::Request, diff --git a/crates/rpc/rpc-eth-api/src/types.rs b/crates/rpc/rpc-eth-api/src/types.rs index 653730ed3c99..1d176dd1e86e 100644 --- a/crates/rpc/rpc-eth-api/src/types.rs +++ b/crates/rpc/rpc-eth-api/src/types.rs @@ -2,16 +2,15 @@ use std::{error::Error, fmt}; -use alloy_network::{AnyNetwork, Network}; +use alloy_network::Network; use alloy_rpc_types::Block; -use reth_rpc_eth_types::EthApiError; use reth_rpc_types_compat::TransactionCompat; use crate::{AsEthApiError, FromEthApiError, FromEvmError}; /// Network specific `eth` API types. pub trait EthApiTypes: Send + Sync + Clone { - /// Extension of [`EthApiError`], with network specific errors. + /// Extension of [`FromEthApiError`], with network specific errors. type Error: Into> + FromEthApiError + AsEthApiError @@ -28,16 +27,6 @@ pub trait EthApiTypes: Send + Sync + Clone { fn tx_resp_builder(&self) -> &Self::TransactionCompat; } -impl EthApiTypes for () { - type Error = EthApiError; - type NetworkTypes = AnyNetwork; - type TransactionCompat = (); - - fn tx_resp_builder(&self) -> &Self::TransactionCompat { - self - } -} - /// Adapter for network specific transaction type. pub type RpcTransaction = ::TransactionResponse; diff --git a/crates/rpc/rpc-eth-types/Cargo.toml b/crates/rpc/rpc-eth-types/Cargo.toml index 46a0d7b5c323..80901bcf8121 100644 --- a/crates/rpc/rpc-eth-types/Cargo.toml +++ b/crates/rpc/rpc-eth-types/Cargo.toml @@ -36,7 +36,6 @@ revm.workspace = true revm-inspectors.workspace = true revm-primitives = { workspace = true, features = ["dev"] } alloy-rpc-types.workspace = true -alloy-serde.workspace = true alloy-eips.workspace = true # rpc diff --git a/crates/rpc/rpc-eth-types/src/lib.rs b/crates/rpc/rpc-eth-types/src/lib.rs index fa36dae4c881..03c23dc34565 100644 --- a/crates/rpc/rpc-eth-types/src/lib.rs +++ b/crates/rpc/rpc-eth-types/src/lib.rs @@ -37,5 +37,5 @@ pub use gas_oracle::{ }; pub use id_provider::EthSubscriptionIdProvider; pub use pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; -pub use receipt::ReceiptBuilder; +pub use receipt::EthReceiptBuilder; pub use transaction::TransactionSource; diff --git a/crates/rpc/rpc-eth-types/src/receipt.rs b/crates/rpc/rpc-eth-types/src/receipt.rs index 198ca79aa2ac..0734b547ec8e 100644 --- a/crates/rpc/rpc-eth-types/src/receipt.rs +++ b/crates/rpc/rpc-eth-types/src/receipt.rs @@ -1,24 +1,101 @@ //! RPC receipt response builder, extends a layer one receipt with layer two data. -use alloy_consensus::Transaction; +use alloy_consensus::{ReceiptEnvelope, Transaction}; use alloy_primitives::{Address, TxKind}; -use alloy_rpc_types::{AnyReceiptEnvelope, Log, ReceiptWithBloom, TransactionReceipt}; -use alloy_serde::OtherFields; -use reth_primitives::{Receipt, TransactionMeta, TransactionSigned}; +use alloy_rpc_types::{Log, ReceiptWithBloom, TransactionReceipt}; +use reth_primitives::{Receipt, TransactionMeta, TransactionSigned, TxType}; use revm_primitives::calc_blob_gasprice; use super::{EthApiError, EthResult}; +/// Builds an [`TransactionReceipt`] obtaining the inner receipt envelope from the given closure. +pub fn build_receipt( + transaction: &TransactionSigned, + meta: TransactionMeta, + receipt: &Receipt, + all_receipts: &[Receipt], + build_envelope: impl FnOnce(ReceiptWithBloom) -> T, +) -> EthResult> { + // Note: we assume this transaction is valid, because it's mined (or part of pending block) + // and we don't need to check for pre EIP-2 + let from = + transaction.recover_signer_unchecked().ok_or(EthApiError::InvalidTransactionSignature)?; + + // get the previous transaction cumulative gas used + let gas_used = if meta.index == 0 { + receipt.cumulative_gas_used + } else { + let prev_tx_idx = (meta.index - 1) as usize; + all_receipts + .get(prev_tx_idx) + .map(|prev_receipt| receipt.cumulative_gas_used - prev_receipt.cumulative_gas_used) + .unwrap_or_default() + }; + + let blob_gas_used = transaction.transaction.blob_gas_used(); + // Blob gas price should only be present if the transaction is a blob transaction + let blob_gas_price = blob_gas_used.and_then(|_| meta.excess_blob_gas.map(calc_blob_gasprice)); + let logs_bloom = receipt.bloom_slow(); + + // get number of logs in the block + let mut num_logs = 0; + for prev_receipt in all_receipts.iter().take(meta.index as usize) { + num_logs += prev_receipt.logs.len(); + } + + let logs: Vec = receipt + .logs + .iter() + .enumerate() + .map(|(tx_log_idx, log)| Log { + inner: log.clone(), + block_hash: Some(meta.block_hash), + block_number: Some(meta.block_number), + block_timestamp: Some(meta.timestamp), + transaction_hash: Some(meta.tx_hash), + transaction_index: Some(meta.index), + log_index: Some((num_logs + tx_log_idx) as u64), + removed: false, + }) + .collect(); + + let rpc_receipt = alloy_rpc_types::Receipt { + status: receipt.success.into(), + cumulative_gas_used: receipt.cumulative_gas_used as u128, + logs, + }; + + let (contract_address, to) = match transaction.transaction.kind() { + TxKind::Create => (Some(from.create(transaction.transaction.nonce())), None), + TxKind::Call(addr) => (None, Some(Address(*addr))), + }; + + Ok(TransactionReceipt { + inner: build_envelope(ReceiptWithBloom { receipt: rpc_receipt, logs_bloom }), + transaction_hash: meta.tx_hash, + transaction_index: Some(meta.index), + block_hash: Some(meta.block_hash), + block_number: Some(meta.block_number), + from, + to, + gas_used: gas_used as u128, + contract_address, + effective_gas_price: transaction.effective_gas_price(meta.base_fee), + // EIP-4844 fields + blob_gas_price, + blob_gas_used: blob_gas_used.map(u128::from), + authorization_list: transaction.authorization_list().map(|l| l.to_vec()), + }) +} + /// Receipt response builder. #[derive(Debug)] -pub struct ReceiptBuilder { +pub struct EthReceiptBuilder { /// The base response body, contains L1 fields. - pub base: TransactionReceipt>, - /// Additional L2 fields. - pub other: OtherFields, + pub base: TransactionReceipt, } -impl ReceiptBuilder { +impl EthReceiptBuilder { /// Returns a new builder with the base response body (L1 fields) set. /// /// Note: This requires _all_ block receipts because we need to calculate the gas used by the @@ -29,88 +106,23 @@ impl ReceiptBuilder { receipt: &Receipt, all_receipts: &[Receipt], ) -> EthResult { - // Note: we assume this transaction is valid, because it's mined (or part of pending block) - // and we don't need to check for pre EIP-2 - let from = transaction - .recover_signer_unchecked() - .ok_or(EthApiError::InvalidTransactionSignature)?; - - // get the previous transaction cumulative gas used - let gas_used = if meta.index == 0 { - receipt.cumulative_gas_used - } else { - let prev_tx_idx = (meta.index - 1) as usize; - all_receipts - .get(prev_tx_idx) - .map(|prev_receipt| receipt.cumulative_gas_used - prev_receipt.cumulative_gas_used) - .unwrap_or_default() - }; - - let blob_gas_used = transaction.transaction.blob_gas_used(); - // Blob gas price should only be present if the transaction is a blob transaction - let blob_gas_price = - blob_gas_used.and_then(|_| meta.excess_blob_gas.map(calc_blob_gasprice)); - let logs_bloom = receipt.bloom_slow(); - - // get number of logs in the block - let mut num_logs = 0; - for prev_receipt in all_receipts.iter().take(meta.index as usize) { - num_logs += prev_receipt.logs.len(); - } - - let logs: Vec = receipt - .logs - .iter() - .enumerate() - .map(|(tx_log_idx, log)| Log { - inner: log.clone(), - block_hash: Some(meta.block_hash), - block_number: Some(meta.block_number), - block_timestamp: Some(meta.timestamp), - transaction_hash: Some(meta.tx_hash), - transaction_index: Some(meta.index), - log_index: Some((num_logs + tx_log_idx) as u64), - removed: false, - }) - .collect(); - - let rpc_receipt = alloy_rpc_types::Receipt { - status: receipt.success.into(), - cumulative_gas_used: receipt.cumulative_gas_used as u128, - logs, - }; - - let (contract_address, to) = match transaction.transaction.kind() { - TxKind::Create => (Some(from.create(transaction.transaction.nonce())), None), - TxKind::Call(addr) => (None, Some(Address(*addr))), - }; - - #[allow(clippy::needless_update)] - let base = TransactionReceipt { - inner: AnyReceiptEnvelope { - inner: ReceiptWithBloom { receipt: rpc_receipt, logs_bloom }, - r#type: transaction.transaction.tx_type().into(), - }, - transaction_hash: meta.tx_hash, - transaction_index: Some(meta.index), - block_hash: Some(meta.block_hash), - block_number: Some(meta.block_number), - from, - to, - gas_used: gas_used as u128, - contract_address, - effective_gas_price: transaction.effective_gas_price(meta.base_fee), - // EIP-4844 fields - blob_gas_price, - blob_gas_used: blob_gas_used.map(u128::from), - authorization_list: transaction.authorization_list().map(|l| l.to_vec()), - }; + let base = build_receipt(transaction, meta, receipt, all_receipts, |receipt_with_bloom| { + match receipt.tx_type { + TxType::Legacy => ReceiptEnvelope::Legacy(receipt_with_bloom), + TxType::Eip2930 => ReceiptEnvelope::Eip2930(receipt_with_bloom), + TxType::Eip1559 => ReceiptEnvelope::Eip1559(receipt_with_bloom), + TxType::Eip4844 => ReceiptEnvelope::Eip4844(receipt_with_bloom), + TxType::Eip7702 => ReceiptEnvelope::Eip7702(receipt_with_bloom), + #[allow(unreachable_patterns)] + _ => unreachable!(), + } + })?; - Ok(Self { base, other: Default::default() }) + Ok(Self { base }) } /// Builds a receipt response from the base response body, and any set additional fields. - pub fn build(self) -> TransactionReceipt> { + pub fn build(self) -> TransactionReceipt { self.base } } diff --git a/crates/rpc/rpc-types-compat/Cargo.toml b/crates/rpc/rpc-types-compat/Cargo.toml index 7d5eac9dbb91..b9a9e5f03614 100644 --- a/crates/rpc/rpc-types-compat/Cargo.toml +++ b/crates/rpc/rpc-types-compat/Cargo.toml @@ -22,7 +22,6 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true alloy-rpc-types.workspace = true alloy-rpc-types-eth = { workspace = true, default-features = false, features = ["serde"] } -alloy-serde.workspace = true alloy-rpc-types-engine.workspace = true alloy-consensus.workspace = true @@ -30,4 +29,4 @@ alloy-consensus.workspace = true serde.workspace = true [dev-dependencies] -serde_json.workspace = true \ No newline at end of file +serde_json.workspace = true diff --git a/crates/rpc/rpc-types-compat/src/transaction/mod.rs b/crates/rpc/rpc-types-compat/src/transaction/mod.rs index 2d92747d4015..16742144f25a 100644 --- a/crates/rpc/rpc-types-compat/src/transaction/mod.rs +++ b/crates/rpc/rpc-types-compat/src/transaction/mod.rs @@ -8,9 +8,8 @@ use std::fmt; use alloy_consensus::Transaction as _; use alloy_rpc_types::{ request::{TransactionInput, TransactionRequest}, - Transaction, TransactionInfo, + TransactionInfo, }; -use alloy_serde::WithOtherFields; use reth_primitives::{TransactionSigned, TransactionSignedEcRecovered, TxType}; use serde::{Deserialize, Serialize}; @@ -89,28 +88,6 @@ pub trait TransactionCompat: Send + Sync + Unpin + Clone + fmt::Debug { fn tx_type(tx: &Self::Transaction) -> u8; } -impl TransactionCompat for () { - // this noop impl depends on integration in `reth_rpc_eth_api::EthApiTypes` noop impl, and - // `alloy_network::AnyNetwork` - type Transaction = WithOtherFields; - - fn fill( - &self, - _tx: TransactionSignedEcRecovered, - _tx_info: TransactionInfo, - ) -> Self::Transaction { - WithOtherFields::default() - } - - fn otterscan_api_truncate_input(tx: &mut Self::Transaction) { - tx.input = tx.input.slice(..4); - } - - fn tx_type(_tx: &Self::Transaction) -> u8 { - 0 - } -} - /// Gas price and max fee per gas for a transaction. Helper type to format transaction RPC response. #[derive(Debug, Default)] pub struct GasPrice { diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index 026c87153f2f..98ac9e9f4094 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -3,7 +3,7 @@ use std::sync::Arc; -use alloy_network::AnyNetwork; +use alloy_network::Ethereum; use alloy_primitives::U256; use derive_more::Deref; use reth_primitives::BlockNumberOrTag; @@ -132,8 +132,7 @@ where Self: Send + Sync, { type Error = EthApiError; - // todo: replace with alloy_network::Ethereum - type NetworkTypes = AnyNetwork; + type NetworkTypes = Ethereum; type TransactionCompat = EthTxBuilder; fn tx_resp_builder(&self) -> &Self::TransactionCompat { diff --git a/crates/rpc/rpc/src/eth/helpers/block.rs b/crates/rpc/rpc/src/eth/helpers/block.rs index d5341d0b22b4..1e2d1802e0d8 100644 --- a/crates/rpc/rpc/src/eth/helpers/block.rs +++ b/crates/rpc/rpc/src/eth/helpers/block.rs @@ -1,14 +1,13 @@ //! Contains RPC handler implementations specific to blocks. -use alloy_rpc_types::{AnyTransactionReceipt, BlockId}; -use alloy_serde::WithOtherFields; +use alloy_rpc_types::{BlockId, TransactionReceipt}; use reth_primitives::TransactionMeta; use reth_provider::{BlockReaderIdExt, HeaderProvider}; use reth_rpc_eth_api::{ helpers::{EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, SpawnBlocking}, RpcReceipt, }; -use reth_rpc_eth_types::{EthApiError, ReceiptBuilder}; +use reth_rpc_eth_types::{EthApiError, EthReceiptBuilder}; use crate::EthApi; @@ -16,7 +15,7 @@ impl EthBlocks for EthApi, + NetworkTypes: alloy_network::Network, Provider: HeaderProvider, >, { @@ -51,9 +50,8 @@ where excess_blob_gas, timestamp, }; - ReceiptBuilder::new(&tx, meta, receipt, &receipts) + EthReceiptBuilder::new(&tx, meta, receipt, &receipts) .map(|builder| builder.build()) - .map(WithOtherFields::new) }) .collect::, Self::Error>>() .map(Some) diff --git a/crates/rpc/rpc/src/eth/helpers/receipt.rs b/crates/rpc/rpc/src/eth/helpers/receipt.rs index d0cb5867eac4..594cffd09f22 100644 --- a/crates/rpc/rpc/src/eth/helpers/receipt.rs +++ b/crates/rpc/rpc/src/eth/helpers/receipt.rs @@ -1,9 +1,8 @@ //! Builds an RPC receipt response w.r.t. data layout of network. -use alloy_serde::WithOtherFields; use reth_primitives::{Receipt, TransactionMeta, TransactionSigned}; use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcNodeCoreExt, RpcReceipt}; -use reth_rpc_eth_types::{EthApiError, ReceiptBuilder}; +use reth_rpc_eth_types::{EthApiError, EthReceiptBuilder}; use crate::EthApi; @@ -26,6 +25,6 @@ where .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(hash.into()))?; - Ok(WithOtherFields::new(ReceiptBuilder::new(&tx, meta, &receipt, &all_receipts)?.build())) + Ok(EthReceiptBuilder::new(&tx, meta, &receipt, &all_receipts)?.build()) } } diff --git a/crates/rpc/rpc/src/eth/helpers/types.rs b/crates/rpc/rpc/src/eth/helpers/types.rs index 848bcdc365a5..0998c057e29c 100644 --- a/crates/rpc/rpc/src/eth/helpers/types.rs +++ b/crates/rpc/rpc/src/eth/helpers/types.rs @@ -1,10 +1,9 @@ //! L1 `eth` API types. use alloy_consensus::Transaction as _; -use alloy_network::{AnyNetwork, Network}; +use alloy_network::{Ethereum, Network}; use alloy_primitives::{Address, TxKind}; use alloy_rpc_types::{Transaction, TransactionInfo}; -use alloy_serde::WithOtherFields; use reth_primitives::TransactionSignedEcRecovered; use reth_rpc_types_compat::{ transaction::{from_primitive_signature, GasPrice}, @@ -19,7 +18,7 @@ impl TransactionCompat for EthTxBuilder where Self: Send + Sync, { - type Transaction = ::TransactionResponse; + type Transaction = ::TransactionResponse; fn fill( &self, @@ -53,41 +52,38 @@ where signed_tx.chain_id(), ); - WithOtherFields { - inner: Transaction { - hash: signed_tx.hash(), - nonce: signed_tx.nonce(), - from: signer, - to, - value: signed_tx.value(), - gas_price, - max_fee_per_gas, - max_priority_fee_per_gas: signed_tx.max_priority_fee_per_gas(), - signature: Some(signature), - gas: signed_tx.gas_limit(), - input, - chain_id, - access_list, - transaction_type: Some(signed_tx.tx_type() as u8), - // These fields are set to None because they are not stored as part of the - // transaction - block_hash, - block_number, - transaction_index, - // EIP-4844 fields - max_fee_per_blob_gas: signed_tx.max_fee_per_blob_gas(), - blob_versioned_hashes, - authorization_list, - }, - ..Default::default() + Transaction { + hash: signed_tx.hash(), + nonce: signed_tx.nonce(), + from: signer, + to, + value: signed_tx.value(), + gas_price, + max_fee_per_gas, + max_priority_fee_per_gas: signed_tx.max_priority_fee_per_gas(), + signature: Some(signature), + gas: signed_tx.gas_limit(), + input, + chain_id, + access_list, + transaction_type: Some(signed_tx.tx_type() as u8), + // These fields are set to None because they are not stored as part of the + // transaction + block_hash, + block_number, + transaction_index, + // EIP-4844 fields + max_fee_per_blob_gas: signed_tx.max_fee_per_blob_gas(), + blob_versioned_hashes, + authorization_list, } } fn otterscan_api_truncate_input(tx: &mut Self::Transaction) { - tx.inner.input = tx.inner.input.slice(..4); + tx.input = tx.input.slice(..4); } fn tx_type(tx: &Self::Transaction) -> u8 { - tx.inner.transaction_type.unwrap_or(0) + tx.transaction_type.unwrap_or(0) } } diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index ac962610ef8a..663ec0b99d6d 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -8,9 +8,8 @@ use alloy_rpc_types::{ Params, PubSubSyncStatus, SubscriptionKind, SubscriptionResult as EthSubscriptionResult, SyncStatusMetadata, }, - FilteredParams, Header, Log, Transaction, + FilteredParams, Header, Log, }; -use alloy_serde::WithOtherFields; use futures::StreamExt; use jsonrpsee::{ server::SubscriptionMessage, types::ErrorObject, PendingSubscriptionSink, SubscriptionSink, @@ -123,11 +122,9 @@ where { match kind { SubscriptionKind::NewHeads => { - let stream = pubsub.new_headers_stream().map(|header| { - EthSubscriptionResult::>::Header(Box::new( - header.into(), - )) - }); + let stream = pubsub + .new_headers_stream() + .map(|header| EthSubscriptionResult::<()>::Header(Box::new(header.into()))); pipe_from_stream(accepted_sink, stream).await } SubscriptionKind::Logs => { @@ -139,9 +136,9 @@ where } _ => FilteredParams::default(), }; - let stream = pubsub.log_stream(filter).map(|log| { - EthSubscriptionResult::>::Log(Box::new(log)) - }); + let stream = pubsub + .log_stream(filter) + .map(|log| EthSubscriptionResult::<()>::Log(Box::new(log))); pipe_from_stream(accepted_sink, stream).await } SubscriptionKind::NewPendingTransactions => { @@ -170,7 +167,7 @@ where let stream = pubsub .pending_transaction_hashes_stream() - .map(EthSubscriptionResult::>::TransactionHash); + .map(EthSubscriptionResult::<()>::TransactionHash); pipe_from_stream(accepted_sink, stream).await } SubscriptionKind::Syncing => { From e374e4bfe9c4779828c73f39ffc30d99850286ec Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Sat, 2 Nov 2024 13:18:31 -0600 Subject: [PATCH 234/239] renamed OptimismEvmConfig to OpEvmConfig (#12284) --- crates/optimism/evm/src/execute.rs | 8 ++++---- crates/optimism/evm/src/lib.rs | 20 ++++++++++---------- crates/optimism/node/src/node.rs | 8 ++++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index c2b006144369..91cdb1bd2c50 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -1,6 +1,6 @@ //! Optimism block execution strategy. -use crate::{l1::ensure_create2_deployer, OptimismBlockExecutionError, OptimismEvmConfig}; +use crate::{l1::ensure_create2_deployer, OpEvmConfig, OptimismBlockExecutionError}; use alloc::{boxed::Box, sync::Arc, vec::Vec}; use alloy_consensus::Transaction as _; use alloy_eips::eip7685::Requests; @@ -29,7 +29,7 @@ use tracing::trace; /// Factory for [`OpExecutionStrategy`]. #[derive(Debug, Clone)] -pub struct OpExecutionStrategyFactory { +pub struct OpExecutionStrategyFactory { /// The chainspec chain_spec: Arc, /// How to create an EVM. @@ -39,7 +39,7 @@ pub struct OpExecutionStrategyFactory { impl OpExecutionStrategyFactory { /// Creates a new default optimism executor strategy factory. pub fn optimism(chain_spec: Arc) -> Self { - Self::new(chain_spec.clone(), OptimismEvmConfig::new(chain_spec)) + Self::new(chain_spec.clone(), OpEvmConfig::new(chain_spec)) } } @@ -339,7 +339,7 @@ mod tests { chain_spec: Arc, ) -> BasicBlockExecutorProvider { let strategy_factory = - OpExecutionStrategyFactory::new(chain_spec.clone(), OptimismEvmConfig::new(chain_spec)); + OpExecutionStrategyFactory::new(chain_spec.clone(), OpEvmConfig::new(chain_spec)); BasicBlockExecutorProvider::new(strategy_factory) } diff --git a/crates/optimism/evm/src/lib.rs b/crates/optimism/evm/src/lib.rs index 03aecf2c83e8..1adb5db7496b 100644 --- a/crates/optimism/evm/src/lib.rs +++ b/crates/optimism/evm/src/lib.rs @@ -38,12 +38,12 @@ use revm_primitives::{ /// Optimism-related EVM configuration. #[derive(Debug, Clone)] -pub struct OptimismEvmConfig { +pub struct OpEvmConfig { chain_spec: Arc, } -impl OptimismEvmConfig { - /// Creates a new [`OptimismEvmConfig`] with the given chain spec. +impl OpEvmConfig { + /// Creates a new [`OpEvmConfig`] with the given chain spec. pub const fn new(chain_spec: Arc) -> Self { Self { chain_spec } } @@ -54,7 +54,7 @@ impl OptimismEvmConfig { } } -impl ConfigureEvmEnv for OptimismEvmConfig { +impl ConfigureEvmEnv for OpEvmConfig { type Header = Header; type Error = DecodeError; @@ -174,7 +174,7 @@ impl ConfigureEvmEnv for OptimismEvmConfig { } } -impl ConfigureEvm for OptimismEvmConfig { +impl ConfigureEvm for OpEvmConfig { type DefaultExternalContext<'a> = (); fn evm(&self, db: DB) -> Evm<'_, Self::DefaultExternalContext<'_>, DB> { @@ -226,8 +226,8 @@ mod tests { sync::Arc, }; - fn test_evm_config() -> OptimismEvmConfig { - OptimismEvmConfig::new(BASE_MAINNET.clone()) + fn test_evm_config() -> OpEvmConfig { + OpEvmConfig::new(BASE_MAINNET.clone()) } #[test] @@ -254,9 +254,9 @@ mod tests { // Define the total difficulty as zero (default) let total_difficulty = U256::ZERO; - // Use the `OptimismEvmConfig` to fill the `cfg_env` and `block_env` based on the ChainSpec, + // Use the `OpEvmConfig` to fill the `cfg_env` and `block_env` based on the ChainSpec, // Header, and total difficulty - OptimismEvmConfig::new(Arc::new(OpChainSpec { inner: chain_spec.clone() })) + OpEvmConfig::new(Arc::new(OpChainSpec { inner: chain_spec.clone() })) .fill_cfg_and_block_env(&mut cfg_env, &mut block_env, &header, total_difficulty); // Assert that the chain ID in the `cfg_env` is correctly set to the chain ID of the @@ -266,7 +266,7 @@ mod tests { #[test] fn test_evm_configure() { - // Create a default `OptimismEvmConfig` + // Create a default `OpEvmConfig` let evm_config = test_evm_config(); // Initialize an empty database wrapped in CacheDB diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 87266767da22..7437ec676ab0 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -20,7 +20,7 @@ use reth_node_builder::{ }; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_consensus::OpBeaconConsensus; -use reth_optimism_evm::{OpExecutionStrategyFactory, OptimismEvmConfig}; +use reth_optimism_evm::{OpEvmConfig, OpExecutionStrategyFactory}; use reth_optimism_rpc::OpEthApi; use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; use reth_primitives::{Block, Header}; @@ -185,14 +185,14 @@ impl ExecutorBuilder for OpExecutorBuilder where Node: FullNodeTypes>, { - type EVM = OptimismEvmConfig; + type EVM = OpEvmConfig; type Executor = BasicBlockExecutorProvider; async fn build_evm( self, ctx: &BuilderContext, ) -> eyre::Result<(Self::EVM, Self::Executor)> { - let evm_config = OptimismEvmConfig::new(ctx.chain_spec()); + let evm_config = OpEvmConfig::new(ctx.chain_spec()); let strategy_factory = OpExecutionStrategyFactory::new(ctx.chain_spec(), evm_config.clone()); let executor = BasicBlockExecutorProvider::new(strategy_factory); @@ -359,7 +359,7 @@ where ctx: &BuilderContext, pool: Pool, ) -> eyre::Result> { - self.spawn(OptimismEvmConfig::new(ctx.chain_spec()), ctx, pool) + self.spawn(OpEvmConfig::new(ctx.chain_spec()), ctx, pool) } } From e167cc9b0976e86963c294b3b102f4e6f4859ff7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 3 Nov 2024 13:51:20 +0000 Subject: [PATCH 235/239] chore(deps): weekly `cargo update` (#12289) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> --- Cargo.lock | 134 ++++++++++++++++++++++++++--------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8aab2c0a8ebc..0b3318c4ae09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feb7c662fd0be3d0c926a456be4ac44e9cf8e05cbd91df6db7f7140b861016a" +checksum = "836cf02383d9ebb35502d379bcd1ae803155094077eaab9c29131d888cd5fa3e" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -415,7 +415,7 @@ checksum = "2b09cae092c27b6f1bde952653a22708691802e57bfef4a2973b80bea21efd3f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -639,7 +639,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -655,7 +655,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "syn-solidity", "tiny-keccak", ] @@ -671,7 +671,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "syn-solidity", ] @@ -828,9 +828,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" @@ -877,7 +877,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1100,7 +1100,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1111,7 +1111,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1149,7 +1149,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1255,7 +1255,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1437,7 +1437,7 @@ checksum = "240f4126219a83519bad05c9a40bfc0303921eeb571fc2d7e44c17ffac99d3f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "synstructure", ] @@ -1559,7 +1559,7 @@ checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -1647,9 +1647,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" dependencies = [ "jobserver", "libc", @@ -1777,7 +1777,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2234,7 +2234,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2258,7 +2258,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2269,7 +2269,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2391,7 +2391,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2402,7 +2402,7 @@ checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2423,7 +2423,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "unicode-xid", ] @@ -2537,7 +2537,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2685,7 +2685,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2696,7 +2696,7 @@ checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -2753,7 +2753,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3309,7 +3309,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3835,7 +3835,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -3985,7 +3985,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -4153,7 +4153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -4401,7 +4401,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -4819,7 +4819,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -4965,7 +4965,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -5213,7 +5213,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -5570,7 +5570,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -5599,7 +5599,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -5771,7 +5771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -5822,7 +5822,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -5920,7 +5920,7 @@ checksum = "6ff7ff745a347b87471d859a377a9a404361e7efc2a971d73424a6d183c0fc77" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -6744,7 +6744,7 @@ dependencies = [ "proc-macro2", "quote", "similar-asserts", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -9904,7 +9904,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -9939,7 +9939,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -9990,7 +9990,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10013,7 +10013,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10299,7 +10299,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10357,9 +10357,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -10375,7 +10375,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10401,7 +10401,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10478,7 +10478,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10517,7 +10517,7 @@ checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10694,7 +10694,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -10895,7 +10895,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11286,7 +11286,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11357,7 +11357,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -11391,7 +11391,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -11547,7 +11547,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11558,7 +11558,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11569,7 +11569,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11580,7 +11580,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11855,7 +11855,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "synstructure", ] @@ -11877,7 +11877,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11897,7 +11897,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", "synstructure", ] @@ -11918,7 +11918,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] @@ -11940,7 +11940,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.86", + "syn 2.0.87", ] [[package]] From 20d3b21904dc5412c3526f72caa999be488a6886 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Sun, 3 Nov 2024 09:26:31 -0600 Subject: [PATCH 236/239] renamed OptimismEngineTypes to OpEngineTypes (#12285) --- crates/optimism/node/src/engine.rs | 18 +++++++++--------- crates/optimism/node/src/lib.rs | 2 +- crates/optimism/node/src/node.rs | 21 +++++++++------------ 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index af517b00ea0e..d956f0cd5cd3 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -22,17 +22,17 @@ use reth_optimism_payload_builder::{ /// The types used in the optimism beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct OptimismEngineTypes { +pub struct OpEngineTypes { _marker: std::marker::PhantomData, } -impl PayloadTypes for OptimismEngineTypes { +impl PayloadTypes for OpEngineTypes { type BuiltPayload = T::BuiltPayload; type PayloadAttributes = T::PayloadAttributes; type PayloadBuilderAttributes = T::PayloadBuilderAttributes; } -impl EngineTypes for OptimismEngineTypes +impl EngineTypes for OpEngineTypes where T::BuiltPayload: TryInto + TryInto @@ -45,7 +45,7 @@ where type ExecutionPayloadEnvelopeV4 = OpExecutionPayloadEnvelopeV4; } -/// A default payload type for [`OptimismEngineTypes`] +/// A default payload type for [`OpEngineTypes`] #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] pub struct OpPayloadTypes; @@ -224,7 +224,7 @@ mod test { let attributes = get_attributes(None, 1799999999); let result = >::ensure_well_formed_attributes( &validator, EngineApiMessageVersion::V3, &attributes ); @@ -237,7 +237,7 @@ mod test { let attributes = get_attributes(None, 1800000000); let result = >::ensure_well_formed_attributes( &validator, EngineApiMessageVersion::V3, &attributes ); @@ -250,7 +250,7 @@ mod test { let attributes = get_attributes(Some(b64!("0000000000000008")), 1800000000); let result = >::ensure_well_formed_attributes( &validator, EngineApiMessageVersion::V3, &attributes ); @@ -263,7 +263,7 @@ mod test { let attributes = get_attributes(Some(b64!("0000000800000008")), 1800000000); let result = >::ensure_well_formed_attributes( &validator, EngineApiMessageVersion::V3, &attributes ); @@ -276,7 +276,7 @@ mod test { let attributes = get_attributes(Some(b64!("0000000000000000")), 1800000000); let result = >::ensure_well_formed_attributes( &validator, EngineApiMessageVersion::V3, &attributes ); diff --git a/crates/optimism/node/src/lib.rs b/crates/optimism/node/src/lib.rs index ff25e7173a66..f2870d0b839b 100644 --- a/crates/optimism/node/src/lib.rs +++ b/crates/optimism/node/src/lib.rs @@ -15,7 +15,7 @@ pub mod args; /// Exports optimism-specific implementations of the [`EngineTypes`](reth_node_api::EngineTypes) /// trait. pub mod engine; -pub use engine::OptimismEngineTypes; +pub use engine::OpEngineTypes; pub mod node; pub use node::OptimismNode; diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 7437ec676ab0..7e6f1d0981c4 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -36,7 +36,7 @@ use crate::{ args::RollupArgs, engine::OptimismEngineValidator, txpool::{OpTransactionPool, OpTransactionValidator}, - OptimismEngineTypes, + OpEngineTypes, }; /// Optimism primitive types. @@ -74,7 +74,7 @@ impl OptimismNode { > where Node: FullNodeTypes< - Types: NodeTypesWithEngine, + Types: NodeTypesWithEngine, >, { let RollupArgs { disable_txpool_gossip, compute_pending_block, discovery_v4, .. } = args; @@ -93,9 +93,7 @@ impl OptimismNode { impl Node for OptimismNode where - N: FullNodeTypes< - Types: NodeTypesWithEngine, - >, + N: FullNodeTypes>, { type ComponentsBuilder = ComponentsBuilder< N, @@ -127,7 +125,7 @@ impl NodeTypes for OptimismNode { } impl NodeTypesWithEngine for OptimismNode { - type Engine = OptimismEngineTypes; + type Engine = OpEngineTypes; } /// Add-ons w.r.t. optimism. @@ -312,10 +310,10 @@ impl OpPayloadBuilder { evm_config: Evm, ctx: &BuilderContext, pool: Pool, - ) -> eyre::Result> + ) -> eyre::Result> where Node: FullNodeTypes< - Types: NodeTypesWithEngine, + Types: NodeTypesWithEngine, >, Pool: TransactionPool + Unpin + 'static, Evm: ConfigureEvm
, @@ -349,16 +347,15 @@ impl OpPayloadBuilder { impl PayloadServiceBuilder for OpPayloadBuilder where - Node: FullNodeTypes< - Types: NodeTypesWithEngine, - >, + Node: + FullNodeTypes>, Pool: TransactionPool + Unpin + 'static, { async fn spawn_payload_service( self, ctx: &BuilderContext, pool: Pool, - ) -> eyre::Result> { + ) -> eyre::Result> { self.spawn(OpEvmConfig::new(ctx.chain_spec()), ctx, pool) } } From 4e3b32c5afe27ee9643d5e6afbc81bf670041f0f Mon Sep 17 00:00:00 2001 From: tedison <76473430+edisontim@users.noreply.github.com> Date: Sun, 3 Nov 2024 10:40:49 -0500 Subject: [PATCH 237/239] feat: add more decimals to database growth (#12288) --- etc/grafana/dashboards/overview.json | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 8c77f5979fe9..25cc280fe03b 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -1953,6 +1953,7 @@ "mode": "off" } }, + "decimals": 4, "mappings": [], "thresholds": { "mode": "absolute", From 61f19ab2dcfe06a23dd7a88acf1ac8ce49ef75dd Mon Sep 17 00:00:00 2001 From: Deon <110722148+DanielEmmanuel1@users.noreply.github.com> Date: Sun, 3 Nov 2024 18:03:45 +0100 Subject: [PATCH 238/239] Refactor: use fully-qualified paths in Compact derives(Deon Branch) (#12279) Co-authored-by: Matthias Seitz --- crates/primitives/src/receipt.rs | 5 ++- crates/primitives/src/transaction/mod.rs | 3 -- crates/primitives/src/transaction/tx_type.rs | 3 -- crates/storage/codecs/derive/src/arbitrary.rs | 20 +++++++++-- .../codecs/derive/src/compact/generator.rs | 33 +++++++++++++++++-- .../storage/codecs/derive/src/compact/mod.rs | 11 ++++--- crates/storage/codecs/derive/src/lib.rs | 4 +-- .../codecs/src/alloy/authorization_list.rs | 3 +- .../codecs/src/alloy/genesis_account.rs | 10 ++++-- crates/storage/codecs/src/alloy/header.rs | 2 ++ .../codecs/src/alloy/transaction/eip1559.rs | 3 +- .../codecs/src/alloy/transaction/eip2930.rs | 3 +- .../codecs/src/alloy/transaction/eip4844.rs | 3 +- .../codecs/src/alloy/transaction/eip7702.rs | 3 +- .../codecs/src/alloy/transaction/legacy.rs | 3 +- .../codecs/src/alloy/transaction/optimism.rs | 3 +- crates/storage/codecs/src/alloy/withdrawal.rs | 3 +- crates/storage/codecs/src/lib.rs | 6 ++-- 18 files changed, 87 insertions(+), 34 deletions(-) diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 21443f482c91..e60bddb9d799 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -11,8 +11,6 @@ use alloy_rlp::{length_of_length, Decodable, Encodable, RlpDecodable, RlpEncodab use bytes::{Buf, BufMut}; use core::{cmp::Ordering, ops::Deref}; use derive_more::{DerefMut, From, IntoIterator}; -#[cfg(feature = "reth-codec")] -use reth_codecs::Compact; use serde::{Deserialize, Serialize}; /// Receipt containing result of transaction execution. @@ -91,7 +89,7 @@ impl Receipts { self.receipt_vec.len() } - /// Returns `true` if the `Receipts` vector is empty. + /// Returns true if the `Receipts` vector is empty. pub fn is_empty(&self) -> bool { self.receipt_vec.is_empty() } @@ -518,6 +516,7 @@ mod tests { use super::*; use crate::revm_primitives::Bytes; use alloy_primitives::{address, b256, bytes, hex_literal::hex}; + use reth_codecs::Compact; #[test] fn test_decode_receipt() { diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 2e0a786fc1a8..59d3b9e12976 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -63,9 +63,6 @@ use tx_type::{ COMPACT_IDENTIFIER_LEGACY, }; -#[cfg(test)] -use reth_codecs::Compact; - use alloc::vec::Vec; /// Either a transaction hash or number. diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index b6e1ecbf226a..eff1c17a71a7 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -6,9 +6,6 @@ use alloy_primitives::{U64, U8}; use alloy_rlp::{Decodable, Encodable}; use serde::{Deserialize, Serialize}; -#[cfg(test)] -use reth_codecs::Compact; - /// Identifier parameter for legacy transaction #[cfg(any(test, feature = "reth-codec"))] pub(crate) const COMPACT_IDENTIFIER_LEGACY: usize = 0; diff --git a/crates/storage/codecs/derive/src/arbitrary.rs b/crates/storage/codecs/derive/src/arbitrary.rs index 8aa44062e21f..753bb1e33a51 100644 --- a/crates/storage/codecs/derive/src/arbitrary.rs +++ b/crates/storage/codecs/derive/src/arbitrary.rs @@ -18,10 +18,26 @@ pub fn maybe_generate_tests( let mut traits = vec![]; let mut roundtrips = vec![]; let mut additional_tests = vec![]; + let mut is_crate = false; - for arg in args { + let mut iter = args.into_iter().peekable(); + + // we check if there's a crate argument which is used from inside the codecs crate directly + if let Some(arg) = iter.peek() { + if arg.to_string() == "crate" { + is_crate = true; + iter.next(); + } + } + + for arg in iter { if arg.to_string() == "compact" { - traits.push(quote! { use super::Compact; }); + let path = if is_crate { + quote! { use crate::Compact; } + } else { + quote! { use reth_codecs::Compact; } + }; + traits.push(path); roundtrips.push(quote! { { let mut buf = vec![]; diff --git a/crates/storage/codecs/derive/src/compact/generator.rs b/crates/storage/codecs/derive/src/compact/generator.rs index 1fb6d40fa2b2..cf9bcc0c6295 100644 --- a/crates/storage/codecs/derive/src/compact/generator.rs +++ b/crates/storage/codecs/derive/src/compact/generator.rs @@ -2,10 +2,12 @@ use super::*; use convert_case::{Case, Casing}; +use syn::{Attribute, LitStr}; /// Generates code to implement the `Compact` trait for a data type. pub fn generate_from_to( ident: &Ident, + attrs: &[Attribute], has_lifetime: bool, fields: &FieldList, is_zstd: bool, @@ -20,6 +22,8 @@ pub fn generate_from_to( let fuzz = format_ident!("fuzz_test_{snake_case_ident}"); let test = format_ident!("fuzz_{snake_case_ident}"); + let reth_codecs = parse_reth_codecs_path(attrs).unwrap(); + let lifetime = if has_lifetime { quote! { 'a } } else { @@ -28,11 +32,11 @@ pub fn generate_from_to( let impl_compact = if has_lifetime { quote! { - impl<#lifetime> Compact for #ident<#lifetime> + impl<#lifetime> #reth_codecs::Compact for #ident<#lifetime> } } else { quote! { - impl Compact for #ident + impl #reth_codecs::Compact for #ident } }; @@ -53,6 +57,7 @@ pub fn generate_from_to( #[allow(dead_code)] #[test_fuzz::test_fuzz] fn #fuzz(obj: #ident) { + use #reth_codecs::Compact; let mut buf = vec![]; let len = obj.clone().to_compact(&mut buf); let (same_obj, buf) = #ident::from_compact(buf.as_ref(), len); @@ -191,7 +196,7 @@ fn generate_to_compact(fields: &FieldList, ident: &Ident, is_zstd: bool) -> Vec< } // Just because a type supports compression, doesn't mean all its values are to be compressed. - // We skip the smaller ones, and thus require a flag `__zstd` to specify if this value is + // We skip the smaller ones, and thus require a flag` __zstd` to specify if this value is // compressed or not. if is_zstd { lines.push(quote! { @@ -232,3 +237,25 @@ fn generate_to_compact(fields: &FieldList, ident: &Ident, is_zstd: bool) -> Vec< lines } + +/// Function to extract the crate path from `reth_codecs(crate = "...")` attribute. +fn parse_reth_codecs_path(attrs: &[Attribute]) -> syn::Result { + // let default_crate_path: syn::Path = syn::parse_str("reth-codecs").unwrap(); + let mut reth_codecs_path: syn::Path = syn::parse_quote!(reth_codecs); + for attr in attrs { + if attr.path().is_ident("reth_codecs") { + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("crate") { + let value = meta.value()?; + let lit: LitStr = value.parse()?; + reth_codecs_path = syn::parse_str(&lit.value())?; + Ok(()) + } else { + Err(meta.error("unsupported attribute")) + } + })?; + } + } + + Ok(reth_codecs_path) +} diff --git a/crates/storage/codecs/derive/src/compact/mod.rs b/crates/storage/codecs/derive/src/compact/mod.rs index e5a79b3fe530..b9d5cf18d6b7 100644 --- a/crates/storage/codecs/derive/src/compact/mod.rs +++ b/crates/storage/codecs/derive/src/compact/mod.rs @@ -43,13 +43,13 @@ pub enum FieldTypes { pub fn derive(input: TokenStream, is_zstd: bool) -> TokenStream { let mut output = quote! {}; - let DeriveInput { ident, data, generics, .. } = parse_macro_input!(input); + let DeriveInput { ident, data, generics, attrs, .. } = parse_macro_input!(input); let has_lifetime = has_lifetime(&generics); let fields = get_fields(&data); output.extend(generate_flag_struct(&ident, has_lifetime, &fields, is_zstd)); - output.extend(generate_from_to(&ident, has_lifetime, &fields, is_zstd)); + output.extend(generate_from_to(&ident, &attrs, has_lifetime, &fields, is_zstd)); output.into() } @@ -233,10 +233,10 @@ mod tests { // Generate code that will impl the `Compact` trait. let mut output = quote! {}; - let DeriveInput { ident, data, .. } = parse2(f_struct).unwrap(); + let DeriveInput { ident, data, attrs, .. } = parse2(f_struct).unwrap(); let fields = get_fields(&data); output.extend(generate_flag_struct(&ident, false, &fields, false)); - output.extend(generate_from_to(&ident, false, &fields, false)); + output.extend(generate_from_to(&ident, &attrs, false, &fields, false)); // Expected output in a TokenStream format. Commas matter! let should_output = quote! { @@ -285,6 +285,7 @@ mod tests { #[allow(dead_code)] #[test_fuzz::test_fuzz] fn fuzz_test_test_struct(obj: TestStruct) { + use reth_codecs::Compact; let mut buf = vec![]; let len = obj.clone().to_compact(&mut buf); let (same_obj, buf) = TestStruct::from_compact(buf.as_ref(), len); @@ -295,7 +296,7 @@ mod tests { pub fn fuzz_test_struct() { fuzz_test_test_struct(TestStruct::default()) } - impl Compact for TestStruct { + impl reth_codecs::Compact for TestStruct { fn to_compact(&self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]> { let mut flags = TestStructFlags::default(); let mut total_length = 0; diff --git a/crates/storage/codecs/derive/src/lib.rs b/crates/storage/codecs/derive/src/lib.rs index 4ffdbfd6ef64..0b4015830f5d 100644 --- a/crates/storage/codecs/derive/src/lib.rs +++ b/crates/storage/codecs/derive/src/lib.rs @@ -49,14 +49,14 @@ mod compact; /// own encoding and do not rely on the bitflag struct. /// - `Bytes` fields and any types containing a `Bytes` field should be placed last to ensure /// efficient decoding. -#[proc_macro_derive(Compact, attributes(maybe_zero))] +#[proc_macro_derive(Compact, attributes(maybe_zero, reth_codecs))] pub fn derive(input: TokenStream) -> TokenStream { let is_zstd = false; compact::derive(input, is_zstd) } /// Adds `zstd` compression to derived [`Compact`]. -#[proc_macro_derive(CompactZstd, attributes(maybe_zero))] +#[proc_macro_derive(CompactZstd, attributes(maybe_zero, reth_codecs))] pub fn derive_zstd(input: TokenStream) -> TokenStream { let is_zstd = true; compact::derive(input, is_zstd) diff --git a/crates/storage/codecs/src/alloy/authorization_list.rs b/crates/storage/codecs/src/alloy/authorization_list.rs index 3fc9518a6376..e17c0fb32a10 100644 --- a/crates/storage/codecs/src/alloy/authorization_list.rs +++ b/crates/storage/codecs/src/alloy/authorization_list.rs @@ -11,12 +11,13 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with `alloy_eips::eip7702::Authorization` #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct Authorization { chain_id: u64, address: Address, diff --git a/crates/storage/codecs/src/alloy/genesis_account.rs b/crates/storage/codecs/src/alloy/genesis_account.rs index b29fe526df4f..a35d4947db79 100644 --- a/crates/storage/codecs/src/alloy/genesis_account.rs +++ b/crates/storage/codecs/src/alloy/genesis_account.rs @@ -11,6 +11,7 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with `alloy_genesis::GenesisAccount` #[derive(Debug, Clone, PartialEq, Eq, Compact)] +#[reth_codecs(crate = "crate")] pub(crate) struct GenesisAccountRef<'a> { /// The nonce of the account at genesis. nonce: Option, @@ -27,12 +28,13 @@ pub(crate) struct GenesisAccountRef<'a> { /// Acts as bridge which simplifies Compact implementation for /// `AlloyGenesisAccount`. #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct GenesisAccount { /// The nonce of the account at genesis. nonce: Option, @@ -47,21 +49,23 @@ pub(crate) struct GenesisAccount { } #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct StorageEntries { entries: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct StorageEntry { key: B256, value: B256, diff --git a/crates/storage/codecs/src/alloy/header.rs b/crates/storage/codecs/src/alloy/header.rs index 78f2029c32ea..04b7d6ab718b 100644 --- a/crates/storage/codecs/src/alloy/header.rs +++ b/crates/storage/codecs/src/alloy/header.rs @@ -18,6 +18,7 @@ use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, B256, U256}; )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] +#[reth_codecs(crate = "crate")] pub(crate) struct Header { parent_hash: B256, ommers_hash: B256, @@ -54,6 +55,7 @@ pub(crate) struct Header { )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] +#[reth_codecs(crate = "crate")] pub(crate) struct HeaderExt { requests_hash: Option, } diff --git a/crates/storage/codecs/src/alloy/transaction/eip1559.rs b/crates/storage/codecs/src/alloy/transaction/eip1559.rs index 0e7f44cdec18..6d910a6900c6 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip1559.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip1559.rs @@ -13,11 +13,12 @@ use alloy_primitives::{Bytes, ChainId, TxKind, U256}; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip1559`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Compact, Default)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] -#[cfg_attr(any(test, feature = "test-utils"), crate::add_arbitrary_tests(compact))] +#[cfg_attr(any(test, feature = "test-utils"), crate::add_arbitrary_tests(crate, compact))] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] pub(crate) struct TxEip1559 { chain_id: ChainId, diff --git a/crates/storage/codecs/src/alloy/transaction/eip2930.rs b/crates/storage/codecs/src/alloy/transaction/eip2930.rs index 75cab9e8a09c..aeb08f361bea 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip2930.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip2930.rs @@ -15,12 +15,13 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip2930`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct TxEip2930 { chain_id: ChainId, nonce: u64, diff --git a/crates/storage/codecs/src/alloy/transaction/eip4844.rs b/crates/storage/codecs/src/alloy/transaction/eip4844.rs index c89e2b0785ba..fac9ab9a1b2a 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip4844.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip4844.rs @@ -16,9 +16,10 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip4844`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr(any(test, feature = "test-utils"), derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct TxEip4844 { chain_id: ChainId, nonce: u64, diff --git a/crates/storage/codecs/src/alloy/transaction/eip7702.rs b/crates/storage/codecs/src/alloy/transaction/eip7702.rs index 8acf59425f2d..eab10af0b663 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip7702.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip7702.rs @@ -16,12 +16,13 @@ use reth_codecs_derive::add_arbitrary_tests; /// /// Notice: Make sure this struct is 1:1 with [`alloy_consensus::TxEip7702`] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct TxEip7702 { chain_id: ChainId, nonce: u64, diff --git a/crates/storage/codecs/src/alloy/transaction/legacy.rs b/crates/storage/codecs/src/alloy/transaction/legacy.rs index c83626aa4cf9..60250ba64af0 100644 --- a/crates/storage/codecs/src/alloy/transaction/legacy.rs +++ b/crates/storage/codecs/src/alloy/transaction/legacy.rs @@ -6,10 +6,11 @@ use alloy_primitives::{Bytes, ChainId, TxKind, U256}; /// Legacy transaction. #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] +#[reth_codecs(crate = "crate")] #[cfg_attr( any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize), - crate::add_arbitrary_tests(compact) + crate::add_arbitrary_tests(crate, compact) )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] pub(crate) struct TxLegacy { diff --git a/crates/storage/codecs/src/alloy/transaction/optimism.rs b/crates/storage/codecs/src/alloy/transaction/optimism.rs index 22f508fd4ceb..bb970b581775 100644 --- a/crates/storage/codecs/src/alloy/transaction/optimism.rs +++ b/crates/storage/codecs/src/alloy/transaction/optimism.rs @@ -19,7 +19,8 @@ use reth_codecs_derive::add_arbitrary_tests; derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] -#[add_arbitrary_tests(compact)] +#[reth_codecs(crate = "crate")] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct TxDeposit { source_hash: B256, from: Address, diff --git a/crates/storage/codecs/src/alloy/withdrawal.rs b/crates/storage/codecs/src/alloy/withdrawal.rs index 0f3347cec1a5..8aa5671798df 100644 --- a/crates/storage/codecs/src/alloy/withdrawal.rs +++ b/crates/storage/codecs/src/alloy/withdrawal.rs @@ -13,8 +13,9 @@ use reth_codecs_derive::add_arbitrary_tests; any(test, feature = "test-utils"), derive(arbitrary::Arbitrary, serde::Serialize, serde::Deserialize) )] +#[reth_codecs(crate = "crate")] #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] -#[add_arbitrary_tests(compact)] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct Withdrawal { /// Monotonically increasing identifier issued by consensus layer. index: u64, diff --git a/crates/storage/codecs/src/lib.rs b/crates/storage/codecs/src/lib.rs index 598f2131bde6..284c6454f838 100644 --- a/crates/storage/codecs/src/lib.rs +++ b/crates/storage/codecs/src/lib.rs @@ -662,7 +662,8 @@ mod tests { } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Compact, arbitrary::Arbitrary)] - #[add_arbitrary_tests(compact)] + #[add_arbitrary_tests(crate, compact)] + #[reth_codecs(crate = "crate")] struct TestStruct { f_u64: u64, f_u256: U256, @@ -714,7 +715,8 @@ mod tests { #[derive( Debug, PartialEq, Clone, Default, Serialize, Deserialize, Compact, arbitrary::Arbitrary, )] - #[add_arbitrary_tests(compact)] + #[add_arbitrary_tests(crate, compact)] + #[reth_codecs(crate = "crate")] enum TestEnum { #[default] Var0, From 21d911abb2cf594834c1818d2f723718b5109f8b Mon Sep 17 00:00:00 2001 From: Kien Trinh <51135161+kien6034@users.noreply.github.com> Date: Mon, 4 Nov 2024 00:49:07 +0700 Subject: [PATCH 239/239] docs: add debug.etherscan in the book (#12293) Co-authored-by: Matthias Seitz --- book/run/mainnet.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/book/run/mainnet.md b/book/run/mainnet.md index 6f1ec144df81..c4908971f69f 100644 --- a/book/run/mainnet.md +++ b/book/run/mainnet.md @@ -84,3 +84,13 @@ In the meantime, consider setting up [observability](./observability.md) to moni ## Running without a Consensus Layer We provide a method for running Reth without a Consensus Layer via the `--debug.tip ` parameter. If you provide that to your node, it will simulate sending an `engine_forkchoiceUpdated` message _once_ and will trigger syncing to the provided block hash. This is useful for testing and debugging purposes, but in order to have a node that can keep up with the tip you'll need to run a CL alongside it. At the moment we have no plans of including a Consensus Layer implementation in Reth, and we are open to including light clients other methods of syncing like importing Lighthouse as a library. + +## Running with Etherscan as Block Source + +You can use `--debug.etherscan` to run Reth with a fake consensus client that advances the chain using recent blocks on Etherscan. This requires an Etherscan API key (set via `ETHERSCAN_API_KEY` environment variable). Optionally, specify a custom API URL with `--debug.etherscan `. + +Example: +```bash +export ETHERSCAN_API_KEY=your_api_key_here +reth node --debug.etherscan +``` \ No newline at end of file