diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml index 9ab0b1d07973..fdcc7be4fb3d 100644 --- a/.github/workflows/sanity.yml +++ b/.github/workflows/sanity.yml @@ -22,7 +22,7 @@ jobs: - uses: dtolnay/rust-toolchain@nightly - uses: taiki-e/install-action@cargo-udeps - name: Check for unused dependencies - run: cargo udeps --features "jemalloc,${{ matrix.network }}" + run: cargo udeps --lib --features "jemalloc,${{ matrix.network }}" - uses: JasonEtco/create-an-issue@v2 if: ${{ failure() }} env: diff --git a/Cargo.lock b/Cargo.lock index e936350bce07..5b1b530de179 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2745,21 +2745,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -3335,19 +3320,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iai" version = "0.1.1" @@ -4281,9 +4253,9 @@ dependencies = [ [[package]] name = "mev-share-sse" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e59928ecfd8f9dd3211f2eb08bdc260c78c2e2af34d1a652445a6287d3c95942" +checksum = "ba263a1c478aade75b60835fbeb6f57c0280fb0953742c3d84de45ea51139ae4" dependencies = [ "async-sse", "bytes", @@ -4396,24 +4368,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nibble_vec" version = "0.1.0" @@ -4665,50 +4619,12 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "openssl" -version = "0.10.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "option-ext" version = "0.2.0" @@ -5572,21 +5488,19 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-tls", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", + "rustls", "serde", "serde_json", "serde_urlencoded", "system-configuration", "tokio", - "tokio-native-tls", "tokio-util", "tower-service", "url", @@ -8020,16 +7934,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -8646,12 +8550,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "vergen" version = "8.2.6" diff --git a/bin/reth/src/debug_cmd/build_block.rs b/bin/reth/src/debug_cmd/build_block.rs index e1d57e4b7d93..4677bd551f15 100644 --- a/bin/reth/src/debug_cmd/build_block.rs +++ b/bin/reth/src/debug_cmd/build_block.rs @@ -277,11 +277,26 @@ impl Command { let state = executor.take_output_state(); debug!(target: "reth::cli", ?state, "Executed block"); + let hashed_state = state.hash_state_slow(); + let (state_root, trie_updates) = state + .state_root_calculator(provider_factory.provider()?.tx_ref(), &hashed_state) + .root_with_updates()?; + + if state_root != block_with_senders.state_root { + eyre::bail!( + "state root mismatch. expected: {}. got: {}", + block_with_senders.state_root, + state_root + ); + } + // Attempt to insert new block without committing let provider_rw = provider_factory.provider_rw()?; - provider_rw.append_blocks_with_bundle_state( + provider_rw.append_blocks_with_state( Vec::from([block_with_senders]), state, + hashed_state, + trie_updates, None, )?; info!(target: "reth::cli", "Successfully appended built block"); diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 8d399dda2599..b510dc9008da 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -6,7 +6,7 @@ use crate::{ state::{BlockChainId, TreeState}, AppendableChain, BlockIndices, BlockchainTreeConfig, BundleStateData, TreeExternals, }; -use reth_db::database::Database; +use reth_db::{database::Database, DatabaseError}; use reth_interfaces::{ blockchain_tree::{ error::{BlockchainTreeError, CanonicalError, InsertBlockError, InsertBlockErrorKind}, @@ -14,17 +14,18 @@ use reth_interfaces::{ }, consensus::{Consensus, ConsensusError}, executor::{BlockExecutionError, BlockValidationError}, - RethResult, + provider::RootMismatch, + RethError, RethResult, }; use reth_primitives::{ - BlockHash, BlockNumHash, BlockNumber, ForkBlock, Hardfork, PruneModes, Receipt, SealedBlock, - SealedBlockWithSenders, SealedHeader, U256, + BlockHash, BlockNumHash, BlockNumber, ForkBlock, GotExpected, Hardfork, PruneModes, Receipt, + SealedBlock, SealedBlockWithSenders, SealedHeader, U256, }; use reth_provider::{ chain::{ChainSplit, ChainSplitTarget}, BlockExecutionWriter, BlockNumReader, BlockWriter, BundleStateWithReceipts, CanonStateNotification, CanonStateNotificationSender, CanonStateNotifications, Chain, - ChainSpecProvider, DisplayBlocksChain, ExecutorFactory, HeaderProvider, + ChainSpecProvider, DisplayBlocksChain, ExecutorFactory, HeaderProvider, ProviderError, }; use reth_stages::{MetricEvent, MetricEventsSender}; use std::{collections::BTreeMap, sync::Arc}; @@ -1104,14 +1105,35 @@ impl BlockchainTree { /// Write the given chain to the database as canonical. fn commit_canonical_to_database(&self, chain: Chain) -> RethResult<()> { - let provider_rw = self.externals.provider_factory.provider_rw()?; + // Compute state root before opening write transaction. + let hashed_state = chain.state().hash_state_slow(); + let (state_root, trie_updates) = chain + .state() + .state_root_calculator( + self.externals.provider_factory.provider()?.tx_ref(), + &hashed_state, + ) + .root_with_updates() + .map_err(Into::::into)?; + let tip = chain.tip(); + if state_root != tip.state_root { + return Err(RethError::Provider(ProviderError::StateRootMismatch(Box::new( + RootMismatch { + root: GotExpected { got: state_root, expected: tip.state_root }, + block_number: tip.number, + block_hash: tip.hash, + }, + )))) + } let (blocks, state) = chain.into_inner(); - + let provider_rw = self.externals.provider_factory.provider_rw()?; provider_rw - .append_blocks_with_bundle_state( + .append_blocks_with_state( blocks.into_blocks().collect(), state, + hashed_state, + trie_updates, self.prune_modes.as_ref(), ) .map_err(|e| BlockExecutionError::CanonicalCommit { inner: e.to_string() })?; diff --git a/crates/primitives/src/account.rs b/crates/primitives/src/account.rs index 94d245828697..c505ee5dfc1b 100644 --- a/crates/primitives/src/account.rs +++ b/crates/primitives/src/account.rs @@ -34,7 +34,7 @@ impl Account { Some(hash) => hash == KECCAK_EMPTY, }; - self.nonce == 0 && self.balance == U256::ZERO && is_bytecode_empty + self.nonce == 0 && self.balance.is_zero() && is_bytecode_empty } /// Returns an account bytecode's hash. diff --git a/crates/primitives/src/revm/env.rs b/crates/primitives/src/revm/env.rs index 26b87a159b40..634cc6db1ce7 100644 --- a/crates/primitives/src/revm/env.rs +++ b/crates/primitives/src/revm/env.rs @@ -92,26 +92,44 @@ pub fn fill_block_env_with_coinbase( /// Return the coinbase address for the given header and chain spec. pub fn block_coinbase(chain_spec: &ChainSpec, header: &Header, after_merge: bool) -> Address { if chain_spec.chain == Chain::goerli() && !after_merge { - recover_header_signer(header).expect("failed to recover signer") + recover_header_signer(header).unwrap_or_else(|err| { + panic!( + "Failed to recover goerli Clique Consensus signer from header ({}, {}) using extradata {}: {:?}", + header.number, header.hash_slow(), header.extra_data, err + ) + }) } else { header.beneficiary } } +/// Error type for recovering Clique signer from a header. +#[derive(Debug, thiserror::Error)] +pub enum CliqueSignerRecoveryError { + /// Header extradata is too short. + #[error("Invalid extra data length")] + InvalidExtraData, + /// Recovery failed. + #[error("Invalid signature: {0}")] + InvalidSignature(#[from] secp256k1::Error), +} + /// Recover the account from signed header per clique consensus rules. -pub fn recover_header_signer(header: &Header) -> Option
{ +pub fn recover_header_signer(header: &Header) -> Result { let extra_data_len = header.extra_data.len(); // Fixed number of extra-data suffix bytes reserved for signer signature. // 65 bytes fixed as signatures are based on the standard secp256k1 curve. // Filled with zeros on genesis block. let signature_start_byte = extra_data_len - 65; - let signature: [u8; 65] = header.extra_data[signature_start_byte..].try_into().ok()?; + let signature: [u8; 65] = header.extra_data[signature_start_byte..] + .try_into() + .map_err(|_| CliqueSignerRecoveryError::InvalidExtraData)?; let seal_hash = { let mut header_to_seal = header.clone(); header_to_seal.extra_data = Bytes::from(header.extra_data[..signature_start_byte].to_vec()); header_to_seal.hash_slow() }; - recover_signer(&signature, &seal_hash.0).ok() + recover_signer(&signature, &seal_hash.0).map_err(CliqueSignerRecoveryError::InvalidSignature) } /// Returns a new [TxEnv] filled with the transaction's data. diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index 76c4a893f6f3..89c30648cd7a 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -72,7 +72,7 @@ impl Signature { #[inline] pub fn v(&self, chain_id: Option) -> u64 { #[cfg(feature = "optimism")] - if self.r == U256::ZERO && self.s == U256::ZERO { + if self.r.is_zero() && self.s.is_zero() { return 0 } diff --git a/crates/revm/revm-inspectors/src/tracing/mod.rs b/crates/revm/revm-inspectors/src/tracing/mod.rs index 5fbc9bb2ccb2..6b4b93636364 100644 --- a/crates/revm/revm-inspectors/src/tracing/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/mod.rs @@ -144,7 +144,7 @@ impl TracingInspector { ) -> bool { if data.precompiles.contains(to) { // only if this is _not_ the root call - return self.is_deep() && value == U256::ZERO + return self.is_deep() && value.is_zero() } false } diff --git a/crates/rpc/rpc-engine-api/tests/it/payload.rs b/crates/rpc/rpc-engine-api/tests/it/payload.rs index 5b012726cee7..0b9ec850a470 100644 --- a/crates/rpc/rpc-engine-api/tests/it/payload.rs +++ b/crates/rpc/rpc-engine-api/tests/it/payload.rs @@ -90,7 +90,7 @@ fn payload_validation() { assert_matches!( try_into_sealed_block(block_with_zero_base_fee,None), - Err(PayloadError::BaseFee(val)) if val == U256::ZERO + Err(PayloadError::BaseFee(val)) if val.is_zero() ); // Invalid encoded transactions diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 44c90408830b..6e50bafbde79 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -40,9 +40,11 @@ jsonrpsee.workspace = true http = "0.2.8" http-body = "0.4.5" hyper = "0.14.24" -reqwest = { version = "0.11.20", optional = true } jsonwebtoken = "8" +## required for optimism sequencer delegation +reqwest = { version = "0.11", default-features = false, features = ["rustls"], optional = true } + # async async-trait.workspace = true tokio = { workspace = true, features = ["sync"] } diff --git a/crates/rpc/rpc/src/eth/revm_utils.rs b/crates/rpc/rpc/src/eth/revm_utils.rs index bbfb3b2d1753..4393272fc8ce 100644 --- a/crates/rpc/rpc/src/eth/revm_utils.rs +++ b/crates/rpc/rpc/src/eth/revm_utils.rs @@ -609,7 +609,7 @@ mod tests { let CallFees { gas_price, .. } = CallFees::ensure_fees(None, None, None, U256::from(99), None, None, Some(U256::ZERO)) .unwrap(); - assert_eq!(gas_price, U256::ZERO); + assert!(gas_price.is_zero()); } #[test] @@ -617,7 +617,7 @@ mod tests { let CallFees { gas_price, max_fee_per_blob_gas, .. } = CallFees::ensure_fees(None, None, None, U256::from(99), None, None, Some(U256::ZERO)) .unwrap(); - assert_eq!(gas_price, U256::ZERO); + assert!(gas_price.is_zero()); assert_eq!(max_fee_per_blob_gas, None); let CallFees { gas_price, max_fee_per_blob_gas, .. } = CallFees::ensure_fees( @@ -630,7 +630,7 @@ mod tests { Some(U256::from(99)), ) .unwrap(); - assert_eq!(gas_price, U256::ZERO); + assert!(gas_price.is_zero()); assert_eq!(max_fee_per_blob_gas, Some(U256::from(99))); } } diff --git a/crates/storage/libmdbx-rs/src/flags.rs b/crates/storage/libmdbx-rs/src/flags.rs index e464a3b20a05..ad88c1fbedc9 100644 --- a/crates/storage/libmdbx-rs/src/flags.rs +++ b/crates/storage/libmdbx-rs/src/flags.rs @@ -128,6 +128,10 @@ impl From for EnvironmentFlags { pub struct EnvironmentFlags { pub no_sub_dir: bool, pub exclusive: bool, + /// Flag is intended to open an existing sub-database which was created with unknown flags + /// In such cases, instead of returning the `MDBX_INCOMPATIBLE` error, the sub-database will be + /// opened with flags which it was created, and then an application could determine the actual + /// flags. pub accede: bool, pub mode: Mode, pub no_rdahead: bool, @@ -137,6 +141,7 @@ pub struct EnvironmentFlags { } impl EnvironmentFlags { + /// Configures the mdbx flags to use when opening the environment. pub(crate) fn make_flags(&self) -> ffi::MDBX_env_flags_t { let mut flags = 0; diff --git a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs index 6077a2f6560f..33dd0590e38e 100644 --- a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs +++ b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs @@ -128,7 +128,6 @@ impl BundleStateWithReceipts { /// /// The hashed post state. pub fn hash_state_slow(&self) -> HashedPostState { - //let mut storages = BTreeMap::default(); let mut hashed_state = HashedPostState::default(); for (address, account) in self.bundle.state() { @@ -136,7 +135,7 @@ impl BundleStateWithReceipts { if let Some(account) = &account.info { hashed_state.insert_account(hashed_address, into_reth_acc(account.clone())) } else { - hashed_state.insert_cleared_account(hashed_address); + hashed_state.insert_destroyed_account(hashed_address); } // insert storage. @@ -144,7 +143,7 @@ impl BundleStateWithReceipts { for (key, value) in account.storage.iter() { let hashed_key = keccak256(B256::new(key.to_be_bytes())); - if value.present_value == U256::ZERO { + if value.present_value.is_zero() { hashed_storage.insert_zero_valued_slot(hashed_key); } else { hashed_storage.insert_non_zero_valued_storage(hashed_key, value.present_value); @@ -155,8 +154,8 @@ impl BundleStateWithReceipts { hashed_state.sorted() } - /// Returns [StateRoot] calculator. - fn state_root_calculator<'a, 'b, TX: DbTx>( + /// Returns [StateRoot] calculator based on database and in-memory state. + pub fn state_root_calculator<'a, 'b, TX: DbTx>( &self, tx: &'a TX, hashed_post_state: &'b HashedPostState, @@ -167,6 +166,7 @@ impl BundleStateWithReceipts { .with_hashed_cursor_factory(hashed_cursor_factory) .with_changed_account_prefixes(account_prefix_set) .with_changed_storage_prefixes(storage_prefix_set) + .with_destroyed_accounts(hashed_post_state.destroyed_accounts()) } /// Calculate the state root for this [BundleState]. diff --git a/crates/storage/provider/src/bundle_state/hashed_state_changes.rs b/crates/storage/provider/src/bundle_state/hashed_state_changes.rs new file mode 100644 index 000000000000..cfdacb973674 --- /dev/null +++ b/crates/storage/provider/src/bundle_state/hashed_state_changes.rs @@ -0,0 +1,128 @@ +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + tables, + transaction::{DbTx, DbTxMut}, + DatabaseError, +}; +use reth_primitives::{Account, StorageEntry, B256, U256}; +use reth_trie::hashed_cursor::HashedPostState; +use std::collections::BTreeMap; + +/// Changes to the hashed state. +#[derive(Debug, Default)] +pub struct HashedStateChanges(pub HashedPostState); + +impl HashedStateChanges { + /// Write the bundle state to the database. + pub fn write_to_db(self, tx: &TX) -> Result<(), DatabaseError> { + // Collect hashed account changes. + let mut hashed_accounts = BTreeMap::>::default(); + for (hashed_address, account) in self.0.accounts() { + hashed_accounts.insert(hashed_address, account); + } + + // Write hashed account updates. + let mut hashed_accounts_cursor = tx.cursor_write::()?; + for (hashed_address, account) in hashed_accounts { + if let Some(account) = account { + hashed_accounts_cursor.upsert(hashed_address, account)?; + } else if hashed_accounts_cursor.seek_exact(hashed_address)?.is_some() { + hashed_accounts_cursor.delete_current()?; + } + } + + // Collect hashed storage changes. + let mut hashed_storages = BTreeMap::)>::default(); + for (hashed_address, storage) in self.0.storages() { + let entry = hashed_storages.entry(*hashed_address).or_default(); + entry.0 |= storage.wiped(); + for (hashed_slot, value) in storage.storage_slots() { + entry.1.insert(hashed_slot, value); + } + } + + // Write hashed storage changes. + let mut hashed_storage_cursor = tx.cursor_dup_write::()?; + for (hashed_address, (wiped, storage)) in hashed_storages { + if wiped && hashed_storage_cursor.seek_exact(hashed_address)?.is_some() { + hashed_storage_cursor.delete_current_duplicates()?; + } + + for (hashed_slot, value) in storage { + let entry = StorageEntry { key: hashed_slot, value }; + if let Some(db_entry) = + hashed_storage_cursor.seek_by_key_subkey(hashed_address, entry.key)? + { + if db_entry.key == entry.key { + hashed_storage_cursor.delete_current()?; + } + } + + if entry.value != U256::ZERO { + hashed_storage_cursor.upsert(hashed_address, entry)?; + } + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::create_test_provider_factory; + use reth_primitives::{keccak256, Address}; + use reth_trie::hashed_cursor::HashedStorage; + + #[test] + fn wiped_entries_are_removed() { + let provider_factory = create_test_provider_factory(); + + let addresses = (0..10).map(|_| Address::random()).collect::>(); + let destroyed_address = *addresses.first().unwrap(); + let destroyed_address_hashed = keccak256(destroyed_address); + let slot = B256::with_last_byte(1); + let hashed_slot = keccak256(slot); + { + let provider_rw = provider_factory.provider_rw().unwrap(); + let mut accounts_cursor = + provider_rw.tx_ref().cursor_write::().unwrap(); + let mut storage_cursor = + provider_rw.tx_ref().cursor_write::().unwrap(); + + for address in addresses { + let hashed_address = keccak256(address); + accounts_cursor + .insert(hashed_address, Account { nonce: 1, ..Default::default() }) + .unwrap(); + storage_cursor + .insert(hashed_address, StorageEntry { key: hashed_slot, value: U256::from(1) }) + .unwrap(); + } + provider_rw.commit().unwrap(); + } + + let mut hashed_state = HashedPostState::default(); + hashed_state.insert_destroyed_account(destroyed_address_hashed); + hashed_state.insert_hashed_storage(destroyed_address_hashed, HashedStorage::new(true)); + + let provider_rw = provider_factory.provider_rw().unwrap(); + assert_eq!(HashedStateChanges(hashed_state).write_to_db(provider_rw.tx_ref()), Ok(())); + provider_rw.commit().unwrap(); + + let provider = provider_factory.provider().unwrap(); + assert_eq!( + provider.tx_ref().get::(destroyed_address_hashed), + Ok(None) + ); + assert_eq!( + provider + .tx_ref() + .cursor_read::() + .unwrap() + .seek_by_key_subkey(destroyed_address_hashed, hashed_slot), + Ok(None) + ); + } +} diff --git a/crates/storage/provider/src/bundle_state/mod.rs b/crates/storage/provider/src/bundle_state/mod.rs index 88b17ad563f0..3f5da6ec6242 100644 --- a/crates/storage/provider/src/bundle_state/mod.rs +++ b/crates/storage/provider/src/bundle_state/mod.rs @@ -1,11 +1,13 @@ //! Bundle state module. //! This module contains all the logic related to bundle state. mod bundle_state_with_receipts; +mod hashed_state_changes; mod state_changes; mod state_reverts; pub use bundle_state_with_receipts::{ AccountRevertInit, BundleStateInit, BundleStateWithReceipts, OriginalValuesKnown, RevertsInit, }; +pub use hashed_state_changes::HashedStateChanges; pub use state_changes::StateChanges; pub use state_reverts::StateReverts; diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 585f73e16be7..b6e436a97248 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,5 +1,5 @@ use crate::{ - bundle_state::{BundleStateInit, BundleStateWithReceipts, RevertsInit}, + bundle_state::{BundleStateInit, BundleStateWithReceipts, HashedStateChanges, RevertsInit}, providers::{database::metrics, SnapshotProvider}, traits::{ AccountExtReader, BlockSource, ChangeSetReader, ReceiptProvider, StageCheckpointWriter, @@ -43,7 +43,9 @@ use reth_primitives::{ TransactionMeta, TransactionSigned, TransactionSignedEcRecovered, TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, B256, U256, }; -use reth_trie::{prefix_set::PrefixSetMut, StateRoot}; +use reth_trie::{ + hashed_cursor::HashedPostState, prefix_set::PrefixSetMut, updates::TrieUpdates, StateRoot, +}; use revm::primitives::{BlockEnv, CfgEnv, SpecId}; use std::{ collections::{hash_map, BTreeMap, BTreeSet, HashMap, HashSet}, @@ -2199,21 +2201,27 @@ impl BlockWriter for DatabaseProvider { let tx_count = block.body.len() as u64; - let senders_len = senders.as_ref().map(|s| s.len()); - let tx_iter = if Some(block.body.len()) == senders_len { - block.body.into_iter().zip(senders.unwrap()).collect::>() - } else { - let senders = TransactionSigned::recover_signers(&block.body, block.body.len()) - .ok_or(ProviderError::SenderRecoveryError)?; - durations_recorder.record_relative(metrics::Action::RecoverSigners); - debug_assert_eq!(senders.len(), block.body.len(), "missing one or more senders"); - block.body.into_iter().zip(senders).collect() + // Ensures we have all the senders for the block's transactions. + let senders = match senders { + Some(senders) if block.body.len() == senders.len() => { + // senders have the correct length as transactions in the block + senders + } + _ => { + // recover senders from transactions + let senders = TransactionSigned::recover_signers(&block.body, block.body.len()) + .ok_or(ProviderError::SenderRecoveryError)?; + durations_recorder.record_relative(metrics::Action::RecoverSigners); + debug_assert_eq!(senders.len(), block.body.len(), "missing one or more senders"); + senders + } }; let mut tx_senders_elapsed = Duration::default(); let mut transactions_elapsed = Duration::default(); let mut tx_hash_numbers_elapsed = Duration::default(); - for (transaction, sender) in tx_iter { + + for (transaction, sender) in block.body.into_iter().zip(senders) { let hash = transaction.hash(); if prune_modes @@ -2287,10 +2295,12 @@ impl BlockWriter for DatabaseProvider { Ok(block_indices) } - fn append_blocks_with_bundle_state( + fn append_blocks_with_state( &self, blocks: Vec, state: BundleStateWithReceipts, + hashed_state: HashedPostState, + trie_updates: TrieUpdates, prune_modes: Option<&PruneModes>, ) -> ProviderResult<()> { if blocks.is_empty() { @@ -2303,8 +2313,6 @@ impl BlockWriter for DatabaseProvider { let last = blocks.last().unwrap(); let last_block_number = last.number; - let last_block_hash = last.hash(); - let expected_state_root = last.state_root; let mut durations_recorder = metrics::DurationsRecorder::default(); @@ -2320,7 +2328,11 @@ impl BlockWriter for DatabaseProvider { state.write_to_db(self.tx_ref(), OriginalValuesKnown::No)?; durations_recorder.record_relative(metrics::Action::InsertState); - self.insert_hashes(first_number..=last_block_number, last_block_hash, expected_state_root)?; + // insert hashes and intermediate merkle nodes + { + HashedStateChanges(hashed_state).write_to_db(&self.tx)?; + trie_updates.flush(&self.tx)?; + } durations_recorder.record_relative(metrics::Action::InsertHashes); self.update_history_indices(first_number..=last_block_number)?; diff --git a/crates/storage/provider/src/traits/block.rs b/crates/storage/provider/src/traits/block.rs index 44951a3fcac8..137d14fbbff7 100644 --- a/crates/storage/provider/src/traits/block.rs +++ b/crates/storage/provider/src/traits/block.rs @@ -10,6 +10,7 @@ use reth_primitives::{ ChainSpec, Header, PruneModes, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, B256, }; +use reth_trie::{hashed_cursor::HashedPostState, updates::TrieUpdates}; use std::ops::RangeInclusive; /// Enum to control transaction hash inclusion. @@ -291,10 +292,12 @@ pub trait BlockWriter: Send + Sync { /// # Returns /// /// Returns `Ok(())` on success, or an error if any operation fails. - fn append_blocks_with_bundle_state( + fn append_blocks_with_state( &self, blocks: Vec, state: BundleStateWithReceipts, + hashed_state: HashedPostState, + trie_updates: TrieUpdates, prune_modes: Option<&PruneModes>, ) -> ProviderResult<()>; } diff --git a/crates/transaction-pool/src/pool/blob.rs b/crates/transaction-pool/src/pool/blob.rs index e7f6fa8f2448..fc0bebbf8991 100644 --- a/crates/transaction-pool/src/pool/blob.rs +++ b/crates/transaction-pool/src/pool/blob.rs @@ -1,7 +1,7 @@ #![allow(dead_code, unused)] use crate::{ identifier::TransactionId, pool::size::SizeTracker, traits::BestTransactionsAttributes, - PoolTransaction, ValidPoolTransaction, + PoolTransaction, SubPoolLimit, ValidPoolTransaction, }; use std::{ cmp::Ordering, @@ -182,6 +182,27 @@ impl BlobTransactions { removed } + /// Removes transactions until the pool satisfies its [SubPoolLimit]. + /// + /// This is done by removing transactions according to their ordering in the pool, defined by + /// the [BlobOrd] struct. + /// + /// Removed transactions are returned in the order they were removed. + pub(crate) fn truncate_pool( + &mut self, + limit: SubPoolLimit, + ) -> Vec>> { + let mut removed = Vec::new(); + + while self.size() > limit.max_size && self.len() > limit.max_txs { + let tx = self.all.last().expect("pool is not empty"); + let id = *tx.transaction.id(); + removed.push(self.remove_transaction(&id).expect("transaction exists")); + } + + removed + } + /// Returns `true` if the transaction with the given id is already included in this pool. #[cfg(test)] #[allow(unused)] @@ -287,6 +308,13 @@ const LOG_2_1_125: f64 = 0.16992500144231237; /// /// This is supposed to get the number of fee jumps required to get from the current fee to the fee /// cap, or where the transaction would not be executable any more. +/// +/// A positive value means that the transaction will remain executable unless the current fee +/// increases. +/// +/// A negative value means that the transaction is currently not executable, and requires the +/// current fee to decrease by some number of jumps before the max fee is greater than the current +/// fee. pub fn fee_delta(max_tx_fee: u128, current_fee: u128) -> i64 { if max_tx_fee == current_fee { // if these are equal, then there's no fee jump @@ -331,16 +359,29 @@ pub fn blob_tx_priority( let delta_blob_fee = fee_delta(blob_fee_cap, blob_fee); let delta_priority_fee = fee_delta(max_priority_fee, base_fee); + // TODO: this could be u64: + // * if all are positive, zero is returned + // * if all are negative, the min negative value is returned + // * if some are positive and some are negative, the min negative value is returned + // + // the BlobOrd could then just be a u64, and higher values represent worse transactions (more + // jumps for one of the fees until the cap satisfies) + // // priority = min(delta-basefee, delta-blobfee, 0) delta_blob_fee.min(delta_priority_fee).min(0) } +/// A struct used to determine the ordering for a specific blob transaction in the pool. This uses +/// a `priority` value to determine the ordering, and uses the `submission_id` to break ties. +/// +/// The `priority` value is calculated using the [blob_tx_priority] function, and should be +/// re-calculated on each block. #[derive(Debug, Clone)] struct BlobOrd { /// Identifier that tags when transaction was submitted in the pool. pub(crate) submission_id: u64, - // The priority for this transaction, calculated using the [`blob_tx_priority`] function, - // taking into account both the blob and priority fee. + /// The priority for this transaction, calculated using the [`blob_tx_priority`] function, + /// taking into account both the blob and priority fee. pub(crate) priority: i64, } @@ -360,6 +401,9 @@ impl PartialOrd for BlobOrd { impl Ord for BlobOrd { fn cmp(&self, other: &Self) -> Ordering { + // order in reverse, so transactions with a lower ordering return Greater - this is + // important because transactions with larger negative values will take more fee jumps and + // it will take longer to become executable, so those should be evicted first let ord = other.priority.cmp(&self.priority); // use submission_id to break ties diff --git a/crates/trie/src/hashed_cursor/post_state.rs b/crates/trie/src/hashed_cursor/post_state.rs index 62e028657fc0..0b739c08849a 100644 --- a/crates/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/src/hashed_cursor/post_state.rs @@ -32,6 +32,19 @@ impl HashedStorage { } } + /// Returns `true` if the storage was wiped. + pub fn wiped(&self) -> bool { + self.wiped + } + + /// Returns all storage slots. + pub fn storage_slots(&self) -> impl Iterator + '_ { + self.zero_valued_slots + .iter() + .map(|slot| (*slot, U256::ZERO)) + .chain(self.non_zero_valued_storage.iter().cloned()) + } + /// Sorts the non zero value storage entries. pub fn sort_storage(&mut self) { if !self.sorted { @@ -58,8 +71,8 @@ impl HashedStorage { pub struct HashedPostState { /// Map of hashed addresses to account info. accounts: Vec<(B256, Account)>, - /// Set of cleared accounts. - cleared_accounts: HashSet, + /// Set of destroyed accounts. + destroyed_accounts: HashSet, /// Map of hashed addresses to hashed storage. storages: HashMap, /// Whether the account and storage entries were sorted or not. @@ -70,7 +83,7 @@ impl Default for HashedPostState { fn default() -> Self { Self { accounts: Vec::new(), - cleared_accounts: HashSet::new(), + destroyed_accounts: HashSet::new(), storages: HashMap::new(), sorted: true, // empty is sorted } @@ -84,6 +97,18 @@ impl HashedPostState { self } + /// Returns all accounts with their state. + pub fn accounts(&self) -> impl Iterator)> + '_ { + self.destroyed_accounts.iter().map(|hashed_address| (*hashed_address, None)).chain( + self.accounts.iter().map(|(hashed_address, account)| (*hashed_address, Some(*account))), + ) + } + + /// Returns all account storages. + pub fn storages(&self) -> impl Iterator { + self.storages.iter() + } + /// Sort account and storage entries. pub fn sort(&mut self) { if !self.sorted { @@ -102,9 +127,9 @@ impl HashedPostState { self.sorted = false; } - /// Insert cleared hashed account key. - pub fn insert_cleared_account(&mut self, hashed_address: B256) { - self.cleared_accounts.insert(hashed_address); + /// Insert destroyed hashed account key. + pub fn insert_destroyed_account(&mut self, hashed_address: B256) { + self.destroyed_accounts.insert(hashed_address); } /// Insert hashed storage entry. @@ -113,6 +138,11 @@ impl HashedPostState { self.storages.insert(hashed_address, hashed_storage); } + /// Returns all destroyed accounts. + pub fn destroyed_accounts(&self) -> HashSet { + self.destroyed_accounts.clone() + } + /// Construct (PrefixSet)[PrefixSet] from hashed post state. /// The prefix sets contain the hashed account and storage keys that have been changed in the /// post state. @@ -125,7 +155,7 @@ impl HashedPostState { for (hashed_address, _) in &self.accounts { account_prefix_set.insert(Nibbles::unpack(hashed_address)); } - for hashed_address in &self.cleared_accounts { + for hashed_address in &self.destroyed_accounts { account_prefix_set.insert(Nibbles::unpack(hashed_address)); } @@ -213,7 +243,7 @@ impl<'b, C> HashedPostStateAccountCursor<'b, C> { /// This function only checks the post state, not the database, because the latter does not /// store destroyed accounts. fn is_account_cleared(&self, account: &B256) -> bool { - self.post_state.cleared_accounts.contains(account) + self.post_state.destroyed_accounts.contains(account) } /// Return the account with the lowest hashed account key. @@ -667,7 +697,7 @@ mod tests { let mut hashed_post_state = HashedPostState::default(); for (hashed_address, account) in accounts.iter().filter(|x| x.0[31] % 2 != 0) { if removed_keys.contains(hashed_address) { - hashed_post_state.insert_cleared_account(*hashed_address); + hashed_post_state.insert_destroyed_account(*hashed_address); } else { hashed_post_state.insert_account(*hashed_address, *account); } @@ -722,7 +752,7 @@ mod tests { if let Some(account) = account { hashed_post_state.insert_account(*hashed_address, *account); } else { - hashed_post_state.insert_cleared_account(*hashed_address); + hashed_post_state.insert_destroyed_account(*hashed_address); } } hashed_post_state.sort(); @@ -886,7 +916,7 @@ mod tests { let wiped = false; let mut hashed_storage = HashedStorage::new(wiped); for (slot, value) in post_state_storage.iter() { - if *value == U256::ZERO { + if value.is_zero() { hashed_storage.insert_zero_valued_slot(*slot); } else { hashed_storage.insert_non_zero_valued_storage(*slot, *value); @@ -1000,7 +1030,7 @@ mod tests { for (address, (wiped, storage)) in &post_state_storages { let mut hashed_storage = HashedStorage::new(*wiped); for (slot, value) in storage { - if *value == U256::ZERO { + if value.is_zero() { hashed_storage.insert_zero_valued_slot(*slot); } else { hashed_storage.insert_non_zero_valued_storage(*slot, *value); diff --git a/deny.toml b/deny.toml index 524ab567d81c..dd8e50780b73 100644 --- a/deny.toml +++ b/deny.toml @@ -19,9 +19,7 @@ wildcards = "allow" highlight = "all" # List of crates to deny deny = [ - # Each entry the name of a crate and a version range. If version is - # not specified, all versions will be matched. - #{ name = "ansi_term", version = "=0.11.0" }, + { name = "openssl" }, ] # Certain crates/versions that will be skipped when doing duplicate detection. skip = [] diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml index 84a97c1f80e6..c5f8e1334d86 100644 --- a/etc/docker-compose.yml +++ b/etc/docker-compose.yml @@ -20,7 +20,7 @@ services: node --chain mainnet --metrics 0.0.0.0:9001 - --log.directory /root/rethlogs + --log.file.directory /root/rethlogs --authrpc.addr 0.0.0.0 --authrpc.port 8551 --authrpc.jwtsecret /root/jwt/jwt.hex diff --git a/examples/beacon-api-sse/Cargo.toml b/examples/beacon-api-sse/Cargo.toml index 892c2e229e73..60bb76c53656 100644 --- a/examples/beacon-api-sse/Cargo.toml +++ b/examples/beacon-api-sse/Cargo.toml @@ -15,4 +15,4 @@ tracing.workspace = true futures-util.workspace = true tokio = { workspace = true, features = ["time"] } -mev-share-sse = "0.1.5" \ No newline at end of file +mev-share-sse = { version = "0.1.6" , default-features = false } \ No newline at end of file diff --git a/examples/db-access.rs b/examples/db-access.rs index 235193a38f64..dd9c74e58016 100644 --- a/examples/db-access.rs +++ b/examples/db-access.rs @@ -67,7 +67,7 @@ fn header_provider_example(provider: T, number: u64) -> eyre: // The header's total difficulty is stored in a separate table, so we have a separate call for // it. This is not needed for post PoS transition chains. let td = provider.header_td_by_number(number)?.ok_or(eyre::eyre!("header td not found"))?; - assert_ne!(td, U256::ZERO); + assert!(!td.is_zero()); // Can query headers by range as well, already sealed! let headers = provider.sealed_headers_range(100..200)?; diff --git a/testing/ef-tests/tests/tests.rs b/testing/ef-tests/tests/tests.rs index 75e6e2a8cd27..e1d18116bccd 100644 --- a/testing/ef-tests/tests/tests.rs +++ b/testing/ef-tests/tests/tests.rs @@ -10,7 +10,7 @@ macro_rules! general_state_test { // std::thread::Builder::new() .stack_size( - 1024 * 1024 * 4, // 4MB + 1024 * 1024 * 8, // 8MB ) .spawn(move || { BlockchainTests::new(format!("GeneralStateTests/{}", stringify!($dir))).run();