From 74c4d52181c11731a5dc162394dec8e97dd4bd2d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 17 Sep 2026 11:49:43 +0200 Subject: [PATCH 001/133] perf(txpool): reserve insertion metadata before locking (#27283) --- crates/transaction-pool/src/pool/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 3cd955f21c6..04c076e6460 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -673,13 +673,14 @@ where Item = (TransactionOrigin, TransactionValidationOutcome), >, ) -> Vec> { + let transactions = transactions.into_iter(); + let mut added_metas = Vec::with_capacity(transactions.size_hint().0); + // Collect results and metadata while holding the pool write lock let (mut results, added_metas, discarded) = { let mut pool = self.pool.write(); - let mut added_metas = Vec::new(); let results = transactions - .into_iter() .map(|(origin, tx)| { let (result, meta) = self.add_transaction(&mut pool, origin, tx); From 8f1b805f17d816ed56e377172b792c85333deceb Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:22:02 +0000 Subject: [PATCH 002/133] chore(mdbx): disable Linux mincore residency probes (#27271) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- crates/storage/libmdbx-rs/mdbx-sys/build.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/storage/libmdbx-rs/mdbx-sys/build.rs b/crates/storage/libmdbx-rs/mdbx-sys/build.rs index 1bd764b87d2..612896a047d 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/build.rs +++ b/crates/storage/libmdbx-rs/mdbx-sys/build.rs @@ -28,6 +28,11 @@ fn main() { // which works correctly on all filesystems. .define("MDBX_USE_FALLOCATE", "0"); + if env::var("CARGO_CFG_TARGET_OS").unwrap() == "linux" { + // Avoid residency syscalls on the write path while retaining MDBX's prefault writes. + cc.define("MDBX_USE_MINCORE", "0"); + } + // Enable debugging on debug builds #[cfg(debug_assertions)] cc.define("MDBX_DEBUG", "1").define("MDBX_ENABLE_PROFGC", "1"); From b59d2ab287cff92d9a0903bd4542550a679551c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:23:45 +0000 Subject: [PATCH 003/133] chore(provider): share insert_headers test helper (#27289) --- crates/net/downloaders/src/bodies/bodies.rs | 4 ++-- crates/net/downloaders/src/bodies/task.rs | 7 ++---- .../net/downloaders/src/bodies/test_utils.rs | 22 ----------------- crates/net/downloaders/src/file_client.rs | 7 ++---- crates/stages/stages/src/stages/headers.rs | 12 ++-------- crates/storage/provider/src/test_utils/mod.rs | 24 ++++++++++++++++--- 6 files changed, 29 insertions(+), 47 deletions(-) diff --git a/crates/net/downloaders/src/bodies/bodies.rs b/crates/net/downloaders/src/bodies/bodies.rs index abe2bdaf834..fd2b7f9b3df 100644 --- a/crates/net/downloaders/src/bodies/bodies.rs +++ b/crates/net/downloaders/src/bodies/bodies.rs @@ -610,13 +610,13 @@ impl BodiesDownloaderBuilder { mod tests { use super::*; use crate::{ - bodies::test_utils::{insert_headers, zip_blocks}, + bodies::test_utils::zip_blocks, test_utils::{generate_bodies, TestBodiesClient}, }; use alloy_primitives::{map::B256Map, B256}; use assert_matches::assert_matches; use reth_consensus::test_utils::TestConsensus; - use reth_provider::test_utils::create_test_provider_factory; + use reth_provider::test_utils::{create_test_provider_factory, insert_headers}; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; // Check that the blocks are emitted in order of block number, not in order of diff --git a/crates/net/downloaders/src/bodies/task.rs b/crates/net/downloaders/src/bodies/task.rs index 48a3cd8d02b..e03f3bba8ee 100644 --- a/crates/net/downloaders/src/bodies/task.rs +++ b/crates/net/downloaders/src/bodies/task.rs @@ -132,16 +132,13 @@ impl Future for SpawnedDownloader { mod tests { use super::*; use crate::{ - bodies::{ - bodies::BodiesDownloaderBuilder, - test_utils::{insert_headers, zip_blocks}, - }, + bodies::{bodies::BodiesDownloaderBuilder, test_utils::zip_blocks}, test_utils::{generate_bodies, TestBodiesClient}, }; use assert_matches::assert_matches; use reth_consensus::test_utils::TestConsensus; use reth_network_p2p::error::DownloadError; - use reth_provider::test_utils::create_test_provider_factory; + use reth_provider::test_utils::{create_test_provider_factory, insert_headers}; use std::sync::Arc; #[tokio::test(flavor = "multi_thread")] diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index ae36c57882e..b6ccf48ed51 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -7,10 +7,6 @@ use alloy_primitives::map::B256Map; use reth_ethereum_primitives::BlockBody; use reth_network_p2p::bodies::response::BlockResponse; use reth_primitives_traits::{Block, SealedBlock, SealedHeader}; -use reth_provider::{ - test_utils::MockNodeTypesWithDB, ProviderFactory, StaticFileProviderFactory, StaticFileSegment, - StaticFileWriter, -}; pub(crate) fn zip_blocks<'a, B: Block>( headers: impl Iterator>, @@ -41,21 +37,3 @@ pub(crate) fn create_raw_bodies( }) .collect() } - -#[inline] -pub(crate) fn insert_headers( - factory: &ProviderFactory, - headers: &[SealedHeader], -) { - let provider_rw = factory.provider_rw().expect("failed to create provider"); - let static_file_provider = provider_rw.static_file_provider(); - let mut writer = static_file_provider - .latest_writer(StaticFileSegment::Headers) - .expect("failed to create writer"); - - for header in headers { - writer.append_header(header.header(), &header.hash()).expect("failed to append header"); - } - drop(writer); - provider_rw.commit().expect("failed to commit"); -} diff --git a/crates/net/downloaders/src/file_client.rs b/crates/net/downloaders/src/file_client.rs index 9b11dcbf5c6..3e3348b1a8e 100644 --- a/crates/net/downloaders/src/file_client.rs +++ b/crates/net/downloaders/src/file_client.rs @@ -701,10 +701,7 @@ pub struct DecodedFileChunk { mod tests { use super::*; use crate::{ - bodies::{ - bodies::BodiesDownloaderBuilder, - test_utils::{insert_headers, zip_blocks}, - }, + bodies::{bodies::BodiesDownloaderBuilder, test_utils::zip_blocks}, headers::{reverse_headers::ReverseHeadersDownloaderBuilder, test_utils::child_header}, test_utils::{generate_bodies, generate_bodies_file}, }; @@ -718,7 +715,7 @@ mod tests { bodies::downloader::BodyDownloader, headers::downloader::{HeaderDownloader, SyncTarget}, }; - use reth_provider::test_utils::create_test_provider_factory; + use reth_provider::test_utils::{create_test_provider_factory, insert_headers}; use std::sync::Arc; use tokio::{ fs::File, diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index 6b8a8c9d361..a128d82c671 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -392,7 +392,7 @@ mod tests { }; use alloy_primitives::B256; use assert_matches::assert_matches; - use reth_provider::{DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory}; + use reth_provider::{test_utils::insert_headers, ProviderFactory, StaticFileProviderFactory}; use reth_stages_api::StageUnitCheckpoint; use reth_testing_utils::generators::{self, random_header, random_header_range}; use std::sync::Arc; @@ -611,15 +611,7 @@ mod tests { tip.hash(), ); - let provider = runner.db().factory.database_provider_rw().unwrap(); - let static_file_provider = provider.static_file_provider(); - let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); - for header in sealed_headers { - writer.append_header(header.header(), &header.hash()).unwrap(); - } - drop(writer); - - provider.commit().unwrap(); + insert_headers(&runner.db().factory, &sealed_headers); // now we can unwind 10 blocks let unwind_input = UnwindInput { diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index 8a5df8d1ea7..d21769cc253 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -3,15 +3,16 @@ use crate::{ NodeTypesForProvider, ProviderNodeTypes, RocksDBBuilder, StaticFileProvider, StaticFileProviderBuilder, }, - HashingWriter, ProviderFactory, TrieWriter, + HashingWriter, ProviderFactory, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, + TrieWriter, }; use alloy_primitives::B256; use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; use reth_db::{mdbx::DatabaseArguments, test_utils::TempDatabase, DatabaseEnv}; use reth_errors::ProviderResult; use reth_ethereum_engine_primitives::EthEngineTypes; -use reth_node_types::NodeTypesWithDBAdapter; -use reth_primitives_traits::{Account, StorageEntry}; +use reth_node_types::{HeaderTy, NodeTypesWithDBAdapter}; +use reth_primitives_traits::{Account, SealedHeader, StorageEntry}; use reth_storage_api::StorageSettingsCache; use reth_trie::StateRoot; use reth_trie_db::DatabaseStateRoot; @@ -174,3 +175,20 @@ pub fn insert_genesis>( Ok(root) } + +/// Appends `headers` to the headers static files and commits them. +pub fn insert_headers( + factory: &ProviderFactory, + headers: &[SealedHeader>], +) { + let provider = factory.provider_rw().expect("failed to create provider"); + let static_file_provider = provider.static_file_provider(); + let mut writer = static_file_provider + .latest_writer(StaticFileSegment::Headers) + .expect("failed to create writer"); + for header in headers { + writer.append_header(header.header(), &header.hash()).expect("failed to append header"); + } + drop(writer); + provider.commit().expect("failed to commit"); +} From 06f542d253374e41b0b5af23c430032f2acd86ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:39:39 +0000 Subject: [PATCH 004/133] perf(evm): encode built BAL once in block builder (#27288) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 1 - crates/engine/util/Cargo.toml | 1 - crates/engine/util/src/reorg.rs | 4 +--- crates/ethereum/node/tests/e2e/rpc.rs | 5 +++-- crates/ethereum/payload/src/lib.rs | 5 ++--- crates/evm/evm/src/execute.rs | 17 +++++++++++------ crates/net/downloaders/Cargo.toml | 1 - crates/rpc/rpc/src/testing.rs | 4 +--- 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a2a28b3901..af63d8b8a72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8652,7 +8652,6 @@ version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", - "alloy-rlp", "alloy-rpc-types-engine", "eyre", "futures", diff --git a/crates/engine/util/Cargo.toml b/crates/engine/util/Cargo.toml index aa2f0ba9d1e..4baad75c063 100644 --- a/crates/engine/util/Cargo.toml +++ b/crates/engine/util/Cargo.toml @@ -25,7 +25,6 @@ reth-payload-primitives.workspace = true # alloy alloy-primitives.workspace = true -alloy-rlp.workspace = true alloy-rpc-types-engine.workspace = true alloy-consensus.workspace = true diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index 8a91827c759..4769c729aac 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -309,7 +309,5 @@ where let BlockBuilderOutcome { block, block_access_list, .. } = builder.finish(&state_provider, None)?; - let encoded_bal: Option = block_access_list.map(|bal| alloy_rlp::encode(&bal).into()); - - Ok((block.into_sealed_block(), encoded_bal)) + Ok((block.into_sealed_block(), block_access_list.map(|bal| bal.split().1))) } diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 3cad83a38b5..12b12593357 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -3,7 +3,7 @@ use alloy_eips::{ eip2718::Encodable2718, eip7910::EthConfig, eip7928::BlockAccessList, BlockNumberOrTag, }; use alloy_genesis::Genesis; -use alloy_primitives::{Address, Bytes, B256, U256}; +use alloy_primitives::{keccak256, Address, Bytes, B256, U256}; use alloy_provider::{ ext::DebugApi, network::{EthereumWallet, TransactionBuilder}, @@ -758,7 +758,8 @@ async fn test_flashbots_validate_v6() -> eyre::Result<()> { inject_blob_transaction(&node, &wallet).await?; let payload = node.new_payload().await?; - assert!(payload.block_access_list().is_some()); + let block_access_list = payload.block_access_list().expect("amsterdam payload has a BAL"); + assert_eq!(Some(keccak256(block_access_list)), payload.block().block_access_list_hash); let envelope = payload.clone().try_into_v6()?; assert!(!envelope.blobs_bundle.blobs.is_empty()); diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 0769c72539b..3cb0ade0fea 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -9,7 +9,7 @@ #![cfg_attr(docsrs, feature(doc_cfg))] use alloy_consensus::{BlockHeader, Transaction}; -use alloy_primitives::{Bytes, U256}; +use alloy_primitives::U256; use alloy_rlp::Encodable; use alloy_rpc_types_engine::PayloadAttributes as EthPayloadAttributes; use reth_basic_payload_builder::{ @@ -497,8 +497,7 @@ where })); } - let block_access_list: Option = - block_access_list.map(|block_access_list| alloy_rlp::encode(&block_access_list).into()); + let block_access_list = block_access_list.map(|bal| bal.split().1); let payload = EthBuiltPayload::new(Arc::new(block), total_fees, requests, block_access_list) // add blob sidecars from the executed txs .with_sidecars(blob_sidecars); diff --git a/crates/evm/evm/src/execute.rs b/crates/evm/evm/src/execute.rs index 8035ac80964..5edee39ead6 100644 --- a/crates/evm/evm/src/execute.rs +++ b/crates/evm/evm/src/execute.rs @@ -3,7 +3,7 @@ use crate::{ConfigureEvm, Database, OnStateHook, TxEnvFor}; use alloc::{boxed::Box, sync::Arc, vec::Vec}; use alloy_consensus::{BlockHeader, Header}; -use alloy_eip7928::{compute_block_access_list_hash, BlockAccessList}; +use alloy_eip7928::{bal::DecodedBal, compute_block_access_list_hash_with_buf, BlockAccessList}; use alloy_eips::eip2718::WithEncoded; pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory, GasOutput}; use alloy_evm::{ @@ -314,8 +314,9 @@ pub struct BlockBuilderOutcome { pub trie_updates: TrieUpdates, /// The built block. pub block: RecoveredBlock, - /// Block access list built during execution (EIP-7928, Amsterdam). - pub block_access_list: Option, + /// Block access list built during execution (EIP-7928, Amsterdam), with its RLP bytes and + /// hash. + pub block_access_list: Option, } /// A type that knows how to execute and build a block. @@ -510,9 +511,13 @@ where // merge all transitions into bundle state db.merge_transitions(BundleRetention::Reverts); - let block_access_list = db.take_built_alloy_bal(); - let block_access_list_hash = - block_access_list.as_ref().map(|bal| compute_block_access_list_hash(bal.as_slice())); + // Encode the built BAL once and keep the bytes, so callers don't re-encode it. + let block_access_list = db.take_built_alloy_bal().map(|bal| { + let mut raw = Vec::new(); + let hash = compute_block_access_list_hash_with_buf(&bal, &mut raw); + DecodedBal::new_unchecked(bal.into(), raw.into(), hash) + }); + let block_access_list_hash = block_access_list.as_ref().map(DecodedBal::hash); let hashed_state = state.hashed_post_state(&db.bundle_state).map_err(BlockExecutionError::other)?; diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml index b9f052962bc..1d451aaaf88 100644 --- a/crates/net/downloaders/Cargo.toml +++ b/crates/net/downloaders/Cargo.toml @@ -25,7 +25,6 @@ reth-trie-common.workspace = true # optional deps for the test-utils feature reth-ethereum-primitives = { workspace = true, optional = true } -reth-provider = { workspace = true, optional = true } reth-testing-utils = { workspace = true, optional = true } # ethereum diff --git a/crates/rpc/rpc/src/testing.rs b/crates/rpc/rpc/src/testing.rs index b166f913a3f..99206114aa8 100644 --- a/crates/rpc/rpc/src/testing.rs +++ b/crates/rpc/rpc/src/testing.rs @@ -272,9 +272,7 @@ where let has_requests = outcome.block.requests_hash().is_some(); let requests = has_requests.then_some(outcome.execution_result.requests); - let block_access_list = outcome - .block_access_list - .map(|block_access_list| alloy_rlp::encode(&block_access_list).into()); + let block_access_list = outcome.block_access_list.map(|bal| bal.split().1); Ok(EthBuiltPayload::new( Arc::new(outcome.block), From 835e42864d2f5cf375ebeacca3d279c3838971dd Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 17 Sep 2026 12:58:50 +0000 Subject: [PATCH 005/133] perf(trie): skip historical trie changesets with complete overlays (#27129) --- .../src/providers/blockchain_provider.rs | 153 +++++++++++++ .../provider/src/providers/state/latest.rs | 25 ++- crates/storage/provider/src/writer/mod.rs | 12 +- crates/storage/storage-overlay/src/builder.rs | 136 +++++++++--- .../storage-overlay/src/changeset_cache.rs | 4 +- crates/storage/storage-overlay/src/manager.rs | 6 +- .../storage/storage-overlay/src/provider.rs | 208 ++++++++++-------- crates/trie/common/src/input.rs | 7 + crates/trie/db/src/proof.rs | 22 +- crates/trie/db/src/storage.rs | 27 ++- 10 files changed, 440 insertions(+), 160 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index ac11538c2fd..318a374421a 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1080,6 +1080,159 @@ mod tests { sync::Arc, }; + #[test] + fn historical_proofs_complete_masked_trie_rows() -> eyre::Result<()> { + use crate::{StaticFileProviderFactory, StaticFileSegment, StaticFileWriter}; + use reth_storage_api::{StorageSettings, StorageSettingsCache, TrieWriter}; + use reth_trie::{test_utils::TrieTestHarness, MultiProofTargetsV2}; + use revm::state::AccountInfo; + + let address = Address::with_last_byte(1); + let other_address = Address::with_last_byte(2); + let hashed_address = keccak256(address); + let account = Account { balance: U256::from(1), ..Default::default() }; + // Branches 00, 01 and 02 share the cached row at 0. The first two change + // in different blocks; the third keeps proof-v2's branch-collapse path inactive. + let slots: Vec = (0..=2u8) + .flat_map(|prefix| { + (0..u64::MAX) + .map(|slot| B256::from(U256::from(slot))) + .filter(move |slot| keccak256(slot)[0] == prefix) + .take(2) + }) + .collect(); + let mut storage: BTreeMap<_, _> = slots.iter().map(|&slot| (slot, U256::from(1))).collect(); + let mut trie = TrieTestHarness::new( + storage.iter().map(|(slot, value)| (keccak256(slot), *value)).collect(), + ); + let mut blocks = Vec::::new(); + let mut storage_roots = Vec::new(); + for number in 0..=2 { + let mut output = (*ExecutedBlock::::default() + .execution_output) + .clone(); + let (state, storage_updates) = if number == 0 { + ( + HashedPostState::default() + .with_accounts([ + (hashed_address, Some(account)), + (keccak256(other_address), Some(account)), + ]) + .with_storages([( + hashed_address, + HashedStorage::from_iter(trie.storage().iter().map(|(k, v)| (*k, *v))), + )]), + trie.storage_trie_updates().clone(), + ) + } else { + let slot = slots[(number as usize - 1) * 2]; + let value = U256::from(number + 1); + let changes = BTreeMap::from([(keccak256(slot), value)]); + let (_, updates) = trie.get_root_with_updates(&changes); + trie.apply_changeset(changes); + storage.insert(slot, value); + let info = AccountInfo::from_balance(account.balance); + output.state = BundleState::builder(number..=number) + .state_original_account_info(address, info.clone()) + .state_present_account_info(address, info) + .state_storage( + address, + std::iter::once((U256::from_be_bytes(slot.0), (U256::from(1), value))) + .collect(), + ) + .revert_storage( + number, + address, + vec![(U256::from_be_bytes(slot.0), U256::from(1))], + ) + .build(); + ( + HashedPostState::from_hashed_storage( + hashed_address, + HashedStorage::from_iter([(keccak256(slot), value)]), + ), + updates, + ) + }; + let mut updates = TrieUpdates::default(); + updates.storage_tries.insert(hashed_address, storage_updates); + let state_root = reth_trie::test_utils::state_root([ + (address, (account, storage.clone())), + (other_address, (account, BTreeMap::new())), + ]); + storage_roots.push(reth_trie::test_utils::storage_root(storage.clone())); + let block = Block { + header: alloy_consensus::Header { + number, + parent_hash: blocks + .last() + .map(|b| b.recovered_block().hash()) + .unwrap_or_default(), + state_root, + ..Default::default() + }, + ..Default::default() + }; + blocks.push(ExecutedBlock::new( + Arc::new(RecoveredBlock::new_unhashed(block, vec![])), + Arc::new(output), + ComputedTrieData::new( + Arc::new(state.into_sorted()), + Arc::new(updates.into_sorted()), + ), + )); + } + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(StorageSettings::v2()); + let writer = factory.provider_rw()?; + writer.insert_block(blocks[0].recovered_block())?; + writer.write_hashed_state(blocks[0].hashed_state_ref())?; + writer.write_trie_updates_sorted(blocks[0].trie_updates_ref())?; + writer.commit()?; + let static_files = factory.static_file_provider(); + static_files.get_writer(0, StaticFileSegment::Receipts)?.increment_block(0)?; + static_files + .get_writer(0, StaticFileSegment::AccountChangeSets)? + .append_account_changeset(vec![], 0)?; + static_files + .get_writer(0, StaticFileSegment::StorageChangeSets)? + .append_storage_changeset(vec![], 0)?; + static_files.commit()?; + let writer = factory.provider_rw()?; + writer.save_blocks(&SaveBlocksInput::new(blocks[1..].to_vec(), 0, 0, 2, 1))?; + writer.commit()?; + factory.overlay_manager().insert_block(blocks[2].clone()); + let provider = BlockchainProvider::new(factory)?; + + for number in [1, 2] { + let state = provider.history_by_block_hash(blocks[number].recovered_block().hash())?; + let root = blocks[number].recovered_block().state_root; + state + .multiproof_v2( + Default::default(), + MultiProofTargetsV2 { + account_targets: vec![keccak256(other_address).into()], + ..Default::default() + }, + )? + .account_proof(other_address, &[])? + .verify(root)?; + assert_eq!(state.state_root(HashedPostState::default())?, root); + assert_eq!( + state.storage_root(address, HashedStorage::default())?, + storage_roots[number] + ); + state + .storage_proof(address, slots[2], HashedStorage::default())? + .verify(storage_roots[number])?; + let proof = state.storage_multiproof(address, &[slots[2]], HashedStorage::default())?; + assert_eq!(proof.root, storage_roots[number]); + proof.storage_proof(slots[2])?.verify(storage_roots[number])?; + } + Ok(()) + } + const TEST_BLOCKS_COUNT: usize = 5; const TEST_TRANSACTIONS_COUNT: u8 = 4; diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index f85b371ea72..788c60e9bd8 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -153,7 +153,14 @@ impl StorageRootProvider hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.0, |A| { - >::overlay_root(self.tx(), address, hashed_storage) + let input = TrieInputSorted::from_state( + HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + ) + .into_sorted(), + ); + >::overlay_root(self.tx(), address, input) .map_err(|err| ProviderError::Database(err.into())) }) } @@ -182,13 +189,15 @@ impl StorageRootProvider hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.0, |A| { - >::overlay_storage_multiproof( - self.tx(), - address, - slots, - hashed_storage, - ) - .map_err(ProviderError::from) + let input = TrieInputSorted::from_state( + HashedPostState::from_hashed_storage( + alloy_primitives::keccak256(address), + hashed_storage, + ) + .into_sorted(), + ); + >::overlay_storage_multiproof(self.tx(), address, slots, input) + .map_err(ProviderError::from) }) } } diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index ef8fb25013b..b5a313c4448 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -20,6 +20,7 @@ mod tests { use reth_trie::{ test_utils::{state_root, storage_root_prehashed}, HashedPostState, HashedStorage, StateRoot, StorageRoot, StorageRootProgress, + TrieInputSorted, }; use reth_trie_db::{DatabaseStateRoot, DatabaseStorageRoot, LegacyKeyAdapter, PackedKeyAdapter}; use revm::database::{ @@ -1186,18 +1187,25 @@ mod tests { reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, >; let is_v2 = provider_rw.cached_storage_settings().is_v2(); + let input = TrieInputSorted::from_state(HashedPostState::from_hashed_storage( + hashed_address, + updated_storage.clone(), + ).into_sorted()); let storage_root = if is_v2 { TestStorageRoot::<_, _, PackedKeyAdapter>::overlay_root( tx, address, - updated_storage.clone(), + input, ) .unwrap() } else { TestStorageRoot::<_, _, LegacyKeyAdapter>::overlay_root( tx, address, - updated_storage.clone(), + TrieInputSorted::from_state(HashedPostState::from_hashed_storage( + hashed_address, + updated_storage.clone(), + ).into_sorted()), ) .unwrap() }; diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index d56fc0b4d7f..b78df88c9d3 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1,4 +1,7 @@ -use crate::{manager::OverlayCacheConfig, OverlayManager}; +use crate::{ + manager::{OverlayCacheConfig, StateTrieOverlayError}, + OverlayManager, +}; use alloy_eips::BlockNumHash; use alloy_primitives::{ map::{AddressMap, AddressSet, B256Map, U256Map}, @@ -16,7 +19,7 @@ use reth_storage_api::{ BlockNumReader, ChangeSetReader, DBProvider, PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; -use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; +use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted}; use reth_trie_db::DatabaseHashedPostState; use revm::{bytecode::Bytecode, database::BundleState, state::AccountInfo}; use std::{ @@ -29,28 +32,23 @@ use tracing::{debug, debug_span, instrument}; /// Contains the trie and hashed-state data required to initialize an overlay state provider. #[derive(Debug, Clone)] pub struct StateTrieOverlay { - /// Trie updates overlay. - pub trie_updates: Arc, - /// Hashed state overlay. - pub hashed_post_state: Arc, + input: TrieInputSorted, /// Whether construction was skipped because a reused sparse trie covers this range. skipped_for_reused_sparse_trie: bool, } impl StateTrieOverlay { - pub(crate) const fn new( - trie_updates: Arc, - hashed_post_state: Arc, - ) -> Self { - Self { trie_updates, hashed_post_state, skipped_for_reused_sparse_trie: false } + pub(crate) const fn new(input: TrieInputSorted) -> Self { + Self { input, skipped_for_reused_sparse_trie: false } } fn empty() -> Self { - Self { - trie_updates: Arc::new(TrieUpdatesSorted::default()), - hashed_post_state: Arc::new(HashedPostStateSorted::default()), - skipped_for_reused_sparse_trie: true, - } + Self { input: TrieInputSorted::default(), skipped_for_reused_sparse_trie: true } + } + + /// Returns the trie input represented by this overlay. + pub const fn input(&self) -> &TrieInputSorted { + &self.input } pub(crate) const fn skipped_for_reused_sparse_trie(&self) -> bool { @@ -393,11 +391,17 @@ impl OverlayBuilder { } /// Builds the effective state trie overlay for the given provider. + /// + /// Set `trie_changesets` only for consumers that produce [`TrieUpdates`], such as + /// [`StateRootProvider::state_root_with_updates`]. Other consumers, including roots, proofs, + /// multiproofs, and witnesses, should leave it false: complete the cached trie at Finish and + /// invalidate hashed-state revert prefixes instead of querying trie changesets. #[cfg(test)] #[instrument(level = "debug", target = "storage::overlay", skip_all)] fn build_state_trie_overlay( &self, provider: &Provider, + trie_changesets: bool, ) -> ProviderResult where Provider: StageCheckpointReader @@ -409,7 +413,12 @@ impl OverlayBuilder { + StorageSettingsCache, { let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - self.build_state_trie_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) + self.build_state_trie_overlay_at_frontiers( + provider, + state_trie_tip_block, + finish_tip_block, + trie_changesets, + ) } /// Builds the effective state trie overlay using frontiers already read from the provider. @@ -426,6 +435,7 @@ impl OverlayBuilder { provider: &Provider, state_trie_tip_block: BlockNumHash, finish_tip_block: BlockNumHash, + trie_changesets: bool, ) -> ProviderResult where Provider: ChangeSetReader @@ -445,7 +455,7 @@ impl OverlayBuilder { self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; // Collect any reverts which are required to bring the DB view back to the anchor hash. - let (trie_updates, hashed_post_state) = match &anchor_for_parent { + let (trie_updates, hashed_post_state, prefix_sets) = match &anchor_for_parent { AnchorForParent::RevertsRequired { anchor, .. } => { let revert_blocks = self.revert_blocks(&anchor_for_parent)?.expect("reverts are required"); @@ -457,7 +467,7 @@ impl OverlayBuilder { "Collecting trie reverts for overlay state provider" ); - let trie_reverts = { + let trie_reverts = if trie_changesets { let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); @@ -470,6 +480,32 @@ impl OverlayBuilder { )?; retrieve_trie_reverts_duration = start.elapsed(); accumulated_reverts + } else if state_trie_tip_block == finish_tip_block { + retrieve_trie_reverts_duration = Duration::ZERO; + Arc::default() + } else { + // Revert prefixes describe changes since the anchor, not stale hashes in + // masked DB rows. Complete the trie at Finish before applying those prefixes. + let start = Instant::now(); + let finish_state = self + .overlay_manager + .block_state(finish_tip_block.hash) + .ok_or_else(|| { + ProviderError::other(StateTrieOverlayError { + tip_hash: finish_tip_block.hash, + anchor_hash: state_trie_tip_block.hash, + }) + })?; + let (nodes, _) = self + .overlay_manager + .overlay_for_parent( + &finish_state, + state_trie_tip_block.hash, + OverlayCacheConfig::default(), + ) + .map_err(ProviderError::other)?; + retrieve_trie_reverts_duration = start.elapsed(); + nodes }; let mut hashed_state_reverts = { @@ -482,6 +518,12 @@ impl OverlayBuilder { res }; + let prefix_sets = if trie_changesets { + Default::default() + } else { + hashed_state_reverts.construct_prefix_sets() + }; + // Resolve overlays and extend reverts with them. If reverts are empty, use overlays // directly to avoid cloning. let (overlay_trie, overlay_state) = @@ -517,7 +559,7 @@ impl OverlayBuilder { "Reverted to anchor block", ); - (trie_updates, hashed_state_updates) + (trie_updates, hashed_state_updates, prefix_sets) } AnchorForParent::NoReverts { anchor } => { // If no reverts are needed, use the manager overlay directly unless the reused @@ -548,7 +590,7 @@ impl OverlayBuilder { "Built overlay directly from durable frontier" ); - (trie_updates, hashed_post_state) + (trie_updates, hashed_post_state, Default::default()) } }; @@ -561,7 +603,11 @@ impl OverlayBuilder { self.metrics.trie_updates_size.record(trie_updates_total_len as f64); self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); - Ok(StateTrieOverlay::new(trie_updates, hashed_post_state)) + Ok(StateTrieOverlay::new(TrieInputSorted::new( + trie_updates, + hashed_post_state, + prefix_sets, + ))) } /// Returns the in-memory execution overlay and the block for historical fallback reads. @@ -892,11 +938,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() + overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -1013,7 +1059,7 @@ mod tests { for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { let overlay = manager .overlay_builder(blocks[parent_index].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap(); assert_eq!( @@ -1041,11 +1087,11 @@ mod tests { let overlay = manager .overlay_builder(blocks[4].recovered_block().hash()) .with_skip_overlay_for_reused_sparse_trie(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap(); - assert!(overlay.hashed_post_state.is_empty()); - assert!(overlay.trie_updates.is_empty()); + assert!(overlay.input().state.is_empty()); + assert!(overlay.input().nodes.is_empty()); } #[test] @@ -1056,7 +1102,7 @@ mod tests { let builder = OverlayManager::::default() .overlay_builder(blocks[1].recovered_block().hash()) .with_no_reverts(); - let error = builder.build_state_trie_overlay(&provider).unwrap_err(); + let error = builder.build_state_trie_overlay(&provider, true).unwrap_err(); assert!(error.to_string().contains("reverts are disabled")); } @@ -1077,11 +1123,37 @@ mod tests { Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash )); assert!(matches!( - builder.build_state_trie_overlay(&provider), + builder.build_state_trie_overlay(&provider, true), Err(ProviderError::BlockHashNotFound(hash)) if hash == parent_hash )); } + #[test] + fn state_trie_overlay_uses_revert_prefix_sets_without_trie_changesets() { + let (factory, blocks) = setup_frontiers(3, 3); + let provider_rw = factory.provider_rw().unwrap(); + provider_rw + .tx_ref() + .put::( + 3, + AccountBeforeTx { + address: Address::with_last_byte(1), + info: Some(Account::default()), + }, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let overlay = OverlayManager::::default() + .overlay_builder(blocks[1].recovered_block().hash()) + .build_state_trie_overlay(&provider, false) + .unwrap(); + + assert!(overlay.input().nodes.is_empty()); + assert!(!overlay.input().prefix_sets.is_empty()); + } + #[test] fn execution_overlay_marks_historical_fallback() { let (factory, blocks) = setup_frontiers(1, 3); @@ -1276,7 +1348,7 @@ mod tests { let provider = factory.provider().unwrap(); let error = OverlayManager::::default() .overlay_builder(blocks[3].recovered_block().hash()) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap_err(); assert!( @@ -1292,7 +1364,7 @@ mod tests { let parent_hash = blocks[3].recovered_block().hash(); let error = OverlayManager::::default() .overlay_builder(parent_hash) - .build_state_trie_overlay(&provider) + .build_state_trie_overlay(&provider, true) .unwrap_err(); assert!(error.to_string().contains("is after partial state trie frontier")); diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index c2e070fd4b1..69191d65ae9 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -403,7 +403,7 @@ impl ChangesetCache { let overlay = overlay_manager .overlay_builder(finish.hash) .with_no_reverts() - .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish)?; + .build_state_trie_overlay_at_frontiers(provider, partial_state_trie, finish, true)?; let state_trie_provider = OverlayStateProvider::<&P, N>::new_with_state_trie( provider, overlay, @@ -644,7 +644,7 @@ mod tests { } fn empty_overlay() -> StateTrieOverlay { - StateTrieOverlay::new(Arc::default(), Arc::default()) + StateTrieOverlay::new(TrieInputSorted::default()) } fn insert_test_changesets( diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 0e942dacdf7..8eb63374718 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -99,7 +99,7 @@ impl OverlayManager { OverlayBuilder::new(parent_hash, self.block_state(parent_hash), self.clone()) } - fn block_state(&self, parent_hash: B256) -> Option> { + pub(crate) fn block_state(&self, parent_hash: B256) -> Option> { let mut blocks = self.parent_chain(parent_hash).collect::>(); blocks.pop().map(|oldest| { blocks.into_iter().rev().fold(BlockState::new(oldest), |parent, block| { @@ -685,9 +685,9 @@ impl Default for OverlayCacheConfig { #[derive(Debug)] pub(crate) struct StateTrieOverlayError { /// Requested in-memory tip hash. - tip_hash: B256, + pub(crate) tip_hash: B256, /// Requested anchor hash. - anchor_hash: B256, + pub(crate) anchor_hash: B256, } impl fmt::Display for StateTrieOverlayError { diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 578e4453560..cf2614c56a7 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -52,7 +52,7 @@ pub struct OverlayStateProviderFactory { factory: F, /// Overlay builder containing the configuration and overlay calculation logic. overlay_builder: OverlayBuilder, - /// A cache mapping `(state_trie_tip, finish_tip)` to [`StateTrieOverlay`]. + /// A cache mapping `(state_trie_tip, finish_tip, trie_changesets)` to [`StateTrieOverlay`]. /// /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. @@ -129,6 +129,7 @@ pub struct OverlayStateProvider { state_trie_overlay_cache: StateTrieOverlayCache, metrics: OverlayStateProviderFactoryMetrics, state_trie_overlay: OnceCell, + state_trie_overlay_with_trie_changesets: OnceCell, execution_overlay: OnceCell, is_v2: bool, } @@ -162,6 +163,7 @@ impl OverlayStateProvider, state_trie_overlay_cache, metrics, state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -179,6 +181,7 @@ impl OverlayStateProvider, state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::from(CachedExecutionOverlay { overlay: execution_overlay, historical_fallback: None, @@ -201,6 +204,7 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { state_trie_overlay_cache: Default::default(), metrics: Default::default(), state_trie_overlay: OnceCell::new(), + state_trie_overlay_with_trie_changesets: OnceCell::new(), execution_overlay: OnceCell::new(), is_v2, } @@ -216,7 +220,8 @@ impl<'a, Provider, N: NodePrimitives> OverlayStateProvider<&'a Provider, N> { overlay_builder: None, state_trie_overlay_cache: Default::default(), metrics: Default::default(), - state_trie_overlay: OnceCell::from(state_trie_overlay), + state_trie_overlay: OnceCell::from(state_trie_overlay.clone()), + state_trie_overlay_with_trie_changesets: OnceCell::from(state_trie_overlay), execution_overlay: OnceCell::new(), is_v2, } @@ -232,7 +237,7 @@ where &self.provider } - fn state_trie_overlay(&self) -> ProviderResult<&StateTrieOverlay> + fn state_trie_overlay(&self, trie_changesets: bool) -> ProviderResult<&StateTrieOverlay> where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -242,15 +247,17 @@ where + BlockNumReader + StorageSettingsCache, { - if let Some(overlay) = self.state_trie_overlay.get() { + let state_trie_overlay = self.state_trie_overlay_mut(trie_changesets); + if let Some(overlay) = state_trie_overlay.get() { return Ok(overlay) } let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(self.provider())?; - let overlay = match self - .state_trie_overlay_cache - .entry((state_trie_tip_block.hash, finish_tip_block.hash)) - { + let overlay = match self.state_trie_overlay_cache.entry(( + state_trie_tip_block.hash, + finish_tip_block.hash, + trie_changesets, + )) { dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { self.metrics.state_trie_overlay_cache_misses.increment(1); @@ -262,6 +269,7 @@ where self.provider(), state_trie_tip_block, finish_tip_block, + trie_changesets, )?; if !overlay.skipped_for_reused_sparse_trie() { entry.insert(overlay.clone()); @@ -269,11 +277,23 @@ where overlay } }; - let _ = self.state_trie_overlay.set(overlay); - Ok(self.state_trie_overlay.get().expect("state trie overlay was just initialized")) + let _ = state_trie_overlay.set(overlay); + Ok(state_trie_overlay.get().expect("state trie overlay was just initialized")) + } + + const fn state_trie_overlay_mut(&self, trie_changesets: bool) -> &OnceCell { + if trie_changesets { + &self.state_trie_overlay_with_trie_changesets + } else { + &self.state_trie_overlay + } } - fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult + fn build_overlay( + &self, + input: TrieInputSorted, + trie_changesets: bool, + ) -> ProviderResult where Provider::Target: StageCheckpointReader + PruneCheckpointReader @@ -283,13 +303,14 @@ where + BlockNumReader + StorageSettingsCache, { - let overlay = self.state_trie_overlay()?; + let overlay = self.state_trie_overlay(trie_changesets)?; if overlay.skipped_for_reused_sparse_trie() { return Err(ProviderError::UnsupportedProvider) } - let TrieInputSorted { nodes: input_nodes, state: input_state, prefix_sets } = input; - let mut nodes = Arc::clone(&overlay.trie_updates); - let mut state = Arc::clone(&overlay.hashed_post_state); + let TrieInputSorted { nodes: input_nodes, state: input_state, mut prefix_sets } = input; + let overlay_input = overlay.input(); + let mut nodes = Arc::clone(&overlay_input.nodes); + let mut state = Arc::clone(&overlay_input.state); if !input_nodes.is_empty() { Arc::make_mut(&mut nodes).extend_ref_and_sort(&input_nodes); @@ -298,6 +319,7 @@ where Arc::make_mut(&mut state).extend_ref_and_sort(&input_state); } + prefix_sets.extend_ref(&overlay_input.prefix_sets); Ok(TrieInputSorted::new(nodes, state, prefix_sets)) } @@ -511,16 +533,15 @@ where { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; + let input = + self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), false)?; Ok(>::overlay_root_from_nodes(self.provider().tx(), input)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes( self.provider().tx(), input, @@ -533,9 +554,8 @@ where hashed_state: HashedPostState, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(hashed_state), - ))?; + let input = + self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), true)?; Ok(>::overlay_root_from_nodes_with_updates( self.provider().tx(), input, @@ -548,7 +568,7 @@ where input: TrieInput, ) -> ProviderResult<(B256, TrieUpdates)> { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes_with_updates( self.provider().tx(), @@ -576,20 +596,15 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); - >::overlay_root(self.provider().tx(), address, hashed_storage) + let hashed_address = alloy_primitives::keccak256(address); + let input = self.build_overlay( + TrieInputSorted::from_state( + HashedPostState::from_hashed_storage(hashed_address, hashed_storage) + .into_sorted(), + ), + false, + )?; + >::overlay_root(self.provider().tx(), address, input) .map_err(|err| ProviderError::Database(err.into())) }) } @@ -600,28 +615,9 @@ where slot: B256, hashed_storage: HashedStorage, ) -> ProviderResult { - reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); - >::overlay_storage_proof( - self.provider().tx(), - address, - slot, - hashed_storage, - ) + self.storage_multiproof(address, &[slot], hashed_storage)? + .storage_proof(slot) .map_err(ProviderError::from) - }) } fn storage_multiproof( @@ -631,24 +627,19 @@ where hashed_storage: HashedStorage, ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted( - TrieInput::from_state(HashedPostState::from_hashed_storage( - alloy_primitives::keccak256(address), - hashed_storage, - )), - ))?; - let hashed_storage = input - .state - .account_storages() - .get(&alloy_primitives::keccak256(address)) - .cloned() - .unwrap_or_default() - .into(); + let hashed_address = alloy_primitives::keccak256(address); + let input = self.build_overlay( + TrieInputSorted::from_state( + HashedPostState::from_hashed_storage(hashed_address, hashed_storage) + .into_sorted(), + ), + false, + )?; >::overlay_storage_multiproof( self.provider().tx(), address, slots, - hashed_storage, + input, ) .map_err(ProviderError::from) }) @@ -674,7 +665,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -692,7 +683,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -710,7 +701,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -729,7 +720,7 @@ where ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input))?; + self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; let witness = TrieWitness::new( InMemoryTrieCursorFactory::new( DatabaseTrieCursorFactory::<_, A>::new(self.provider().tx()), @@ -778,7 +769,7 @@ where return Ok(hashed_state) } - let overlay_state = self.build_overlay(TrieInputSorted::default())?.state; + let overlay_state = self.build_overlay(TrieInputSorted::default(), false)?.state; zero_destroyed_account_storage( &HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), @@ -893,7 +884,7 @@ where Self: 'a; fn account_trie_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseAccountTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_read::()?, @@ -903,14 +894,14 @@ where self.provider().tx().cursor_read::()?, )) }; - Ok(InMemoryTrieCursor::new_account(cursor, &overlay.trie_updates)) + Ok(InMemoryTrieCursor::new_account(cursor, &overlay.input().nodes)) } fn storage_trie_cursor( &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; let cursor: Box = if self.is_v2 { Box::new(DatabaseStorageTrieCursor::<_, PackedKeyAdapter>::new( self.provider().tx().cursor_dup_read::()?, @@ -922,7 +913,7 @@ where hashed_address, )) }; - Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.trie_updates, hashed_address)) + Ok(InMemoryTrieCursor::new_storage(cursor, &overlay.input().nodes, hashed_address)) } } @@ -954,10 +945,10 @@ where Self: 'a; fn hashed_account_cursor(&self) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.hashed_post_state, + &overlay.input().state, ) .hashed_account_cursor() } @@ -966,10 +957,10 @@ where &self, hashed_address: B256, ) -> Result, DatabaseError> { - let overlay = self.state_trie_overlay().map_err(into_database_error)?; + let overlay = self.state_trie_overlay(true).map_err(into_database_error)?; HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), - &overlay.hashed_post_state, + &overlay.input().state, ) .hashed_storage_cursor(hashed_address) } @@ -987,7 +978,7 @@ pub(crate) struct OverlayStateProviderFactoryMetrics { state_trie_overlay_cache_misses: Counter, } -type StateTrieOverlayCache = Arc>; +type StateTrieOverlayCache = Arc>; #[derive(Clone, Debug)] struct CachedExecutionOverlay { @@ -1122,11 +1113,11 @@ mod tests { } fn account_keys(overlay: &StateTrieOverlay) -> Vec { - overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() + overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() } fn account_node_paths(overlay: &StateTrieOverlay) -> Vec { - overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() + overlay.input().nodes.account_nodes_ref().iter().map(|(path, _)| *path).collect() } #[test] @@ -1142,7 +1133,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - let first = provider.state_trie_overlay().unwrap().clone(); + let first = provider.state_trie_overlay(false).unwrap().clone(); assert_eq!(account_keys(&first), vec![B256::with_last_byte(3), B256::with_last_byte(4)]); drop(provider); @@ -1158,7 +1149,7 @@ mod tests { provider_rw.commit().unwrap(); let provider = state_provider_factory.database_provider_ro().unwrap(); - let second = provider.state_trie_overlay().unwrap().clone(); + let second = provider.state_trie_overlay(false).unwrap().clone(); assert_eq!(account_keys(&second), vec![B256::with_last_byte(4)]); assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); @@ -1191,6 +1182,39 @@ mod tests { assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 1); } + #[test] + fn state_trie_overlay_cache_is_keyed_by_trie_changesets() { + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + for block in &blocks[2..=3] { + manager.insert_block(block.clone()); + } + let state_provider_factory = OverlayStateProviderFactory::new( + factory, + manager.overlay_builder(blocks[3].recovered_block().hash()), + ); + let provider = state_provider_factory.database_provider_ro().unwrap(); + + provider.state_trie_overlay(false).unwrap(); + provider.state_trie_overlay(true).unwrap(); + + assert_eq!(state_provider_factory.state_trie_overlay_cache.len(), 2); + } + + #[test] + fn supplied_state_trie_overlay_is_available_in_both_modes() { + let (factory, _) = setup_frontiers(1, 1); + let provider = factory.provider().unwrap(); + let provider = OverlayStateProvider::<&_, EthPrimitives>::new_with_state_trie( + &provider, + StateTrieOverlay::new(TrieInputSorted::default()), + false, + ); + + assert!(provider.state_trie_overlay(false).is_ok()); + assert!(provider.state_trie_overlay(true).is_ok()); + } + #[test] fn lazy_overlays_survive_manager_block_removal() { let (factory, blocks) = setup_frontiers(1, 1); @@ -1212,7 +1236,7 @@ mod tests { [blocks[2].recovered_block().num_hash(), blocks[3].recovered_block().num_hash()] ); assert_eq!( - account_keys(provider.state_trie_overlay().unwrap()), + account_keys(provider.state_trie_overlay(false).unwrap()), vec![B256::with_last_byte(3), B256::with_last_byte(4)] ); } @@ -1230,7 +1254,7 @@ mod tests { ); let provider = state_provider_factory.database_provider_ro().unwrap(); - assert!(provider.state_trie_overlay().unwrap().skipped_for_reused_sparse_trie()); + assert!(provider.state_trie_overlay(false).unwrap().skipped_for_reused_sparse_trie()); assert!(state_provider_factory.state_trie_overlay_cache.is_empty()); assert!(matches!( provider.state_root(HashedPostState::default()), diff --git a/crates/trie/common/src/input.rs b/crates/trie/common/src/input.rs index 3d3bc2349d3..d9f8f41f46b 100644 --- a/crates/trie/common/src/input.rs +++ b/crates/trie/common/src/input.rs @@ -160,6 +160,13 @@ impl TrieInputSorted { Self { nodes, state, prefix_sets } } + /// Create new sorted trie input from sorted in-memory state. The prefix sets will be + /// constructed and set automatically. + pub fn from_state(state: HashedPostStateSorted) -> Self { + let prefix_sets = state.construct_prefix_sets(); + Self { nodes: Default::default(), state: Arc::new(state), prefix_sets } + } + /// Create from unsorted [`TrieInput`] by sorting. pub fn from_unsorted(input: TrieInput) -> Self { Self { diff --git a/crates/trie/db/src/proof.rs b/crates/trie/db/src/proof.rs index f362692963d..934c3fecd4e 100644 --- a/crates/trie/db/src/proof.rs +++ b/crates/trie/db/src/proof.rs @@ -7,7 +7,7 @@ use reth_trie::{ proof::{Proof, StorageProof}, trie_cursor::InMemoryTrieCursorFactory, AccountProof, DecodedMultiProofV2, HashedPostStateSorted, HashedStorage, MultiProof, - MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, TrieInput, + MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, TrieInput, TrieInputSorted, }; /// Extends [`Proof`] with operations specific for working with a database transaction. @@ -114,7 +114,7 @@ pub trait DatabaseStorageProof<'a, TX> { tx: &'a TX, address: Address, slots: &[B256], - storage: HashedStorage, + input: TrieInputSorted, ) -> Result; } @@ -158,21 +158,21 @@ impl<'a, TX: DbTx, A: TrieTableAdapter> DatabaseStorageProof<'a, TX> tx: &'a TX, address: Address, slots: &[B256], - storage: HashedStorage, + mut input: TrieInputSorted, ) -> Result { let hashed_address = keccak256(address); let targets = slots.iter().map(keccak256).collect(); - let prefix_set = storage.construct_prefix_set(); - let state_sorted = HashedPostStateSorted::new( - Default::default(), - HashMap::from_iter([(hashed_address, storage.into_sorted())]), - ); StorageProof::new( - DatabaseTrieCursorFactory::<_, A>::new(tx), - HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &state_sorted), + InMemoryTrieCursorFactory::new( + DatabaseTrieCursorFactory::<_, A>::new(tx), + &input.nodes, + ), + HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &input.state), address, ) - .with_prefix_set_mut(prefix_set) + .with_prefix_set_mut( + input.prefix_sets.storage_prefix_sets.remove(&hashed_address).unwrap_or_default(), + ) .storage_multiproof(targets) } } diff --git a/crates/trie/db/src/storage.rs b/crates/trie/db/src/storage.rs index 884d6f15565..e634723f1b2 100644 --- a/crates/trie/db/src/storage.rs +++ b/crates/trie/db/src/storage.rs @@ -5,7 +5,8 @@ use reth_execution_errors::StorageRootError; use reth_storage_api::{BlockNumReader, StorageChangeSetReader}; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - hashed_cursor::HashedPostStateCursorFactory, HashedPostState, HashedStorage, StorageRoot, + hashed_cursor::HashedPostStateCursorFactory, trie_cursor::InMemoryTrieCursorFactory, + HashedStorage, StorageRoot, TrieInputSorted, }; #[cfg(feature = "metrics")] @@ -19,11 +20,11 @@ pub trait DatabaseStorageRoot<'a, TX> { /// Create a new storage root calculator from database transaction and hashed address. fn from_tx_hashed(tx: &'a TX, hashed_address: B256) -> Self; - /// Calculates the storage root for this [`HashedStorage`] and returns it. + /// Calculates the storage root with the given trie input. fn overlay_root( tx: &'a TX, address: Address, - hashed_storage: HashedStorage, + input: TrieInputSorted, ) -> Result; } @@ -85,16 +86,22 @@ impl<'a, TX: DbTx, A: TrieTableAdapter> DatabaseStorageRoot<'a, TX> fn overlay_root( tx: &'a TX, address: Address, - hashed_storage: HashedStorage, + mut input: TrieInputSorted, ) -> Result { - let prefix_set = hashed_storage.construct_prefix_set().freeze(); - let state_sorted = - HashedPostState::from_hashed_storage(keccak256(address), hashed_storage).into_sorted(); + let hashed_address = keccak256(address); StorageRoot::new( - DatabaseTrieCursorFactory::<_, A>::new(tx), - HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &state_sorted), + InMemoryTrieCursorFactory::new( + DatabaseTrieCursorFactory::<_, A>::new(tx), + &input.nodes, + ), + HashedPostStateCursorFactory::new(DatabaseHashedCursorFactory::new(tx), &input.state), address, - prefix_set, + input + .prefix_sets + .storage_prefix_sets + .remove(&hashed_address) + .unwrap_or_default() + .freeze(), #[cfg(feature = "metrics")] TrieRootMetrics::new(reth_trie::TrieType::Storage), ) From 277b0198b8f73d0c36659e3c650c8590e8e67b33 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 17 Sep 2026 15:20:14 +0000 Subject: [PATCH 006/133] fix(engine): lower minimum backpressure threshold (#27295) --- crates/engine/primitives/src/config.rs | 10 +++++----- crates/node/core/src/args/engine.rs | 11 ++++++----- docs/vocs/docs/pages/cli/reth/node.mdx | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index 763f56cc16a..7493e0f0594 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -9,9 +9,9 @@ pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 50; /// Number of persisted blocks whose state/trie writes are masked by an in-memory suffix. pub const DEFAULT_NUM_STATE_MASKING_BLOCKS: u64 = 30; -/// Maximum number of blocks beyond the in-memory buffer target awaiting persistence before engine -/// API processing is stalled. -pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = DEFAULT_PERSISTENCE_THRESHOLD * 2; +/// The engine stalls API processing when this many blocks are awaiting persistence beyond the +/// in-memory buffer target. +pub const MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 16; /// How close to the canonical head we persist blocks. pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 5; @@ -225,7 +225,7 @@ impl Default for TreeConfig { fn default() -> Self { assert_backpressure_threshold_invariant( DEFAULT_PERSISTENCE_THRESHOLD, - DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, + DEFAULT_PERSISTENCE_THRESHOLD * 2, ); assert_state_masking_invariant( DEFAULT_PERSISTENCE_THRESHOLD, @@ -236,7 +236,7 @@ impl Default for TreeConfig { persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, num_state_masking_blocks: DEFAULT_NUM_STATE_MASKING_BLOCKS, memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, - persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, + persistence_backpressure_threshold: DEFAULT_PERSISTENCE_THRESHOLD * 2, block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT, max_invalid_header_cache_length: DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH, invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index f93a4236031..6a688686615 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -8,7 +8,7 @@ use eyre::ensure; use reth_cli_util::{parse_duration_from_secs_or_ms, parsers::format_duration_as_secs_or_ms}; use reth_engine_primitives::{ TreeConfig, DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, - DEFAULT_NUM_STATE_MASKING_BLOCKS, DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, + DEFAULT_NUM_STATE_MASKING_BLOCKS, MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD, }; use std::{sync::OnceLock, time::Duration}; @@ -75,7 +75,7 @@ impl DefaultEngineValues { self } - /// Set the default persistence backpressure threshold + /// Set the minimum default persistence backpressure threshold. pub const fn with_persistence_backpressure_threshold(mut self, v: u64) -> Self { self.persistence_backpressure_threshold = v; self @@ -263,7 +263,7 @@ impl Default for DefaultEngineValues { fn default() -> Self { Self { persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, - persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, + persistence_backpressure_threshold: MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD, num_state_masking_blocks: DEFAULT_NUM_STATE_MASKING_BLOCKS, memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, @@ -323,7 +323,7 @@ pub struct EngineArgs { /// Configure the maximum number of blocks beyond the in-memory buffer target that may await /// persistence before engine API processing stalls. /// - /// If omitted, this defaults to the larger of the default backpressure threshold and twice + /// If omitted, this defaults to the larger of the minimum backpressure threshold and twice /// `--engine.persistence-threshold`. /// /// This value must be greater than `--engine.persistence-threshold`. @@ -846,7 +846,7 @@ mod tests { } #[test] - fn default_backpressure_threshold_uses_global_default_when_larger() { + fn default_backpressure_threshold_uses_minimum_when_larger() { let args = CommandParser::::parse_from([ "reth", "--engine.persistence-threshold", @@ -858,6 +858,7 @@ mod tests { args.persistence_backpressure_threshold(), DefaultEngineValues::get_global().persistence_backpressure_threshold ); + assert_eq!(args.persistence_backpressure_threshold(), 16); } #[test] diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 79414e8cbc7..867d8f62b5d 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1016,7 +1016,7 @@ Engine: --engine.persistence-backpressure-threshold Configure the maximum number of blocks beyond the in-memory buffer target that may await persistence before engine API processing stalls. - If omitted, this defaults to the larger of the default backpressure threshold and twice `--engine.persistence-threshold`. + If omitted, this defaults to the larger of the minimum backpressure threshold and twice `--engine.persistence-threshold`. This value must be greater than `--engine.persistence-threshold`. From 25993190fae484f6d93a247a935d25db30b3d6fc Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:54:06 +0000 Subject: [PATCH 007/133] chore(deps): update imbl and remove bitmaps exception (#27286) Co-authored-by: Sergei Shulepov <2205845+pepyakin@users.noreply.github.com> --- Cargo.lock | 18 ++++-------------- deny.toml | 2 -- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af63d8b8a72..7a8572520bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1756,12 +1756,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "bitmaps" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" - [[package]] name = "bitvec" version = "1.0.1" @@ -5014,13 +5008,12 @@ dependencies = [ [[package]] name = "imbl" -version = "7.0.1" +version = "7.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ea8d4c37ee560727e824d62804183624d371e632019b0e9e3532bce64a33e5" +checksum = "46bad832b9b463ed9398b8506488cc2e3b897a9d44f13af115f382aac71f4fec" dependencies = [ "arbitrary", "archery", - "bitmaps", "equivalent", "imbl-sized-chunks", "rand_core 0.9.5", @@ -5032,12 +5025,9 @@ dependencies = [ [[package]] name = "imbl-sized-chunks" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4241005618a62f8d57b2febd02510fb96e0137304728543dfc5fd6f052c22d" -dependencies = [ - "bitmaps", -] +checksum = "2a0813be332553f857953298749fa19549e8b61b80589757c29b4e2a804fa9c6" [[package]] name = "impl-codec" diff --git a/deny.toml b/deny.toml index 133b02e82a1..be45b22db44 100644 --- a/deny.toml +++ b/deny.toml @@ -12,8 +12,6 @@ ignore = [ "RUSTSEC-2023-0089", # https://rustsec.org/advisories/RUSTSEC-2026-0173 proc-macro-error2 is unmaintained "RUSTSEC-2026-0173", - # https://rustsec.org/advisories/RUSTSEC-2026-0247 bitmaps is unmaintained, transitive dep via imbl - "RUSTSEC-2026-0247", ] # This section is considered when running `cargo deny check bans`. From 3abd6762607486fe5bf3d5fad0d213b0b819e69e Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 17 Sep 2026 17:18:51 +0000 Subject: [PATCH 008/133] fix(bench): report measured elapsed time and throughput (#27291) Co-authored-by: Emma Jamieson-Hoare <21029500+emmajam@users.noreply.github.com> --- .github/scripts/bench-reth-summary.py | 147 ++++++++++++++--- .github/scripts/bench-upload-clickhouse.py | 9 +- .github/scripts/bench-utils.js | 17 +- .github/scripts/test_bench_elapsed_metrics.py | 150 ++++++++++++++++++ 4 files changed, 293 insertions(+), 30 deletions(-) create mode 100644 .github/scripts/test_bench_elapsed_metrics.py diff --git a/.github/scripts/bench-reth-summary.py b/.github/scripts/bench-reth-summary.py index 7d1ad07a2f4..e766fbdda94 100755 --- a/.github/scripts/bench-reth-summary.py +++ b/.github/scripts/bench-reth-summary.py @@ -14,7 +14,8 @@ Generates a statistical comparison between baseline and feature. Point estimates use pooled baseline and feature rows. Confidence intervals use whole-run cluster bootstrapping when multiple runs are available. Fails if baseline or feature CSV -is missing or empty. +is missing or empty. Elapsed time and end-to-end throughput use per-run means +from sibling txgen report.json files; missing measured durations remain unavailable. """ from __future__ import annotations @@ -49,6 +50,7 @@ "p90": 1.35, "p99": 5.0, "mgas_s": 1.20, + "end_to_end_mgas_s": 1.20, "wall_clock": 0.70, "persist_wait": 5.0, } @@ -243,7 +245,7 @@ def compute_stats(combined: list[dict]) -> dict: mean_mgas_s = sum(mgas_s_values) / len(mgas_s_values) if mgas_s_values else 0 total_latencies_ms = [r["total_latency_us"] / 1_000 for r in combined] - wall_clock_s = sum(total_latencies_ms) / 1_000 + processing_time_s = sum(total_latencies_ms) / 1_000 mean_total_lat_ms = sum(total_latencies_ms) / n # Persistence wait mean (for main table) @@ -262,7 +264,9 @@ def compute_stats(combined: list[dict]) -> dict: "p90_ms": percentile(sorted_lat, 90), "p99_ms": percentile(sorted_lat, 99), "mean_mgas_s": mean_mgas_s, - "wall_clock_s": wall_clock_s, + "processing_time_s": processing_time_s, + "wall_clock_s": None, + "end_to_end_mgas_s": None, "mean_total_lat_ms": mean_total_lat_ms, "mean_persist_ms": mean_persist_ms, } @@ -286,6 +290,84 @@ def compute_point_stats(runs: list[list[dict]]) -> dict: return stats +def load_elapsed_run(csv_path: str, rows: list[dict]) -> dict | None: + """Read measured run totals, never reconstruct elapsed time from server timings.""" + report_path = Path(csv_path).with_name("report.json") + if not report_path.exists(): + return None + with report_path.open() as f: + report = json.load(f) + totals = report.get("run_stats", {}) + duration_ms = totals.get("duration_ms") + if duration_ms is None: + return None + if ( + isinstance(duration_ms, bool) + or not isinstance(duration_ms, (int, float)) + or not math.isfinite(duration_ms) + or duration_ms <= 0 + ): + raise ValueError(f"Invalid measured duration in {report_path}") + gas = sum(row["gas_used"] for row in rows) + expected = { + "total_blocks": len(rows), + "total_gas": gas, + "start_block": min(row["block_number"] for row in rows), + "end_block": max(row["block_number"] for row in rows), + } + if any(totals.get(key) != value for key, value in expected.items()): + raise ValueError(f"Run totals in {report_path} do not match {csv_path}") + duration_s = duration_ms / 1_000 + return { + "wall_clock_s": duration_s, + "end_to_end_mgas_s": gas / duration_s / 1_000_000, + } + + +def add_elapsed_stats( + baseline_paths: list[str], + feature_paths: list[str], + baseline_runs: list[list[dict]], + feature_runs: list[list[dict]], + baseline_stats: dict, + feature_stats: dict, + ci_stats: dict, +) -> None: + """Average measured run metrics and bootstrap whole runs in matching units. + + A single run has no run-to-run noise estimate. Its elapsed metrics remain + informational; block-level resampling cannot estimate that uncertainty. + """ + baseline = [ + load_elapsed_run(path, rows) + for path, rows in zip(baseline_paths, baseline_runs, strict=True) + ] + feature = [ + load_elapsed_run(path, rows) + for path, rows in zip(feature_paths, feature_runs, strict=True) + ] + if any(run is None for run in baseline + feature): + print("Measured run duration unavailable; elapsed metrics are n/a", file=sys.stderr) + return + ci_stats["elapsed_informational"] = len(baseline) < 2 or len(feature) < 2 + rng = random.Random(42) + for key, ci_key in ( + ("wall_clock_s", "wall_clock_ci_s"), + ("end_to_end_mgas_s", "end_to_end_mgas_ci"), + ): + b = [run[key] for run in baseline] + f = [run[key] for run in feature] + baseline_stats[key] = _mean(b) + feature_stats[key] = _mean(f) + ci_stats[ci_key] = 0.0 + if not ci_stats["elapsed_informational"]: + samples = [ + _mean(rng.choices(f, k=len(f))) - _mean(rng.choices(b, k=len(b))) + for _ in range(BOOTSTRAP_ITERATIONS) + ] + ci_stats[ci_key] = _ci_half_width(samples) + + def compute_wait_stats(combined: list[dict], field: str) -> dict: """Compute mean/p50/p95 for a wait time field (in ms).""" values_ms = [] @@ -407,7 +489,6 @@ def _per_run_metric_values(runs: list[list[dict]]) -> dict[str, list[float]]: "p90_ms": [], "p99_ms": [], "mgas": [], - "wall_clock_ms": [], "persist_ms": [], } for run in runs: @@ -417,7 +498,6 @@ def _per_run_metric_values(runs: list[list[dict]]) -> dict[str, list[float]]: values["p90_ms"].append(stats["p90_ms"]) values["p99_ms"].append(stats["p99_ms"]) values["mgas"].append(stats["mean_mgas_s"]) - values["wall_clock_ms"].append(stats["mean_total_lat_ms"]) values["persist_ms"].append(stats["mean_persist_ms"]) return values @@ -434,7 +514,7 @@ def _cluster_bootstrap_ci( replacement. This estimates run-to-run noise without expanding reused baseline/feature runs into independent block-level datapoints. """ - metrics = ("mean_ms", "p50_ms", "p90_ms", "p99_ms", "mgas", "wall_clock_ms", "persist_ms") + metrics = ("mean_ms", "p50_ms", "p90_ms", "p99_ms", "mgas", "persist_ms") empty = {metric: 0.0 for metric in metrics} if len(baseline_runs) < 2 or len(feature_runs) < 2: return empty @@ -479,10 +559,9 @@ def compute_ci_stats( p90_ci = cluster_ci["p90_ms"] p99_ci = cluster_ci["p99_ms"] mgas_ci = cluster_ci["mgas"] - wall_clock_ci_ms = cluster_ci["wall_clock_ms"] persist_ci_ms = cluster_ci["persist_ms"] else: - pairs, all_lat_diffs, all_mgas_diffs, all_total_lat_diffs, all_persist_diffs = ( + pairs, all_lat_diffs, all_mgas_diffs, _all_total_lat_diffs, all_persist_diffs = ( _paired_data(baseline_runs[0], feature_runs[0]) ) if not all_lat_diffs: @@ -492,7 +571,6 @@ def compute_ci_stats( p90_ci = _bootstrap_percentile_ci(rng, pairs, 90) p99_ci = _bootstrap_percentile_ci(rng, pairs, 99) mgas_ci = _bootstrap_ci(rng, all_mgas_diffs) if all_mgas_diffs else 0.0 - wall_clock_ci_ms = _bootstrap_ci(rng, all_total_lat_diffs) if all_total_lat_diffs else 0.0 persist_ci_ms = _bootstrap_ci(rng, all_persist_diffs) if all_persist_diffs else 0.0 return { @@ -501,7 +579,6 @@ def compute_ci_stats( "p90_ci_ms": p90_ci, "p99_ci_ms": p99_ci, "mgas_ci": mgas_ci, - "wall_clock_ci_ms": wall_clock_ci_ms, "persist_ci_ms": persist_ci_ms, "blocks": blocks, } @@ -578,6 +655,8 @@ def informational_reason( baseline_stats: dict, feature_stats: dict, ) -> str | None: + if metric in ("wall_clock", "end_to_end_mgas_s") and ci_stats.get("elapsed_informational"): + return "informational <2 measured runs per side" if metric == "p99" and ci_stats["blocks"] < P99_MIN_VERDICT_BLOCKS: return f"informational <{P99_MIN_VERDICT_BLOCKS} blocks" if ( @@ -627,9 +706,13 @@ def ci_pct(ci_ms: float, base_ms: float) -> float: ("p90", "p90_ms", "p90_ci_ms", "p90_ms", True), ("p99", "p99_ms", "p99_ci_ms", "p99_ms", True), ("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False), - ("wall_clock", "wall_clock_s", "wall_clock_ci_ms", "mean_total_lat_ms", True), ("persist_wait", "mean_persist_ms", "persist_ci_ms", "mean_persist_ms", True), ] + if baseline_stats.get("wall_clock_s") is not None and feature_stats.get("wall_clock_s") is not None: + metrics.extend([ + ("wall_clock", "wall_clock_s", "wall_clock_ci_s", "wall_clock_s", True), + ("end_to_end_mgas_s", "end_to_end_mgas_s", "end_to_end_mgas_ci", "end_to_end_mgas_s", False), + ]) changes = {} for name, stat_key, ci_key, base_key, lower_is_better in metrics: p = pct(baseline_stats[stat_key], feature_stats[stat_key]) @@ -1671,7 +1754,6 @@ def pct(base: float, feat: float) -> float: return (feat - base) / base * 100.0 if base > 0 else 0.0 gas_pct = pct(run1["mean_mgas_s"], run2["mean_mgas_s"]) - wall_pct = pct(run1["wall_clock_s"], run2["wall_clock_s"]) mean_pct = pct(run1["mean_ms"], run2["mean_ms"]) p50_pct = pct(run1["p50_ms"], run2["p50_ms"]) @@ -1688,7 +1770,6 @@ def pct(base: float, feat: float) -> float: # CI as a percentage of baseline mgas_ci_pct = ci_stats["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0 - wall_ci_pct = ci_stats["wall_clock_ci_ms"] / run1["mean_total_lat_ms"] * 100.0 if run1["mean_total_lat_ms"] > 0 else 0.0 persist_ci_pct = ci_stats["persist_ci_ms"] / run1["mean_persist_ms"] * 100.0 if run1["mean_persist_ms"] > 0 else 0.0 mean_floor = practical_floor_pct("mean", run1["mean_ms"]) @@ -1696,7 +1777,6 @@ def pct(base: float, feat: float) -> float: p90_floor = practical_floor_pct("p90", run1["p90_ms"]) p99_floor = practical_floor_pct("p99", run1["p99_ms"]) mgas_floor = practical_floor_pct("mgas_s", run1["mean_mgas_s"]) - wall_floor = practical_floor_pct("wall_clock", run1["mean_total_lat_ms"]) persist_floor = practical_floor_pct("persist_wait", run1["mean_persist_ms"]) p99_informational = informational_reason("p99", ci_stats, run1, run2) persist_informational = informational_reason("persist_wait", ci_stats, run1, run2) @@ -1708,15 +1788,35 @@ def pct(base: float, feat: float) -> float: lines = [ f"| Metric | {baseline_label} | {feature_label} | Change |", "|--------|------|--------|--------|", - f"| Mean | {fmt_ms(run1['mean_ms'])} | {fmt_ms(run2['mean_ms'])} | {change_str(mean_pct, mean_ci_pct, mean_floor, lower_is_better=True)} |", - f"| P50 | {fmt_ms(run1['p50_ms'])} | {fmt_ms(run2['p50_ms'])} | {change_str(p50_pct, p50_ci_pct, p50_floor, lower_is_better=True)} |", - f"| P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, p90_floor, lower_is_better=True)} |", - f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, p99_floor, lower_is_better=True, informational=p99_informational)} |", - f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, mgas_floor, lower_is_better=False)} |", - f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, wall_floor, lower_is_better=True)} |", + f"| Execution Mean | {fmt_ms(run1['mean_ms'])} | {fmt_ms(run2['mean_ms'])} | {change_str(mean_pct, mean_ci_pct, mean_floor, lower_is_better=True)} |", + f"| Execution P50 | {fmt_ms(run1['p50_ms'])} | {fmt_ms(run2['p50_ms'])} | {change_str(p50_pct, p50_ci_pct, p50_floor, lower_is_better=True)} |", + f"| Execution P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, p90_floor, lower_is_better=True)} |", + f"| Execution P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, p99_floor, lower_is_better=True, informational=p99_informational)} |", + f"| Execution Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, mgas_floor, lower_is_better=False)} |", f"| Persist Wait | {fmt_ms(run1['mean_persist_ms'])} | {fmt_ms(run2['mean_persist_ms'])} | {change_str(persist_pct, persist_ci_pct, persist_floor, lower_is_better=True, informational=persist_informational)} |", - "", ] + changes = compute_changes(run1, run2, ci_stats) + for label, key, metric, formatter, lower_is_better in ( + ("Wall Clock", "wall_clock_s", "wall_clock", fmt_s, True), + ("End-to-end Mgas/s", "end_to_end_mgas_s", "end_to_end_mgas_s", fmt_mgas, False), + ): + if metric not in changes: + lines.append(f"| {label} | n/a | n/a | |") + continue + change = changes[metric] + description = change_str( + change["pct"], change["ci_pct"], change["floor_pct"], lower_is_better, + change.get("informational_reason"), + ) + lines.append(f"| {label} | {formatter(run1[key])} | {formatter(run2[key])} | {description} |") + lines.extend([ + "", + "Execution metrics use server processing time (local block time in RPC mode). " + "Wall Clock and end-to-end throughput use measured run duration, including persistence " + "waits and any configured pacing; values are averaged per run. " + "Missing measured durations are shown as n/a.", + "", + ]) meta_parts = [f"{n} {'big blocks' if big_blocks else 'blocks'}", f"mode: {mode}"] if warmup_blocks: meta_parts.append(f"{warmup_blocks} warmup") @@ -2021,6 +2121,11 @@ def main(): print("No comparable baseline and feature results", file=sys.stderr) sys.exit(1) + add_elapsed_stats( + args.baseline_csv, args.feature_csv, baseline_runs, feature_runs, + baseline_stats, feature_stats, ci_stats, + ) + baseline_ref = args.baseline_ref or "main" baseline_name = args.baseline_name or "baseline" feature_name = args.feature_name or "feature" diff --git a/.github/scripts/bench-upload-clickhouse.py b/.github/scripts/bench-upload-clickhouse.py index 6fe36c57922..d4b6fe32627 100755 --- a/.github/scripts/bench-upload-clickhouse.py +++ b/.github/scripts/bench-upload-clickhouse.py @@ -71,6 +71,11 @@ def wait_mean(field): b_gas_per_second = b_stats["mean_mgas_s"] * 1_000_000 f_gas_per_second = f_stats["mean_mgas_s"] * 1_000_000 + # Keep these legacy columns as summed processing latency, alongside the + # execution-only gas/s columns. wall_clock_s now means measured elapsed time. + b_processing_ms = b_stats.get("processing_time_s", b_stats.get("wall_clock_s", 0)) * 1000 + f_processing_ms = f_stats.get("processing_time_s", f_stats.get("wall_clock_s", 0)) * 1000 + mean_change = changes.get("mean", {}).get("pct", 0.0) gas_change = changes.get("mgas_s", {}).get("pct", 0.0) latency_improved = 1 if mean_change < 0 else 0 @@ -107,10 +112,10 @@ def esc(s): '{esc(baseline["ref"])}', '{esc(baseline["ref"])}', '{esc(feature["ref"])}', '{esc(feature["ref"])}', {blocks}, - {b_stats.get("wall_clock_s", 0) * 1000}, {b_gas_per_second}, + {b_processing_ms}, {b_gas_per_second}, {b_stats["mean_ms"]}, {b_stats["p50_ms"]}, {b_stats["p90_ms"]}, {b_stats["p99_ms"]}, - {f_stats.get("wall_clock_s", 0) * 1000}, {f_gas_per_second}, + {f_processing_ms}, {f_gas_per_second}, {f_stats["mean_ms"]}, {f_stats["p50_ms"]}, {f_stats["p90_ms"]}, {f_stats["p99_ms"]}, {mean_change}, {gas_change}, diff --git a/.github/scripts/bench-utils.js b/.github/scripts/bench-utils.js index b07e317eb23..877be57807f 100644 --- a/.github/scripts/bench-utils.js +++ b/.github/scripts/bench-utils.js @@ -134,14 +134,17 @@ function metricRows(summary) { { label: 'Error rate', baseline: optPct(b.error_rate_pct), feature: optPct(f.error_rate_pct), change: '' }, ]; } + const optS = v => (Number.isFinite(v) ? fmtS(v) : 'n/a'); + const optMgas = v => (Number.isFinite(v) ? fmtMgas(v) : 'n/a'); return [ - { label: 'Mean', baseline: fmtMs(b.mean_ms), feature: fmtMs(f.mean_ms), change: fmtChange(c.mean) }, - { label: 'StdDev', baseline: fmtMs(b.stddev_ms), feature: fmtMs(f.stddev_ms), change: '' }, - { label: 'P50', baseline: fmtMs(b.p50_ms), feature: fmtMs(f.p50_ms), change: fmtChange(c.p50) }, - { label: 'P90', baseline: fmtMs(b.p90_ms), feature: fmtMs(f.p90_ms), change: fmtChange(c.p90) }, - { label: 'P99', baseline: fmtMs(b.p99_ms), feature: fmtMs(f.p99_ms), change: fmtChange(c.p99) }, - { label: 'Mgas/s', baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) }, - { label: 'Wall Clock', baseline: fmtS(b.wall_clock_s), feature: fmtS(f.wall_clock_s), change: fmtChange(c.wall_clock) }, + { label: 'Execution Mean', baseline: fmtMs(b.mean_ms), feature: fmtMs(f.mean_ms), change: fmtChange(c.mean) }, + { label: 'Execution StdDev', baseline: fmtMs(b.stddev_ms), feature: fmtMs(f.stddev_ms), change: '' }, + { label: 'Execution P50', baseline: fmtMs(b.p50_ms), feature: fmtMs(f.p50_ms), change: fmtChange(c.p50) }, + { label: 'Execution P90', baseline: fmtMs(b.p90_ms), feature: fmtMs(f.p90_ms), change: fmtChange(c.p90) }, + { label: 'Execution P99', baseline: fmtMs(b.p99_ms), feature: fmtMs(f.p99_ms), change: fmtChange(c.p99) }, + { label: 'Execution Mgas/s', baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) }, + { label: 'Wall Clock', baseline: optS(b.wall_clock_s), feature: optS(f.wall_clock_s), change: fmtChange(c.wall_clock) }, + { label: 'End-to-end Mgas/s', baseline: optMgas(b.end_to_end_mgas_s), feature: optMgas(f.end_to_end_mgas_s), change: fmtChange(c.end_to_end_mgas_s) }, { label: 'Persist Wait', baseline: fmtMs(b.mean_persist_ms || 0), feature: fmtMs(f.mean_persist_ms || 0), change: fmtChange(c.persist_wait) }, ]; } diff --git a/.github/scripts/test_bench_elapsed_metrics.py b/.github/scripts/test_bench_elapsed_metrics.py new file mode 100644 index 00000000000..0df455a5eaf --- /dev/null +++ b/.github/scripts/test_bench_elapsed_metrics.py @@ -0,0 +1,150 @@ +"""Regression tests for measured elapsed metrics and the shared Slack renderer. + +Run with: uv run python -m unittest discover -s .github/scripts +""" + +import csv +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + + +SCRIPTS = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("bench_summary", SCRIPTS / "bench-reth-summary.py") +summary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(summary) + + +class ElapsedMetricsTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def run_fixture(self, name, execution_us, duration_ms, blocks=2): + directory = self.root / name + directory.mkdir() + path = directory / "combined_latency.csv" + with path.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=[ + "block_number", "gas_used", "new_payload_latency", "total_latency", + "persistence_wait", "execution_cache_wait", "sparse_trie_wait", + ]) + writer.writeheader() + for number in range(100, 100 + blocks): + writer.writerow({ + "block_number": number, "gas_used": 1_000_000, + "new_payload_latency": execution_us, "total_latency": execution_us, + "persistence_wait": 0, "execution_cache_wait": 0, "sparse_trie_wait": 0, + }) + if duration_ms is not None: + (directory / "report.json").write_text(json.dumps({"run_stats": { + "duration_ms": duration_ms, "total_gas": blocks * 1_000_000, + "total_blocks": blocks, "start_block": 100, "end_block": 99 + blocks, + }})) + return str(path) + + def compare(self, baseline_paths, feature_paths): + b_runs = [summary.parse_combined_csv(path) for path in baseline_paths] + f_runs = [summary.parse_combined_csv(path) for path in feature_paths] + b = summary.compute_point_stats(b_runs) + f = summary.compute_point_stats(f_runs) + ci = summary.compute_ci_stats(b_runs, f_runs) + summary.add_elapsed_stats(baseline_paths, feature_paths, b_runs, f_runs, b, f, ci) + return b, f, ci, summary.compute_changes(b, f, ci) + + def render_rows(self, b, f, changes): + result = subprocess.run([ + "node", "-e", + "const u=require(process.argv[1]); let s=''; " + "process.stdin.on('data', d=>s+=d); " + "process.stdin.on('end', ()=>{const x=JSON.parse(s); " + "process.stdout.write(JSON.stringify({rows:u.metricRows(x),verdict:u.verdict(x.changes)}));});", + str(SCRIPTS / "bench-utils.js"), + ], input=json.dumps({"baseline": {"stats": b}, "feature": {"stats": f}, "changes": changes}), + text=True, capture_output=True, check=True) + return json.loads(result.stdout) + + def test_persistence_savings_survive_slower_execution(self): + baseline = [self.run_fixture(f"b{i}", 10_000, 1000) for i in range(2)] + feature = [self.run_fixture(f"f{i}", 20_000, 500) for i in range(2)] + b, f, ci, changes = self.compare(baseline, feature) + self.assertEqual(b["wall_clock_s"], 1.0) # Mean per run, not the sum of two runs. + self.assertEqual(f["wall_clock_s"], 0.5) + self.assertEqual(b["end_to_end_mgas_s"], 2.0) + self.assertEqual(f["end_to_end_mgas_s"], 4.0) + self.assertEqual(changes["mean"]["sig"], "bad") + self.assertEqual(changes["mgas_s"]["sig"], "bad") + self.assertEqual(changes["wall_clock"]["sig"], "good") + self.assertEqual(changes["end_to_end_mgas_s"]["sig"], "good") + self.assertEqual(changes["wall_clock"]["pct"], -50.0) + self.assertEqual(changes["end_to_end_mgas_s"]["pct"], 100.0) + rendered = self.render_rows(b, f, changes) + rows = {row["label"]: row for row in rendered["rows"]} + self.assertEqual(rows["Wall Clock"]["baseline"], "1.00s") + self.assertEqual(rows["End-to-end Mgas/s"]["feature"], "4.00") + self.assertIn("❌", rows["Execution Mean"]["change"]) + self.assertIn("✅", rows["Wall Clock"]["change"]) + self.assertEqual(rendered["verdict"]["label"], "Mixed Results") + markdown = summary.generate_comparison_table(b, f, ci, "owner/repo", "base", "base", "feat", "sha") + self.assertIn("| Wall Clock | 1.00s | 0.50s |", markdown) + self.assertIn("| End-to-end Mgas/s | 2.00 | 4.00 |", markdown) + + def test_average_run_throughput_not_ratio_of_averages(self): + baseline = [self.run_fixture("b1", 10_000, 1000), self.run_fixture("b2", 10_000, 3000)] + feature = [self.run_fixture("f1", 10_000, 2000), self.run_fixture("f2", 10_000, 2000)] + b, f, ci, changes = self.compare(baseline, feature) + self.assertEqual(b["wall_clock_s"], 2) + self.assertAlmostEqual(b["end_to_end_mgas_s"], (2 + 2 / 3) / 2) + self.assertAlmostEqual(ci["wall_clock_ci_s"], 1) + self.assertAlmostEqual(changes["wall_clock"]["ci_pct"], 50) + self.assertEqual(changes["wall_clock"]["sig"], "neutral") + + def test_run_count_does_not_inflate_elapsed_time(self): + baseline = [self.run_fixture(f"b{i}", 10_000, 1000) for i in range(2)] + feature = [self.run_fixture(f"f{i}", 10_000, 1000) for i in range(3)] + b, f, ci, changes = self.compare(baseline, feature) + self.assertEqual(b["wall_clock_s"], f["wall_clock_s"]) + self.assertEqual(changes["wall_clock"]["pct"], 0) + + def test_single_run_elapsed_verdict_is_informational(self): + b, f, ci, changes = self.compare( + [self.run_fixture("b", 10_000, 1000)], [self.run_fixture("f", 10_000, 500)], + ) + for name in ("wall_clock", "end_to_end_mgas_s"): + self.assertTrue(changes[name]["informational"]) + self.assertEqual(changes[name]["sig"], "neutral") + self.assertEqual(self.render_rows(b, f, changes)["verdict"]["label"], "No Difference") + + def test_missing_report_does_not_fabricate_wall_clock(self): + baseline = [self.run_fixture("b1", 10_000, 1000), self.run_fixture("b2", 10_000, None)] + feature = [self.run_fixture(f"f{i}", 10_000, 500) for i in range(2)] + b, f, ci, changes = self.compare(baseline, feature) + self.assertIsNone(b["wall_clock_s"]) + self.assertIsNone(f["end_to_end_mgas_s"]) + self.assertNotIn("wall_clock", changes) + rows = {row["label"]: row for row in self.render_rows(b, f, changes)["rows"]} + self.assertEqual(rows["Wall Clock"]["baseline"], "n/a") + markdown = summary.generate_comparison_table(b, f, ci, "owner/repo", "base", "base", "feat", "sha") + self.assertIn("| Wall Clock | n/a | n/a |", markdown) + + def test_invalid_duration_rejected(self): + for i, duration in enumerate((0, -1, float("nan"), float("inf"), True, "1000")): + with self.subTest(duration=duration): + path = self.run_fixture(f"bad{i}", 10_000, duration) + with self.assertRaisesRegex(ValueError, "Invalid measured duration"): + summary.load_elapsed_run(path, summary.parse_combined_csv(path)) + + def test_mismatched_report_rejected(self): + path = self.run_fixture("bad", 10_000, 1000) + rows = summary.parse_combined_csv(path) + rows[0]["gas_used"] += 1 + with self.assertRaisesRegex(ValueError, "do not match"): + summary.load_elapsed_run(path, rows) + + +if __name__ == "__main__": + unittest.main() From 8db774c86c9a555efc9c17728549bb042b3d4333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:11:22 +0000 Subject: [PATCH 009/133] feat(snap-sync): catch up downloaded state with block access lists (#27264) Co-authored-by: Matthias Seitz --- crates/snap-sync/src/attempt.rs | 57 +- crates/snap-sync/src/bytecode/download.rs | 5 +- crates/snap-sync/src/bytecode/store.rs | 6 +- .../src/{bal.rs => catch_up/apply.rs} | 127 ++-- crates/snap-sync/src/catch_up/download.rs | 565 +++++++++++++++ crates/snap-sync/src/catch_up/mod.rs | 20 + crates/snap-sync/src/catch_up/store.rs | 665 ++++++++++++++++++ crates/snap-sync/src/error.rs | 54 +- crates/snap-sync/src/lib.rs | 7 +- crates/snap-sync/src/storage/download.rs | 5 +- crates/snap-sync/src/storage/store.rs | 4 +- crates/snap-sync/src/test_utils.rs | 176 ++++- 12 files changed, 1583 insertions(+), 108 deletions(-) rename crates/snap-sync/src/{bal.rs => catch_up/apply.rs} (80%) create mode 100644 crates/snap-sync/src/catch_up/download.rs create mode 100644 crates/snap-sync/src/catch_up/mod.rs create mode 100644 crates/snap-sync/src/catch_up/store.rs diff --git a/crates/snap-sync/src/attempt.rs b/crates/snap-sync/src/attempt.rs index fec3d65158d..8ef36ec9611 100644 --- a/crates/snap-sync/src/attempt.rs +++ b/crates/snap-sync/src/attempt.rs @@ -4,9 +4,9 @@ //! attempt produced it. Every write presents a [`SnapWrite`]; ones that no longer match are //! refused. -use crate::{SnapGeneration, SnapSyncError}; +use crate::{CatchUpProgress, SnapCatchUpStore, SnapGeneration, SnapSyncError}; use reth_storage_api::{ - MetadataProvider, MetadataWriter, SnapAttempt, SnapAttemptId, StorageSettings, + BlockHashReader, MetadataProvider, MetadataWriter, SnapAttempt, SnapAttemptId, StorageSettings, }; /// Persistence for the attempt that owns downloaded snap state. @@ -14,7 +14,8 @@ use reth_storage_api::{ /// Blanket-implemented over node metadata access, so these writes join the caller's transaction: /// state, bytecode and the attempt record commit together or not at all. pub trait SnapAttemptStore { - /// Starts an attempt anchored to `generation`, superseding any already recorded. + /// Starts an attempt anchored to `generation`, superseding any already recorded, with its + /// catch-up progress at that pivot. fn start_snap_attempt(&self, generation: SnapGeneration) -> Result where Self: MetadataWriter; @@ -25,14 +26,28 @@ pub trait SnapAttemptStore { /// Returns the attempt when `write` still owns the persisted state, rejecting it otherwise. fn authorize_snap_write(&self, write: SnapWrite) -> Result; + /// Returns the attempt when `write` owns its state and its pivot is still canonical. + /// + /// Downloaded ranges are proved against the pivot, so they describe the fork it sits on, not + /// the canonical chain the applied blocks were checked against. + fn authorize_canonical_snap_write( + &self, + write: SnapWrite, + ) -> Result + where + Self: BlockHashReader; + /// Re-anchors the attempt to `generation`, refusing writes proved against the previous root. + /// + /// The old pivot must still be canonical, and the new pivot must not be below the last block + /// whose list is applied. An orphaned pivot requires recovery or a fresh attempt. fn advance_snap_pivot( &self, write: SnapWrite, generation: SnapGeneration, ) -> Result where - Self: MetadataWriter; + Self: MetadataWriter + BlockHashReader; /// Marks the attempt's downloaded state verified. fn verify_snap_attempt(&self, write: SnapWrite) -> Result<(), SnapSyncError> @@ -84,6 +99,8 @@ impl SnapAttemptStore for T { let attempt = SnapAttempt::start(self.snap_attempt()?, generation.target(), generation.state_root()); self.write_snap_attempt(&attempt)?; + // Ranges committed at this pivot need every list after it, however far the pivot moves. + CatchUpProgress::at_pivot(attempt.pivot()).write(self, attempt.id())?; Ok(SnapWrite::of(&attempt)) } @@ -101,15 +118,38 @@ impl SnapAttemptStore for T { } Ok(attempt) } + + fn authorize_canonical_snap_write(&self, write: SnapWrite) -> Result + where + Self: BlockHashReader, + { + let attempt = self.authorize_snap_write(write)?; + let pivot = attempt.pivot(); + if self.block_hash(pivot.number)? != Some(pivot.hash) { + return Err(SnapSyncError::NonCanonicalBlock { block: pivot.number, hash: pivot.hash }) + } + Ok(attempt) + } + fn advance_snap_pivot( &self, write: SnapWrite, generation: SnapGeneration, ) -> Result where - Self: MetadataWriter, + Self: MetadataWriter + BlockHashReader, { - let mut attempt = self.authorize_snap_write(write)?; + // Replacing an orphaned pivot would hide the fork that downloaded ranges belong to. + let mut attempt = self.authorize_canonical_snap_write(write)?; + // Ranges downloaded at the new pivot would already hold the applied blocks' changes. + if let Some(progress) = self.catch_up_progress(write)? && + generation.target().number < progress.applied().number + { + return Err(SnapSyncError::PivotBelowApplied { + applied: progress.applied().number, + pivot: generation.target().number, + }) + } attempt.re_anchor(generation.target(), generation.state_root()); self.write_snap_attempt(&attempt)?; Ok(SnapWrite::of(&attempt)) @@ -142,6 +182,7 @@ impl SnapAttemptStore for T { #[cfg(test)] mod tests { use super::*; + use crate::test_utils::insert_generation_headers; use alloy_eips::BlockNumHash; use alloy_primitives::{Bytes, B256}; use reth_db_api::{tables, transaction::DbTx}; @@ -167,12 +208,14 @@ mod tests { ) } - // A database using the hashed state layout snap writes into. + // A database using the hashed state layout snap writes into, holding the blocks the + // generations above anchor to. fn factory() -> ProviderFactory { let factory = create_test_provider_factory(); let provider = factory.database_provider_rw().unwrap(); provider.write_storage_settings(StorageSettings::v2()).unwrap(); provider.commit().unwrap(); + insert_generation_headers(&factory); factory } diff --git a/crates/snap-sync/src/bytecode/download.rs b/crates/snap-sync/src/bytecode/download.rs index 07d94efd9c0..cafe6c8b1ea 100644 --- a/crates/snap-sync/src/bytecode/download.rs +++ b/crates/snap-sync/src/bytecode/download.rs @@ -128,8 +128,8 @@ mod tests { use super::*; use crate::{ test_utils::{ - account, byte_codes, generation, hashed_factory, key, state_root, verified_range, - ScriptedSnapClient, + account, byte_codes, generation, hashed_factory, insert_generation_headers, key, + state_root, verified_range, ScriptedSnapClient, }, SnapAccountStore, SnapAttemptStore, }; @@ -168,6 +168,7 @@ mod tests { // An attempt that fetched all of `accounts` as one range, not yet committed. fn started(accounts: &[(B256, TrieAccount)]) -> (Factory, VerifiedRange) { let factory = hashed_factory(); + insert_generation_headers(&factory); let provider = factory.database_provider_rw().unwrap(); let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); provider.start_account_coverage(write).unwrap(); diff --git a/crates/snap-sync/src/bytecode/store.rs b/crates/snap-sync/src/bytecode/store.rs index b3eeaa7d215..0c5a33fec18 100644 --- a/crates/snap-sync/src/bytecode/store.rs +++ b/crates/snap-sync/src/bytecode/store.rs @@ -98,7 +98,10 @@ impl SnapBytecodeStore for T { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{account, generation, hashed_factory, key, state_root, verified_range}; + use crate::test_utils::{ + account, generation, hashed_factory, insert_generation_headers, key, state_root, + verified_range, + }; use alloy_primitives::bytes; use reth_provider::{ test_utils::MockNodeTypesWithDB, DatabaseProviderFactory, ProviderFactory, @@ -120,6 +123,7 @@ mod tests { fn started(accounts: &[(B256, TrieAccount)]) -> (Factory, SnapWrite) { let factory = hashed_factory(); + insert_generation_headers(&factory); let provider = factory.database_provider_rw().unwrap(); let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); provider.commit().unwrap(); diff --git a/crates/snap-sync/src/bal.rs b/crates/snap-sync/src/catch_up/apply.rs similarity index 80% rename from crates/snap-sync/src/bal.rs rename to crates/snap-sync/src/catch_up/apply.rs index a0063b8d83c..b5b5aae3502 100644 --- a/crates/snap-sync/src/bal.rs +++ b/crates/snap-sync/src/catch_up/apply.rs @@ -1,78 +1,25 @@ -//! Carries downloaded state forward through verified block access lists. +//! State updates implied by verified block access lists. //! //! [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189#synchronization-algorithm) advances the //! pivot by applying later blocks' lists to the state downloaded so far. Each list records the //! final value of every field it changes, so untouched fields come from the downloaded account. -use crate::SnapSyncError; -use alloy_eip7928::AccountChanges; -use alloy_primitives::{keccak256, map::B256Map, Bytes, B256, KECCAK256_EMPTY}; +use alloy_primitives::{map::B256Map, Bytes, B256}; use reth_primitives_traits::Account; -use reth_trie_common::{HashedPostState, HashedStorage}; +use reth_trie_common::HashedPostState; /// Changes one block access list makes to the downloaded state. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct BalStateUpdate { // Post-state of every entry whose account is downloaded. - state: HashedPostState, + pub(super) state: HashedPostState, // Final code of accounts whose code changed, keyed by code hash. - bytecodes: B256Map, + pub(super) bytecodes: B256Map, // Entries left out because their account is not downloaded yet. - unresolved: Vec, + pub(super) unresolved: Vec, } impl BalStateUpdate { - /// Applies `bal` on top of the downloaded state, resolved by hashed address through `base`. - /// - /// The list must already be verified against its header. - pub fn from_block_access_list( - bal: &[AccountChanges], - mut base: impl FnMut(B256) -> Result, - ) -> Result { - let mut update = Self::default(); - for account_changes in bal { - let account_info = account_changes.account_info(); - // Read-only entries record accesses, not changes. - if !account_info.changes_state_root(account_changes) { - continue - } - - let hashed_address = keccak256(account_changes.address()); - let mut account = match base(hashed_address)? { - // Its range is downloaded against a later root, which includes this change. - DownloadedAccount::Unknown => { - update.unresolved.push(hashed_address); - continue - } - DownloadedAccount::Absent => Account::default(), - DownloadedAccount::Present(account) => account, - }; - - account.apply_bal_info(account_info); - // Stored accounts represent empty code with no code hash. - account.bytecode_hash = account.bytecode_hash.filter(|hash| *hash != KECCAK256_EMPTY); - // Execution removes accounts a block leaves empty, see EIP-161. - update.state.accounts.insert(hashed_address, (!account.is_empty()).then_some(account)); - if account_changes.has_storage_changes() { - update.state.storages.insert( - hashed_address, - HashedStorage::from_iter( - account_changes - .storage_post_states() - .map(|(slot, value)| (keccak256(B256::from(slot)), value)), - ), - ); - } - if let Some((code_hash, code)) = account_info - .code_hash - .zip(account_changes.code_post_state().filter(|code| !code.is_empty())) - { - update.bytecodes.insert(code_hash, code.clone()); - } - } - Ok(update) - } - /// Post-state of every entry whose account is downloaded, keyed by hashed address. pub const fn state(&self) -> &HashedPostState { &self.state @@ -87,6 +34,12 @@ impl BalStateUpdate { pub fn unresolved(&self) -> &[B256] { &self.unresolved } + + /// Consumes this update into the state it writes, the code it stores and its unresolved + /// entries. + pub fn into_parts(self) -> (HashedPostState, B256Map, Vec) { + (self.state, self.bytecodes, self.unresolved) + } } /// What the downloaded state holds for an account a list changes. @@ -103,9 +56,11 @@ pub enum DownloadedAccount { #[cfg(test)] mod tests { use super::*; + use crate::{test_utils::hashed_factory, AccountCoverage, SnapCatchUpStore}; use alloy_consensus::{Header, TxLegacy}; use alloy_eip7928::{ - BalanceChange, BlockAccessIndex, CodeChange, NonceChange, SlotChanges, StorageChange, + AccountChanges, BalanceChange, BlockAccessIndex, CodeChange, NonceChange, SlotChanges, + StorageChange, }; use alloy_eips::{ eip2935::{HISTORY_STORAGE_ADDRESS, HISTORY_STORAGE_CODE}, @@ -113,13 +68,15 @@ mod tests { eip4895::Withdrawal, eip7002::{WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, WITHDRAWAL_REQUEST_PREDEPLOY_CODE}, }; - use alloy_primitives::{bytes, Address, Signature, TxKind, U256}; + use alloy_primitives::{bytes, keccak256, Address, Signature, TxKind, U256}; use reth_chainspec::ChainSpecBuilder; + use reth_db_api::{tables, transaction::DbTxMut}; use reth_ethereum_primitives::{Block, BlockBody, Transaction, TransactionSigned}; use reth_evm::{execute::BlockExecutor, ConfigureEvm, Evm}; use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{Block as _, Recovered}; - use reth_trie_common::KeccakKeyHasher; + use reth_storage_api::DatabaseProviderFactory; + use reth_trie_common::{HashedStorage, KeccakKeyHasher}; use revm::{ database::{states::bundle_state::BundleRetention, CacheDB, EmptyDB, State}, state::{AccountInfo, Bytecode}, @@ -133,18 +90,37 @@ mod tests { BlockAccessIndex::new(value) } + fn state_update( + bal: &[AccountChanges], + coverage: AccountCoverage, + accounts: impl IntoIterator, + ) -> BalStateUpdate { + let factory = hashed_factory(); + let provider = factory.database_provider_rw().unwrap(); + for (address, account) in accounts { + provider.tx_ref().put::(address, account).unwrap(); + } + provider.block_access_list_update(coverage, bal).unwrap() + } + fn apply(changes: &AccountChanges, base: DownloadedAccount) -> BalStateUpdate { - BalStateUpdate::from_block_access_list(std::slice::from_ref(changes), |_| Ok(base)).unwrap() + let (coverage, account) = match base { + DownloadedAccount::Unknown => (AccountCoverage::START, None), + DownloadedAccount::Absent => (AccountCoverage::COMPLETE, None), + DownloadedAccount::Present(account) => (AccountCoverage::COMPLETE, Some(account)), + }; + state_update( + std::slice::from_ref(changes), + coverage, + account.map(|account| (keccak256(changes.address()), account)), + ) } #[test] fn read_only_entries_write_nothing() { let changes = AccountChanges::new(ACCOUNT).with_storage_read(U256::from(1)); - let update = BalStateUpdate::from_block_access_list(&[changes], |_| { - panic!("read-only entries need no downloaded account") - }) - .unwrap(); + let update = apply(&changes, DownloadedAccount::Unknown); assert_eq!(update, BalStateUpdate::default()); } @@ -154,10 +130,7 @@ mod tests { let changes = AccountChanges::new(ACCOUNT) .with_storage_change(SlotChanges::new(U256::from(1), vec![])); - let update = BalStateUpdate::from_block_access_list(&[changes], |_| { - panic!("empty slot entries need no downloaded account") - }) - .unwrap(); + let update = apply(&changes, DownloadedAccount::Unknown); assert_eq!(update, BalStateUpdate::default()); } @@ -418,13 +391,11 @@ mod tests { state.merge_transitions(BundleRetention::PlainState); let bundle = state.take_bundle(); - let update = BalStateUpdate::from_block_access_list(&bal, |hashed_address| { - Ok(pre - .0 - .get(&hashed_address) - .map_or(DownloadedAccount::Absent, |account| DownloadedAccount::Present(*account))) - }) - .unwrap(); + let update = state_update( + &bal, + AccountCoverage::COMPLETE, + pre.0.iter().map(|(address, account)| (*address, *account)), + ); let executed = HashedPostState::from_bundle_state::(bundle.state()); let post = fold(pre.clone(), &update.state); diff --git a/crates/snap-sync/src/catch_up/download.rs b/crates/snap-sync/src/catch_up/download.rs new file mode 100644 index 00000000000..8fc74f25f1c --- /dev/null +++ b/crates/snap-sync/src/catch_up/download.rs @@ -0,0 +1,565 @@ +//! Requests the lists of the canonical blocks between the applied one and the catch-up target. +//! +//! Headers are read again each request, so a reorged anchor is caught before peers are asked. + +use crate::{ + common::DownloadContext, CatchUpProgress, SnapAttemptStore, SnapCatchUpStore, SnapSyncError, + SnapWrite, +}; +use alloy_eips::BlockNumHash; +use reth_db_api::transaction::DbTxMut; +use reth_downloaders::snap::{BlockAccessListDownloader, BlockAccessListOutcome}; +use reth_eth_wire_types::snap::GetBlockAccessListsMessage; +use reth_network_p2p::snap::client::SnapClient; +use reth_network_peers::PeerId; +use reth_primitives_traits::{AlloyBlockHeader, SealedHeader}; +use reth_storage_api::{ + BlockHashReader, DBProvider, DatabaseProviderFactory, HeaderProvider, MetadataProvider, + MetadataWriter, StateWriter, +}; +use reth_tasks::Runtime; +use std::fmt; + +/// Default soft response limit for block access list requests, as EIP-8189 recommends. +pub const DEFAULT_BAL_RESPONSE_BYTES: u64 = 2 * 1024 * 1024; + +/// Default number of blocks asked for per request, chosen so that many average 60M gas lists +/// still fit under [`DEFAULT_BAL_RESPONSE_BYTES`]. +pub const DEFAULT_CATCH_UP_BLOCKS: u64 = 28; + +/// Applies the lists of the canonical blocks after the applied one, one request at a time. +pub struct BlockAccessListCatchUp { + context: DownloadContext, + // Blocks asked for per request. + max_blocks: u64, +} + +impl BlockAccessListCatchUp { + /// Creates a catch-up that requests the lists the applied state does not carry yet. + pub const fn new(client: C, factory: F, runtime: Runtime) -> Self { + let mut context = DownloadContext::new(client, factory, runtime); + context.set_response_bytes(DEFAULT_BAL_RESPONSE_BYTES); + Self { context, max_blocks: DEFAULT_CATCH_UP_BLOCKS } + } + + /// Returns this catch-up asking peers for at most `response_bytes` per response. + pub const fn with_response_bytes(mut self, response_bytes: u64) -> Self { + self.context.set_response_bytes(response_bytes); + self + } + + /// Returns this catch-up asking for at most `max_blocks` blocks per request, at least one. + pub const fn with_max_blocks(mut self, max_blocks: u64) -> Self { + self.max_blocks = if max_blocks == 0 { 1 } else { max_blocks }; + self + } +} + +impl BlockAccessListCatchUp +where + C: SnapClient + Clone + Unpin, + F: DatabaseProviderFactory + Clone + 'static, + F::Provider: HeaderProvider + MetadataProvider + BlockHashReader, + F::ProviderRW: + BlockHashReader + MetadataProvider + MetadataWriter + StateWriter + DBProvider, +{ + /// Requests the lists of the canonical blocks between the applied one and `target`, and + /// commits those continuing it, in order. + /// + /// A list a peer leaves out ends the run this call commits, leaving it and every block after + /// it to the next one. + pub async fn next( + &mut self, + write: SnapWrite, + target: u64, + ) -> Result { + let max_blocks = self.max_blocks; + let headers = self + .context + .read(move |provider| pending_headers(provider, write, target, max_blocks)) + .await?; + if headers.is_empty() { + return Ok(CatchUpStep::Complete) + } + + let request = GetBlockAccessListsMessage { + request_id: self.context.next_request_id(), + block_hashes: headers.iter().map(SealedHeader::hash).collect(), + response_bytes: self.context.response_bytes(), + }; + let downloader = BlockAccessListDownloader::new( + self.context.client().clone(), + request, + &headers, + self.context.runtime().clone(), + )?; + let verified = match downloader.await? { + BlockAccessListOutcome::Verified(verified) => verified, + BlockAccessListOutcome::Unavailable { peer_id } => { + return Ok(CatchUpStep::Unavailable { peer_id }) + } + }; + + let peer_id = verified.peer_id(); + // A list applies only once every earlier block's has, so the run stops at the first block + // the peer left out. + let applied = verified + .into_block_access_lists() + .into_iter() + .zip(&headers) + .map_while(|((_, list), header)| { + let block = BlockNumHash::new(header.number(), header.hash()); + list.map(|list| (block, header.parent_hash(), list)) + }) + .collect::>(); + if applied.is_empty() { + return Ok(CatchUpStep::Unavailable { peer_id }) + } + + let blocks = applied.len(); + let progress = self + .context + .commit(move |provider| { + let mut progress = None; + for (block, parent, list) in applied { + progress = Some(provider.commit_block_access_list( + write, + block, + parent, + list.as_bal(), + )?); + } + Ok(progress.expect("the applied run is not empty")) + }) + .await?; + Ok(CatchUpStep::Applied { progress, blocks }) + } +} + +impl fmt::Debug for BlockAccessListCatchUp { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BlockAccessListCatchUp") + .field("context", &self.context) + .field("max_blocks", &self.max_blocks) + .finish() + } +} + +/// What one request of a [`BlockAccessListCatchUp`] produced. +#[derive(Debug)] +pub enum CatchUpStep { + /// Lists were committed, carrying the downloaded state this far. + Applied { + /// How far the state is carried now. + progress: CatchUpProgress, + /// Blocks this request carried it past. + blocks: usize, + }, + /// The peer held no list for the next block, so the state stays where it is. + Unavailable { + /// Peer that answered without it, so the retry can go elsewhere. + peer_id: PeerId, + }, + /// Every block through the target is applied. + Complete, +} + +// The canonical headers continuing the applied block, at most `max_blocks` of them and none past +// `target` or the pivot. +// +// Both anchors must remain canonical: downloaded ranges can contain changes from the pivot +// even when catch-up has only reached an earlier block. +fn pending_headers( + provider: &P, + write: SnapWrite, + target: u64, + max_blocks: u64, +) -> Result>, SnapSyncError> { + let progress = provider.catch_up_progress(write)?.ok_or(SnapSyncError::NoCatchUpProgress)?; + let applied = progress.applied(); + let canonical = provider + .sealed_header(applied.number)? + .ok_or(SnapSyncError::MissingHeader { block: applied.number })?; + if canonical.hash() != applied.hash { + return Err(SnapSyncError::ForkedBlock { expected: applied.hash, got: canonical.hash() }) + } + // Pending ranges are proved against the pivot, so no list past it applies. + let target = target.min(provider.authorize_canonical_snap_write(write)?.pivot().number); + if target <= applied.number { + return Ok(Vec::new()) + } + + let end = target.min(applied.number.saturating_add(max_blocks)); + let headers = provider.sealed_headers_range(progress.next()..=end)?; + if headers.is_empty() { + return Err(SnapSyncError::MissingHeader { block: progress.next() }) + } + // A list is only authenticated by the header of the block it belongs to, so a gap in them + // ends the run as surely as a gap in the lists themselves. + for (number, header) in (progress.next()..=end).zip(&headers) { + if header.number() != number { + return Err(SnapSyncError::MissingHeader { block: number }) + } + } + Ok(headers) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + test_utils::{ + account, hashed_factory, key, state_root, verified_range, BalChain, ScriptedSnapClient, + SnapStateSnapshot, + }, + SnapAccountStore, SnapGeneration, + }; + use alloy_eip7928::{AccountChanges, BalanceChange, BlockAccessIndex}; + use alloy_primitives::{keccak256, Address, B256, U256}; + use reth_db_api::{tables, transaction::DbTx}; + use reth_network_p2p::{error::PeerRequestResult, snap::client::SnapResponse}; + use reth_provider::{ + test_utils::{insert_headers, MockNodeTypesWithDB}, + ProviderFactory, + }; + use reth_trie_common::TrieAccount; + use std::sync::Arc; + + type Factory = ProviderFactory; + type CatchUp = BlockAccessListCatchUp, Factory>; + + // The account every fixture list credits. + const CHANGED: Address = Address::repeat_byte(0xaa); + // Pivot the fixture chain is anchored to. + const PIVOT: u64 = 2; + + fn accounts() -> Vec<(B256, TrieAccount)> { + let mut accounts = vec![(key(1), account(1)), (keccak256(CHANGED), account(2))]; + accounts.sort_by_key(|(hashed_address, _)| *hashed_address); + accounts + } + + // A list crediting `CHANGED` with `balance`. + fn credit(balance: u64) -> Vec { + vec![AccountChanges::new(CHANGED) + .with_balance_change(BalanceChange::new(BlockAccessIndex::new(1), U256::from(balance)))] + } + + // Three blocks after the pivot, crediting the account 10, 20 and 30. + fn chain() -> BalChain { + BalChain::new(PIVOT, [credit(10), credit(20), credit(30)]) + } + + // An attempt started at `chain`'s pivot, with every account downloaded and nothing applied, + // then moved to the chain's last block. + fn started(chain: &BalChain, accounts: &[(B256, TrieAccount)]) -> (Factory, SnapWrite) { + let factory = hashed_factory(); + insert_headers(&factory, &chain.headers); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(chain.generation(state_root(accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + let range = verified_range(accounts, 0..accounts.len(), B256::ZERO, &[]); + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + let tip = SnapGeneration::new(chain.tip(), state_root(accounts)); + let write = provider.advance_snap_pivot(write, tip).unwrap(); + provider.commit().unwrap(); + (factory, write) + } + + // Applied block 2 is shared, but ranges downloaded at pivot 3 contain an old-branch-only + // balance change. The replacement block's empty BAL cannot undo that change. + fn pivot_reorg_fixture() -> (Factory, SnapWrite, BalChain, BalChain) { + let chain = BalChain::new(PIVOT, [credit(30)]); + let replacement = BalChain::new(PIVOT, [Vec::new()]); + assert_eq!(chain.block(0), replacement.block(0)); + let mut accounts = accounts(); + let factory = hashed_factory(); + insert_headers(&factory, &chain.headers); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(chain.generation(state_root(&accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + accounts.iter_mut().find(|(key, _)| *key == keccak256(CHANGED)).unwrap().1.balance = + U256::from(30); + let write = provider + .advance_snap_pivot(write, SnapGeneration::new(chain.tip(), state_root(&accounts))) + .unwrap(); + let range = verified_range(&accounts, 0..accounts.len(), B256::ZERO, &[]); + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + provider.commit().unwrap(); + (factory, write, chain, replacement) + } + + fn catch_up( + responses: impl IntoIterator>, + factory: Factory, + ) -> (Arc, CatchUp) { + catch_up_with(responses, factory, DEFAULT_CATCH_UP_BLOCKS) + } + + fn catch_up_with( + responses: impl IntoIterator>, + factory: Factory, + max_blocks: u64, + ) -> (Arc, CatchUp) { + let client = Arc::new(ScriptedSnapClient::new(responses)); + let catch_up = BlockAccessListCatchUp::new(Arc::clone(&client), factory, Runtime::test()); + (client, catch_up.with_max_blocks(max_blocks)) + } + + async fn applied(catch_up: &mut CatchUp, write: SnapWrite, target: u64) -> CatchUpProgress { + match catch_up.next(write, target).await.unwrap() { + CatchUpStep::Applied { progress, .. } => progress, + step => panic!("expected an application, got {step:?}"), + } + } + + fn balance(factory: &Factory) -> U256 { + let provider = factory.database_provider_ro().unwrap(); + provider + .tx_ref() + .get::(keccak256(CHANGED)) + .unwrap() + .unwrap() + .balance + } + + #[tokio::test] + async fn every_list_a_response_carries_is_applied_in_block_order() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + let (client, mut catch_up) = + catch_up([chain.response(1, [Some(1), Some(2), Some(3)])], factory.clone()); + + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + + assert_eq!(progress.applied(), chain.block(3)); + // The last block's credit is the one that survives. + assert_eq!(balance(&factory), U256::from(30)); + assert_eq!( + *client.block_requests(), + [(1..=3).map(|nth| chain.block(nth).hash).collect::>()] + ); + assert!(matches!(catch_up.next(write, PIVOT + 3).await.unwrap(), CatchUpStep::Complete)); + } + + #[tokio::test] + async fn a_list_a_peer_leaves_out_holds_back_the_blocks_after_it() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + let (client, mut catch_up) = catch_up( + [chain.response(1, [Some(1), None, Some(3)]), chain.response(2, [Some(2), Some(3)])], + factory.clone(), + ); + + // Only the block before the gap applies, although a later list was authenticated. + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + assert_eq!(progress.applied(), chain.block(1)); + assert_eq!(balance(&factory), U256::from(10)); + + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + assert_eq!(progress.applied(), chain.block(3)); + assert_eq!(balance(&factory), U256::from(30)); + // The gap is asked for again, and nothing before it is. + assert_eq!(client.block_requests()[1], [chain.block(2).hash, chain.block(3).hash]); + } + + #[tokio::test] + async fn a_response_cut_short_leaves_the_rest_pending() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + // A peer that stops at its soft byte limit answers fewer blocks than it was asked for. + let (client, mut catch_up) = catch_up( + [chain.response(1, [Some(1), Some(2)]), chain.response(2, [Some(3)])], + factory.clone(), + ); + + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + assert_eq!(progress.applied(), chain.block(2)); + + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + assert_eq!(progress.applied(), chain.block(3)); + assert_eq!(balance(&factory), U256::from(30)); + // Only the suffix the response left off is asked for again. + assert_eq!(client.block_requests()[1], [chain.block(3).hash]); + } + + #[tokio::test] + async fn a_reply_repeating_an_applied_list_cannot_apply_it_again() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + // The first block's list, served again for every attempt at the second block. + let duplicate = std::iter::repeat_with(|| chain.response(2, [Some(1)])).take(4); + let (_, mut catch_up) = catch_up_with( + std::iter::once(chain.response(1, [Some(1)])).chain(duplicate), + factory.clone(), + 1, + ); + applied(&mut catch_up, write, PIVOT + 3).await; + + // It belongs to a block the applied state already covers, so it authenticates against + // nothing the request asked for. + let repeated = catch_up.next(write, PIVOT + 3).await; + + assert!(matches!(repeated, Err(SnapSyncError::Request(_)))); + assert_eq!(balance(&factory), U256::from(10)); + } + + #[tokio::test] + async fn a_peer_holding_no_list_for_the_next_block_applies_nothing() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + let (_, mut catch_up) = catch_up([chain.response(1, [None, None, None])], factory.clone()); + + let step = catch_up.next(write, PIVOT + 3).await.unwrap(); + + assert!(matches!(step, CatchUpStep::Unavailable { .. })); + assert_eq!(balance(&factory), U256::from(1)); + } + + #[tokio::test] + async fn a_block_that_changes_nothing_is_still_carried_past() { + let chain = BalChain::new(PIVOT, [Vec::new()]); + let (factory, write) = started(&chain, &accounts()); + let (_, mut catch_up) = catch_up([chain.response(1, [Some(1)])], factory.clone()); + + let progress = applied(&mut catch_up, write, PIVOT + 1).await; + + assert_eq!(progress.applied(), chain.block(1)); + assert_eq!(balance(&factory), U256::from(1)); + } + + #[tokio::test] + async fn only_the_blocks_a_request_can_carry_are_asked_for() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + let (client, mut catch_up) = catch_up_with([chain.response(1, [Some(1)])], factory, 1); + + applied(&mut catch_up, write, PIVOT + 3).await; + + assert_eq!(*client.block_requests(), [vec![chain.block(1).hash]]); + } + + #[tokio::test] + async fn an_unbounded_request_asks_for_every_block_through_the_target() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + let (client, mut catch_up) = + catch_up_with([chain.response(1, [Some(1), Some(2), Some(3)])], factory, u64::MAX); + + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + + assert_eq!(progress.applied(), chain.block(3)); + assert_eq!( + *client.block_requests(), + [(1..=3).map(|nth| chain.block(nth).hash).collect::>()] + ); + } + + #[tokio::test] + async fn a_target_already_applied_needs_no_request() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + let (client, mut catch_up) = catch_up([], factory); + + assert!(matches!(catch_up.next(write, PIVOT).await.unwrap(), CatchUpStep::Complete)); + assert!(client.block_requests().is_empty()); + } + + #[tokio::test] + async fn a_target_past_the_pivot_is_held_at_it() { + let chain = chain(); + let factory = hashed_factory(); + insert_headers(&factory, &chain.headers); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(chain.generation(state_root(&accounts()))).unwrap(); + provider.commit().unwrap(); + let (client, mut catch_up) = catch_up([], factory); + + assert!(matches!(catch_up.next(write, PIVOT + 3).await.unwrap(), CatchUpStep::Complete)); + assert!(client.block_requests().is_empty()); + } + + #[tokio::test] + async fn an_applied_block_the_canonical_chain_no_longer_holds_is_reported() { + let chain = chain(); + let accounts = accounts(); + // The chain the node holds instead, forking before the pivot as a reorg leaves it. + let reorged = BalChain::new(0, [credit(1), credit(2), credit(3)]); + let factory = hashed_factory(); + insert_headers(&factory, &reorged.headers); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(chain.generation(state_root(&accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider.commit().unwrap(); + let (_, mut catch_up) = catch_up([], factory); + + let forked = catch_up.next(write, PIVOT + 3).await; + + assert!(matches!(forked, Err(SnapSyncError::ForkedBlock { .. }))); + } + + #[tokio::test] + async fn a_pivot_the_canonical_chain_no_longer_holds_is_reported() { + let (factory, write, chain, replacement) = pivot_reorg_fixture(); + let before = SnapStateSnapshot::read(&factory.database_provider_ro().unwrap()); + replacement.replace_tip(&factory); + let (client, mut catch_up) = catch_up([], factory.clone()); + + // Even an already-applied target must reject the orphan instead of reporting completion. + for target in [PIVOT, PIVOT + 1] { + assert!(matches!( + catch_up.next(write, target).await, + Err(SnapSyncError::NonCanonicalBlock { block, hash }) + if block == PIVOT + 1 && hash == chain.tip().hash + )); + } + assert!(client.block_requests().is_empty()); + let provider = factory.database_provider_ro().unwrap(); + assert_eq!(provider.block_hash(PIVOT).unwrap(), Some(chain.block(0).hash)); + assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied(), chain.block(0)); + assert_eq!(SnapStateSnapshot::read(&provider), before); + } + + #[tokio::test] + async fn a_pivot_reorg_during_a_bal_request_changes_nothing() { + let (factory, write, chain, replacement) = pivot_reorg_fixture(); + let before = SnapStateSnapshot::read(&factory.database_provider_ro().unwrap()); + let reorg_factory = factory.clone(); + let replacement_hash = replacement.tip().hash; + let client = Arc::new( + ScriptedSnapClient::new([chain.response(1, [Some(1)])]) + .on_block_request(move || replacement.replace_tip(&reorg_factory)), + ); + let mut catch_up = + BlockAccessListCatchUp::new(client.clone(), factory.clone(), Runtime::test()); + + assert!(matches!( + catch_up.next(write, PIVOT + 1).await, + Err(SnapSyncError::NonCanonicalBlock { block, hash }) + if block == PIVOT + 1 && hash == chain.tip().hash + )); + + assert_eq!(*client.block_requests(), [vec![chain.tip().hash]]); + let provider = factory.database_provider_ro().unwrap(); + assert_eq!(provider.block_hash(PIVOT).unwrap(), Some(chain.block(0).hash)); + assert_eq!(provider.block_hash(PIVOT + 1).unwrap(), Some(replacement_hash)); + assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied(), chain.block(0)); + assert_eq!(SnapStateSnapshot::read(&provider), before); + } + + #[tokio::test] + async fn a_reply_to_an_earlier_request_is_ignored() { + let chain = chain(); + let (factory, write) = started(&chain, &accounts()); + // A reply carrying another request's id, as a delayed one does. + let stale = chain.response(99, [Some(1), Some(2), Some(3)]); + let (_, mut catch_up) = + catch_up([stale, chain.response(1, [Some(1), Some(2), Some(3)])], factory.clone()); + + // The retry is what the authenticated lists arrive on. + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + + assert_eq!(progress.applied(), chain.block(3)); + assert_eq!(balance(&factory), U256::from(30)); + } +} diff --git a/crates/snap-sync/src/catch_up/mod.rs b/crates/snap-sync/src/catch_up/mod.rs new file mode 100644 index 00000000000..03846394120 --- /dev/null +++ b/crates/snap-sync/src/catch_up/mod.rs @@ -0,0 +1,20 @@ +//! Carries the downloaded state forward through verified block access lists. +//! +//! The chain moves on while ranges download, so state anchored at the pivot is already behind by +//! the time it is complete. [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189#application-order) +//! closes that distance by applying the lists of the blocks after the pivot in strict block order, +//! each recording the final value of every field its block changes. +//! +//! Progress is the last block applied, kept as a number and hash so the next list is only taken +//! from its child, and committed with the state that list changes. A block whose list no peer +//! serves holds back every block after it rather than leaving a hole in the applied sequence. + +mod apply; +mod download; +mod store; + +pub use apply::{BalStateUpdate, DownloadedAccount}; +pub use download::{ + BlockAccessListCatchUp, CatchUpStep, DEFAULT_BAL_RESPONSE_BYTES, DEFAULT_CATCH_UP_BLOCKS, +}; +pub use store::{CatchUpProgress, SnapCatchUpStore}; diff --git a/crates/snap-sync/src/catch_up/store.rs b/crates/snap-sync/src/catch_up/store.rs new file mode 100644 index 00000000000..204da37ed9c --- /dev/null +++ b/crates/snap-sync/src/catch_up/store.rs @@ -0,0 +1,665 @@ +//! Applies verified block access lists to the downloaded state, in strict block order. +//! +//! A list commits with the progress it advances, so recorded progress never runs ahead of the +//! state it describes, and a block whose list is still missing leaves every later one pending. + +use crate::{ + common::SnapRecord, AccountCoverage, BalStateUpdate, DownloadedAccount, SnapAccountStore, + SnapAttemptStore, SnapBytecodeStore, SnapSyncError, SnapWrite, +}; +use alloy_eip7928::AccountChanges; +use alloy_eips::BlockNumHash; +use alloy_primitives::{keccak256, B256, KECCAK256_EMPTY}; +use reth_db_api::{ + tables, + transaction::{DbTx, DbTxMut}, +}; +use reth_primitives_traits::Account; +use reth_storage_api::{ + BlockHashReader, DBProvider, MetadataProvider, MetadataWriter, SnapAttemptId, StateWriter, +}; +use reth_trie_common::HashedStorage; +use serde::{Deserialize, Serialize}; + +/// Persistence for the block access lists an attempt applies to its downloaded state. +/// +/// Blanket-implemented over the node's writers, so the state a list changes, the code it deploys +/// and the progress it advances join the caller's transaction and commit together or not at all. +pub trait SnapCatchUpStore { + /// Returns the progress recorded for the attempt `write` belongs to, if any. + fn catch_up_progress(&self, write: SnapWrite) + -> Result, SnapSyncError>; + + /// Returns what the downloaded state holds for `hashed_address` under `coverage`. + fn downloaded_account( + &self, + coverage: AccountCoverage, + hashed_address: B256, + ) -> Result + where + Self: DBProvider; + + /// Returns the state update implied by `bal`, reading downloaded accounts from this provider. + /// + /// The list must already be verified against its header. Accounts outside `coverage` remain + /// unresolved, and no state is written. + fn block_access_list_update( + &self, + coverage: AccountCoverage, + bal: &[AccountChanges], + ) -> Result + where + Self: DBProvider; + + /// Applies `bal` to the downloaded state and records `block` as the last one applied. + /// + /// The list must be authenticated against `block`'s header commitment, and `block` must be + /// canonical and the child of the last applied one. + fn commit_block_access_list( + &self, + write: SnapWrite, + block: BlockNumHash, + parent: B256, + bal: &[AccountChanges], + ) -> Result + where + Self: BlockHashReader + MetadataWriter + StateWriter + DBProvider; +} + +/// How far past the pivot the downloaded state has been carried. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatchUpProgress { + // Last block whose list is applied, the pivot until one is. + applied: BlockNumHash, +} + +impl CatchUpProgress { + /// Nothing applied past `pivot`. + pub const fn at_pivot(pivot: BlockNumHash) -> Self { + Self { applied: pivot } + } + + /// Last block whose list is applied. + pub const fn applied(&self) -> BlockNumHash { + self.applied + } + + /// Number of the block whose list comes next. + pub const fn next(&self) -> u64 { + self.applied.number + 1 + } + + // The progress after applying `block`, which must be the child of the applied one. + fn advance(&self, block: BlockNumHash, parent: B256) -> Result { + if block.number != self.next() { + return Err(SnapSyncError::OutOfOrderBlock { expected: self.next(), got: block.number }) + } + if parent != self.applied.hash { + return Err(SnapSyncError::ForkedBlock { expected: self.applied.hash, got: parent }) + } + Ok(Self { applied: block }) + } + + /// Persists this progress as `attempt`'s record. + pub(crate) fn write( + &self, + provider: &impl MetadataWriter, + attempt: SnapAttemptId, + ) -> Result<(), SnapSyncError> { + StoredCatchUpProgress::new(attempt, *self).write(provider) + } +} + +// The progress record as persisted, tied to the attempt that recorded it. +#[derive(Serialize, Deserialize)] +struct StoredCatchUpProgress { + // Encoding version, checked before the rest is decoded. + version: u32, + // Attempt the progress belongs to. + attempt: SnapAttemptId, + // Number of the last block whose list is applied. + block: u64, + // Hash of that block, which the next one builds on. + hash: B256, +} + +impl SnapRecord for StoredCatchUpProgress { + const KEY: &'static str = "snap_catch_up_progress"; + const VERSION: u32 = 1; +} + +impl StoredCatchUpProgress { + // `progress` for `attempt` at this build's version. + const fn new(attempt: SnapAttemptId, progress: CatchUpProgress) -> Self { + Self { + version: Self::VERSION, + attempt, + block: progress.applied.number, + hash: progress.applied.hash, + } + } + + const fn progress(&self) -> CatchUpProgress { + CatchUpProgress::at_pivot(BlockNumHash::new(self.block, self.hash)) + } +} + +impl SnapCatchUpStore for T { + // A record left by another attempt reads as no progress. + fn catch_up_progress( + &self, + write: SnapWrite, + ) -> Result, SnapSyncError> { + self.authorize_snap_write(write)?; + let Some(stored) = StoredCatchUpProgress::read(self)? else { return Ok(None) }; + Ok((stored.attempt == write.attempt()).then(|| stored.progress())) + } + + fn downloaded_account( + &self, + coverage: AccountCoverage, + hashed_address: B256, + ) -> Result + where + Self: DBProvider, + { + // Ranges cover the key space in order, so anything from the cursor on is still pending. + if coverage.next().is_some_and(|next| hashed_address >= next) { + return Ok(DownloadedAccount::Unknown) + } + Ok(self + .tx_ref() + .get::(hashed_address)? + .map_or(DownloadedAccount::Absent, DownloadedAccount::Present)) + } + + fn block_access_list_update( + &self, + coverage: AccountCoverage, + bal: &[AccountChanges], + ) -> Result + where + Self: DBProvider, + { + let mut update = BalStateUpdate::default(); + for account_changes in bal { + let account_info = account_changes.account_info(); + // Read-only entries record accesses, not changes. + if !account_info.changes_state_root(account_changes) { + continue + } + + let hashed_address = keccak256(account_changes.address()); + let mut account = match self.downloaded_account(coverage, hashed_address)? { + // Its range is downloaded against a later root, which includes this change. + DownloadedAccount::Unknown => { + update.unresolved.push(hashed_address); + continue + } + DownloadedAccount::Absent => Account::default(), + DownloadedAccount::Present(account) => account, + }; + + account.apply_bal_info(account_info); + // Stored accounts represent empty code with no code hash. + account.bytecode_hash = account.bytecode_hash.filter(|hash| *hash != KECCAK256_EMPTY); + // Execution removes accounts a block leaves empty, see EIP-161. + update.state.accounts.insert(hashed_address, (!account.is_empty()).then_some(account)); + if account_changes.has_storage_changes() { + update.state.storages.insert( + hashed_address, + HashedStorage::from_iter( + account_changes + .storage_post_states() + .map(|(slot, value)| (keccak256(B256::from(slot)), value)), + ), + ); + } + if let Some((code_hash, code)) = account_info + .code_hash + .zip(account_changes.code_post_state().filter(|code| !code.is_empty())) + { + update.bytecodes.insert(code_hash, code.clone()); + } + } + Ok(update) + } + + // Every check runs before the first write, so a refused list changes nothing. + fn commit_block_access_list( + &self, + write: SnapWrite, + block: BlockNumHash, + parent: B256, + bal: &[AccountChanges], + ) -> Result + where + Self: BlockHashReader + MetadataWriter + StateWriter + DBProvider, + { + // Downloaded ranges are proved against the pivot, so it must still be canonical too. + let attempt = self.authorize_canonical_snap_write(write)?; + // The chain may have reorged since the list was requested. + if self.block_hash(block.number)? != Some(block.hash) { + return Err(SnapSyncError::NonCanonicalBlock { block: block.number, hash: block.hash }) + } + let advanced = self + .catch_up_progress(write)? + .ok_or(SnapSyncError::NoCatchUpProgress)? + .advance(block, parent)?; + // Pending ranges are proved against the pivot, so they hold no change past it. + if block.number > attempt.pivot().number { + return Err(SnapSyncError::BlockPastPivot { + pivot: attempt.pivot().number, + block: block.number, + }) + } + let coverage = self.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; + let update = self.block_access_list_update(coverage, bal)?; + + // Entries left unresolved are downloaded whole against the pivot, which already includes + // this block, so nothing here has to remember them. + let (state, bytecodes, _unresolved) = update.into_parts(); + for (hashed_address, account) in &state.accounts { + // Execution clears the storage of an account it removes, which the list leaves to + // whoever applies it. + if account.is_none() { + self.remove::(*hashed_address..=*hashed_address)?; + } + } + self.commit_bytecodes(write, bytecodes.into_iter().collect())?; + self.write_hashed_state(&state.into_sorted())?; + advanced.write(self, write.attempt())?; + Ok(advanced) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + test_utils::{ + account, hashed_factory, key, state_root, storage_root_of, BalChain, SnapStateSnapshot, + }, + SnapAccountStore, SnapGeneration, + }; + use alloy_eip7928::{BalanceChange, BlockAccessIndex, CodeChange, SlotChanges, StorageChange}; + use alloy_primitives::{bytes, keccak256, map::B256Map, Address, Bytes, U256}; + use reth_primitives_traits::Account; + use reth_provider::{ + test_utils::{insert_headers, MockNodeTypesWithDB}, + DatabaseProviderFactory, ProviderFactory, + }; + use reth_trie_common::{HashedStorage, TrieAccount}; + + type Factory = ProviderFactory; + + // The account the fixture lists change, holding one storage slot. + const CHANGED: Address = Address::repeat_byte(0xaa); + const SLOT: U256 = U256::from_limbs([1, 0, 0, 0]); + + fn hashed_slot() -> B256 { + keccak256(B256::from(SLOT)) + } + + fn code() -> Bytes { + bytes!("6001") + } + + // The downloaded trie: filler accounts a partial range can stop inside, and the account the + // lists change, holding one slot and nothing but a balance, so a list can empty it. + fn accounts() -> Vec<(B256, TrieAccount)> { + let mut changed = account(0); + changed.storage_root = storage_root_of(&[(hashed_slot(), U256::from(7))]); + let mut accounts: Vec<_> = (1..=3).map(|nonce| (key(nonce), account(nonce))).collect(); + accounts.push((keccak256(CHANGED), changed)); + accounts.sort_by_key(|(hashed_address, _)| *hashed_address); + accounts + } + + // Position of the changed account in the trie. + fn changed_index(accounts: &[(B256, TrieAccount)]) -> usize { + accounts + .iter() + .position(|(hashed_address, _)| *hashed_address == keccak256(CHANGED)) + .unwrap() + } + + // An attempt started at block 1, with `served` of the trie's accounts downloaded and its + // block access list progress recorded, then moved to pivot 2. + fn started(accounts: &[(B256, TrieAccount)], served: usize) -> (Factory, SnapWrite) { + let factory = hashed_factory(); + insert_headers(&factory, &chain().headers); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + // A partial range needs a proof placing the accounts it leaves out. + let targets = (served < accounts.len()).then(|| accounts[served - 1].0); + let range = + crate::test_utils::verified_range(accounts, 0..served, B256::ZERO, targets.as_slice()); + let storages = B256Map::from_iter([( + keccak256(CHANGED), + HashedStorage::from_iter([(hashed_slot(), U256::from(7))]), + )]); + provider + .commit_account_range( + write, + &range, + if served == accounts.len() { storages } else { B256Map::default() }, + Vec::new(), + ) + .unwrap(); + let write = + provider.advance_snap_pivot(write, generation(2, state_root(accounts))).unwrap(); + provider.commit().unwrap(); + (factory, write) + } + + // Canonical blocks 0 through 3, pivoted at block 1. The lists the tests apply are their own, as + // the store does not authenticate them. + fn chain() -> BalChain { + BalChain::new(1, [Vec::new(), Vec::new()]) + } + + // Generation anchored to block `number` of the fixture chain. + fn generation(number: u64, state_root: B256) -> SnapGeneration { + SnapGeneration::new(chain().block(number as usize - 1), state_root) + } + + // Block `number` of the fixture chain, with the hash of its parent. + fn block(number: u64) -> (BlockNumHash, B256) { + let chain = chain(); + (chain.block(number as usize - 1), chain.block(number as usize - 2).hash) + } + + fn index(value: u64) -> BlockAccessIndex { + BlockAccessIndex::new(value) + } + + // A list crediting `CHANGED` with `balance`. + fn credit(balance: u64) -> Vec { + vec![AccountChanges::new(CHANGED) + .with_balance_change(BalanceChange::new(index(1), U256::from(balance)))] + } + + fn stored(provider: &impl DBProvider, hashed_address: B256) -> Option { + provider.tx_ref().get::(hashed_address).unwrap() + } + + #[test] + fn progress_starts_at_the_first_pivot_and_resumes_where_it_left_off() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(2); + + // The pivot moved to block 2, but ranges committed at block 1 still need its list. + assert_eq!( + provider.catch_up_progress(write).unwrap().unwrap().applied(), + generation(1, B256::ZERO).target() + ); + + let progress = + provider.commit_block_access_list(write, block, parent, &credit(10)).unwrap(); + assert_eq!(progress.applied(), block); + assert_eq!(progress.next(), 3); + // Restarting the attempt re-applies nothing. + assert_eq!(provider.catch_up_progress(write).unwrap(), Some(progress)); + } + + #[test] + fn a_pivot_below_the_applied_block_is_refused() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(2); + provider.commit_block_access_list(write, block, parent, &credit(10)).unwrap(); + + let refused = provider.advance_snap_pivot(write, generation(1, state_root(&accounts))); + + assert!(matches!(refused, Err(SnapSyncError::PivotBelowApplied { applied: 2, pivot: 1 }))); + provider.authorize_snap_write(write).unwrap(); + } + + #[test] + fn a_list_commits_with_the_block_it_carries_the_state_to() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let (block, parent) = block(2); + let changes = vec![AccountChanges::new(CHANGED) + .with_balance_change(BalanceChange::new(index(1), U256::from(10))) + .with_code_change(CodeChange::new(index(1), code())) + .with_storage_change(SlotChanges::new( + SLOT, + vec![StorageChange::new(index(1), U256::from(9))], + ))]; + + let provider = factory.database_provider_rw().unwrap(); + provider.commit_block_access_list(write, block, parent, &changes).unwrap(); + provider.commit().unwrap(); + + let provider = factory.database_provider_ro().unwrap(); + let applied = stored(&provider, keccak256(CHANGED)).unwrap(); + assert_eq!(applied.balance, U256::from(10)); + assert_eq!(applied.bytecode_hash, Some(keccak256(code()))); + // The code the list deploys is stored with the account referencing it. + assert!(provider.tx_ref().get::(keccak256(code())).unwrap().is_some()); + assert_eq!( + crate::test_utils::stored_slots(&provider, keccak256(CHANGED)), + [(hashed_slot(), U256::from(9))] + ); + assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied(), block); + } + + #[test] + fn an_interrupted_application_leaves_nothing_behind() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let (block, parent) = block(2); + + let provider = factory.database_provider_rw().unwrap(); + provider.commit_block_access_list(write, block, parent, &credit(10)).unwrap(); + drop(provider); + + let provider = factory.database_provider_ro().unwrap(); + assert_eq!(stored(&provider, keccak256(CHANGED)).unwrap().balance, U256::from(1)); + assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied().number, 1); + } + + #[test] + fn a_block_already_applied_is_refused() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(2); + provider.commit_block_access_list(write, block, parent, &credit(10)).unwrap(); + + let duplicate = provider.commit_block_access_list(write, block, parent, &credit(20)); + + assert!(matches!(duplicate, Err(SnapSyncError::OutOfOrderBlock { expected: 3, got: 2 }))); + assert_eq!(stored(&provider, keccak256(CHANGED)).unwrap().balance, U256::from(10)); + } + + #[test] + fn a_block_the_applied_state_has_not_reached_is_refused() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(3); + + let gap = provider.commit_block_access_list(write, block, parent, &credit(10)); + + assert!(matches!(gap, Err(SnapSyncError::OutOfOrderBlock { expected: 2, got: 3 }))); + assert_eq!(stored(&provider, keccak256(CHANGED)).unwrap().balance, U256::from(1)); + } + + #[test] + fn a_block_the_canonical_chain_no_longer_holds_is_refused() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (_, parent) = block(2); + let orphaned = BlockNumHash::new(2, B256::repeat_byte(0xee)); + + let refused = provider.commit_block_access_list(write, orphaned, parent, &credit(10)); + + assert!(matches!(refused, Err(SnapSyncError::NonCanonicalBlock { block: 2, .. }))); + assert_eq!(stored(&provider, keccak256(CHANGED)).unwrap().balance, U256::from(1)); + } + + #[test] + fn a_list_proved_against_an_orphaned_pivot_is_refused() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (applied, parent) = block(2); + provider.commit_block_access_list(write, applied, parent, &[]).unwrap(); + // A reorg replaced the pivot, leaving the applied block and the one after it canonical. + let orphaned = BlockNumHash::new(3, B256::repeat_byte(0xee)); + let write = provider + .advance_snap_pivot(write, SnapGeneration::new(orphaned, state_root(&accounts))) + .unwrap(); + let (block, parent) = block(3); + let before = SnapStateSnapshot::read(&provider); + + let refused = provider.commit_block_access_list(write, block, parent, &credit(10)); + + assert!(matches!( + refused, + Err(SnapSyncError::NonCanonicalBlock { block: 3, hash }) if hash == orphaned.hash + )); + assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied(), applied); + assert_eq!(SnapStateSnapshot::read(&provider), before); + provider.commit().unwrap(); + assert_eq!(SnapStateSnapshot::read(&factory.database_provider_ro().unwrap()), before); + } + + #[test] + fn an_orphaned_pivot_cannot_be_replaced() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (applied, parent) = block(2); + provider.commit_block_access_list(write, applied, parent, &[]).unwrap(); + let orphaned = BlockNumHash::new(3, B256::repeat_byte(0xee)); + let write = provider + .advance_snap_pivot(write, SnapGeneration::new(orphaned, state_root(&accounts))) + .unwrap(); + + let before = SnapStateSnapshot::read(&provider); + + let refused = provider.advance_snap_pivot(write, generation(3, state_root(&accounts))); + + // Re-anchoring would leave the ranges downloaded against the orphan looking canonical. + assert!(matches!( + refused, + Err(SnapSyncError::NonCanonicalBlock { block: 3, hash }) if hash == orphaned.hash + )); + assert_eq!(provider.authorize_snap_write(write).unwrap().pivot(), orphaned); + assert_eq!(SnapStateSnapshot::read(&provider), before); + provider.commit().unwrap(); + assert_eq!(SnapStateSnapshot::read(&factory.database_provider_ro().unwrap()), before); + } + + #[test] + fn a_block_building_on_another_chain_is_refused() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, _) = block(2); + + let forked = + provider.commit_block_access_list(write, block, B256::repeat_byte(0xee), &credit(10)); + + assert!(matches!(forked, Err(SnapSyncError::ForkedBlock { .. }))); + assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied().number, 1); + } + + #[test] + fn a_block_that_changes_nothing_still_carries_the_state_past_it() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(2); + // A list a peer holds but that touches no state, as against one it does not hold. + let read_only = vec![AccountChanges::new(CHANGED).with_storage_read(SLOT)]; + + let progress = provider.commit_block_access_list(write, block, parent, &read_only).unwrap(); + + assert_eq!(progress.applied(), block); + assert_eq!(stored(&provider, keccak256(CHANGED)).unwrap().balance, U256::from(1)); + } + + #[test] + fn an_account_the_list_empties_loses_its_storage() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(2); + + provider.commit_block_access_list(write, block, parent, &credit(0)).unwrap(); + + assert_eq!(stored(&provider, keccak256(CHANGED)), None); + assert!(crate::test_utils::stored_slots(&provider, keccak256(CHANGED)).is_empty()); + } + + #[test] + fn a_block_past_the_pivot_is_refused() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(2); + provider.commit_block_access_list(write, block, parent, &credit(10)).unwrap(); + let (block, parent) = self::block(3); + + let refused = provider.commit_block_access_list(write, block, parent, &credit(20)); + + assert!(matches!(refused, Err(SnapSyncError::BlockPastPivot { pivot: 2, block: 3 }))); + assert_eq!(stored(&provider, keccak256(CHANGED)).unwrap().balance, U256::from(10)); + } + + #[test] + fn an_account_outside_the_coverage_is_left_to_its_range() { + let accounts = accounts(); + let index = changed_index(&accounts); + let (factory, write) = started(&accounts, index); + let provider = factory.database_provider_rw().unwrap(); + let (block, parent) = block(2); + + let progress = + provider.commit_block_access_list(write, block, parent, &credit(10)).unwrap(); + + // The range still to download authenticates against a root that already holds the change. + assert_eq!(stored(&provider, keccak256(CHANGED)), None); + assert_eq!(progress.applied(), block); + } + + #[test] + fn a_list_from_a_replaced_attempt_changes_nothing() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + provider.advance_snap_pivot(write, generation(2, B256::repeat_byte(0xcc))).unwrap(); + let (block, parent) = block(2); + + let refused = provider.commit_block_access_list(write, block, parent, &credit(10)); + + assert!(matches!(refused, Err(SnapSyncError::StaleWrite { .. }))); + assert!(matches!(provider.catch_up_progress(write), Err(SnapSyncError::StaleWrite { .. }))); + } + + #[test] + fn a_record_this_build_cannot_read_is_reported() { + let accounts = accounts(); + let (factory, write) = started(&accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + provider.write_metadata(StoredCatchUpProgress::KEY, br#"{"version":0}"#.to_vec()).unwrap(); + + assert!(matches!( + provider.catch_up_progress(write), + Err(SnapSyncError::UnsupportedRecord { + key: StoredCatchUpProgress::KEY, + version: Some(0) + }) + )); + } +} diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index 22ba20c9d55..8dcc10f572a 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -1,7 +1,7 @@ //! Failures raised while assembling a snap state generation. use alloy_primitives::B256; -use reth_downloaders::snap::InvalidStorageRangeRequest; +use reth_downloaders::snap::{InvalidBlockAccessListRequest, InvalidStorageRangeRequest}; use reth_network_p2p::error::RequestError; use reth_storage_api::SnapAttemptId; use reth_storage_errors::{db::DatabaseError, provider::ProviderError}; @@ -18,6 +18,9 @@ pub enum SnapSyncError { /// A storage request did not match the accounts it was built from. #[error(transparent)] StorageRequest(#[from] InvalidStorageRangeRequest), + /// A block access list request did not match the headers it was built from. + #[error(transparent)] + BlockAccessListRequest(#[from] InvalidBlockAccessListRequest), /// The storage layout keys state by address, which snap cannot fill in without preimages. #[error("snap synchronization requires the hashed state layout")] UnsupportedStorage, @@ -40,6 +43,55 @@ pub enum SnapSyncError { /// Root the range was proved against. got: B256, }, + /// No catch-up progress is recorded for the attempt. + #[error("no catch-up progress is recorded for the attempt")] + NoCatchUpProgress, + /// A block the catch-up needs has no header. + #[error("block {block} has no header to authenticate its access list against")] + MissingHeader { + /// Block the header is missing for. + block: u64, + }, + /// A list was applied for a block other than the one continuing the applied sequence. + #[error("block access list for block {got}, the applied state continues at {expected}")] + OutOfOrderBlock { + /// Block the applied state continues at. + expected: u64, + /// Block the list was applied for. + got: u64, + }, + /// A list was applied for a block past the pivot. + #[error("block access list for block {block} past pivot {pivot}")] + BlockPastPivot { + /// Block the attempt is anchored to. + pivot: u64, + /// Block the list was applied for. + block: u64, + }, + /// The pivot was moved below the last block whose list is applied. + #[error("pivot {pivot} is below applied block {applied}")] + PivotBelowApplied { + /// Last block whose list is applied. + applied: u64, + /// Block the pivot was moved to. + pivot: u64, + }, + /// A list was applied for a block the canonical chain no longer holds. + #[error("block {block} ({hash}) is no longer canonical")] + NonCanonicalBlock { + /// Number of the block. + block: u64, + /// Hash of the block the list belongs to. + hash: B256, + }, + /// A list was applied for a block building on another chain than the applied state. + #[error("block access list for a block building on {got}, the applied state is at {expected}")] + ForkedBlock { + /// Hash of the last applied block. + expected: B256, + /// Hash the block the list belongs to builds on. + got: B256, + }, /// No account coverage is recorded for the attempt. #[error("no account coverage is recorded for the attempt")] NoCoverage, diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index b28cda7b815..2610bb65073 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -38,8 +38,8 @@ mod account; mod attempt; -mod bal; mod bytecode; +mod catch_up; mod common; mod error; mod generation; @@ -54,8 +54,11 @@ pub use account::{ AccountCoverage, AccountRangeDownload, AccountRangeStep, SnapAccountStore, VerifiedRange, }; pub use attempt::{SnapAttemptStore, SnapWrite}; -pub use bal::{BalStateUpdate, DownloadedAccount}; pub use bytecode::{BytecodeDownload, BytecodeStep, SnapBytecodeStore, DEFAULT_CODE_HASHES}; +pub use catch_up::{ + BalStateUpdate, BlockAccessListCatchUp, CatchUpProgress, CatchUpStep, DownloadedAccount, + SnapCatchUpStore, DEFAULT_BAL_RESPONSE_BYTES, DEFAULT_CATCH_UP_BLOCKS, +}; pub use common::{DEFAULT_RESPONSE_BYTES, MAX_HASH}; pub use error::SnapSyncError; pub use generation::{SnapGeneration, SnapPhase}; diff --git a/crates/snap-sync/src/storage/download.rs b/crates/snap-sync/src/storage/download.rs index f140ada9a05..f0db0168ec0 100644 --- a/crates/snap-sync/src/storage/download.rs +++ b/crates/snap-sync/src/storage/download.rs @@ -169,8 +169,8 @@ mod tests { use super::*; use crate::{ test_utils::{ - account, generation, hashed_factory, key, state_root, storage_ranges, storage_root_of, - stored_slots, verified_range, ScriptedSnapClient, + account, generation, hashed_factory, insert_generation_headers, key, state_root, + storage_ranges, storage_root_of, stored_slots, verified_range, ScriptedSnapClient, }, SnapAccountStore, SnapAttemptStore, }; @@ -215,6 +215,7 @@ mod tests { // An attempt that fetched all of `accounts` as one range, not yet committed. fn started(accounts: &[(B256, TrieAccount)]) -> (Factory, VerifiedRange) { let factory = hashed_factory(); + insert_generation_headers(&factory); let provider = factory.database_provider_rw().unwrap(); let write = provider.start_snap_attempt(generation(1, state_root(accounts))).unwrap(); provider.start_account_coverage(write).unwrap(); diff --git a/crates/snap-sync/src/storage/store.rs b/crates/snap-sync/src/storage/store.rs index 0bf957d62e3..1103bd1f7ec 100644 --- a/crates/snap-sync/src/storage/store.rs +++ b/crates/snap-sync/src/storage/store.rs @@ -241,7 +241,8 @@ pub(crate) fn persisted_storage_root(tx: &impl DbTx, account: B256) -> Result (ProviderFactory, SnapWrite) { let factory = hashed_factory(); + insert_generation_headers(&factory); let provider = factory.database_provider_rw().unwrap(); let mut contract = account(1); contract.storage_root = root(); diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs index c75c3b42f49..27b468e53bb 100644 --- a/crates/snap-sync/src/test_utils.rs +++ b/crates/snap-sync/src/test_utils.rs @@ -5,15 +5,24 @@ use crate::{ SnapGeneration, SnapPivotPolicy, }; use alloy_consensus::Header; -use alloy_eips::BlockNumHash; +use alloy_eip7928::{compute_block_access_list_hash, AccountChanges}; +use alloy_eips::{eip7928::bal::Bal, BlockNumHash}; use alloy_primitives::{Bytes, B256, KECCAK256_EMPTY, U256}; use futures::future::{ready, Ready}; -use reth_db_api::{cursor::DbDupCursorRO, tables, transaction::DbTx}; +use reth_db_api::{ + cursor::{DbCursorRO, DbDupCursorRO}, + table::Table, + tables, + transaction::DbTx, +}; use reth_downloaders::snap::{AccountRangeDownloader, AccountRangeOutcome, VerifiedAccountRange}; -use reth_eth_wire_types::snap::{ - AccountData, AccountRangeMessage, ByteCodesMessage, GetAccountRangeMessage, - GetBlockAccessListsMessage, GetByteCodesMessage, GetStorageRangesMessage, StorageData, - StorageRangesMessage, +use reth_eth_wire_types::{ + snap::{ + AccountData, AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage, + GetAccountRangeMessage, GetBlockAccessListsMessage, GetByteCodesMessage, + GetStorageRangesMessage, StorageData, StorageRangesMessage, + }, + BlockAccessLists, }; use reth_network_p2p::{ download::DownloadClient, @@ -22,9 +31,13 @@ use reth_network_p2p::{ snap::client::{SnapClient, SnapResponse}, }; use reth_network_peers::{PeerId, WithPeerId}; +use reth_primitives_traits::{Account, AlloyBlockHeader, Bytecode, SealedHeader, StorageEntry}; use reth_provider::{ - test_utils::{create_test_provider_factory, MockEthProvider, MockNodeTypesWithDB}, - DatabaseProviderFactory, ProviderFactory, + test_utils::{ + create_test_provider_factory, insert_headers, MockEthProvider, MockNodeTypesWithDB, + }, + DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory, StaticFileSegment, + StaticFileWriter, }; use reth_storage_api::{DBProvider, MetadataWriter, StorageSettings, StorageSettingsCache}; use reth_tasks::Runtime; @@ -84,6 +97,18 @@ pub(crate) fn generation(block: u64, state_root: B256) -> SnapGeneration { SnapGeneration::new(BlockNumHash::new(block, B256::repeat_byte(block as u8)), state_root) } +/// Writes blocks `0..=3` hashed the way [`generation`] anchors them, so an attempt started from +/// one finds its pivot on the canonical chain. +pub(crate) fn insert_generation_headers(factory: &ProviderFactory) { + let headers: Vec<_> = (0..=3u64) + .map(|number| { + let parent = number.checked_sub(1).map_or(B256::ZERO, |n| B256::repeat_byte(n as u8)); + SealedHeader::new(header(number, parent, None), B256::repeat_byte(number as u8)) + }) + .collect(); + insert_headers(factory, &headers); +} + /// A database using the hashed state layout snap writes into. pub(crate) fn hashed_factory() -> ProviderFactory { let factory = create_test_provider_factory(); @@ -217,6 +242,90 @@ pub(crate) fn byte_codes(request_id: u64, codes: &[Bytes]) -> PeerRequestResult< Ok(WithPeerId::new(PeerId::random(), SnapResponse::ByteCodes(message))) } +/// A canonical chain from genesis through the blocks a catch-up applies, each of those carrying +/// the list its header commits to. +pub(crate) struct BalChain { + /// Headers from genesis through the last block after the pivot. + pub(crate) headers: Vec>, + // Block the downloaded state is anchored to. + pivot: u64, + // Encoded lists of the blocks after the pivot, as peers serve them. + lists: Vec, +} + +impl BalChain { + /// A chain anchored at `pivot`, carrying one block per entry of `lists` after it. + pub(crate) fn new(pivot: u64, lists: impl IntoIterator>) -> Self { + let (commitments, lists): (Vec, Vec) = lists + .into_iter() + .map(|changes| { + ( + compute_block_access_list_hash(&changes), + alloy_rlp::encode(Bal::from(changes)).into(), + ) + }) + .unzip(); + let mut headers = Vec::new(); + let mut parent = B256::ZERO; + for number in 0..=pivot { + headers.push(SealedHeader::seal_slow(header(number, parent, None))); + parent = headers[number as usize].hash(); + } + for (index, commitment) in commitments.into_iter().enumerate() { + let sealed = + SealedHeader::seal_slow(header(pivot + index as u64 + 1, parent, Some(commitment))); + parent = sealed.hash(); + headers.push(sealed); + } + Self { headers, pivot, lists } + } + + /// Generation anchored to this chain's pivot, downloading against `state_root`. + pub(crate) fn generation(&self, state_root: B256) -> SnapGeneration { + SnapGeneration::new(self.block(0), state_root) + } + + /// The chain's last block. + pub(crate) fn tip(&self) -> BlockNumHash { + self.block(self.lists.len()) + } + + /// Replaces the canonical tip with this chain's tip, retaining its ancestors. + pub(crate) fn replace_tip(&self, factory: &ProviderFactory) { + let static_files = factory.static_file_provider(); + let mut writer = static_files.latest_writer(StaticFileSegment::Headers).unwrap(); + writer.prune_headers(1).unwrap(); + writer.commit().unwrap(); + let tip = self.headers.last().unwrap(); + writer.append_header(tip.header(), &tip.hash()).unwrap(); + writer.commit().unwrap(); + } + + /// The block `nth` after the pivot, which is the pivot itself at zero. + pub(crate) fn block(&self, nth: usize) -> BlockNumHash { + let header = &self.headers[self.pivot as usize + nth]; + BlockNumHash::new(header.number(), header.hash()) + } + + /// A peer's answer serving the list of each block `served` names, holding none where it names + /// no block. + pub(crate) fn response( + &self, + request_id: u64, + served: impl IntoIterator>, + ) -> PeerRequestResult { + let block_access_lists = served + .into_iter() + .map(|nth| nth.map(|nth| self.lists[nth - 1].clone())) + .collect::>(); + let message = BlockAccessListsMessage { + request_id, + block_access_lists: BlockAccessLists(block_access_lists), + }; + Ok(WithPeerId::new(PeerId::random(), SnapResponse::BlockAccessLists(message))) + } +} + /// Slots persisted for `account`, in key order. pub(crate) fn stored_slots(provider: &impl DBProvider, account: B256) -> Vec<(B256, U256)> { let mut cursor = provider.tx_ref().cursor_dup_read::().unwrap(); @@ -230,12 +339,38 @@ pub(crate) fn stored_slots(provider: &impl DBProvider, account: B256) -> Vec<(B2 .collect() } +/// Persisted snap state and all metadata, including attempt, coverage and catch-up progress. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct SnapStateSnapshot { + accounts: Vec<(B256, Account)>, + storages: Vec<(B256, StorageEntry)>, + bytecodes: Vec<(B256, Bytecode)>, + metadata: Vec<(String, Vec)>, +} + +impl SnapStateSnapshot { + pub(crate) fn read(provider: &impl DBProvider) -> Self { + Self { + accounts: table_entries::(provider), + storages: table_entries::(provider), + bytecodes: table_entries::(provider), + metadata: table_entries::(provider), + } + } +} + +fn table_entries(provider: &impl DBProvider) -> Vec<(T::Key, T::Value)> { + provider.tx_ref().cursor_read::().unwrap().walk(None).unwrap().map(Result::unwrap).collect() +} + /// Serves scripted answers in request order, recording what each request asked for. pub(crate) struct ScriptedSnapClient { responses: Mutex>>, origins: Mutex>, storage_requests: Mutex, B256)>>, code_requests: Mutex>>, + block_requests: Mutex>>, + on_block_request: Mutex>>, } impl ScriptedSnapClient { @@ -247,9 +382,17 @@ impl ScriptedSnapClient { origins: Mutex::new(Vec::new()), storage_requests: Mutex::new(Vec::new()), code_requests: Mutex::new(Vec::new()), + block_requests: Mutex::new(Vec::new()), + on_block_request: Mutex::new(None), } } + /// Runs `hook` after the next BAL request is recorded, before returning its response. + pub(crate) fn on_block_request(self, hook: impl FnOnce() + Send + 'static) -> Self { + *self.on_block_request.lock().unwrap() = Some(Box::new(hook)); + self + } + /// Origins of the account range requests sent so far. pub(crate) fn origins(&self) -> MutexGuard<'_, Vec> { self.origins.lock().unwrap() @@ -265,6 +408,11 @@ impl ScriptedSnapClient { self.code_requests.lock().unwrap() } + /// Block hashes of the block access list requests sent so far. + pub(crate) fn block_requests(&self) -> MutexGuard<'_, Vec>> { + self.block_requests.lock().unwrap() + } + fn next_response(&self) -> Ready> { let response = self.responses.lock().unwrap().pop_front(); ready(response.unwrap_or(Err(RequestError::ChannelClosed))) @@ -326,13 +474,13 @@ impl SnapClient for ScriptedSnapClient { fn get_block_access_lists_with_priority( &self, - _request: GetBlockAccessListsMessage, + request: GetBlockAccessListsMessage, _priority: Priority, ) -> Self::Output { - unsupported() + self.block_requests.lock().unwrap().push(request.block_hashes); + if let Some(hook) = self.on_block_request.lock().unwrap().take() { + hook(); + } + self.next_response() } } - -fn unsupported() -> Ready> { - ready(Err(RequestError::UnsupportedCapability)) -} From 9415efd4beae124c66ce8c3e0788d7ce261bb87f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 18 Sep 2026 14:44:17 +0200 Subject: [PATCH 010/133] feat: carry the prepared BAL on ExecutedBlock and Chain (#27297) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- Cargo.lock | 6 +- crates/chain-state/src/in_memory.rs | 78 ++++++++++- crates/engine/tree/src/persistence.rs | 10 +- crates/engine/tree/src/tree/mod.rs | 9 +- .../src/tree/payload_processor/bal/execute.rs | 10 +- .../engine/tree/src/tree/payload_validator.rs | 87 ++++++++----- crates/engine/tree/src/tree/tests.rs | 15 ++- crates/engine/tree/src/tree/types.rs | 12 +- crates/evm/execution-types/Cargo.toml | 3 + crates/evm/execution-types/src/bal.rs | 10 ++ crates/evm/execution-types/src/chain.rs | 59 ++++++++- crates/evm/execution-types/src/lib.rs | 3 + crates/net/network/tests/it/requests.rs | 8 +- crates/rpc/rpc-eth-types/src/cache/mod.rs | 68 ++++++++-- crates/storage/provider/Cargo.toml | 2 - crates/storage/provider/src/bal.rs | 121 +----------------- crates/storage/provider/src/bal/rocksdb.rs | 38 +----- crates/storage/provider/src/lib.rs | 6 +- crates/storage/storage-api/Cargo.toml | 4 - crates/storage/storage-api/src/bal.rs | 62 --------- 20 files changed, 294 insertions(+), 317 deletions(-) create mode 100644 crates/evm/execution-types/src/bal.rs diff --git a/Cargo.lock b/Cargo.lock index 7a8572520bb..f59ec947be7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9068,6 +9068,7 @@ name = "reth-execution-types" version = "2.5.2" dependencies = [ "alloy-consensus", + "alloy-eip7928", "alloy-eips", "alloy-evm", "alloy-primitives", @@ -9925,7 +9926,6 @@ dependencies = [ "reth-storage-overlay", "reth-tasks", "reth-testing-utils", - "reth-tokio-util", "reth-tracing", "reth-trie", "reth-trie-db", @@ -9935,7 +9935,6 @@ dependencies = [ "strum 0.27.2", "tempfile", "tokio", - "tokio-stream", "tracing", ] @@ -10604,12 +10603,9 @@ dependencies = [ "reth-prune-types", "reth-stages-types", "reth-storage-errors", - "reth-tokio-util", "reth-trie-common", "revm", "serde_json", - "tokio", - "tokio-stream", ] [[package]] diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index ef0fa4fabb3..29ca16a9682 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -10,7 +10,9 @@ use alloy_primitives::{map::B256Map, BlockNumber, TxHash, B256}; use parking_lot::RwLock; use reth_chainspec::ChainInfo; use reth_ethereum_primitives::EthPrimitives; -use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome}; +use reth_execution_types::{ + BlockExecutionOutput, BlockExecutionResult, Chain, DecodedRevmBal, ExecutionOutcome, +}; use reth_metrics::{metrics::Gauge, Metrics}; use reth_primitives_traits::{ BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, @@ -768,6 +770,18 @@ pub struct ExecutedBlock { /// This allows deferring the computation of the trie data which can be expensive. /// The data can be populated asynchronously after the block was validated. pub trie_data: LazyTrieData, + /// The prepared block access list of the block, if one is available. + /// + /// `None` means no BAL was available when the block was constructed, not that the block + /// has none: only blocks the engine validated from a payload that carried a BAL have it + /// attached. Blocks from before the BAL fork, blocks built or loaded outside engine + /// validation (payload builder, persistence, tests) and downloaded blocks without a BAL + /// sidecar leave it unset; the BAL store is the source of truth in that case. + /// + /// When present, this carries the raw RLP (for the BAL store) together with the revm + /// representation (for consumers like the RPC state cache), so that neither has to be + /// re-derived after validation. + pub bal: Option>, } impl Default for ExecutedBlock { @@ -784,13 +798,15 @@ impl Default for ExecutedBlock { state: Default::default(), }), trie_data: LazyTrieData::ready(ComputedTrieData::default()), + bal: None, } } } impl PartialEq for ExecutedBlock { fn eq(&self, other: &Self) -> bool { - // Trie data is computed asynchronously and doesn't define block identity. + // Trie data is computed asynchronously and the block access list is derived data; neither + // defines block identity. self.recovered_block == other.recovered_block && self.execution_output == other.execution_output } @@ -806,7 +822,12 @@ impl ExecutedBlock { execution_output: Arc>, trie_data: ComputedTrieData, ) -> Self { - Self { recovered_block, execution_output, trie_data: LazyTrieData::ready(trie_data) } + Self { + recovered_block, + execution_output, + trie_data: LazyTrieData::ready(trie_data), + bal: None, + } } /// Create a new [`ExecutedBlock`] with deferred trie data. @@ -827,7 +848,21 @@ impl ExecutedBlock { execution_output: Arc>, trie_data: LazyTrieData, ) -> Self { - Self { recovered_block, execution_output, trie_data } + Self { recovered_block, execution_output, trie_data, bal: None } + } + + /// Attaches the prepared block access list of the block, or clears it with `None`. + pub fn with_bal(mut self, bal: Option>) -> Self { + self.bal = bal; + self + } + + /// Returns the prepared block access list of the block, if one is available. + /// + /// `None` only means no BAL was attached to this block; see the `bal` field for details. + #[inline] + pub const fn bal(&self) -> Option<&Arc> { + self.bal.as_ref() } /// Returns a reference to an inner [`SealedBlock`] @@ -970,7 +1005,7 @@ impl> NewCanonicalChain { /// Converts a slice of executed blocks into a [`Chain`]. fn blocks_to_chain(blocks: &[ExecutedBlock]) -> Chain { - match blocks { + let mut chain = match blocks { [] => Chain::default(), [first, rest @ ..] => { let mut chain = Chain::from_block( @@ -993,7 +1028,13 @@ impl> NewCanonicalChain { } chain } + }; + for exec in blocks { + if let Some(bal) = exec.bal() { + chain.insert_bal(exec.block_number(), Arc::clone(bal)); + } } + chain } /// Returns the new tip of the chain. @@ -1642,4 +1683,31 @@ mod tests { } ); } + + #[test] + fn test_to_chain_notification_carries_prepared_bal() { + let mut test_block_builder: TestBlockBuilder = TestBlockBuilder::default(); + let block0 = test_block_builder.get_executed_block_with_number(0, B256::random()); + let block1 = test_block_builder + .get_executed_block_with_number(1, block0.recovered_block.hash()) + .with_bal(Some(Arc::new(DecodedRevmBal::new( + Arc::new(revm::state::bal::Bal::default()), + Bytes::from_static(&[0xc0]), + )))); + + let chain = NewCanonicalChain::Commit { new: vec![block0, block1.clone()] }; + let CanonStateNotification::Commit { new } = chain.to_chain_notification() else { + panic!("expected a commit notification") + }; + + // Only the block whose payload carried a BAL contributes one. + assert_eq!(new.bals().len(), 1); + assert_eq!(new.bal_at(1), block1.bal()); + + let mut blocks_and_bals = new.blocks_and_bals(); + let (block, bal) = blocks_and_bals.next().expect("block with BAL"); + assert_eq!(block.hash(), block1.recovered_block.hash()); + assert_eq!(Some(bal), block1.bal()); + assert!(blocks_and_bals.next().is_none()); + } } diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 8db05e6c0e8..3c6261945c1 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -420,9 +420,9 @@ mod tests { use reth_provider::{ providers::{BlockchainProvider, ProviderFactoryBuilder, ReadOnlyConfig}, test_utils::{create_test_provider_factory, MockNodeTypes}, - AccountReader, BalConfig, BalNotificationStream, BalStore, BalStoreHandle, - ChainSpecProvider, HeaderProvider, InMemoryBalStore, ProviderError, ProviderResult, RawBal, - StateProviderFactory, StorageSettingsCache, + AccountReader, BalConfig, BalStore, BalStoreHandle, ChainSpecProvider, HeaderProvider, + InMemoryBalStore, ProviderError, ProviderResult, RawBal, StateProviderFactory, + StorageSettingsCache, }; use reth_prune::Pruner; use reth_prune_types::PruneMode; @@ -539,10 +539,6 @@ mod tests { fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> ProviderResult>> { Ok(vec![None; block_hashes.len()]) } - - fn bal_stream(&self) -> BalNotificationStream { - BalStoreHandle::noop().bal_stream() - } } #[test] diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index d45cfcab271..df80919699f 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3213,13 +3213,10 @@ where let start = Instant::now(); - let ValidationOutput { - executed_block: executed, - execution_timing_stats: timing_stats, - raw_bal, - } = execute(&mut self.payload_validator, input, ctx)?; + let ValidationOutput { executed_block: executed, execution_timing_stats: timing_stats } = + execute(&mut self.payload_validator, input, ctx)?; - if let Some(raw_bal) = raw_bal { + if let Some(raw_bal) = executed.bal().map(|bal| bal.as_raw_bal().clone()) { let num_hash = executed.recovered_block().num_hash(); if let Err(err) = self.provider.bal_store().insert(num_hash, raw_bal) { warn!( diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index b8dac4edd67..41063f32ae3 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -54,6 +54,9 @@ use crate::tree::payload_processor::receipt_root_task::IndexedReceipt; /// /// The ordered commit loop applies Ethereum block-level gas admission. Executors with different /// admission rules, such as segment-scoped gas budgets, must align those checks before using it. +/// +/// Returns the execution output, the recovered senders, the BAL rebuilt from this execution, and +/// the received BAL in the revm representation the workers consumed. #[expect(clippy::too_many_arguments, clippy::type_complexity)] pub fn execute_block<'a, Evm, Tx, Err, DB, MakeDb>( runtime: &Runtime, @@ -66,7 +69,7 @@ pub fn execute_block<'a, Evm, Tx, Err, DB, MakeDb>( txs: Receiver<(usize, Result)>, receipt_tx: Sender>>, ) -> Result< - (BlockExecutionOutput>, Vec
, BlockAccessList), + (BlockExecutionOutput>, Vec
, BlockAccessList, Arc), BalExecutionError, > where @@ -109,7 +112,7 @@ fn execute_block_inner<'scope, Evm, Tx, Err, DB, MakeDb>( receipt_tx: Sender>>, worker_count: usize, ) -> Result< - (BlockExecutionOutput>, Vec
, BlockAccessList), + (BlockExecutionOutput>, Vec
, BlockAccessList, Arc), BalExecutionError, > where @@ -221,6 +224,7 @@ where BlockExecutionOutput { state: canonical_state.take_bundle(), result: block_result }, senders, built_bal, + input_bal_revm, )) } @@ -608,7 +612,7 @@ mod tests { tx_stream(txs), receipt_tx, ) - .map(|(output, _, built_bal)| (output, built_bal)) + .map(|(output, _, built_bal, _)| (output, built_bal)) } /// Inserts `AccountInfo { nonce: 0, balance }` for `addr` into the canonical DB. diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 4655ab978a0..e2ba09e7bae 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -141,6 +141,7 @@ use reth_evm::{ OnStateHook, SpecFor, }; use reth_execution_cache::{CacheFillMode, CacheStats}; +use reth_execution_types::DecodedRevmBal; use reth_network_p2p::full_block::SealedBlockWithAccessList; use reth_payload_builder::{PayloadBuilderLease, PayloadBuilderResources}; use reth_payload_primitives::{ @@ -164,6 +165,7 @@ use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, KeccakKeyHasher, LazyTrieData, }; +use revm::state::bal::Bal as RevmBal; use std::{ sync::{ atomic::{AtomicUsize, Ordering}, @@ -739,7 +741,9 @@ where if let (Some(metrics), Some(stats)) = (&state_provider_metrics, &state_provider_stats) { metrics.record_totals(stats); } - let (output, senders, receipt_root_rx, built_bal) = ensure_ok!(execution_result); + let (output, senders, receipt_root_rx, executed_bal) = ensure_ok!(execution_result); + let (built_bal, revm_bal) = + executed_bal.map(|ExecutedBal { alloy, revm }| (alloy, revm)).unzip(); // After executing the block we can stop prewarming transactions handle.stop_prewarming_execution(); @@ -914,10 +918,17 @@ where let _ = valid_block_tx.send(()); } - let executed_block = - self.spawn_deferred_trie_task(Arc::new(block), output, hashed_state, trie_output); - let raw_bal = decoded_bal.map(|decoded_bal| decoded_bal.as_raw_bal().clone()); - Ok(ValidationOutput::new(executed_block, timing_stats).with_raw_bal(raw_bal)) + // The payload's raw bytes are already bound to this header: payload-to-block conversion + // and downloaded-sidecar verification both check their hash against it, so pairing them + // with the BAL this execution produced is sound. The zip leaves the BAL unset for a + // downloaded block that carried only a BAL hash and no sidecar. + let bal = revm_bal.zip(decoded_bal).map(|(revm_bal, decoded_bal)| { + Arc::new(DecodedRevmBal::with_raw_bal(revm_bal, decoded_bal.as_raw_bal().clone())) + }); + let executed_block = self + .spawn_deferred_trie_task(Arc::new(block), output, hashed_state, trie_output) + .with_bal(bal); + Ok(ValidationOutput::new(executed_block, timing_stats)) } /// Spawns a background task to convert a [`BlockOrPayload`] into a [`SealedBlock`] and perform @@ -1008,12 +1019,7 @@ where handle: &mut PayloadHandle, Err, N::Receipt>, state_hook: Option>, ) -> Result< - ( - BlockExecutionOutput, - Vec
, - ReceiptRootReceiver, - Option, - ), + (BlockExecutionOutput, Vec
, ReceiptRootReceiver, Option), InsertBlockErrorKind, > where @@ -1094,7 +1100,13 @@ where debug_span!(target: "engine::tree", "merge_transitions") .in_scope(|| db.merge_transitions(BundleRetention::Reverts)); - let built_bal = if has_bal { db.take_built_alloy_bal() } else { None }; + // The builder only exists when the block declares a BAL. The revm form is the one shared + // with the executed block, so it must survive here; the clone only feeds the + // post-execution consensus checks, the sole consumer of the alloy form. + let built_bal = db.take_built_bal().map(|revm_bal| ExecutedBal { + alloy: revm_bal.clone().into_alloy_bal(), + revm: Arc::new(revm_bal), + }); let output = BlockExecutionOutput { result, state: db.take_bundle() }; let execution_duration = execution_start.elapsed(); @@ -1134,7 +1146,8 @@ where /// 2. Relies on BAL prewarm to stream state-root updates and optional state prefetches. /// 3. Spawns the receipt-root task. /// 4. Calls [`crate::tree::payload_processor::bal::execute_block`]. - /// 5. Returns the rebuilt BAL for post-execution consensus validation. + /// 5. Returns the rebuilt BAL for post-execution consensus validation, paired with the revm + /// representation the workers consumed. #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] #[expect(clippy::type_complexity)] fn execute_block_bal( @@ -1144,12 +1157,7 @@ where handle: &PayloadHandle, make_state_provider: &MakeStateProvider, ) -> Result< - ( - BlockExecutionOutput, - Vec
, - ReceiptRootReceiver, - Option, - ), + (BlockExecutionOutput, Vec
, ReceiptRootReceiver, Option), InsertBlockErrorKind, > where @@ -1175,17 +1183,18 @@ where let execution_start = Instant::now(); let ctx = self.execution_ctx_for(input).map_err(|e| InsertBlockErrorKind::Other(Box::new(e)))?; - let (output, senders, built_bal) = crate::tree::payload_processor::bal::execute_block( - &self.runtime, - &self.evm_config, - &make_db, - input_bal, - env.evm_env, - ctx, - env.transaction_count, - handle.clone_transaction_receiver(), - receipt_tx, - )?; + let (output, senders, built_bal, received_bal_revm) = + crate::tree::payload_processor::bal::execute_block( + &self.runtime, + &self.evm_config, + &make_db, + input_bal, + env.evm_env, + ctx, + env.transaction_count, + handle.clone_transaction_receiver(), + receipt_tx, + )?; let execution_duration = execution_start.elapsed(); self.metrics.record_block_execution(&output, execution_duration); @@ -1196,7 +1205,15 @@ where "Executed block via BAL path", ); - Ok((output, senders, result_rx, Some(built_bal))) + // The received BAL was already converted to the revm form to drive the workers. Once the + // post-execution hash check against the rebuilt BAL passes, both have the same content, so + // the converted one can be reused instead of converting the rebuilt BAL again. + Ok(( + output, + senders, + result_rx, + Some(ExecutedBal { alloy: built_bal, revm: received_bal_revm }), + )) } fn spawn_receipt_root_task( @@ -2106,3 +2123,11 @@ impl BlockOrPayload { } } } + +/// Block access list produced by executing a block. +struct ExecutedBal { + /// Alloy form, only needed for the post-execution consensus checks (hash and gas limit). + alloy: BlockAccessList, + /// Revm form, shared with the executed block so consumers can reuse it. + revm: Arc, +} diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 6830500088a..2eeddcac802 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -10,6 +10,7 @@ use crate::{ }; use reth_storage_overlay::OverlayManager; +use alloy_eip7928::bal::DecodedBal; use alloy_eips::eip1898::BlockWithParent; use alloy_primitives::{ map::{B256Map, B256Set}, @@ -33,6 +34,7 @@ use reth_primitives_traits::Block as _; use reth_provider::{test_utils::MockEthProvider, BalStoreHandle, InMemoryBalStore, RawBal}; use reth_tasks::spawn_os_thread; use reth_trie_common::ComputedTrieData; +use revm::state::bal::Bal as RevmBal; use std::{ collections::BTreeMap, str::FromStr, @@ -1114,10 +1116,15 @@ fn test_validated_payload_bal_is_inserted_into_store() { child_block.block_with_parent(), child, |_, executed, _| { - Ok::<_, InsertPayloadError>( - ValidationOutput::new(executed, None) - .with_raw_bal(Some(RawBal::from(raw_bal.clone()))), - ) + // `raw_bal` is the empty-list RLP, so the empty revm BAL is its decoded value. + let bal = DecodedBal::with_raw_bal( + Arc::new(RevmBal::default()), + RawBal::from(raw_bal.clone()), + ); + Ok::<_, InsertPayloadError>(ValidationOutput::new( + executed.with_bal(Some(Arc::new(bal))), + None, + )) }, |_, executed| Ok(executed.recovered_block().clone_sealed_block().into()), ) diff --git a/crates/engine/tree/src/tree/types.rs b/crates/engine/tree/src/tree/types.rs index effab8bd947..9e11a88426a 100644 --- a/crates/engine/tree/src/tree/types.rs +++ b/crates/engine/tree/src/tree/types.rs @@ -1,7 +1,7 @@ //! Shared types for blockchain tree validation. use crate::tree::error::InsertPayloadError; -use alloy_eip7928::bal::{DecodedBal, RawBal}; +use alloy_eip7928::bal::DecodedBal; use alloy_eips::eip4895::Withdrawal; use alloy_primitives::B256; use reth_chain_state::{ExecutedBlock, ExecutionTimingStats}; @@ -76,8 +76,6 @@ pub struct ValidationOutput { pub executed_block: ExecutedBlock, /// Optional execution timing stats collected during validation. pub execution_timing_stats: Option>, - /// Validated raw block access list carried by the payload. - pub raw_bal: Option, } impl ValidationOutput { @@ -86,12 +84,6 @@ impl ValidationOutput { executed_block: ExecutedBlock, execution_timing_stats: Option>, ) -> Self { - Self { executed_block, execution_timing_stats, raw_bal: None } - } - - /// Sets the validated raw block access list carried by the payload. - pub fn with_raw_bal(mut self, raw_bal: Option) -> Self { - self.raw_bal = raw_bal; - self + Self { executed_block, execution_timing_stats } } } diff --git a/crates/evm/execution-types/Cargo.toml b/crates/evm/execution-types/Cargo.toml index 9f7300d178d..ad30e87601b 100644 --- a/crates/evm/execution-types/Cargo.toml +++ b/crates/evm/execution-types/Cargo.toml @@ -21,6 +21,7 @@ revm.workspace = true alloy-evm.workspace = true alloy-consensus.workspace = true alloy-primitives.workspace = true +alloy-eip7928.workspace = true alloy-eips.workspace = true alloy-rlp = { workspace = true, optional = true } serde = { workspace = true, optional = true } @@ -49,6 +50,7 @@ serde = [ "alloy-consensus/serde", "reth-trie-common/serde", "reth-ethereum-primitives/serde", + "alloy-eip7928/serde", ] serde-bincode-compat = [ "serde", @@ -59,6 +61,7 @@ serde-bincode-compat = [ "alloy-eips/serde-bincode-compat", ] std = [ + "alloy-eip7928/std", "alloy-eips/std", "alloy-primitives/std", "revm/std", diff --git a/crates/evm/execution-types/src/bal.rs b/crates/evm/execution-types/src/bal.rs new file mode 100644 index 00000000000..e357c354c7f --- /dev/null +++ b/crates/evm/execution-types/src/bal.rs @@ -0,0 +1,10 @@ +//! Types for EIP-7928 block access lists produced by block execution. + +use alloc::sync::Arc; + +/// A decoded EIP-7928 block access list in the representation revm consumes, paired with the +/// raw RLP payload it was decoded from. +/// +/// Produced once during block validation and shared with consumers such as the RPC state +/// cache, so the RLP payload does not have to be decoded and converted a second time. +pub type DecodedRevmBal = alloy_eip7928::bal::DecodedBal>; diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index a09a517de81..79d078938b9 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -1,6 +1,6 @@ //! Contains [Chain], a chain of blocks and their final state. -use crate::ExecutionOutcome; +use crate::{DecodedRevmBal, ExecutionOutcome}; use alloc::{borrow::Cow, collections::BTreeMap, sync::Arc, vec::Vec}; use alloy_consensus::{ transaction::{Recovered, TxHashRef}, @@ -41,6 +41,15 @@ pub struct Chain { /// /// Contains handles to lazily-initialized sorted trie updates and hashed state. trie_data: BTreeMap, + /// Block access lists prepared during block validation, keyed by block number. + /// + /// A missing entry means the BAL was not available for that block, not that the block has + /// none: only blocks the engine validated from a payload that carried a BAL have one, so + /// chains built from storage or received over the wire have no entries at all. Consumers + /// that need the BAL of an arbitrary block must read it from the BAL store. This is derived + /// cache data and not part of the serialized representation. + #[cfg_attr(feature = "serde", serde(skip))] + bals: BTreeMap>, } type ChainTxReceiptMeta<'a, N> = ( @@ -56,6 +65,7 @@ impl Default for Chain { blocks: Default::default(), execution_outcome: Default::default(), trie_data: Default::default(), + bals: Default::default(), } } } @@ -80,7 +90,7 @@ impl Chain { .collect::>(); debug_assert!(!blocks.is_empty(), "Chain should have at least one block"); - Self { blocks, execution_outcome, trie_data } + Self { blocks, execution_outcome, trie_data, bals: Default::default() } } /// Create new Chain from a single block and its state. @@ -124,6 +134,29 @@ impl Chain { self.trie_data.clear(); } + /// Get all prepared block access lists for this chain. + /// + /// Blocks without an available BAL have no entry; see the `bals` field for details. + pub const fn bals(&self) -> &BTreeMap> { + &self.bals + } + + /// Get the prepared block access list for a specific block number, if one is available. + /// + /// `None` only means no BAL was attached for that block; see the `bals` field for details. + pub fn bal_at(&self, block_number: BlockNumber) -> Option<&Arc> { + self.bals.get(&block_number) + } + + /// Attach a prepared block access list to a block of this chain. + pub fn insert_bal(&mut self, block_number: BlockNumber, bal: Arc) { + debug_assert!( + self.blocks.contains_key(&block_number), + "BAL must belong to a block of this chain" + ); + self.bals.insert(block_number, bal); + } + /// Get execution outcome of this chain pub const fn execution_outcome(&self) -> &ExecutionOutcome { &self.execution_outcome @@ -237,6 +270,18 @@ impl Chain { self.blocks_iter().zip(self.block_receipts_iter()) } + /// Returns an iterator over the blocks of this chain that have a prepared block access list, + /// paired with that BAL. + /// + /// Blocks without an available BAL are skipped; see the `bals` field for details. + pub fn blocks_and_bals( + &self, + ) -> impl Iterator>, &Arc)> + '_ { + self.bals + .iter() + .filter_map(|(number, bal)| self.blocks.get(number).map(|block| (block, bal))) + } + /// Finds a transaction by hash and returns it along with its corresponding receipt data. /// /// Returns `None` if the transaction is not found in this chain. @@ -363,6 +408,7 @@ impl Chain { self.blocks.extend(other.blocks); self.execution_outcome.extend(other.execution_outcome); self.trie_data.extend(other.trie_data); + self.bals.extend(other.bals); Ok(()) } @@ -628,7 +674,14 @@ pub(super) mod serde_bincode_compat { }) .collect(); - Self { blocks, execution_outcome: value.execution_outcome.into(), trie_data } + // BALs are transient cache data recovered from the BAL store, not carried over the + // wire. + Self { + blocks, + execution_outcome: value.execution_outcome.into(), + trie_data, + bals: BTreeMap::new(), + } } } diff --git a/crates/evm/execution-types/src/lib.rs b/crates/evm/execution-types/src/lib.rs index 8b795981fb5..12a6aad1340 100644 --- a/crates/evm/execution-types/src/lib.rs +++ b/crates/evm/execution-types/src/lib.rs @@ -11,6 +11,9 @@ extern crate alloc; +mod bal; +pub use bal::*; + mod chain; pub use chain::*; diff --git a/crates/net/network/tests/it/requests.rs b/crates/net/network/tests/it/requests.rs index 81f6adfb612..905c23d9a49 100644 --- a/crates/net/network/tests/it/requests.rs +++ b/crates/net/network/tests/it/requests.rs @@ -20,8 +20,8 @@ use reth_network_p2p::{ BlockAccessListsClient, }; use reth_provider::{ - test_utils::MockEthProvider, BalNotificationStream, BalStore, BalStoreHandle, InMemoryBalStore, - ProviderError, ProviderResult, RawBal, + test_utils::MockEthProvider, BalStore, BalStoreHandle, InMemoryBalStore, ProviderError, + ProviderResult, RawBal, }; use reth_transaction_pool::test_utils::{TestPool, TransactionGenerator}; use std::sync::Arc; @@ -725,10 +725,6 @@ impl BalStore for FailingLookupBalStore { fn get_by_hashes(&self, _block_hashes: &[BlockHash]) -> ProviderResult>> { Err(ProviderError::other(std::io::Error::other("BAL lookup failed"))) } - - fn bal_stream(&self) -> BalNotificationStream { - BalStoreHandle::noop().bal_stream() - } } // Sends a GetBlockAccessLists request from peer 0 to peer 1. diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index ee168055d04..3d07789b7b9 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -9,7 +9,7 @@ use alloy_primitives::{Address, Bytes, TxHash, B256}; use futures::{Stream, StreamExt}; use reth_chain_state::CanonStateNotification; use reth_errors::{ProviderError, ProviderResult}; -use reth_execution_types::Chain; +use reth_execution_types::{Chain, DecodedRevmBal}; use reth_primitives_traits::{Block, BlockBody, InMemorySize, NodePrimitives, RecoveredBlock}; use reth_revm::{ bytecode::Bytecode, @@ -596,6 +596,13 @@ where Ok(Some(block_receipts.receipts)), ); } + + for (block_hash, bal) in chain_change.bals { + this.on_new_bal( + block_hash, + Ok(Some(CachedRevmBal::from_shared(bal))), + ); + } } CacheAction::RemoveReorgedChain { chain_change } => { for block in chain_change.blocks { @@ -693,6 +700,7 @@ struct BlockReceipts { struct ChainChange { blocks: Vec>>, receipts: Vec>, + bals: Vec<(B256, Arc)>, } impl ChainChange { @@ -710,7 +718,11 @@ impl ChainChange { (Arc::clone(block), block_receipts) }) .unzip(); - Self { blocks, receipts } + // Blocks without an attached BAL are absent here; their BAL is fetched from the store on + // demand. + let bals = + chain.blocks_and_bals().map(|(block, bal)| (block.hash(), Arc::clone(bal))).collect(); + Self { blocks, receipts, bals } } } @@ -809,15 +821,22 @@ pub async fn cache_new_blocks_task( /// Cached decoded revm BAL. #[derive(Clone, Debug)] -pub(crate) struct CachedRevmBal(Arc>>); +pub(crate) struct CachedRevmBal(Arc); impl CachedRevmBal { /// Creates a cached revm BAL from an owned decoded BAL. #[inline] - fn new(bal: DecodedBal>) -> Self { + fn new(bal: DecodedRevmBal) -> Self { Self(Arc::new(bal)) } + /// Creates a cached revm BAL from a BAL that is already shared, for example one prepared + /// during block validation. + #[inline] + const fn from_shared(bal: Arc) -> Self { + Self(bal) + } + /// Decodes raw BAL bytes into the representation used by revm. fn try_from_raw(raw: Bytes) -> ProviderResult { DecodedBal::from_rlp_bytes(raw) @@ -837,10 +856,8 @@ impl InMemorySize for CachedRevmBal { } } -fn decoded_revm_bal_size(bal: &DecodedBal>) -> usize { - core::mem::size_of::>>() + - bal.as_raw().len() + - revm_bal_size(bal.as_bal()) +fn decoded_revm_bal_size(bal: &DecodedRevmBal) -> usize { + core::mem::size_of::() + bal.as_raw().len() + revm_bal_size(bal.as_bal()) } fn revm_bal_size(bal: &Arc) -> usize { @@ -894,6 +911,7 @@ mod tests { use reth_ethereum_primitives::{ Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned, }; + use reth_execution_types::ExecutionOutcome; use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_storage_api::{ noop::NoopProvider, BalProvider, BalStore, BalStoreHandle, BlockBodyIndicesProvider, @@ -1102,6 +1120,36 @@ mod tests { assert_eq!(fetches.load(Ordering::SeqCst), 0); } + #[tokio::test] + async fn canonical_chain_notification_caches_prepared_bal() { + let fetches = Arc::new(AtomicUsize::default()); + let provider = TestBalProvider::new(fetches.clone()); + let cache = EthStateCache::::spawn_with( + provider, + EthStateCacheConfig { max_blocks: 4, max_bals: 4, ..Default::default() }, + Runtime::test(), + ); + + let block = test_block(); + let block_hash = block.hash(); + let block_number = block.number; + let mut chain: Chain = Chain::new( + [block], + ExecutionOutcome::new(Default::default(), vec![vec![]], block_number, vec![]), + Default::default(), + ); + chain.insert_bal(block_number, Arc::new(test_decoded_revm_bal())); + + cache_new_blocks_task( + cache.clone(), + tokio_stream::iter([CanonStateNotification::Commit { new: Arc::new(chain) }]), + ) + .await; + + assert!(cache.get_bal(block_hash).await.unwrap().is_some()); + assert_eq!(fetches.load(Ordering::SeqCst), 0); + } + #[tokio::test] async fn concurrent_get_bal_requests_share_fetch() { let fetches = Arc::new(AtomicUsize::default()); @@ -1169,10 +1217,6 @@ mod tests { self.fetches.fetch_add(1, Ordering::SeqCst); Ok(block_hashes.iter().map(|_| Some(Bytes::from_static(&[0xc0]))).collect()) } - - fn bal_stream(&self) -> reth_storage_api::BalNotificationStream { - reth_storage_api::NoopBalStore.bal_stream() - } } impl BlockHashReader for TestBalProvider { diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index dfe79de19fe..c31ecfaa331 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -58,7 +58,6 @@ parking_lot.workspace = true smallvec.workspace = true strum.workspace = true eyre.workspace = true -reth-tokio-util.workspace = true tokio = { workspace = true, features = ["sync"], optional = true } # test-utils @@ -87,7 +86,6 @@ assert_matches.workspace = true rand.workspace = true tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] } -tokio-stream = { workspace = true, features = ["sync"] } [features] jemalloc = ["rocksdb/jemalloc"] diff --git a/crates/storage/provider/src/bal.rs b/crates/storage/provider/src/bal.rs index 8ffc049d82e..b75f2c66291 100644 --- a/crates/storage/provider/src/bal.rs +++ b/crates/storage/provider/src/bal.rs @@ -3,11 +3,8 @@ use alloy_eips::NumHash; use alloy_primitives::{BlockHash, BlockNumber, Bytes}; use parking_lot::RwLock; use reth_prune_types::PruneMode; -use reth_storage_api::{ - BalNotification, BalNotificationStream, BalStore, GetBlockAccessListLimit, RawBal, -}; +use reth_storage_api::{BalStore, GetBlockAccessListLimit, RawBal}; use reth_storage_errors::provider::ProviderResult; -use reth_tokio_util::EventSender; use std::{ collections::{BTreeMap, HashMap}, sync::Arc, @@ -21,25 +18,15 @@ pub use rocksdb::RocksDBBalStore; pub struct InMemoryBalStore { config: BalConfig, inner: Arc>, - notifications: EventSender, } impl InMemoryBalStore { /// Creates a new in-memory BAL store with the given config. pub fn new(config: BalConfig) -> Self { - let notifications = EventSender::new(DEFAULT_BAL_NOTIFICATION_CHANNEL_SIZE); - Self { - config, - inner: Arc::new(RwLock::new(InMemoryBalStoreInner::default())), - notifications, - } + Self { config, inner: Arc::new(RwLock::new(InMemoryBalStoreInner::default())) } } } -// Match the canonical state broadcast buffer so BAL subscriptions behave like the existing -// in-memory notification path. This is a bounded best-effort channel, not a durability boundary. -const DEFAULT_BAL_NOTIFICATION_CHANNEL_SIZE: usize = 256; - impl Default for InMemoryBalStore { fn default() -> Self { Self::new(BalConfig::default()) @@ -139,7 +126,6 @@ impl BalStore for InMemoryBalStore { // This preserves insert-time cleanup based on the highest inserted BAL block. inner.prune(self.config.in_memory_retention, highest_block_number); } - self.notifications.notify(BalNotification::new(num_hash, bal)); Ok(()) } @@ -156,11 +142,6 @@ impl BalStore for InMemoryBalStore { if let Some(highest_block_number) = inner.highest_block_number { inner.prune(self.config.in_memory_retention, highest_block_number); } - drop(inner); - - for (num_hash, bal) in entries { - self.notifications.notify(BalNotification::new(num_hash, bal)); - } Ok(()) } @@ -204,17 +185,12 @@ impl BalStore for InMemoryBalStore { Ok(()) } - - fn bal_stream(&self) -> BalNotificationStream { - self.notifications.new_listener() - } } #[cfg(test)] mod tests { use super::*; use alloy_primitives::B256; - use tokio_stream::StreamExt; #[test] fn insert_and_lookup_by_hash() { @@ -404,97 +380,4 @@ mod tests { assert_eq!(store.get_by_hashes(&[hash]).unwrap(), vec![Some(bal)]); } - - #[tokio::test] - async fn insert_notifies_subscribers() { - let store = InMemoryBalStore::default(); - let hash = B256::random(); - let block_number = 7; - let bal = Bytes::from_static(b"bal"); - let mut stream = store.bal_stream(); - - let raw_bal = RawBal::from(bal); - - store.insert(NumHash::new(block_number, hash), raw_bal.clone()).unwrap(); - - assert_eq!( - stream.next().await.unwrap(), - BalNotification::new(NumHash::new(block_number, hash), raw_bal) - ); - } - - #[tokio::test] - async fn insert_many_notifies_subscribers() { - let store = InMemoryBalStore::default(); - let mut stream = store.bal_stream(); - let hash0 = B256::random(); - let hash1 = B256::random(); - let bal0 = RawBal::from(Bytes::from_static(b"bal0")); - let bal1 = RawBal::from(Bytes::from_static(b"bal1")); - - store - .insert_many(vec![ - (NumHash::new(1, hash0), bal0.clone()), - (NumHash::new(2, hash1), bal1.clone()), - ]) - .unwrap(); - - assert_eq!( - stream.next().await.unwrap(), - BalNotification::new(NumHash::new(1, hash0), bal0) - ); - assert_eq!( - stream.next().await.unwrap(), - BalNotification::new(NumHash::new(2, hash1), bal1) - ); - } - - #[test] - fn insert_without_subscribers_still_succeeds() { - let store = InMemoryBalStore::default(); - - assert!(store - .insert(NumHash::new(1, B256::random()), RawBal::from(Bytes::from_static(b"bal"))) - .is_ok()); - } - - #[tokio::test] - async fn bal_stream_skips_lagged_notifications() { - let store = InMemoryBalStore::new(BalConfig::unbounded()); - let mut stream = store.bal_stream(); - - for number in 0..=DEFAULT_BAL_NOTIFICATION_CHANNEL_SIZE as u64 { - store - .insert( - NumHash::new(number, B256::random()), - RawBal::from(Bytes::from(vec![number as u8])), - ) - .unwrap(); - } - - let first = stream.next().await.unwrap(); - let second = stream.next().await.unwrap(); - - assert_eq!(first.num_hash.number, 1); - assert_eq!(second.num_hash.number, 2); - } - - #[tokio::test] - async fn cloned_store_shares_notification_channel() { - let store = InMemoryBalStore::default(); - let clone = store.clone(); - let hash = B256::random(); - let block_number = 9; - let bal = Bytes::from_static(b"bal"); - let mut stream = clone.bal_stream(); - - let raw_bal = RawBal::from(bal); - - store.insert(NumHash::new(block_number, hash), raw_bal.clone()).unwrap(); - - assert_eq!( - stream.next().await.unwrap(), - BalNotification::new(NumHash::new(block_number, hash), raw_bal) - ); - } } diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index 99765824b5a..34494ba3a34 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -9,11 +9,8 @@ use reth_db_api::{ tables, DatabaseError, }; use reth_prune_types::PruneMode; -use reth_storage_api::{ - BalNotification, BalNotificationStream, BalStore, GetBlockAccessListLimit, RawBal, -}; +use reth_storage_api::{BalStore, GetBlockAccessListLimit, RawBal}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; -use reth_tokio_util::EventSender; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, sync::Arc, @@ -36,8 +33,6 @@ pub struct RocksDBBalStore { rocksdb: RocksDBProvider, /// Shared recent-read cache and pending-write state. buffer: Arc>, - /// Broadcasts BAL insert notifications. - notifications: EventSender, } impl RocksDBBalStore { @@ -54,7 +49,6 @@ impl RocksDBBalStore { buffer_retention_distance: blocks, rocksdb, buffer: Arc::new(RwLock::new(RocksDBBalStoreBuffer::default())), - notifications: EventSender::new(super::DEFAULT_BAL_NOTIFICATION_CHANNEL_SIZE), } } @@ -131,11 +125,7 @@ impl std::fmt::Debug for RocksDBBalStore { impl BalStore for RocksDBBalStore { fn insert(&self, block: NumHash, bal: RawBal) -> ProviderResult<()> { - let mut buffer = self.buffer.write(); - buffer.insert(block, bal.clone()); - drop(buffer); - - self.notifications.notify(BalNotification::new(block, bal)); + self.buffer.write().insert(block, bal); Ok(()) } @@ -146,13 +136,8 @@ impl BalStore for RocksDBBalStore { let mut buffer = self.buffer.write(); buffer.entries.reserve(entries.len()); - for (block, bal) in &entries { - buffer.insert(*block, bal.clone()); - } - drop(buffer); - for (block, bal) in entries { - self.notifications.notify(BalNotification::new(block, bal)); + buffer.insert(block, bal); } Ok(()) } @@ -210,10 +195,6 @@ impl BalStore for RocksDBBalStore { } Ok(()) } - - fn bal_stream(&self) -> BalNotificationStream { - self.notifications.new_listener() - } } /// Shared in-memory state for recent reads and writes awaiting canonical confirmation. @@ -354,7 +335,6 @@ mod tests { use super::*; use crate::providers::{RocksDBBuilder, RocksDBProvider}; use alloy_primitives::B256; - use tokio_stream::StreamExt; fn test_rocksdb(dir: &tempfile::TempDir) -> RocksDBProvider { RocksDBBuilder::new(dir.path()) @@ -593,16 +573,4 @@ mod tests { vec![first, second] ); } - - #[tokio::test] - async fn insert_notifies_subscribers() { - let (_dir, store) = test_store(); - let mut stream = store.bal_stream(); - let block = NumHash::new(1, B256::with_last_byte(1)); - let bal = RawBal::from(Bytes::from_static(&[0xc0])); - - store.insert(block, bal.clone()).unwrap(); - - assert_eq!(stream.next().await.unwrap(), BalNotification::new(block, bal)); - } } diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 4460bbc8736..ce9f8d7ba44 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -50,9 +50,9 @@ pub use revm::database::states::OriginalValuesKnown; // reexport traits to avoid breaking changes pub use reth_static_file_types as static_file; pub use reth_storage_api::{ - BalNotification, BalNotificationStream, BalProvider, BalStore, BalStoreHandle, - GetBlockAccessListLimit, HistoryWriter, MetadataProvider, MetadataWriter, NoopBalStore, RawBal, - StateWriteConfig, StatsReader, StorageSettings, StorageSettingsCache, + BalProvider, BalStore, BalStoreHandle, GetBlockAccessListLimit, HistoryWriter, + MetadataProvider, MetadataWriter, NoopBalStore, RawBal, StateWriteConfig, StatsReader, + StorageSettings, StorageSettingsCache, }; /// Re-export provider error. pub use reth_storage_errors::provider::{ProviderError, ProviderResult}; diff --git a/crates/storage/storage-api/Cargo.toml b/crates/storage/storage-api/Cargo.toml index 862f6805bae..5c28a941fc4 100644 --- a/crates/storage/storage-api/Cargo.toml +++ b/crates/storage/storage-api/Cargo.toml @@ -34,12 +34,9 @@ alloy-rpc-types-engine.workspace = true auto_impl.workspace = true serde_json = { workspace = true, optional = true } -reth-tokio-util = { workspace = true, optional = true } [dev-dependencies] alloy-primitives = { workspace = true, features = ["getrandom"] } -tokio = { workspace = true, features = ["macros", "rt"] } -tokio-stream.workspace = true [features] default = ["std"] @@ -59,7 +56,6 @@ std = [ "reth-db-models/std", "reth-trie-common/std", "serde_json?/std", - "dep:reth-tokio-util", "alloy-eip7928/std", ] diff --git a/crates/storage/storage-api/src/bal.rs b/crates/storage/storage-api/src/bal.rs index 791b129cbc9..ca2cd456d27 100644 --- a/crates/storage/storage-api/src/bal.rs +++ b/crates/storage/storage-api/src/bal.rs @@ -5,33 +5,6 @@ use alloy_eips::NumHash; use alloy_primitives::{BlockHash, BlockNumber, Bytes}; use reth_storage_errors::provider::ProviderResult; -/// Notification emitted when a new BAL is inserted into the store. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct BalNotification { - /// Number and hash of the block the BAL belongs to. - pub num_hash: NumHash, - /// Raw BAL RLP payload. - pub bal: RawBal, -} - -impl BalNotification { - /// Creates a new [`BalNotification`]. - pub const fn new(num_hash: NumHash, bal: RawBal) -> Self { - Self { num_hash, bal } - } -} - -#[cfg(feature = "std")] -pub use self::subscriptions::BalNotificationStream; - -#[cfg(feature = "std")] -mod subscriptions { - use super::BalNotification; - - /// A stream of [`BalNotification`]s. - pub type BalNotificationStream = reth_tokio_util::EventStream; -} - /// Store for Block Access Lists (BALs). /// /// This abstraction intentionally does not prescribe where BALs live. Implementations may keep @@ -121,13 +94,6 @@ pub trait BalStore: Send + Sync + 'static { } Ok(()) } - - /// Returns a stream of BAL insert notifications. - /// - /// Notifications are emitted only after a BAL has been successfully inserted into the store. - /// They do not imply canonicality. - #[cfg(feature = "std")] - fn bal_stream(&self) -> BalNotificationStream; } /// The limit to enforce for [`BalStore::get_by_hashes_with_limit`]. @@ -230,13 +196,6 @@ impl BalStoreHandle { ) -> ProviderResult<()> { self.inner.append_by_hashes_with_limit(block_hashes, limit, out) } - - /// Returns a stream of BAL insert notifications. - #[cfg(feature = "std")] - #[inline] - pub fn bal_stream(&self) -> BalNotificationStream { - self.inner.bal_stream() - } } impl Default for BalStoreHandle { @@ -316,19 +275,12 @@ impl BalStore for NoopBalStore { } Ok(()) } - - #[cfg(feature = "std")] - fn bal_stream(&self) -> BalNotificationStream { - reth_tokio_util::EventSender::new(1).new_listener() - } } #[cfg(test)] mod tests { use super::*; use alloy_primitives::B256; - #[cfg(feature = "std")] - use tokio_stream::StreamExt; const EMPTY_LIST_CODE: u8 = 0xc0; @@ -412,15 +364,6 @@ mod tests { assert!(size_limit_2mb.exceeds(3 * 1024 * 1024)); } - #[cfg(feature = "std")] - #[tokio::test] - async fn noop_store_stream_is_empty() { - let store = BalStoreHandle::default(); - let mut stream = store.bal_stream(); - - assert!(stream.next().await.is_none()); - } - #[derive(Debug)] struct TestBalStore { hash: B256, @@ -442,10 +385,5 @@ mod tests { .map(|hash| (*hash == self.hash).then(|| self.raw_bal.clone())) .collect()) } - - #[cfg(feature = "std")] - fn bal_stream(&self) -> BalNotificationStream { - reth_tokio_util::EventSender::new(1).new_listener() - } } } From 2e7be08ba8d23b3df50cfb501cc4f29d060703a9 Mon Sep 17 00:00:00 2001 From: Bruce0X <109469941+Bruce039@users.noreply.github.com> Date: Fri, 18 Sep 2026 09:57:53 -0300 Subject: [PATCH 011/133] fix(rpc): keep unreverted blocks buffered for finalized chain notifications on reorg (#27301) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc/src/reth.rs | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc/src/reth.rs b/crates/rpc/rpc/src/reth.rs index 8bd446de179..2b9b1889f6e 100644 --- a/crates/rpc/rpc/src/reth.rs +++ b/crates/rpc/rpc/src/reth.rs @@ -12,7 +12,7 @@ use reth_chain_state::{ }; use reth_errors::{RethError, RethResult}; use reth_evm::{execute::Executor, ConfigureEvm}; -use reth_execution_types::ExecutionOutcome; +use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives_traits::{NodePrimitives, SealedHeader}; use reth_rpc_api::{RethApiServer, RethJitAction}; use reth_rpc_eth_types::{EthApiError, EthResult}; @@ -338,8 +338,31 @@ async fn finalized_chain_notifications( CanonStateNotification::Commit { .. } => { buffered.push(notification); } - CanonStateNotification::Reorg { .. } => { - buffered.clear(); + CanonStateNotification::Reorg { old, new } => { + let first_reverted = old.first().number(); + buffered.retain_mut(|notification| { + let chain = notification.committed(); + if chain.first().number() >= first_reverted { + return false + } + // Preserve the canonical prefix of a segment crossing the fork. + if chain.tip().number() >= first_reverted { + let (blocks, mut outcome, mut trie_data) = (*chain).clone().into_inner(); + outcome.revert_to(first_reverted - 1); + trie_data.split_off(&first_reverted); + *notification = CanonStateNotification::Commit { + new: Arc::new(Chain::new( + blocks.into_blocks().take_while(|b| b.number() < first_reverted), + outcome, + trie_data, + )), + }; + } + true + }); + if !new.is_empty() { + buffered.push(CanonStateNotification::Commit { new: new.clone() }); + } } } } From bf776999dceaf6384904211ecb2454db7ccdc89b Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:22:07 +0200 Subject: [PATCH 012/133] fix(ci): migrate deny check to gh-actions (#27309) Co-authored-by: grandizzy <38490174+grandizzy@users.noreply.github.com> --- .github/workflows/lint.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b52ca66799c..1176e354bcf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -335,7 +335,10 @@ jobs: deny: permissions: contents: read - uses: tempoxyz/ci/.github/workflows/deny.yml@main + id-token: write + uses: tempoxyz/gh-actions/.github/workflows/rust-deny.yml@6cd9d3a68b9cb0b52c7a0fc9a523f4806921354f + with: + rust-toolchain: nightly lint-success: name: lint success From 46c1e16ec219ab608b5a0ed6a89f0449423585a9 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:04:25 +0200 Subject: [PATCH 013/133] fix(trie): decode inline witness children (#27267) Co-authored-by: Alexey Shekhirin Co-authored-by: Brian Picciano --- crates/trie/common/src/proofs.rs | 187 +++++++++++++++++++++++++++---- 1 file changed, 164 insertions(+), 23 deletions(-) diff --git a/crates/trie/common/src/proofs.rs b/crates/trie/common/src/proofs.rs index 3610ae5ec68..99233b668cf 100644 --- a/crates/trie/common/src/proofs.rs +++ b/crates/trie/common/src/proofs.rs @@ -526,8 +526,8 @@ impl DecodedMultiProofV2 { /// Builds a `DecodedMultiProofV2` from a flat witness map (hash → RLP-encoded trie node). /// - /// This performs a BFS traversal starting from `state_root`, decoding each witness entry - /// as a trie node and organizing them into account and storage proof vectors. This is the + /// This performs a BFS traversal starting from `state_root`, following hashed and inline + /// children and organizing decoded nodes into account and storage proof vectors. This is the /// inverse of witness generation — it reconstructs the structured multiproof from the flat /// format used in `ExecutionWitness`. pub fn from_witness( @@ -538,41 +538,58 @@ impl DecodedMultiProofV2 { let mut storage_nodes: B256Map)>> = B256Map::default(); - let mut queue: VecDeque<(B256, Nibbles, Option)> = - VecDeque::from([(state_root, Nibbles::default(), None)]); - - while let Some((hash, path, maybe_account)) = queue.pop_front() { - let Some(rlp_bytes) = witness.get(&hash) else { continue }; - let trie_node = TrieNode::decode(&mut rlp_bytes.as_ref())?; + let Some(root) = witness.get(&state_root) else { return Ok(Self::default()) }; + let mut queue = + VecDeque::from([(TrieNode::decode(&mut root.as_ref())?, Nibbles::default(), None)]); + while let Some((trie_node, path, maybe_account)) = queue.pop_front() { match &trie_node { TrieNode::Branch(branch) => { + if path.len() >= 64 { + return Err(alloy_rlp::Error::Custom("branch path exceeds 64 nibbles")); + } for (idx, maybe_child) in branch.as_ref().children() { - if let Some(child_hash) = - maybe_child.and_then(alloy_trie::nodes::RlpNode::as_hash) - { + if let Some(child) = maybe_child { let mut child_path = path; child_path.push_unchecked(idx); - queue.push_back((child_hash, child_path, maybe_account)); + let child = if let Some(hash) = child.as_hash() { + let Some(bytes) = witness.get(&hash) else { continue }; + TrieNode::decode(&mut bytes.as_ref())? + } else { + TrieNode::decode(&mut child.as_slice())? + }; + queue.push_back((child, child_path, maybe_account)); } } } TrieNode::Extension(ext) => { - if let Some(child_hash) = ext.child.as_hash() { - let mut child_path = path; - child_path.extend(&ext.key); - queue.push_back((child_hash, child_path, maybe_account)); + if path.len() + ext.key.len() > 64 { + return Err(alloy_rlp::Error::Custom("extension path exceeds 64 nibbles")); } + let mut child_path = path; + child_path.extend(&ext.key); + let child = if let Some(hash) = ext.child.as_hash() { + let Some(bytes) = witness.get(&hash) else { continue }; + TrieNode::decode(&mut bytes.as_ref())? + } else { + TrieNode::decode(&mut ext.child.as_slice())? + }; + queue.push_back((child, child_path, maybe_account)); } TrieNode::Leaf(leaf) => { + if path.len() + leaf.key.len() != 64 { + return Err(alloy_rlp::Error::Custom("leaf path must contain 64 nibbles")); + } if maybe_account.is_none() { let mut full_path = path; full_path.extend(&leaf.key); let hashed_address = B256::from_slice(&full_path.pack()); let account = TrieAccount::decode(&mut &leaf.value[..])?; - if account.storage_root != EMPTY_ROOT_HASH { + if account.storage_root != EMPTY_ROOT_HASH && + let Some(bytes) = witness.get(&account.storage_root) + { queue.push_back(( - account.storage_root, + TrieNode::decode(&mut bytes.as_ref())?, Nibbles::default(), Some(hashed_address), )); @@ -629,17 +646,20 @@ fn matching_v2_proof_nodes<'a>( /// Encodes V2 proof nodes as standard MPT proof nodes. /// -/// V2 combines an extension and its child branch in one node. Both nodes need to be emitted for -/// consumers that reconstruct a sparse trie from the returned proof. +/// Inline children are already encoded in their parent and must not be emitted separately. +/// V2 combines extensions with their child branches, so hashed branches are emitted separately. fn encode_v2_proof_nodes<'a>(nodes: impl Iterator) -> Vec { let mut proof = Vec::new(); for proof_node in nodes { let mut encoded = Vec::new(); proof_node.node.encode(&mut encoded); - proof.push(Bytes::from(encoded)); + if proof_node.path.is_empty() || encoded.len() >= B256::len_bytes() { + proof.push(Bytes::from(encoded)); + } if let TrieNodeV2::Branch(branch) = &proof_node.node && - !branch.key.is_empty() + !branch.key.is_empty() && + branch.branch_rlp_node.as_ref().is_some_and(|node| node.is_hash()) { let mut encoded = Vec::new(); BranchNodeRef::new(&branch.stack, branch.state_mask).encode(&mut encoded); @@ -1151,7 +1171,7 @@ pub mod triehash { mod tests { use super::*; use alloy_trie::{ - nodes::{BranchNode, LeafNode, RlpNode}, + nodes::{BranchNode, ExtensionNode, LeafNode, RlpNode}, TrieMask, }; @@ -1241,6 +1261,127 @@ mod tests { ); } + #[test] + fn witness_nodes_follow_inline_children() { + let leaf_key = Nibbles::from_nibbles([0]); + let leaf_0 = + alloy_rlp::encode(LeafNode::new(leaf_key, encode_fixed_size(&U256::from(1)).to_vec())); + let leaf_1 = + alloy_rlp::encode(LeafNode::new(leaf_key, encode_fixed_size(&U256::from(2)).to_vec())); + assert!(leaf_0.len() < B256::len_bytes()); + assert!(leaf_1.len() < B256::len_bytes()); + + let branch = alloy_rlp::encode(BranchNode::new( + vec![RlpNode::from_rlp(&leaf_0), RlpNode::from_rlp(&leaf_1)], + TrieMask::new(0b11), + )); + assert!(branch.len() < B256::len_bytes()); + + let extension_key = Nibbles::from_nibbles([0; 62]); + let storage_root_node = + alloy_rlp::encode(ExtensionNode::new(extension_key, RlpNode::from_rlp(&branch))); + let storage_root = keccak256(&storage_root_node); + + let account_root_node = alloy_rlp::encode(LeafNode::new( + Nibbles::from_nibbles([0; 64]), + alloy_rlp::encode(TrieAccount { storage_root, ..Default::default() }), + )); + let state_root = keccak256(&account_root_node); + let witness = B256Map::from_iter([ + (state_root, Bytes::from(account_root_node)), + (storage_root, Bytes::from(storage_root_node)), + ]); + + let proof = DecodedMultiProofV2::from_witness(state_root, &witness).unwrap(); + let mut leaf_0_path = extension_key; + leaf_0_path.push_unchecked(0); + let mut leaf_1_path = extension_key; + leaf_1_path.push_unchecked(1); + + assert_eq!( + proof.storage_proofs[&B256::ZERO].iter().map(|node| node.path).collect::>(), + [leaf_0_path, leaf_1_path, Nibbles::default()] + ); + + let nodes = &proof.storage_proofs[&B256::ZERO]; + for (path, value) in [(leaf_0_path, 1), (leaf_1_path, 2)] { + let key = path.join(&leaf_key); + let encoded = encode_v2_proof_nodes(matching_v2_proof_nodes(nodes, &key)); + assert_eq!(encoded, vec![witness[&storage_root].clone()]); + verify_proof(storage_root, key, Some(alloy_rlp::encode(U256::from(value))), &encoded) + .unwrap(); + } + } + + #[test] + fn v2_proof_encoding_keeps_short_root() { + let leaf = LeafNode::new(Nibbles::default(), vec![1]); + let encoded = alloy_rlp::encode(&leaf); + assert!(encoded.len() < 32); + let nodes = [ProofTrieNodeV2 { + path: Nibbles::default(), + node: TrieNodeV2::Leaf(leaf), + masks: None, + }]; + assert_eq!(encode_v2_proof_nodes(nodes.iter()), vec![Bytes::from(encoded)]); + } + + #[test] + fn witness_rejects_invalid_path_lengths() { + let leaf = alloy_rlp::encode(LeafNode::new(Nibbles::default(), vec![1])); + let branch = alloy_rlp::encode(BranchNode::new( + vec![RlpNode::from_rlp(&leaf), RlpNode::from_rlp(&leaf)], + TrieMask::new(0b11), + )); + let extension = alloy_rlp::encode(ExtensionNode::new( + Nibbles::from_nibbles([0]), + RlpNode::from_rlp(&branch), + )); + + let long_leaf = alloy_rlp::encode(LeafNode::new(Nibbles::from_nibbles([0]), vec![1])); + for (child, depth, expected) in [ + (&branch, 64, "branch path exceeds 64 nibbles"), + (&extension, 64, "extension path exceeds 64 nibbles"), + (&leaf, 63, "leaf path must contain 64 nibbles"), + (&long_leaf, 64, "leaf path must contain 64 nibbles"), + ] { + let storage = alloy_rlp::encode(ExtensionNode::new( + Nibbles::from_nibbles(vec![0; depth]), + RlpNode::from_rlp(child), + )); + let storage_root = keccak256(&storage); + let account = alloy_rlp::encode(LeafNode::new( + Nibbles::from_nibbles([0; 64]), + alloy_rlp::encode(TrieAccount { storage_root, ..Default::default() }), + )); + let state_root = keccak256(&account); + let witness = B256Map::from_iter([(state_root, account), (storage_root, storage)]); + assert_eq!( + DecodedMultiProofV2::from_witness(state_root, &witness), + Err(alloy_rlp::Error::Custom(expected)), + ); + } + + for length in [63, 65] { + // A branch adds one nibble to the account leaf's path. + let account = alloy_rlp::encode(LeafNode::new( + Nibbles::from_nibbles(vec![0; length - 1]), + alloy_rlp::encode(TrieAccount::default()), + )); + let hash = keccak256(&account); + let root = alloy_rlp::encode(BranchNode::new( + vec![RlpNode::word_rlp(&hash), RlpNode::word_rlp(&hash)], + TrieMask::new(0b11), + )); + let state_root = keccak256(&root); + let witness = B256Map::from_iter([(state_root, root), (hash, account)]); + assert_eq!( + DecodedMultiProofV2::from_witness(state_root, &witness), + Err(alloy_rlp::Error::Custom("leaf path must contain 64 nibbles")), + ); + } + } + #[test] fn test_multiproof_extend_account_proofs() { let mut proof1 = MultiProof::default(); From 0032bec310b1531dc24d6b5bfeffa690eb35531a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 18 Sep 2026 19:20:06 +0200 Subject: [PATCH 014/133] perf(txpool): reuse insertion timestamp across a batch (#27284) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/transaction-pool/src/pool/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 04c076e6460..f23a7669adc 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -580,6 +580,7 @@ where pool: &mut RwLockWriteGuard<'_, TxPool>, origin: TransactionOrigin, tx: TransactionValidationOutcome, + timestamp: Instant, ) -> (PoolResult, Option>) { match tx { TransactionValidationOutcome::Valid { @@ -609,7 +610,7 @@ where transaction, transaction_id, propagate, - timestamp: Instant::now(), + timestamp, origin, authority_ids: authorities.map(|auths| self.get_sender_ids(auths)), }; @@ -679,10 +680,11 @@ where // Collect results and metadata while holding the pool write lock let (mut results, added_metas, discarded) = { let mut pool = self.pool.write(); + let timestamp = Instant::now(); let results = transactions .map(|(origin, tx)| { - let (result, meta) = self.add_transaction(&mut pool, origin, tx); + let (result, meta) = self.add_transaction(&mut pool, origin, tx, timestamp); // Only collect metadata for successful insertions if result.is_ok() && From 82a44a67647f434382f6b9caa0b91778a777614a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Sat, 19 Sep 2026 01:10:17 +0200 Subject: [PATCH 015/133] feat(snap-sync): advance the pivot while state downloads remain unfinished (#27299) --- crates/snap-sync/src/account/store.rs | 21 +- crates/snap-sync/src/attempt.rs | 91 +++++++- crates/snap-sync/src/catch_up/apply.rs | 21 +- crates/snap-sync/src/catch_up/download.rs | 165 ++++++++++++-- crates/snap-sync/src/catch_up/store.rs | 184 ++++++++++++---- crates/snap-sync/src/error.rs | 16 +- crates/snap-sync/src/session.rs | 128 ++++++++++- crates/snap-sync/src/storage/download.rs | 86 +++++++- crates/snap-sync/src/storage/store.rs | 249 +++++++++++++++++----- 9 files changed, 821 insertions(+), 140 deletions(-) diff --git a/crates/snap-sync/src/account/store.rs b/crates/snap-sync/src/account/store.rs index ff2ee065501..c52a5cca7bb 100644 --- a/crates/snap-sync/src/account/store.rs +++ b/crates/snap-sync/src/account/store.rs @@ -3,8 +3,8 @@ //! Each range replaces its key interval, and ranges commit in key order. use crate::{ - common::SnapRecord, storage::persisted_storage_root, SnapAttemptStore, SnapStorageStore, - SnapSyncError, SnapWrite, StorageProgress, + common::SnapRecord, storage::persisted_storage_root, SnapAttemptStore, SnapCatchUpStore, + SnapStorageStore, SnapSyncError, SnapWrite, StorageProgress, }; use alloy_primitives::{ map::{B256Map, B256Set}, @@ -43,7 +43,8 @@ pub trait SnapAccountStore { /// Persists `range` with its storage and code, replacing its key interval. /// /// Storage must match each account's root, supplied or persisted ahead of the range by - /// [`SnapStorageStore`], and code must be supplied or already stored. + /// [`SnapStorageStore`], and code must be supplied or already stored. Persisted storage only + /// counts once catch-up has carried it to the pivot. fn commit_account_range( &self, write: SnapWrite, @@ -175,6 +176,16 @@ impl SnapAccountStore for T { let coverage = self.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; let advanced = coverage.advance(range)?; let progress = self.storage_progress(write, range.origin())?; + // Slots persisted before the pivot moved hold its old values until the lists reach it. + if progress != StorageProgress::START && + let Some(catch_up) = self.catch_up_progress(write)? && + catch_up.applied().number < attempt.pivot().number + { + return Err(SnapSyncError::CatchUpBehindPivot { + applied: catch_up.applied().number, + pivot: attempt.pivot().number, + }) + } let dependencies = RangeDependencies::new(range.accounts(), storages, bytecodes); let persisted = dependencies.verify(range.accounts(), &progress, self.tx_ref())?; @@ -183,6 +194,10 @@ impl SnapAccountStore for T { self.remove_storages_except(interval, &persisted)?; dependencies.write(self)?; StoredCoverage::new(write.attempt(), advanced).write(self)?; + // A page ending before contracts with persisted storage leaves them to the next range. + if let Some(next) = advanced.next() { + progress.carry_to(self, write, next)?; + } Ok(advanced) } diff --git a/crates/snap-sync/src/attempt.rs b/crates/snap-sync/src/attempt.rs index 8ef36ec9611..afcb0b5b6c4 100644 --- a/crates/snap-sync/src/attempt.rs +++ b/crates/snap-sync/src/attempt.rs @@ -4,7 +4,7 @@ //! attempt produced it. Every write presents a [`SnapWrite`]; ones that no longer match are //! refused. -use crate::{CatchUpProgress, SnapCatchUpStore, SnapGeneration, SnapSyncError}; +use crate::{CatchUpProgress, SnapGeneration, SnapSyncError}; use reth_storage_api::{ BlockHashReader, MetadataProvider, MetadataWriter, SnapAttempt, SnapAttemptId, StorageSettings, }; @@ -39,8 +39,8 @@ pub trait SnapAttemptStore { /// Re-anchors the attempt to `generation`, refusing writes proved against the previous root. /// - /// The old pivot must still be canonical, and the new pivot must not be below the last block - /// whose list is applied. An orphaned pivot requires recovery or a fresh attempt. + /// Both pivots must be canonical and the new one past the old, so it descends from all + /// downloaded state. An orphaned pivot requires recovery or a fresh attempt. fn advance_snap_pivot( &self, write: SnapWrite, @@ -141,16 +141,18 @@ impl SnapAttemptStore for T { { // Replacing an orphaned pivot would hide the fork that downloaded ranges belong to. let mut attempt = self.authorize_canonical_snap_write(write)?; - // Ranges downloaded at the new pivot would already hold the applied blocks' changes. - if let Some(progress) = self.catch_up_progress(write)? && - generation.target().number < progress.applied().number - { - return Err(SnapSyncError::PivotBelowApplied { - applied: progress.applied().number, - pivot: generation.target().number, + let (pivot, target) = (attempt.pivot(), generation.target()); + // Ranges already committed hold the old pivot's changes, which no list can undo. + if target.number <= pivot.number { + return Err(SnapSyncError::PivotNotAdvanced { + pivot: pivot.number, + target: target.number, }) } - attempt.re_anchor(generation.target(), generation.state_root()); + if self.block_hash(target.number)? != Some(target.hash) { + return Err(SnapSyncError::NonCanonicalBlock { block: target.number, hash: target.hash }) + } + attempt.re_anchor(target, generation.state_root()); self.write_snap_attempt(&attempt)?; Ok(SnapWrite::of(&attempt)) } @@ -363,6 +365,73 @@ mod tests { provider.authorize_snap_write(after).unwrap(); } + #[test] + fn the_pivot_advances_repeatedly_within_one_attempt() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let first = provider.start_snap_attempt(generation(1)).unwrap(); + + let second = provider.advance_snap_pivot(first, generation(2)).unwrap(); + let third = provider.advance_snap_pivot(second, generation(3)).unwrap(); + + assert_eq!(third.attempt(), first.attempt()); + assert_eq!(third.state_version(), first.state_version() + 2); + assert_eq!(provider.snap_attempt().unwrap().unwrap().pivot().number, 3); + for stale in [first, second] { + assert!(matches!( + provider.authorize_snap_write(stale), + Err(SnapSyncError::StaleWrite { .. }) + )); + } + } + + #[test] + fn the_pivot_only_moves_forward() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(2)).unwrap(); + + // Committed ranges hold block 2's changes, which a lower pivot's lists cannot undo. + for target in [1, 2] { + assert!(matches!( + provider.advance_snap_pivot(write, generation(target)), + Err(SnapSyncError::PivotNotAdvanced { pivot: 2, target: refused }) + if refused == target + )); + } + provider.authorize_snap_write(write).unwrap(); + } + + #[test] + fn the_pivot_cannot_move_off_the_canonical_chain() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1)).unwrap(); + let orphan = SnapGeneration::new(BlockNumHash::new(2, B256::repeat_byte(0xee)), B256::ZERO); + + let refused = provider.advance_snap_pivot(write, orphan); + + assert!(matches!(refused, Err(SnapSyncError::NonCanonicalBlock { block: 2, .. }))); + provider.authorize_snap_write(write).unwrap(); + } + + #[test] + fn an_interrupted_advance_keeps_the_previous_pivot() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1)).unwrap(); + provider.commit().unwrap(); + + let provider = factory.database_provider_rw().unwrap(); + provider.advance_snap_pivot(write, generation(2)).unwrap(); + drop(provider); + + // Downloads still in flight against the old root keep committing. + let provider = factory.database_provider_rw().unwrap(); + assert_eq!(provider.active_snap_write().unwrap(), Some(write)); + assert_eq!(provider.authorize_snap_write(write).unwrap().pivot().number, 1); + } + #[test] fn a_verified_attempt_accepts_no_further_writes() { let factory = factory(); diff --git a/crates/snap-sync/src/catch_up/apply.rs b/crates/snap-sync/src/catch_up/apply.rs index b5b5aae3502..32d8807c97c 100644 --- a/crates/snap-sync/src/catch_up/apply.rs +++ b/crates/snap-sync/src/catch_up/apply.rs @@ -4,14 +4,15 @@ //! pivot by applying later blocks' lists to the state downloaded so far. Each list records the //! final value of every field it changes, so untouched fields come from the downloaded account. -use alloy_primitives::{map::B256Map, Bytes, B256}; +use alloy_eip7928::AccountChanges; +use alloy_primitives::{keccak256, map::B256Map, Bytes, B256}; use reth_primitives_traits::Account; -use reth_trie_common::HashedPostState; +use reth_trie_common::{HashedPostState, HashedStorage}; /// Changes one block access list makes to the downloaded state. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct BalStateUpdate { - // Post-state of every entry whose account is downloaded. + // Post-state of every downloaded account, and slots persisted ahead of their range. pub(super) state: HashedPostState, // Final code of accounts whose code changed, keyed by code hash. pub(super) bytecodes: B256Map, @@ -20,7 +21,7 @@ pub struct BalStateUpdate { } impl BalStateUpdate { - /// Post-state of every entry whose account is downloaded, keyed by hashed address. + /// Post-state of every downloaded account, and slots persisted ahead of their range. pub const fn state(&self) -> &HashedPostState { &self.state } @@ -40,6 +41,14 @@ impl BalStateUpdate { pub fn into_parts(self) -> (HashedPostState, B256Map, Vec) { (self.state, self.bytecodes, self.unresolved) } + + // Records the final values of the slots `changes` writes for `hashed_address`. + pub(super) fn insert_storage(&mut self, hashed_address: B256, changes: &AccountChanges) { + let storage = HashedStorage::from_iter( + changes.storage_post_states().map(|(slot, value)| (keccak256(B256::from(slot)), value)), + ); + self.state.storages.insert(hashed_address, storage); + } } /// What the downloaded state holds for an account a list changes. @@ -56,7 +65,7 @@ pub enum DownloadedAccount { #[cfg(test)] mod tests { use super::*; - use crate::{test_utils::hashed_factory, AccountCoverage, SnapCatchUpStore}; + use crate::{test_utils::hashed_factory, AccountCoverage, SnapCatchUpStore, StorageProgress}; use alloy_consensus::{Header, TxLegacy}; use alloy_eip7928::{ AccountChanges, BalanceChange, BlockAccessIndex, CodeChange, NonceChange, SlotChanges, @@ -100,7 +109,7 @@ mod tests { for (address, account) in accounts { provider.tx_ref().put::(address, account).unwrap(); } - provider.block_access_list_update(coverage, bal).unwrap() + provider.block_access_list_update(coverage, StorageProgress::START, bal).unwrap() } fn apply(changes: &AccountChanges, base: DownloadedAccount) -> BalStateUpdate { diff --git a/crates/snap-sync/src/catch_up/download.rs b/crates/snap-sync/src/catch_up/download.rs index 8fc74f25f1c..8baebfc3164 100644 --- a/crates/snap-sync/src/catch_up/download.rs +++ b/crates/snap-sync/src/catch_up/download.rs @@ -216,13 +216,13 @@ mod tests { }; use alloy_eip7928::{AccountChanges, BalanceChange, BlockAccessIndex}; use alloy_primitives::{keccak256, Address, B256, U256}; - use reth_db_api::{tables, transaction::DbTx}; + use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; use reth_network_p2p::{error::PeerRequestResult, snap::client::SnapResponse}; use reth_provider::{ test_utils::{insert_headers, MockNodeTypesWithDB}, ProviderFactory, }; - use reth_trie_common::TrieAccount; + use reth_trie_common::{TrieAccount, EMPTY_ROOT_HASH}; use std::sync::Arc; type Factory = ProviderFactory; @@ -266,12 +266,14 @@ mod tests { (factory, write) } - // Applied block 2 is shared, but ranges downloaded at pivot 3 contain an old-branch-only - // balance change. The replacement block's empty BAL cannot undo that change. + // The reorg replaces the pivot alone: the applied block and the one the lists carry the state + // through stay canonical, so only the pivot check can refuse what follows. Ranges downloaded + // at that pivot hold a balance change the replacement branch never made. fn pivot_reorg_fixture() -> (Factory, SnapWrite, BalChain, BalChain) { - let chain = BalChain::new(PIVOT, [credit(30)]); - let replacement = BalChain::new(PIVOT, [Vec::new()]); - assert_eq!(chain.block(0), replacement.block(0)); + let chain = BalChain::new(PIVOT, [Vec::new(), credit(30)]); + let replacement = BalChain::new(PIVOT, [Vec::new(), Vec::new()]); + assert_eq!(chain.block(1), replacement.block(1)); + assert_ne!(chain.tip(), replacement.tip()); let mut accounts = accounts(); let factory = hashed_factory(); insert_headers(&factory, &chain.headers); @@ -323,6 +325,77 @@ mod tests { .balance } + // The fixture accounts with `CHANGED` holding `balance`, plus `extra`. + fn accounts_with( + balance: u64, + extra: impl IntoIterator, + ) -> Vec<(B256, TrieAccount)> { + let mut accounts: Vec<_> = accounts().into_iter().chain(extra).collect(); + accounts.iter_mut().find(|(key, _)| *key == keccak256(CHANGED)).unwrap().1.balance = + U256::from(balance); + accounts.sort_by_key(|(hashed_address, _)| *hashed_address); + accounts + } + + // Root of the account trie the downloaded state holds, none of the fixture accounts having + // storage. + fn downloaded_root(factory: &Factory) -> B256 { + let provider = factory.database_provider_ro().unwrap(); + let mut cursor = provider.tx_ref().cursor_read::().unwrap(); + let accounts: Vec<_> = cursor + .walk(None) + .unwrap() + .map(|entry| { + let (hashed_address, account) = entry.unwrap(); + (hashed_address, account.into_trie_account(EMPTY_ROOT_HASH)) + }) + .collect(); + state_root(&accounts) + } + + // An attempt at `chain`'s pivot holding `accounts[..served]`, proved against `accounts`. + fn partially_downloaded( + chain: &BalChain, + accounts: &[(B256, TrieAccount)], + served: usize, + ) -> (Factory, SnapWrite) { + let factory = hashed_factory(); + insert_headers(&factory, &chain.headers); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(chain.generation(state_root(accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + let range = + verified_range(accounts, 0..served, B256::ZERO, &[B256::ZERO, accounts[served - 1].0]); + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + provider.commit().unwrap(); + (factory, write) + } + + // Moves the attempt to the block `nth` after the chain's pivot, whose state is `accounts`. + fn advance( + factory: &Factory, + write: SnapWrite, + chain: &BalChain, + nth: usize, + accounts: &[(B256, TrieAccount)], + ) -> SnapWrite { + let provider = factory.database_provider_rw().unwrap(); + let generation = SnapGeneration::new(chain.block(nth), state_root(accounts)); + let write = provider.advance_snap_pivot(write, generation).unwrap(); + provider.commit().unwrap(); + write + } + + // Commits the rest of the trie from the coverage cursor, proved against `accounts`. + fn download_rest(factory: &Factory, write: SnapWrite, accounts: &[(B256, TrieAccount)]) { + let provider = factory.database_provider_rw().unwrap(); + let origin = provider.account_coverage(write).unwrap().unwrap().next().unwrap(); + let served = accounts.iter().position(|(key, _)| *key >= origin).unwrap()..accounts.len(); + let range = verified_range(accounts, served, origin, &[origin, accounts.last().unwrap().0]); + provider.commit_account_range(write, &range, Default::default(), Vec::new()).unwrap(); + provider.commit().unwrap(); + } + #[tokio::test] async fn every_list_a_response_carries_is_applied_in_block_order() { let chain = chain(); @@ -506,15 +579,17 @@ mod tests { let (client, mut catch_up) = catch_up([], factory.clone()); // Even an already-applied target must reject the orphan instead of reporting completion. - for target in [PIVOT, PIVOT + 1] { + for target in [PIVOT, PIVOT + 2] { assert!(matches!( catch_up.next(write, target).await, Err(SnapSyncError::NonCanonicalBlock { block, hash }) - if block == PIVOT + 1 && hash == chain.tip().hash + if block == PIVOT + 2 && hash == chain.tip().hash )); } assert!(client.block_requests().is_empty()); let provider = factory.database_provider_ro().unwrap(); + // The block whose list would apply next is untouched by the reorg. + assert_eq!(provider.block_hash(PIVOT + 1).unwrap(), Some(chain.block(1).hash)); assert_eq!(provider.block_hash(PIVOT).unwrap(), Some(chain.block(0).hash)); assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied(), chain.block(0)); assert_eq!(SnapStateSnapshot::read(&provider), before); @@ -533,16 +608,18 @@ mod tests { let mut catch_up = BlockAccessListCatchUp::new(client.clone(), factory.clone(), Runtime::test()); + // The list the peer served belongs to a block the reorg left canonical, so refusing it is + // the pivot check's doing. assert!(matches!( - catch_up.next(write, PIVOT + 1).await, + catch_up.next(write, PIVOT + 2).await, Err(SnapSyncError::NonCanonicalBlock { block, hash }) - if block == PIVOT + 1 && hash == chain.tip().hash + if block == PIVOT + 2 && hash == chain.tip().hash )); - assert_eq!(*client.block_requests(), [vec![chain.tip().hash]]); + assert_eq!(*client.block_requests(), [vec![chain.block(1).hash, chain.tip().hash]]); let provider = factory.database_provider_ro().unwrap(); - assert_eq!(provider.block_hash(PIVOT).unwrap(), Some(chain.block(0).hash)); - assert_eq!(provider.block_hash(PIVOT + 1).unwrap(), Some(replacement_hash)); + assert_eq!(provider.block_hash(PIVOT + 1).unwrap(), Some(chain.block(1).hash)); + assert_eq!(provider.block_hash(PIVOT + 2).unwrap(), Some(replacement_hash)); assert_eq!(provider.catch_up_progress(write).unwrap().unwrap().applied(), chain.block(0)); assert_eq!(SnapStateSnapshot::read(&provider), before); } @@ -562,4 +639,64 @@ mod tests { assert_eq!(progress.applied(), chain.block(3)); assert_eq!(balance(&factory), U256::from(30)); } + + #[tokio::test] + async fn downloads_split_across_pivots_converge_on_the_latest_pivot_state() { + const FAR: B256 = B256::repeat_byte(0xfe); + let chain = chain(); + let at = |balance| accounts_with(balance, [(FAR, account(3))]); + assert_eq!(at(1).last().unwrap().0, FAR); + // The pivot serves every account but the last. + let (factory, write) = partially_downloaded(&chain, &at(1), 2); + let (_, mut catch_up) = + catch_up([chain.response(1, [Some(1)]), chain.response(2, [Some(2)])], factory.clone()); + + let write = advance(&factory, write, &chain, 1, &at(10)); + applied(&mut catch_up, write, PIVOT + 3).await; + assert_eq!(balance(&factory), U256::from(10)); + let write = advance(&factory, write, &chain, 2, &at(20)); + + // A range proved against the previous root cannot complete the new one's coverage. + let provider = factory.database_provider_rw().unwrap(); + let origin = provider.account_coverage(write).unwrap().unwrap().next().unwrap(); + let stale = verified_range(&at(10), 2..3, origin, &[origin, FAR]); + assert!(matches!( + provider.commit_account_range(write, &stale, Default::default(), Vec::new()), + Err(SnapSyncError::RootMismatch { .. }) + )); + drop(provider); + download_rest(&factory, write, &at(20)); + + // Lists stop at the latest pivot, where the downloaded state now sits. + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + assert_eq!(progress.applied(), chain.block(2)); + assert!(matches!(catch_up.next(write, PIVOT + 3).await.unwrap(), CatchUpStep::Complete)); + assert_eq!(downloaded_root(&factory), state_root(&at(20))); + } + + #[tokio::test] + async fn an_account_lists_reach_before_its_range_is_taken_from_the_new_root() { + let chain = chain(); + let at = |balance| accounts_with(balance, []); + // Only the account before the changed one is downloaded at the pivot. + assert_eq!(at(1)[1].0, keccak256(CHANGED)); + let (factory, write) = partially_downloaded(&chain, &at(1), 1); + let write = advance(&factory, write, &chain, 2, &at(20)); + let (_, mut catch_up) = catch_up([chain.response(1, [Some(1), Some(2)])], factory.clone()); + + // The lists have no base to change yet, so they leave the account to its range. + let progress = applied(&mut catch_up, write, PIVOT + 3).await; + assert_eq!(progress.applied(), chain.block(2)); + let provider = factory.database_provider_ro().unwrap(); + assert_eq!( + provider.tx_ref().get::(keccak256(CHANGED)).unwrap(), + None + ); + drop(provider); + + download_rest(&factory, write, &at(20)); + + assert_eq!(balance(&factory), U256::from(20)); + assert_eq!(downloaded_root(&factory), state_root(&at(20))); + } } diff --git a/crates/snap-sync/src/catch_up/store.rs b/crates/snap-sync/src/catch_up/store.rs index 204da37ed9c..63a411c2b85 100644 --- a/crates/snap-sync/src/catch_up/store.rs +++ b/crates/snap-sync/src/catch_up/store.rs @@ -5,7 +5,8 @@ use crate::{ common::SnapRecord, AccountCoverage, BalStateUpdate, DownloadedAccount, SnapAccountStore, - SnapAttemptStore, SnapBytecodeStore, SnapSyncError, SnapWrite, + SnapAttemptStore, SnapBytecodeStore, SnapStorageStore, SnapSyncError, SnapWrite, + StorageProgress, }; use alloy_eip7928::AccountChanges; use alloy_eips::BlockNumHash; @@ -18,7 +19,6 @@ use reth_primitives_traits::Account; use reth_storage_api::{ BlockHashReader, DBProvider, MetadataProvider, MetadataWriter, SnapAttemptId, StateWriter, }; -use reth_trie_common::HashedStorage; use serde::{Deserialize, Serialize}; /// Persistence for the block access lists an attempt applies to its downloaded state. @@ -42,10 +42,11 @@ pub trait SnapCatchUpStore { /// Returns the state update implied by `bal`, reading downloaded accounts from this provider. /// /// The list must already be verified against its header. Accounts outside `coverage` remain - /// unresolved, and no state is written. + /// unresolved, except for slots of contracts `storage` holds ahead of their range. fn block_access_list_update( &self, coverage: AccountCoverage, + storage: StorageProgress, bal: &[AccountChanges], ) -> Result where @@ -176,6 +177,7 @@ impl SnapCatchUpStore for T { fn block_access_list_update( &self, coverage: AccountCoverage, + storage: StorageProgress, bal: &[AccountChanges], ) -> Result where @@ -191,8 +193,12 @@ impl SnapCatchUpStore for T { let hashed_address = keccak256(account_changes.address()); let mut account = match self.downloaded_account(coverage, hashed_address)? { - // Its range is downloaded against a later root, which includes this change. + // Its range is downloaded against a later root, which includes this change. Slots + // persisted ahead of it may predate the change, so they follow the list. DownloadedAccount::Unknown => { + if storage.has_slots(hashed_address) && account_changes.has_storage_changes() { + update.insert_storage(hashed_address, account_changes); + } update.unresolved.push(hashed_address); continue } @@ -206,14 +212,7 @@ impl SnapCatchUpStore for T { // Execution removes accounts a block leaves empty, see EIP-161. update.state.accounts.insert(hashed_address, (!account.is_empty()).then_some(account)); if account_changes.has_storage_changes() { - update.state.storages.insert( - hashed_address, - HashedStorage::from_iter( - account_changes - .storage_post_states() - .map(|(slot, value)| (keccak256(B256::from(slot)), value)), - ), - ); + update.insert_storage(hashed_address, account_changes); } if let Some((code_hash, code)) = account_info .code_hash @@ -254,7 +253,11 @@ impl SnapCatchUpStore for T { }) } let coverage = self.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; - let update = self.block_access_list_update(coverage, bal)?; + let storage = match coverage.next() { + Some(origin) => self.storage_progress(write, origin)?, + None => StorageProgress::START, + }; + let update = self.block_access_list_update(coverage, storage, bal)?; // Entries left unresolved are downloaded whole against the pivot, which already includes // this block, so nothing here has to remember them. @@ -280,7 +283,7 @@ mod tests { test_utils::{ account, hashed_factory, key, state_root, storage_root_of, BalChain, SnapStateSnapshot, }, - SnapAccountStore, SnapGeneration, + SnapAccountStore, SnapGeneration, StorageChunk, }; use alloy_eip7928::{BalanceChange, BlockAccessIndex, CodeChange, SlotChanges, StorageChange}; use alloy_primitives::{bytes, keccak256, map::B256Map, Address, Bytes, U256}; @@ -371,6 +374,27 @@ mod tests { (chain.block(number as usize - 1), chain.block(number as usize - 2).hash) } + // The fixture chain with its last block replaced by one changing a balance. + fn replacement() -> BalChain { + BalChain::new(1, [Vec::new(), credit(1)]) + } + + // The attempt carried through block 2 and anchored to block 3, which a reorg then replaced, + // leaving the applied block canonical. Returns the orphaned pivot. + fn orphaned_pivot(accounts: &[(B256, TrieAccount)]) -> (Factory, SnapWrite, BlockNumHash) { + let (factory, write) = started(accounts, accounts.len()); + let provider = factory.database_provider_rw().unwrap(); + let (applied, parent) = block(2); + provider.commit_block_access_list(write, applied, parent, &[]).unwrap(); + let write = + provider.advance_snap_pivot(write, generation(3, state_root(accounts))).unwrap(); + provider.commit().unwrap(); + let replacement = replacement(); + assert_eq!(replacement.block(1), applied); + replacement.replace_tip(&factory); + (factory, write, block(3).0) + } + fn index(value: u64) -> BlockAccessIndex { BlockAccessIndex::new(value) } @@ -406,20 +430,6 @@ mod tests { assert_eq!(provider.catch_up_progress(write).unwrap(), Some(progress)); } - #[test] - fn a_pivot_below_the_applied_block_is_refused() { - let accounts = accounts(); - let (factory, write) = started(&accounts, accounts.len()); - let provider = factory.database_provider_rw().unwrap(); - let (block, parent) = block(2); - provider.commit_block_access_list(write, block, parent, &credit(10)).unwrap(); - - let refused = provider.advance_snap_pivot(write, generation(1, state_root(&accounts))); - - assert!(matches!(refused, Err(SnapSyncError::PivotBelowApplied { applied: 2, pivot: 1 }))); - provider.authorize_snap_write(write).unwrap(); - } - #[test] fn a_list_commits_with_the_block_it_carries_the_state_to() { let accounts = accounts(); @@ -509,16 +519,11 @@ mod tests { #[test] fn a_list_proved_against_an_orphaned_pivot_is_refused() { let accounts = accounts(); - let (factory, write) = started(&accounts, accounts.len()); + let (factory, write, orphaned) = orphaned_pivot(&accounts); let provider = factory.database_provider_rw().unwrap(); - let (applied, parent) = block(2); - provider.commit_block_access_list(write, applied, parent, &[]).unwrap(); - // A reorg replaced the pivot, leaving the applied block and the one after it canonical. - let orphaned = BlockNumHash::new(3, B256::repeat_byte(0xee)); - let write = provider - .advance_snap_pivot(write, SnapGeneration::new(orphaned, state_root(&accounts))) - .unwrap(); - let (block, parent) = block(3); + let (applied, _) = block(2); + // The replacement block continues the applied one, so only the pivot check refuses it. + let (block, parent) = (replacement().block(2), applied.hash); let before = SnapStateSnapshot::read(&provider); let refused = provider.commit_block_access_list(write, block, parent, &credit(10)); @@ -536,18 +541,12 @@ mod tests { #[test] fn an_orphaned_pivot_cannot_be_replaced() { let accounts = accounts(); - let (factory, write) = started(&accounts, accounts.len()); + let (factory, write, orphaned) = orphaned_pivot(&accounts); let provider = factory.database_provider_rw().unwrap(); - let (applied, parent) = block(2); - provider.commit_block_access_list(write, applied, parent, &[]).unwrap(); - let orphaned = BlockNumHash::new(3, B256::repeat_byte(0xee)); - let write = provider - .advance_snap_pivot(write, SnapGeneration::new(orphaned, state_root(&accounts))) - .unwrap(); - let before = SnapStateSnapshot::read(&provider); + let replacement = SnapGeneration::new(replacement().block(2), state_root(&accounts)); - let refused = provider.advance_snap_pivot(write, generation(3, state_root(&accounts))); + let refused = provider.advance_snap_pivot(write, replacement); // Re-anchoring would leave the ranges downloaded against the orphan looking canonical. assert!(matches!( @@ -633,12 +632,103 @@ mod tests { assert_eq!(progress.applied(), block); } + #[test] + fn slots_persisted_ahead_of_their_range_follow_the_lists() { + let factory = hashed_factory(); + insert_headers(&factory, &chain().headers); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(generation(1, state_root(&accounts()))).unwrap(); + provider.start_account_coverage(write).unwrap(); + let (changed, stored_slot) = (keccak256(CHANGED), (hashed_slot(), U256::from(7))); + let chunk = StorageChunk::new( + changed, + storage_root_of(&[stored_slot]), + B256::ZERO, + vec![stored_slot], + None, + ); + provider.commit_storage_chunk(write, B256::ZERO, chunk).unwrap(); + let mut moved = accounts(); + let position = changed_index(&moved); + moved[position].1.storage_root = storage_root_of(&[(hashed_slot(), U256::from(9))]); + let write = provider.advance_snap_pivot(write, generation(2, state_root(&moved))).unwrap(); + let (block, parent) = block(2); + let changes = vec![AccountChanges::new(CHANGED).with_storage_change(SlotChanges::new( + SLOT, + vec![StorageChange::new(index(1), U256::from(9))], + ))]; + + provider.commit_block_access_list(write, block, parent, &changes).unwrap(); + + // Only the slots follow: the account itself comes whole with its range. + assert_eq!(stored(&provider, changed), None); + assert_eq!( + crate::test_utils::stored_slots(&provider, changed), + [(hashed_slot(), U256::from(9))] + ); + let range = crate::test_utils::verified_range(&moved, 0..moved.len(), B256::ZERO, &[]); + let coverage = + provider.commit_account_range(write, &range, B256Map::default(), Vec::new()).unwrap(); + assert!(coverage.is_complete()); + } + + #[test] + fn skipped_storage_from_an_abandoned_attempt_is_not_reused() { + let factory = hashed_factory(); + insert_headers(&factory, &chain().headers); + let provider = factory.database_provider_rw().unwrap(); + let (changed, later) = (keccak256(CHANGED), B256::repeat_byte(0xff)); + let stale = (B256::ZERO, U256::from(7)); + let mut contract = account(1); + contract.storage_root = storage_root_of(&[stale]); + let mut accounts = vec![(changed, contract), (later, contract)]; + let write = provider.start_snap_attempt(generation(1, state_root(&accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider + .commit_storage_chunk( + write, + B256::ZERO, + StorageChunk::new(changed, contract.storage_root, B256::ZERO, vec![stale], None), + ) + .unwrap(); + + accounts[0].1.storage_root = reth_trie_common::EMPTY_ROOT_HASH; + let write = provider.start_snap_attempt(generation(1, state_root(&accounts))).unwrap(); + provider.start_account_coverage(write).unwrap(); + provider + .commit_storage_chunk( + write, + B256::ZERO, + StorageChunk::new(later, contract.storage_root, B256::ZERO, vec![stale], None), + ) + .unwrap(); + + let new_slot = (hashed_slot(), U256::from(9)); + accounts[0].1.storage_root = storage_root_of(&[new_slot]); + let write = + provider.advance_snap_pivot(write, generation(2, state_root(&accounts))).unwrap(); + let (block, parent) = block(2); + let changes = vec![AccountChanges::new(CHANGED).with_storage_change(SlotChanges::new( + SLOT, + vec![StorageChange::new(index(1), new_slot.1)], + ))]; + provider.commit_block_access_list(write, block, parent, &changes).unwrap(); + + assert_eq!(crate::test_utils::stored_slots(&provider, changed), [new_slot]); + let range = + crate::test_utils::verified_range(&accounts, 0..accounts.len(), B256::ZERO, &[]); + assert!(provider + .commit_account_range(write, &range, B256Map::default(), Vec::new()) + .unwrap() + .is_complete()); + } + #[test] fn a_list_from_a_replaced_attempt_changes_nothing() { let accounts = accounts(); let (factory, write) = started(&accounts, accounts.len()); let provider = factory.database_provider_rw().unwrap(); - provider.advance_snap_pivot(write, generation(2, B256::repeat_byte(0xcc))).unwrap(); + provider.advance_snap_pivot(write, generation(3, B256::repeat_byte(0xcc))).unwrap(); let (block, parent) = block(2); let refused = provider.commit_block_access_list(write, block, parent, &credit(10)); diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index 8dcc10f572a..b87b71ebb1b 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -68,12 +68,20 @@ pub enum SnapSyncError { /// Block the list was applied for. block: u64, }, - /// The pivot was moved below the last block whose list is applied. - #[error("pivot {pivot} is below applied block {applied}")] - PivotBelowApplied { + /// The pivot was moved to a block not past the current one. + #[error("pivot {pivot} cannot move to block {target}, which is not past it")] + PivotNotAdvanced { + /// Block the attempt is anchored to. + pivot: u64, + /// Block the pivot was moved to. + target: u64, + }, + /// Storage persisted ahead of a range was committed before catch-up reached the pivot. + #[error("catch-up applied block {applied}, the pivot is {pivot}")] + CatchUpBehindPivot { /// Last block whose list is applied. applied: u64, - /// Block the pivot was moved to. + /// Block the attempt is anchored to. pivot: u64, }, /// A list was applied for a block the canonical chain no longer holds. diff --git a/crates/snap-sync/src/session.rs b/crates/snap-sync/src/session.rs index 9a51d486901..7a34404564e 100644 --- a/crates/snap-sync/src/session.rs +++ b/crates/snap-sync/src/session.rs @@ -1,7 +1,7 @@ //! Drives one snap synchronization attempt: what it targets, and when it stops. -use crate::{SnapGeneration, SnapPivotPolicy, SnapSyncError}; -use reth_storage_api::HeaderProvider; +use crate::{SnapAttemptStore, SnapGeneration, SnapPivotPolicy, SnapSyncError, SnapWrite}; +use reth_storage_api::{BlockHashReader, HeaderProvider, MetadataProvider, MetadataWriter}; use tokio_util::sync::CancellationToken; /// One snap synchronization attempt, from the pivot it targets to the work it owns. @@ -73,6 +73,40 @@ impl SnapSyncSession { Some((generation, self.cancellation.clone())) } + /// Re-anchors the downloaded target once it lags too far behind `head`, returning the new + /// write. + /// + /// The target is re-read from the attempt record first, so an advance whose transaction was + /// dropped is retried rather than assumed. + pub fn advance

( + &mut self, + provider: &P, + write: SnapWrite, + head: u64, + finalized: Option, + ) -> Result, SnapSyncError> + where + P: HeaderProvider + BlockHashReader + MetadataProvider + MetadataWriter, + { + let SnapSyncSessionState::Downloading(mut current) = self.state else { return Ok(None) }; + let attempt = provider.authorize_snap_write(write)?; + if attempt.pivot() != current.target() { + current = SnapGeneration::new(attempt.pivot(), attempt.state_root()); + self.state = SnapSyncSessionState::Downloading(current); + } + if !self.policy.needs_advance(current, head) { + return Ok(None) + } + let Some(next) = self.policy.select(provider, head, finalized)? else { return Ok(None) }; + // A policy anchoring further back than it re-anchors can select a block behind the pivot. + if next.target().number <= current.target().number { + return Ok(None) + } + let write = provider.advance_snap_pivot(write, next)?; + self.state = SnapSyncSessionState::Downloading(next); + Ok(Some(write)) + } + /// Signals outstanding work to stop and ends the session. /// /// Terminal: a later attempt needs a new session. @@ -108,12 +142,32 @@ impl SnapSyncSessionState { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{chain, policy, provider_with}; + use crate::test_utils::{chain, hashed_factory, policy, provider_with}; + use reth_primitives_traits::SealedHeader; + use reth_provider::{ + test_utils::{insert_headers, MockNodeTypesWithDB}, + DBProvider, DatabaseProviderFactory, ProviderFactory, + }; fn session() -> SnapSyncSession { SnapSyncSession::new(policy()) } + // A session downloading block 1 under head 2, re-anchoring once its pivot lags by two blocks, + // with the attempt recorded in a database holding blocks 0 through 3. + fn downloading() -> (ProviderFactory, SnapSyncSession, SnapWrite) { + let factory = hashed_factory(); + let headers: Vec<_> = chain(Some(0)).into_iter().map(SealedHeader::seal_slow).collect(); + insert_headers(&factory, &headers); + let provider = factory.database_provider_rw().unwrap(); + let mut session = SnapSyncSession::new(policy().with_advance_after(1)); + session.select(&provider, 2, None).unwrap(); + let (generation, _) = session.start().unwrap(); + let write = provider.start_snap_attempt(generation).unwrap(); + provider.commit().unwrap(); + (factory, session, write) + } + #[test] fn waits_while_no_block_is_eligible() { // The only block access list commitment is at the head, past the head distance. @@ -210,4 +264,72 @@ mod tests { assert_eq!(session.select(&provider, 3, None).unwrap(), &SnapSyncSessionState::Cancelled); assert!(session.start().is_none()); } + + #[test] + fn a_lagging_target_advances_with_the_attempt() { + let (factory, mut session, write) = downloading(); + let provider = factory.database_provider_rw().unwrap(); + + let advanced = session.advance(&provider, write, 3, None).unwrap().unwrap(); + + let target = *session.target().unwrap(); + assert_eq!(target.target().number, 2); + assert!(matches!(session.state(), SnapSyncSessionState::Downloading(_))); + let attempt = provider.authorize_snap_write(advanced).unwrap(); + assert_eq!((attempt.pivot(), attempt.state_root()), (target.target(), target.state_root())); + assert!(matches!( + provider.authorize_snap_write(write), + Err(SnapSyncError::StaleWrite { .. }) + )); + } + + #[test] + fn a_target_within_the_window_is_kept() { + let (factory, mut session, write) = downloading(); + let provider = factory.database_provider_rw().unwrap(); + let before = *session.target().unwrap(); + + assert_eq!(session.advance(&provider, write, 2, None).unwrap(), None); + + assert_eq!(session.target(), Some(&before)); + provider.authorize_snap_write(write).unwrap(); + } + + #[test] + fn a_refused_advance_keeps_the_target() { + let (factory, mut session, write) = downloading(); + let provider = factory.database_provider_rw().unwrap(); + let before = *session.target().unwrap(); + provider.abandon_snap_attempt().unwrap(); + + let refused = session.advance(&provider, write, 3, None); + + assert!(matches!(refused, Err(SnapSyncError::StaleWrite { .. }))); + assert_eq!(session.state(), &SnapSyncSessionState::Downloading(before)); + } + + #[test] + fn a_rolled_back_advance_is_retried() { + let (factory, mut session, write) = downloading(); + let provider = factory.database_provider_rw().unwrap(); + session.advance(&provider, write, 3, None).unwrap().unwrap(); + drop(provider); + + let provider = factory.database_provider_rw().unwrap(); + let advanced = session.advance(&provider, write, 3, None).unwrap().unwrap(); + + assert_eq!(session.target().unwrap().target().number, 2); + assert_eq!(provider.authorize_snap_write(advanced).unwrap().pivot().number, 2); + } + + #[test] + fn a_target_no_work_took_is_not_advanced() { + let (factory, _, write) = downloading(); + let provider = factory.database_provider_rw().unwrap(); + let mut session = SnapSyncSession::new(policy().with_advance_after(1)); + session.select(&provider, 2, None).unwrap(); + + assert_eq!(session.advance(&provider, write, 3, None).unwrap(), None); + assert_eq!(session.target().unwrap().target().number, 1); + } } diff --git a/crates/snap-sync/src/storage/download.rs b/crates/snap-sync/src/storage/download.rs index f0db0168ec0..258dd8371fa 100644 --- a/crates/snap-sync/src/storage/download.rs +++ b/crates/snap-sync/src/storage/download.rs @@ -74,7 +74,7 @@ where else { return Ok(StorageRangeStep::Complete) }; - let end = contracts.accounts().len().min(first.saturating_add(self.max_accounts)); + let end = progress.request_end(contracts.accounts(), first, self.max_accounts); let batch = contracts.range(first..end).expect("positions are inside the batch"); let from = progress.resume_at(batch.accounts()[0].0).expect("first contract is incomplete"); @@ -172,8 +172,9 @@ mod tests { account, generation, hashed_factory, insert_generation_headers, key, state_root, storage_ranges, storage_root_of, stored_slots, verified_range, ScriptedSnapClient, }, - SnapAccountStore, SnapAttemptStore, + SnapAccountStore, SnapAttemptStore, SnapCatchUpStore, }; + use alloy_eips::BlockNumHash; use alloy_primitives::U256; use reth_eth_wire_types::snap::StorageRangesMessage; use reth_network_p2p::{error::PeerRequestResult, snap::client::SnapResponse}; @@ -383,4 +384,85 @@ mod tests { assert!(matches!(download.next(&range).await, Err(SnapSyncError::StaleWrite { .. }))); assert_eq!(client.storage_requests().len(), 1); } + + #[tokio::test] + async fn storage_part_way_through_resumes_at_the_new_root() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let (large, small) = (large(), small()); + // The new pivot turned the first account into a contract, ahead of the carried one. + let mut moved = accounts.clone(); + moved[0].1 = contract(1, &small); + let responses = [ + storage_ranges(1, &[&large[..1]], &large, &[B256::ZERO, key(1)]), + storage_ranges(2, &[&small[..]], &small, &[]), + storage_ranges(3, &[&large[1..]], &large, &[key(2), key(3)]), + storage_ranges(4, &[&small[..]], &small, &[]), + ]; + let (client, mut download) = download(responses, factory.clone()); + committed(&mut download, &range).await; + + let provider = factory.database_provider_rw().unwrap(); + let write = + provider.advance_snap_pivot(range.write(), generation(2, state_root(&moved))).unwrap(); + provider.commit().unwrap(); + let range = + VerifiedRange::new(write, verified_range(&moved, 0..moved.len(), B256::ZERO, &[])); + for _ in 0..3 { + committed(&mut download, &range).await; + } + assert!(matches!(download.next(&range).await.unwrap(), StorageRangeStep::Complete)); + assert_eq!( + *client.storage_requests(), + [ + (vec![key(2), key(3)], B256::ZERO), + (vec![key(1)], B256::ZERO), + (vec![key(2), key(3)], key(2)), + (vec![key(3)], B256::ZERO), + ] + ); + + // The carried slot holds pivot 1's value until the lists reach the new pivot. + let provider = factory.database_provider_rw().unwrap(); + assert!(matches!( + provider.commit_account_range(write, range.range(), Default::default(), Vec::new()), + Err(SnapSyncError::CatchUpBehindPivot { applied: 1, pivot: 2 }) + )); + let block = BlockNumHash::new(2, B256::repeat_byte(2)); + provider.commit_block_access_list(write, block, B256::repeat_byte(1), &[]).unwrap(); + let coverage = provider + .commit_account_range(write, range.range(), Default::default(), Vec::new()) + .unwrap(); + provider.commit().unwrap(); + assert!(coverage.is_complete()); + assert_eq!(slots_of(&factory, key(1)), small); + assert_eq!(slots_of(&factory, key(2)), large); + } + + #[tokio::test] + async fn a_page_ending_before_a_carried_contract_hands_it_to_the_next_range() { + let accounts = accounts(); + let (factory, range) = started(&accounts); + let large = large(); + let first = storage_ranges(1, &[&large[..1]], &large, &[B256::ZERO, key(1)]); + let (_, mut download) = download([first], factory.clone()); + committed(&mut download, &range).await; + let provider = factory.database_provider_rw().unwrap(); + let write = provider + .advance_snap_pivot(range.write(), generation(2, state_root(&accounts))) + .unwrap(); + let block = BlockNumHash::new(2, B256::repeat_byte(2)); + provider.commit_block_access_list(write, block, B256::repeat_byte(1), &[]).unwrap(); + + // The new root's page stops before the contract part way through. + let page = verified_range(&accounts, 0..1, B256::ZERO, &[B256::ZERO, key(1)]); + let next = provider + .commit_account_range(write, &page, Default::default(), Vec::new()) + .unwrap() + .next() + .unwrap(); + + assert!(next <= key(2)); + assert_eq!(provider.storage_progress(write, next).unwrap().resume_at(key(2)), Some(key(2))); + } } diff --git a/crates/snap-sync/src/storage/store.rs b/crates/snap-sync/src/storage/store.rs index 1103bd1f7ec..d0e5b24abba 100644 --- a/crates/snap-sync/src/storage/store.rs +++ b/crates/snap-sync/src/storage/store.rs @@ -1,7 +1,7 @@ //! Persists contract storage response by response, ahead of the account range committing to it. //! -//! Progress is tied to the attempt, its pivot and the range, so slots proved against a superseded -//! root are never resumed. +//! Progress is tied to the attempt and the range, and survives pivot moves: catch-up carries the +//! persisted slots forward with block access lists, and downloads resume at the new root. use crate::{common::SnapRecord, SnapAccountStore, SnapAttemptStore, SnapSyncError, SnapWrite}; use alloy_primitives::{B256, U256}; @@ -20,6 +20,8 @@ use serde::{Deserialize, Serialize}; /// transaction and commit together or not at all. pub trait SnapStorageStore { /// Returns how far the storage of the range requested from `origin` is persisted. + /// + /// A contract part way through when the pivot moved resumes with its root unproved. fn storage_progress( &self, write: SnapWrite, @@ -70,65 +72,121 @@ impl StorageChunk { /// How far the contracts of the account range being downloaded have their storage persisted. /// -/// Contracts complete in key order, so only one is ever part way through. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +/// Contracts complete in key order, so only one is part way through per pivot, plus those carried +/// across pivot moves. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct StorageProgress { // Contracts up to this key have all their storage persisted. complete: Option, - // The contract part way through, if any. - partial: Option, + // Contracts part way through, in key order. + partial: Vec, } impl StorageProgress { /// No storage persisted yet. - pub const START: Self = Self { complete: None, partial: None }; + pub const START: Self = Self { complete: None, partial: Vec::new() }; /// Returns whether all of `account`'s storage is persisted. pub fn is_complete(&self, account: B256) -> bool { self.complete.is_some_and(|complete| account <= complete) } + /// Returns whether `account` has storage persisted ahead of its range. + pub fn has_slots(&self, account: B256) -> bool { + self.is_complete(account) || self.partial_of(account).is_some() + } + + // Progress of `account` if it is part way through. + fn partial_of(&self, account: B256) -> Option<&PartialStorage> { + self.partial.iter().find(|partial| partial.account == account) + } + + // End of a request for at most `max` of `contracts` from `first`. Contracts after the first + // are requested whole, so one resuming part way must lead its own request. + pub(crate) fn request_end( + &self, + contracts: &[(B256, T)], + first: usize, + max: usize, + ) -> usize { + let end = contracts.len().min(first.saturating_add(max)); + contracts[first + 1..end] + .iter() + .position(|(account, _)| self.partial_of(*account).is_some()) + .map_or(end, |offset| first + 1 + offset) + } + + // Hands the progress of contracts past a committed range to the range continuing at `next`. + pub(crate) fn carry_to( + mut self, + provider: &impl MetadataWriter, + write: SnapWrite, + next: B256, + ) -> Result<(), SnapSyncError> { + self.complete = self.complete.filter(|complete| *complete >= next); + self.partial.retain(|partial| partial.account >= next); + if self != Self::START { + StoredProgress::new(write, next, self).write(provider)?; + } + Ok(()) + } + /// Slot `account`'s storage resumes at: zero before it starts, `None` once it is complete. pub fn resume_at(&self, account: B256) -> Option { if self.is_complete(account) { return None } - let partial = self.partial.filter(|partial| partial.account == account); - Some(partial.map_or(B256::ZERO, |partial| partial.next)) + Some(self.partial_of(account).map_or(B256::ZERO, |partial| partial.next)) + } + + // The same progress with every partial contract's root unproved, after the pivot moved. + fn carried(mut self) -> Self { + for partial in &mut self.partial { + partial.storage_root = None; + } + self } // The progress after `chunk`, which must continue its contract or start the next one. - fn advance(&self, chunk: &StorageChunk) -> Result { - let another = self.partial.is_some_and(|partial| partial.account != chunk.account); - if another || self.resume_at(chunk.account) != Some(chunk.from) { + // + // A contract carried across a pivot move does not block others: the new root can hold + // contracts before it, or no longer hold it at all. + fn advance(mut self, chunk: &StorageChunk) -> Result { + let blocked = self + .partial + .iter() + .any(|partial| partial.account < chunk.account && partial.storage_root.is_some()); + if blocked || self.resume_at(chunk.account) != Some(chunk.from) { return Err(SnapSyncError::OutOfOrderStorage { account: chunk.account, from: chunk.from, }) } - if let Some(partial) = self.partial && - partial.storage_root != chunk.storage_root + if let Some(partial) = self.partial_of(chunk.account) && + let Some(expected) = partial.storage_root && + expected != chunk.storage_root { return Err(SnapSyncError::StorageRootMismatch { account: chunk.account, - expected: partial.storage_root, + expected, got: chunk.storage_root, }) } if chunk.next.is_some_and(|next| next <= chunk.from) { return Err(SnapSyncError::NoProgress { origin: chunk.from }) } - Ok(match chunk.next { - Some(next) => Self { - complete: self.complete, - partial: Some(PartialStorage { - account: chunk.account, - storage_root: chunk.storage_root, - next, - }), - }, - None => Self { complete: Some(chunk.account), partial: None }, - }) + // Carried contracts past this one stay resumable, those before it were skipped, so the new + // root no longer holds them. + self.partial.retain(|partial| partial.account > chunk.account); + match chunk.next { + Some(next) => { + let storage_root = Some(chunk.storage_root); + self.partial + .insert(0, PartialStorage { account: chunk.account, storage_root, next }); + } + None => self.complete = Some(chunk.account), + } + Ok(self) } } @@ -137,8 +195,8 @@ impl StorageProgress { struct PartialStorage { // Hashed address of the contract. account: B256, - // Root its slots were proved against. - storage_root: B256, + // Root its slots were proved against, none once the pivot moved past it. + storage_root: Option, // Slot its storage resumes at. next: B256, } @@ -150,7 +208,7 @@ struct StoredProgress { version: u32, // Attempt the slots belong to. attempt: SnapAttemptId, - // Pivot generation the slots were proved against. + // Pivot generation the last chunk was proved against. state_version: u64, // Key the account range was requested from. origin: B256, @@ -160,7 +218,7 @@ struct StoredProgress { impl SnapRecord for StoredProgress { const KEY: &'static str = "snap_storage_progress"; - const VERSION: u32 = 1; + const VERSION: u32 = 2; } impl StoredProgress { @@ -175,15 +233,21 @@ impl StoredProgress { } } - fn belongs_to(&self, write: SnapWrite, origin: B256) -> bool { - self.attempt == write.attempt() && - self.state_version == write.state_version() && - self.origin == origin + // The recorded progress, if it belongs to `write`'s attempt and the range at `origin`. + fn into_progress(self, write: SnapWrite, origin: B256) -> Option { + if self.attempt != write.attempt() || self.origin != origin { + return None + } + Some(if self.state_version == write.state_version() { + self.progress + } else { + self.progress.carried() + }) } } impl SnapStorageStore for T { - // A record left by another attempt, pivot or range reads as nothing persisted. + // A record left by another attempt or range reads as nothing persisted. fn storage_progress( &self, write: SnapWrite, @@ -191,7 +255,7 @@ impl SnapStorageStore for T { ) -> Result { self.authorize_snap_write(write)?; let Some(stored) = StoredProgress::read(self)? else { return Ok(StorageProgress::START) }; - Ok(if stored.belongs_to(write, origin) { stored.progress } else { StorageProgress::START }) + Ok(stored.into_progress(write, origin).unwrap_or(StorageProgress::START)) } // Every check runs before the first write, so a refused chunk changes nothing. @@ -211,8 +275,17 @@ impl SnapStorageStore for T { if chunk.account < origin || chunk.storage_root == EMPTY_ROOT_HASH { return Err(SnapSyncError::UnexpectedStorage { account: chunk.account }) } - let progress = self.storage_progress(write, origin)?.advance(&chunk)?; - + let progress = self.storage_progress(write, origin)?; + let complete = progress.complete; + let progress = progress.advance(&chunk)?; + + if chunk.next.is_none() { + // Skipped accounts have empty storage at this pivot. Clear earlier attempts' slots + // before the completion watermark makes them eligible for reuse and BAL updates. + let start = complete + .map_or(origin, |account| (U256::from_be_bytes(account.0) + U256::from(1)).into()); + self.remove::(start..chunk.account)?; + } if chunk.from == B256::ZERO { self.remove::(chunk.account..=chunk.account)?; } @@ -220,8 +293,9 @@ impl SnapStorageStore for T { .with_storages([(chunk.account, HashedStorage::from_iter(chunk.slots))]) .into_sorted(); self.write_hashed_state(&state)?; - StoredProgress::new(write, origin, progress).write(self)?; - Ok(progress) + let stored = StoredProgress::new(write, origin, progress); + stored.write(self)?; + Ok(stored.progress) } } @@ -358,26 +432,101 @@ mod tests { assert_eq!(persisted_storage_root(provider.tx_ref(), CONTRACT).unwrap(), root()); } - #[test] - fn progress_belongs_to_the_pivot_and_range_it_was_recorded_for() { + // `CONTRACT` part way through at pivot 1, then the pivot moved to block 2. + fn carried() -> (ProviderFactory, SnapWrite, SnapWrite) { let (factory, write) = started(); let provider = factory.database_provider_rw().unwrap(); let first = chunk(0..1, B256::ZERO, Some(slot(2))); provider.commit_storage_chunk(write, B256::ZERO, first).unwrap(); + let advanced = + provider.advance_snap_pivot(write, generation(2, B256::repeat_byte(0xcc))).unwrap(); + provider.commit().unwrap(); + (factory, write, advanced) + } - assert_eq!(provider.storage_progress(write, slot(9)).unwrap(), StorageProgress::START); + // Whole storage of `account`, proved against `root()`. + fn whole(account: B256) -> StorageChunk { + StorageChunk::new(account, root(), B256::ZERO, slots(), None) + } - // Slots proved against the previous root are not resumed once the pivot moves. - let moved = generation(2, B256::repeat_byte(0xcc)); - let advanced = provider.advance_snap_pivot(write, moved).unwrap(); - assert_eq!( - provider.storage_progress(advanced, B256::ZERO).unwrap(), - StorageProgress::START - ); + #[test] + fn progress_survives_the_pivot_moving_with_its_root_unproved() { + let (factory, write, advanced) = carried(); + let provider = factory.database_provider_rw().unwrap(); + + assert_eq!(provider.storage_progress(advanced, slot(9)).unwrap(), StorageProgress::START); + let progress = provider.storage_progress(advanced, B256::ZERO).unwrap(); + assert_eq!(progress.resume_at(CONTRACT), Some(slot(2))); assert!(matches!( provider.commit_storage_chunk(write, B256::ZERO, chunk(1..3, slot(2), None)), Err(SnapSyncError::StaleWrite { .. }) )); + + // The new root's slots continue the carried ones, whatever root those were proved against. + let rest = StorageChunk::new( + CONTRACT, + B256::repeat_byte(0x33), + slot(2), + slots()[1..].to_vec(), + None, + ); + let progress = provider.commit_storage_chunk(advanced, B256::ZERO, rest).unwrap(); + assert!(progress.is_complete(CONTRACT)); + assert_eq!(stored_slots(&provider, CONTRACT), slots()); + } + + #[test] + fn a_carried_contract_does_not_block_the_contracts_the_new_root_holds() { + let (factory, _, advanced) = carried(); + let provider = factory.database_provider_rw().unwrap(); + + // A contract created before it since the pivot moved finishes first, keeping it resumable. + let before = B256::repeat_byte(0x11); + let progress = provider.commit_storage_chunk(advanced, B256::ZERO, whole(before)).unwrap(); + assert!(progress.is_complete(before)); + assert_eq!(progress.resume_at(CONTRACT), Some(slot(2))); + + // One after it means the new root no longer holds it as a contract. + let after = B256::repeat_byte(0x44); + let progress = provider.commit_storage_chunk(advanced, B256::ZERO, whole(after)).unwrap(); + assert!(progress.is_complete(after)); + assert!(progress.partial.is_empty()); + } + + #[test] + fn a_new_contract_part_way_keeps_the_carried_one_resumable() { + let (factory, _, advanced) = carried(); + let provider = factory.database_provider_rw().unwrap(); + let before = B256::repeat_byte(0x11); + let first = + StorageChunk::new(before, root(), B256::ZERO, slots()[..1].to_vec(), Some(slot(2))); + provider.commit_storage_chunk(advanced, B256::ZERO, first).unwrap(); + + // Both stay resumable, even across another pivot move. + let advanced = + provider.advance_snap_pivot(advanced, generation(3, B256::repeat_byte(0xdd))).unwrap(); + let progress = provider.storage_progress(advanced, B256::ZERO).unwrap(); + assert_eq!(progress.resume_at(before), Some(slot(2))); + assert_eq!(progress.resume_at(CONTRACT), Some(slot(2))); + + let rest = StorageChunk::new(before, root(), slot(2), slots()[1..].to_vec(), None); + let progress = provider.commit_storage_chunk(advanced, B256::ZERO, rest).unwrap(); + assert!(progress.is_complete(before)); + assert_eq!(progress.resume_at(CONTRACT), Some(slot(2))); + } + + #[test] + fn a_contract_resuming_part_way_leads_its_request() { + let progress = StorageProgress { + complete: None, + partial: vec![PartialStorage { account: CONTRACT, storage_root: None, next: slot(2) }], + }; + let contracts = [(slot(1), ()), (CONTRACT, ()), (B256::repeat_byte(0x33), ())]; + + assert_eq!(progress.request_end(&contracts, 0, 10), 1); + assert_eq!(progress.request_end(&contracts, 1, 10), 3); + assert_eq!(progress.request_end(&contracts, 1, 1), 2); + assert_eq!(StorageProgress::START.request_end(&contracts, 0, 10), 3); } #[test] From 3aaa27000ec9f515fe9b07a714de5b6d44ce3fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Sat, 19 Sep 2026 01:10:21 +0200 Subject: [PATCH 016/133] fix(net): encode snap account bodies as nested RLP lists (#27311) --- crates/net/downloaders/src/snap/mod.rs | 17 +- crates/net/eth-wire-types/src/snap.rs | 226 +++++++++++++++++------- crates/net/network/src/eth_requests.rs | 10 +- crates/net/network/tests/it/snap/mod.rs | 2 +- 4 files changed, 174 insertions(+), 81 deletions(-) diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index 78bc3a04683..28eeedea617 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -6,7 +6,7 @@ use alloy_primitives::{map::B256Set, B256, KECCAK256_EMPTY}; use futures::Future; use reth_eth_wire_types::snap::{ - AccountRangeMessage, GetAccountRangeMessage, GetStorageRangesMessage, + AccountData, AccountRangeMessage, GetAccountRangeMessage, GetStorageRangesMessage, }; use reth_network_p2p::{ error::RequestError, @@ -298,17 +298,8 @@ fn verify_account_range( return Err(RequestError::BadResponse) } - // Decode first so malformed account values are attributed to the responder. - let mut accounts = response - .accounts - .into_iter() - .map(|data| { - data.into_trie_entry().map_err(|error| { - debug!(target: "downloaders::snap", %error, "Invalid account data"); - RequestError::BadResponse - }) - }) - .collect::, _>>()?; + let mut accounts = + response.accounts.into_iter().map(AccountData::into_trie_entry).collect::>(); let next = verify_proof(request, &accounts, &response.proof)?; // Authenticate the boundary account before removing it from the requested range. Once @@ -345,7 +336,7 @@ fn verify_proof( mod tests { use super::{request::MAX_RETRIES, test_utils::TestSnapClient, *}; use alloy_primitives::{Bytes, KECCAK256_EMPTY, U256}; - use reth_eth_wire_types::snap::{AccountData, ByteCodesMessage}; + use reth_eth_wire_types::snap::ByteCodesMessage; use reth_network_p2p::{error::PeerRequestResult, priority::Priority}; use reth_network_peers::WithPeerId; use reth_trie_common::{proof::ProofRetainer, HashBuilder, Nibbles}; diff --git a/crates/net/eth-wire-types/src/snap.rs b/crates/net/eth-wire-types/src/snap.rs index e86e1efb855..6e9641254df 100644 --- a/crates/net/eth-wire-types/src/snap.rs +++ b/crates/net/eth-wire-types/src/snap.rs @@ -8,7 +8,7 @@ use crate::BlockAccessLists; use alloc::vec::Vec; use alloy_primitives::{Bytes, B256, KECCAK256_EMPTY, U256}; -use alloy_rlp::{BufMut, Decodable, Encodable, RlpDecodable, RlpEncodable}; +use alloy_rlp::{BufMut, Decodable, Encodable, Header, RlpDecodable, RlpEncodable}; use alloy_trie::{TrieAccount, EMPTY_ROOT_HASH}; use reth_codecs_derive::add_arbitrary_tests; @@ -110,41 +110,26 @@ pub struct GetAccountRangeMessage { pub struct AccountData { /// Hash of the account address (trie path) pub hash: B256, - /// Account body in slim format - pub body: Bytes, + /// Account fields encoded as a nested RLP list with default hashes elided. + pub body: SlimAccountBody, } impl AccountData { - /// Encodes `account` in snap/2's slim format. + /// Converts `account` to snap/2's slim representation. pub fn from_trie_account(hash: B256, account: &TrieAccount) -> Self { - let body = alloy_rlp::encode(SlimAccountBodyRef { - nonce: account.nonce, - balance: account.balance, - storage_root: SlimAccountBodyRef::shorten(&account.storage_root, EMPTY_ROOT_HASH), - code_hash: SlimAccountBodyRef::shorten(&account.code_hash, KECCAK256_EMPTY), - }); - Self { hash, body: body.into() } + Self { hash, body: account.into() } } - /// Decodes the slim body into the account the trie leaf commits to. + /// Returns the account the trie leaf commits to. /// - /// Range proofs are verified against the full encoding, so the omitted storage root and code - /// hash are restored to their defaults here. - pub fn trie_account(&self) -> alloy_rlp::Result { - let slim = alloy_rlp::decode_exact::(&self.body)?; - - Ok(TrieAccount { - nonce: slim.nonce, - balance: slim.balance, - storage_root: SlimAccountBody::restore(&slim.storage_root, EMPTY_ROOT_HASH)?, - code_hash: SlimAccountBody::restore(&slim.code_hash, KECCAK256_EMPTY)?, - }) + /// Default storage roots and code hashes are restored when decoding the slim body. + pub const fn trie_account(&self) -> TrieAccount { + self.body.0 } /// Consumes the wire value and returns its hashed key with the decoded trie account. - pub fn into_trie_entry(self) -> alloy_rlp::Result<(B256, TrieAccount)> { - let account = self.trie_account()?; - Ok((self.hash, account)) + pub const fn into_trie_entry(self) -> (B256, TrieAccount) { + (self.hash, self.body.0) } } @@ -526,20 +511,20 @@ impl SnapProtocolMessage { } } -/// Like a trie account, with empty code and storage hashes omitted to reduce transfer size. -#[derive(RlpDecodable)] -struct SlimAccountBody { - /// The account's nonce. - nonce: u64, - /// The account's balance. - balance: U256, - /// Empty when the account has no storage. - storage_root: Bytes, - /// Empty when the account has no code. - code_hash: Bytes, -} +/// A trie account encoded with default storage and code hashes replaced by empty byte strings. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SlimAccountBody(TrieAccount); impl SlimAccountBody { + fn as_rlp(&self) -> SlimAccountBodyRef<'_> { + SlimAccountBodyRef { + nonce: self.0.nonce, + balance: self.0.balance, + storage_root: SlimAccountBodyRef::shorten(&self.0.storage_root, EMPTY_ROOT_HASH), + code_hash: SlimAccountBodyRef::shorten(&self.0.code_hash, KECCAK256_EMPTY), + } + } + /// Restores a dropped field to `empty`, rejecting any length the encoding never produces. fn restore(value: &[u8], empty: B256) -> alloy_rlp::Result { match value { @@ -549,6 +534,51 @@ impl SlimAccountBody { } } +impl From<&TrieAccount> for SlimAccountBody { + fn from(account: &TrieAccount) -> Self { + Self(*account) + } +} + +impl Encodable for SlimAccountBody { + fn encode(&self, out: &mut dyn BufMut) { + self.as_rlp().encode(out); + } + + fn length(&self) -> usize { + self.as_rlp().length() + } +} + +impl Decodable for SlimAccountBody { + fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { + let mut payload = Header::decode_bytes(buf, true)?; + let nonce = u64::decode(&mut payload)?; + let balance = U256::decode(&mut payload)?; + let storage_root = + Self::restore(Header::decode_bytes(&mut payload, false)?, EMPTY_ROOT_HASH)?; + let code_hash = Self::restore(Header::decode_bytes(&mut payload, false)?, KECCAK256_EMPTY)?; + if !payload.is_empty() { + return Err(alloy_rlp::Error::UnexpectedLength) + } + Ok(Self(TrieAccount { nonce, balance, storage_root, code_hash })) + } +} + +#[cfg(any(test, feature = "arbitrary"))] +impl<'a> arbitrary::Arbitrary<'a> for SlimAccountBody { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let storage_root = if u.arbitrary()? { u.arbitrary()? } else { EMPTY_ROOT_HASH }; + let code_hash = if u.arbitrary()? { u.arbitrary()? } else { KECCAK256_EMPTY }; + Ok(Self(TrieAccount { + nonce: u.arbitrary()?, + balance: u.arbitrary()?, + storage_root, + code_hash, + })) + } +} + /// Borrowed encode twin of [`SlimAccountBody`]. #[derive(RlpEncodable)] struct SlimAccountBodyRef<'a> { @@ -624,10 +654,10 @@ mod tests { test_roundtrip(SnapProtocolMessage::AccountRange(AccountRangeMessage { request_id: 42, - accounts: vec![AccountData { - hash: b256_from_u64(123), - body: Bytes::from(vec![1, 2, 3]), - }], + accounts: vec![AccountData::from_trie_account( + b256_from_u64(123), + &trie_account(EMPTY_ROOT_HASH, KECCAK256_EMPTY), + )], proof: vec![Bytes::from(vec![4, 5, 6])], })); @@ -888,17 +918,70 @@ mod tests { TrieAccount { nonce: 7, balance: U256::from(42), storage_root, code_hash } } + #[test] + fn account_range_matches_nested_account_wire_encoding() { + // [request_id, [[hash, [nonce, balance, storage_root, code_hash]]], proof] + let wire = alloy_primitives::hex!( + "ea01e7e6a00101010101010101010101010101010101010101010101010101010101010101c4072a8080c0" + ); + let message = AccountRangeMessage { + request_id: 1, + accounts: vec![AccountData::from_trie_account( + B256::repeat_byte(1), + &trie_account(EMPTY_ROOT_HASH, KECCAK256_EMPTY), + )], + proof: vec![], + }; + + assert_eq!(alloy_rlp::encode(&message), wire); + assert_eq!(alloy_rlp::decode_exact::(&wire).unwrap(), message); + } + + #[test] + fn account_data_rejects_byte_string_wrapped_body() { + let wire = alloy_primitives::hex!( + "e7a0010101010101010101010101010101010101010101010101010101010101010185c4072a8080" + ); + assert!(alloy_rlp::decode_exact::(&wire).is_err()); + } + + #[test] + fn account_data_consumes_only_its_own_list() { + let account = AccountData::from_trie_account( + B256::repeat_byte(1), + &trie_account(B256::repeat_byte(2), B256::repeat_byte(3)), + ); + let mut wire = alloy_rlp::encode(&account); + assert_eq!(wire.len(), account.length()); + wire.push(0x80); + let mut input = wire.as_slice(); + assert_eq!(AccountData::decode(&mut input).unwrap(), account); + assert_eq!(input, &[0x80]); + } + + #[test] + fn account_data_rejects_missing_truncated_and_extra_body_items() { + for wire in [ + "e1a00101010101010101010101010101010101010101010101010101010101010101", + "e4a00101010101010101010101010101010101010101010101010101010101010101c4072a", + "e7a00101010101010101010101010101010101010101010101010101010101010101c4072a808080", + ] { + let bytes = alloy_primitives::hex::decode(wire).unwrap(); + assert!(alloy_rlp::decode_exact::(&bytes).is_err()); + } + } + #[test] fn slim_body_elides_empty_storage_and_code() { let account = trie_account(EMPTY_ROOT_HASH, KECCAK256_EMPTY); let hash = B256::repeat_byte(1); let encoded = AccountData::from_trie_account(hash, &account); - let body = SlimAccountBody::decode(&mut encoded.body.as_ref()).unwrap(); - assert!(body.storage_root.is_empty()); - assert!(body.code_hash.is_empty()); - assert_eq!(encoded.trie_account().unwrap(), account); - assert_eq!(encoded.into_trie_entry().unwrap(), (hash, account)); + let body = alloy_rlp::encode(encoded.body); + assert_eq!(body, alloy_primitives::hex!("c4072a8080")); + assert_eq!(alloy_rlp::decode_exact::(&body).unwrap(), encoded.body); + assert_eq!(encoded.trie_account(), account); + assert_eq!(encoded.into_trie_entry(), (hash, account)); } #[test] @@ -906,28 +989,41 @@ mod tests { let account = trie_account(B256::repeat_byte(2), B256::repeat_byte(3)); let encoded = AccountData::from_trie_account(B256::repeat_byte(1), &account); - let body = SlimAccountBody::decode(&mut encoded.body.as_ref()).unwrap(); - assert_eq!(body.storage_root.len(), 32); - assert_eq!(body.code_hash.len(), 32); - assert_eq!(encoded.trie_account().unwrap(), account); + let body = alloy_rlp::encode(encoded.body); + assert_eq!(body, alloy_rlp::encode(account)); + assert_eq!(alloy_rlp::decode_exact::(&body).unwrap(), encoded.body); + assert_eq!(encoded.trie_account(), account); } - #[test] - fn slim_body_rejects_field_lengths_the_encoding_never_produces() { - // A 16-byte field is neither an elided default nor a hash, so accepting it would let a - // peer smuggle a value that hashes differently than the one it claims to serve. - let truncated = Bytes::from_static(&[0xaa; 16]); - - assert!(SlimAccountBody::restore(&truncated, EMPTY_ROOT_HASH).is_err()); + #[test_case(1)] + #[test_case(16)] + #[test_case(31)] + #[test_case(33)] + fn slim_body_rejects_invalid_hash_lengths(len: usize) { + let invalid = vec![0xaa; len]; + for (storage_root, code_hash) in + [(invalid.as_slice(), &[][..]), (&[][..], invalid.as_slice())] + { + let body = alloy_rlp::encode(SlimAccountBodyRef { + nonce: 7, + balance: U256::from(42), + storage_root, + code_hash, + }); + assert_eq!( + alloy_rlp::decode_exact::(&body), + Err(alloy_rlp::Error::UnexpectedLength) + ); + } } - #[test] - fn slim_body_rejects_trailing_bytes() { - let account = trie_account(EMPTY_ROOT_HASH, KECCAK256_EMPTY); - let mut encoded = AccountData::from_trie_account(B256::repeat_byte(1), &account); - encoded.body = [encoded.body.as_ref(), &[0x00]].concat().into(); - - assert!(encoded.trie_account().is_err()); + #[test_case("c0")] + #[test_case("c3072a80")] + #[test_case("c5072a808080")] + #[test_case("c4072a808000")] + fn slim_body_rejects_missing_and_extra_fields(wire: &str) { + let bytes = alloy_primitives::hex::decode(wire).unwrap(); + assert!(alloy_rlp::decode_exact::(&bytes).is_err()); } #[test] diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index ae1957ba56e..3a51bbae4c6 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -1273,8 +1273,14 @@ mod tests { Ok(SnapResponse::AccountRange(AccountRangeMessage { request_id: 1, accounts: vec![ - AccountData { hash: first_hash, body: full_body.into() }, - AccountData { hash: second_hash, body: empty_body }, + AccountData { + hash: first_hash, + body: alloy_rlp::decode_exact(&full_body).unwrap() + }, + AccountData { + hash: second_hash, + body: alloy_rlp::decode_exact(&empty_body).unwrap() + }, ], proof, })) diff --git a/crates/net/network/tests/it/snap/mod.rs b/crates/net/network/tests/it/snap/mod.rs index cd78270e634..b84d8d482ad 100644 --- a/crates/net/network/tests/it/snap/mod.rs +++ b/crates/net/network/tests/it/snap/mod.rs @@ -268,7 +268,7 @@ async fn account_range_roundtrip_carries_slim_encoding_and_proof() { assert_eq!(returned.len(), expected.len()); for (account, (expected_hash, expected_account)) in returned.iter().zip(&expected) { assert_eq!(&account.hash, expected_hash); - let decoded = account.trie_account().unwrap(); + let decoded = account.trie_account(); assert_eq!(decoded.nonce, expected_account.nonce); assert_eq!(decoded.balance, expected_account.balance); // Freshly generated EOAs have no storage/code, so the slim encoding elides both fields From d0cb379bea606268ac70383e1faf66da676c15b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Sat, 19 Sep 2026 01:11:04 +0200 Subject: [PATCH 017/133] fix(tasks): shut down the owned runtime even when clones are still alive (#27313) --- crates/cli/runner/src/lib.rs | 30 ++++++++++++-- crates/tasks/src/runtime.rs | 78 ++++++++++++++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/crates/cli/runner/src/lib.rs b/crates/cli/runner/src/lib.rs index 6de8430942e..68c3a5a98b5 100644 --- a/crates/cli/runner/src/lib.rs +++ b/crates/cli/runner/src/lib.rs @@ -298,14 +298,14 @@ const DEFAULT_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); /// Shut down the given [`Runtime`](reth_tasks::Runtime), and wait for it if `wait` is set. /// -/// Dropping the runtime on the current thread could block due to tokio pool teardown. -/// Instead, we drop it on a separate thread and optionally wait for completion. +/// Shuts down the owned tokio runtime even if tasks retain clones of the shared handle. +/// Pool teardown can block, so it runs on a separate thread. fn runtime_shutdown(rt: reth_tasks::Runtime, wait: bool) { let (tx, rx) = mpsc::channel(); std::thread::Builder::new() .name("rt-shutdown".to_string()) .spawn(move || { - drop(rt); + rt.shutdown_timeout(DEFAULT_RUNTIME_SHUTDOWN_TIMEOUT); let _ = tx.send(()); }) .unwrap(); @@ -316,3 +316,27 @@ fn runtime_shutdown(rt: reth_tasks::Runtime, wait: bool) { }); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn runtime_shutdown_stops_tasks_with_live_runtime_clones() { + let runtime = reth_tasks::Runtime::test(); + let retained = runtime.clone(); + let task_runtime = runtime.clone(); + let (started, ready) = mpsc::channel(); + let task = runtime.handle().spawn(async move { + let _runtime = task_runtime; + started.send(()).unwrap(); + std::future::pending::<()>().await; + }); + ready.recv_timeout(Duration::from_secs(5)).unwrap(); + + runtime_shutdown(runtime, true); + + assert!(task.is_finished(), "shutdown returned while a task still owned the runtime"); + assert!(retained.handle().block_on(task).unwrap_err().is_cancelled()); + } +} diff --git a/crates/tasks/src/runtime.rs b/crates/tasks/src/runtime.rs index 52af910bda5..666a40c6e77 100644 --- a/crates/tasks/src/runtime.rs +++ b/crates/tasks/src/runtime.rs @@ -256,8 +256,8 @@ pub enum RuntimeBuildError { // ── RuntimeInner ────────────────────────────────────────────────────── struct RuntimeInner { - /// Owned tokio runtime, if we built one. Kept alive via the `Arc`. - _tokio_runtime: Option, + /// Owned tokio runtime, taken during shutdown even while other handles remain alive. + tokio_runtime: Mutex>, /// Handle to the tokio runtime. handle: Handle, /// Receiver of the shutdown signal. @@ -304,6 +304,18 @@ struct RuntimeInner { task_manager_handle: Mutex>>>, } +impl Drop for RuntimeInner { + fn drop(&mut self) { + // The last handle can belong to an async task, including one running on this runtime. + // Without explicit shutdown, let blocking tasks finish in the background in that case. + if Handle::try_current().is_ok() && + let Some(runtime) = self.tokio_runtime.get_mut().unwrap().take() + { + runtime.shutdown_background(); + } + } +} + // ── Runtime ─────────────────────────────────────────────────────────── /// A cheaply cloneable handle to the runtime resources. @@ -823,6 +835,17 @@ impl Runtime { self.do_graceful_shutdown(Some(timeout)) } + /// Shuts down the owned tokio runtime for all clones, waiting at most `timeout` for blocking + /// tasks. Externally supplied runtimes are left running. + /// + /// Panics if called from an async context while an owned runtime remains. + pub fn shutdown_timeout(self, timeout: Duration) { + let runtime = self.0.tokio_runtime.lock().unwrap().take(); + if let Some(runtime) = runtime { + runtime.shutdown_timeout(timeout); + } + } + fn do_graceful_shutdown(&self, timeout: Option) -> bool { let _ = self.0.task_events_tx.send(TaskEvent::GracefulShutdown); let deadline = timeout.map(|t| Instant::now() + t); @@ -981,7 +1004,7 @@ impl RuntimeBuilder { }); let inner = RuntimeInner { - _tokio_runtime: owned_runtime, + tokio_runtime: Mutex::new(owned_runtime), handle, on_shutdown, task_events_tx, @@ -1048,6 +1071,55 @@ mod tests { let _ = runtime.handle(); } + #[tokio::test] + async fn last_runtime_clone_can_drop_on_its_worker() { + let runtime = RuntimeBuilder::new(Runtime::test_config()).build().unwrap(); + let task_runtime = runtime.clone(); + let (release, wait) = tokio::sync::oneshot::channel(); + let task = runtime.handle().spawn(async move { + wait.await.unwrap(); + drop(task_runtime); + }); + + thread::spawn(move || drop(runtime)).join().unwrap(); + release.send(()).unwrap(); + task.await.unwrap(); + } + + #[tokio::test] + async fn shutdown_with_live_runtime_clones() { + let runtime = RuntimeBuilder::new(Runtime::test_config()).build().unwrap(); + let retained = runtime.clone(); + let task_runtime = runtime.clone(); + let (started, ready) = tokio::sync::oneshot::channel(); + let task = runtime.handle().spawn(async move { + let _runtime = task_runtime; + started.send(()).unwrap(); + std::future::pending::<()>().await; + }); + ready.await.unwrap(); + + thread::spawn(move || runtime.shutdown_timeout(Duration::from_secs(5))).join().unwrap(); + + assert!(task.await.unwrap_err().is_cancelled()); + assert!(retained.handle().spawn(async {}).await.unwrap_err().is_cancelled()); + retained.shutdown_timeout(Duration::from_secs(5)); + } + + #[tokio::test] + async fn shutdown_preserves_external_runtime() { + let runtime = RuntimeBuilder::new( + Runtime::test_config().with_tokio(TokioConfig::existing_handle(Handle::current())), + ) + .build() + .unwrap(); + let retained = runtime.clone(); + + runtime.shutdown_timeout(Duration::from_secs(5)); + + assert_eq!(retained.handle().spawn(async { 42 }).await.unwrap(), 42); + } + #[test] fn critical_os_thread_uses_requested_name() { let runtime = Runtime::test(); From 334a4067aad15dac42e2bfa3594bfb665e2a9c43 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 19 Sep 2026 03:29:52 +0200 Subject: [PATCH 018/133] refactor(txpool): expose sender-id transaction lookups (#27316) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/transaction-pool/src/pool/mod.rs | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index f23a7669adc..3b1ec21404d 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -1221,6 +1221,14 @@ where sender: Address, ) -> Vec>> { let Some(sender_id) = self.sender_id(&sender) else { return Vec::new() }; + self.get_pending_transactions_by_sender_id(sender_id) + } + + /// Returns all pending transactions for a resolved sender ID. + pub fn get_pending_transactions_by_sender_id( + &self, + sender_id: SenderId, + ) -> Vec>> { self.get_pool_data().pending_txs_by_sender(sender_id) } @@ -1230,6 +1238,14 @@ where sender: Address, ) -> Option>> { let sender_id = self.sender_id(&sender)?; + self.get_highest_transaction_by_sender_id(sender_id) + } + + /// Returns the highest transaction for a resolved sender ID. + pub fn get_highest_transaction_by_sender_id( + &self, + sender_id: SenderId, + ) -> Option>> { self.get_pool_data().get_highest_transaction_by_sender(sender_id) } @@ -1240,6 +1256,15 @@ where on_chain_nonce: u64, ) -> Option>> { let sender_id = self.sender_id(&sender)?; + self.get_highest_consecutive_transaction_by_sender_id(sender_id, on_chain_nonce) + } + + /// Returns the highest consecutive transaction for a resolved sender ID and on-chain nonce. + pub fn get_highest_consecutive_transaction_by_sender_id( + &self, + sender_id: SenderId, + on_chain_nonce: u64, + ) -> Option>> { self.get_pool_data().get_highest_consecutive_transaction_by_sender( sender_id.into_transaction_id(on_chain_nonce), ) @@ -1771,6 +1796,20 @@ mod tests { assert_eq!(nonces(&txs.pending), [0, 1]); assert_eq!(nonces(&txs.queued), [9]); + let sender_id = pool.inner().sender_id(&sender).unwrap(); + assert_eq!(nonces(&pool.inner().get_pending_transactions_by_sender_id(sender_id)), [0, 1]); + assert_eq!( + pool.inner().get_highest_transaction_by_sender_id(sender_id).unwrap().nonce(), + 9 + ); + assert_eq!( + pool.inner() + .get_highest_consecutive_transaction_by_sender_id(sender_id, 0) + .unwrap() + .nonce(), + 1 + ); + // a higher base fee reclassifies the pending transactions as queued; the snapshot must // report each of them on exactly one side pool.set_block_info(BlockInfo { From 9d5861ba1dfe4c7c54f3fcc58c2b560d50b17215 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Sat, 19 Sep 2026 04:27:06 +0200 Subject: [PATCH 019/133] fix(ci): repin deny workflow for paradigmxyz STS support (#27312) Co-authored-by: grandizzy <38490174+grandizzy@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1176e354bcf..5f30c2d041a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -336,7 +336,7 @@ jobs: permissions: contents: read id-token: write - uses: tempoxyz/gh-actions/.github/workflows/rust-deny.yml@6cd9d3a68b9cb0b52c7a0fc9a523f4806921354f + uses: tempoxyz/gh-actions/.github/workflows/rust-deny.yml@d672d5e512852c71bdff2ba3bc48495fae05090b with: rust-toolchain: nightly From 2f46c5e2f91df484f03e770b3177049f7d83e8d9 Mon Sep 17 00:00:00 2001 From: CPerezz <37264926+CPerezz@users.noreply.github.com> Date: Sat, 19 Sep 2026 07:49:56 +0200 Subject: [PATCH 020/133] test(ef-tests): persist trie updates between blocks (#27307) --- testing/ef-tests/src/cases/blockchain_test.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index f48ccab2283..9bc59749f40 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -18,7 +18,7 @@ use reth_provider::{ test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, DatabaseProviderFactory, ExecutionOutcome, HashedPostStateProvider, HistoryWriter, OriginalValuesKnown, StateWriteConfig, StateWriter, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, - StorageSettingsCache, + StorageSettingsCache, TrieWriter, }; use reth_revm::database::StateProviderDatabase; use reth_trie::StateRoot; @@ -261,7 +261,7 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { .hashed_post_state(&output.state) .map_err(|err| Error::block_failed(block_number, err))?; let sorted = hashed_state.clone_into_sorted(); - let (computed_state_root, _) = reth_trie_db::with_adapter!(provider, |A| { + let (computed_state_root, trie_updates) = reth_trie_db::with_adapter!(provider, |A| { StateRoot::, _>::overlay_root_with_updates( provider.tx_ref(), &sorted, @@ -287,6 +287,10 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { provider .write_hashed_state(&hashed_state.into_sorted()) .map_err(|err| Error::block_failed(block_number, err))?; + // Persist the trie so later blocks read stored nodes. + provider + .write_trie_updates(trie_updates) + .map_err(|err| Error::block_failed(block_number, err))?; provider .update_history_indices(block.number..=block.number) .map_err(|err| Error::block_failed(block_number, err))?; From 4dd0cc021a7293b66659468f52957b90947e6252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Sat, 19 Sep 2026 08:17:47 +0000 Subject: [PATCH 021/133] feat(snap-sync): verify downloaded state against the pivot's state root (#27318) --- Cargo.lock | 3 + crates/snap-sync/Cargo.toml | 3 + crates/snap-sync/src/attempt.rs | 3 +- crates/snap-sync/src/error.rs | 15 +- crates/snap-sync/src/lib.rs | 2 + crates/snap-sync/src/verify.rs | 501 ++++++++++++++++++++++++++++++++ 6 files changed, 525 insertions(+), 2 deletions(-) create mode 100644 crates/snap-sync/src/verify.rs diff --git a/Cargo.lock b/Cargo.lock index f59ec947be7..795aa758aaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10418,6 +10418,9 @@ dependencies = [ "reth-network-peers", "reth-primitives-traits", "reth-provider", + "reth-stages", + "reth-stages-api", + "reth-stages-types", "reth-storage-api", "reth-storage-errors", "reth-tasks", diff --git a/crates/snap-sync/Cargo.toml b/crates/snap-sync/Cargo.toml index 56e660796e4..6d1470960fe 100644 --- a/crates/snap-sync/Cargo.toml +++ b/crates/snap-sync/Cargo.toml @@ -20,6 +20,7 @@ reth-network-p2p.workspace = true reth-network-peers.workspace = true reth-primitives-traits = { workspace = true, features = ["eip7928"] } reth-storage-api = { workspace = true, features = ["db-api"] } +reth-stages-types.workspace = true reth-storage-errors.workspace = true reth-tasks.workspace = true reth-trie-common.workspace = true @@ -47,4 +48,6 @@ reth-ethereum-primitives.workspace = true reth-evm.workspace = true reth-evm-ethereum.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } +reth-stages.workspace = true +reth-stages-api.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/snap-sync/src/attempt.rs b/crates/snap-sync/src/attempt.rs index afcb0b5b6c4..c26fff0901b 100644 --- a/crates/snap-sync/src/attempt.rs +++ b/crates/snap-sync/src/attempt.rs @@ -8,6 +8,7 @@ use crate::{CatchUpProgress, SnapGeneration, SnapSyncError}; use reth_storage_api::{ BlockHashReader, MetadataProvider, MetadataWriter, SnapAttempt, SnapAttemptId, StorageSettings, }; +use serde::{Deserialize, Serialize}; /// Persistence for the attempt that owns downloaded snap state. /// @@ -61,7 +62,7 @@ pub trait SnapAttemptStore { } /// What a write presents to prove it belongs to the attempt owning the persisted state. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct SnapWrite { // Attempt this write belongs to. attempt: SnapAttemptId, diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index b87b71ebb1b..2e48563d655 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -76,7 +76,8 @@ pub enum SnapSyncError { /// Block the pivot was moved to. target: u64, }, - /// Storage persisted ahead of a range was committed before catch-up reached the pivot. + /// Catch-up has not carried the downloaded state to the pivot, which committing storage + /// persisted ahead of its range or completing the attempt requires. #[error("catch-up applied block {applied}, the pivot is {pivot}")] CatchUpBehindPivot { /// Last block whose list is applied. @@ -169,6 +170,18 @@ pub enum SnapSyncError { /// Hash of the supplied code. got: B256, }, + /// Account ranges remain to be downloaded. + #[error("accounts from {next} are not downloaded yet")] + IncompleteAccounts { + /// Key the next range is requested from. + next: B256, + }, + /// The pivot is the genesis block, whose trie the merkle stage never rebuilds. + #[error("snap synchronization cannot anchor to the genesis block")] + GenesisPivot, + /// Work stopped because its session was cancelled. + #[error("snap synchronization was cancelled")] + Cancelled, } impl From for SnapSyncError { diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index 2610bb65073..c3eb8ec83b3 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -46,6 +46,7 @@ mod generation; mod pivot; mod session; mod storage; +mod verify; #[cfg(test)] mod test_utils; @@ -68,3 +69,4 @@ pub use storage::{ SnapStorageStore, StorageChunk, StorageProgress, StorageRangeDownload, StorageRangeStep, DEFAULT_STORAGE_ACCOUNTS, }; +pub use verify::{SnapStateVerifier, VerifiedSnapState, DEFAULT_SCAN_CHUNK}; diff --git a/crates/snap-sync/src/verify.rs b/crates/snap-sync/src/verify.rs new file mode 100644 index 00000000000..4e9938700c8 --- /dev/null +++ b/crates/snap-sync/src/verify.rs @@ -0,0 +1,501 @@ +//! Decides when downloaded state is complete, and accepts it once its trie matches the pivot. +//! +//! The trie itself is rebuilt by the merkle stage, which commits its progress in chunks and checks +//! the root against the target header, so a rebuild survives restarts on any state size. + +use crate::{ + common::SnapRecord, SnapAccountStore, SnapAttemptStore, SnapCatchUpStore, SnapSyncError, + SnapWrite, +}; +use alloy_eips::BlockNumHash; +use alloy_primitives::{B256, KECCAK256_EMPTY}; +use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx, RawKey, RawTable}; +use reth_primitives_traits::{AlloyBlockHeader, GotExpected}; +use reth_stages_types::{StageCheckpoint, StageId}; +use reth_storage_api::{ + BlockHashReader, DBProvider, HeaderProvider, MetadataProvider, MetadataWriter, SnapAttemptId, + StageCheckpointReader, StageCheckpointWriter, +}; +use reth_storage_errors::provider::{ProviderError, RootMismatch}; +use serde::{Deserialize, Serialize}; +use tokio_util::sync::CancellationToken; + +/// Accounts scanned between cancellation checks, bounding how long a cancelled session keeps +/// running. +pub const DEFAULT_SCAN_CHUNK: u64 = 100_000; + +/// Decides whether downloaded state can be trusted as the node's state. +/// +/// Writes join the caller's transaction, so a refused check changes nothing. +pub trait SnapStateVerifier { + /// Hands complete state to the merkle stage, which rebuilds its trie from scratch. + /// + /// Progress the stage recorded before the hand-off describes other state, so it is discarded. + /// The hand-off is recorded for this attempt and pivot, so only a rebuild of this state counts. + fn start_trie_rebuild( + &self, + write: SnapWrite, + chunk: u64, + cancel: &CancellationToken, + ) -> Result<(), SnapSyncError> + where + Self: MetadataWriter + StageCheckpointWriter + DBProvider; + + /// Refuses to finish while any downloaded state is still pending. Storage commits with its + /// accounts, so it needs no separate check. + fn verify_completeness( + &self, + write: SnapWrite, + chunk: u64, + cancel: &CancellationToken, + ) -> Result<(), SnapSyncError> + where + Self: DBProvider; + + /// Accepts the state once the merkle stage has rebuilt its trie up to the pivot. + /// + /// The stage checks the root against the pivot header, which must commit to the root the + /// state was downloaded against. + fn verify_state_root(&self, write: SnapWrite) -> Result + where + Self: BlockHashReader + HeaderProvider + MetadataWriter + StageCheckpointReader; +} + +/// Downloaded state whose trie root matched the header of the block it is anchored to. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VerifiedSnapState { + // Attempt that downloaded the state. + attempt: SnapAttemptId, + // Block whose header commits to the state. + target: BlockNumHash, + // Root the target header commits to. + state_root: B256, +} + +impl VerifiedSnapState { + /// Attempt that downloaded the state. + pub const fn attempt(&self) -> SnapAttemptId { + self.attempt + } + + /// Block whose header commits to the state. + pub const fn target(&self) -> BlockNumHash { + self.target + } + + /// Root the target header commits to. + pub const fn state_root(&self) -> B256 { + self.state_root + } +} + +impl SnapStateVerifier for T { + fn start_trie_rebuild( + &self, + write: SnapWrite, + chunk: u64, + cancel: &CancellationToken, + ) -> Result<(), SnapSyncError> + where + Self: MetadataWriter + StageCheckpointWriter + DBProvider, + { + // The stage treats a checkpoint at genesis as already rebuilt, so it would never check + // state anchored there. + if self.authorize_snap_write(write)?.pivot().number == 0 { + return Err(SnapSyncError::GenesisPivot) + } + self.verify_completeness(write, chunk, cancel)?; + // Starting from block zero makes the stage clear the trie tables and rebuild them, since + // downloaded state has no changesets to update an existing trie from. + self.save_stage_checkpoint(StageId::MerkleExecute, StageCheckpoint::default())?; + self.save_stage_checkpoint_progress(StageId::MerkleExecute, Vec::new())?; + StoredRebuild::new(write).write(self)?; + Ok(()) + } + + fn verify_completeness( + &self, + write: SnapWrite, + chunk: u64, + cancel: &CancellationToken, + ) -> Result<(), SnapSyncError> + where + Self: DBProvider, + { + let attempt = self.authorize_snap_write(write)?; + let coverage = self.account_coverage(write)?.ok_or(SnapSyncError::NoCoverage)?; + if let Some(next) = coverage.next() { + return Err(SnapSyncError::IncompleteAccounts { next }) + } + let applied = + self.catch_up_progress(write)?.ok_or(SnapSyncError::NoCatchUpProgress)?.applied(); + if applied != attempt.pivot() { + return Err(SnapSyncError::CatchUpBehindPivot { + applied: applied.number, + pivot: attempt.pivot().number, + }) + } + ensure_code_present(self.tx_ref(), chunk, cancel) + } + + fn verify_state_root(&self, write: SnapWrite) -> Result + where + Self: BlockHashReader + HeaderProvider + MetadataWriter + StageCheckpointReader, + { + let attempt = self.authorize_canonical_snap_write(write)?; + let target = attempt.pivot(); + let header = self + .sealed_header(target.number)? + .filter(|header| header.hash() == target.hash) + .ok_or(SnapSyncError::MissingHeader { block: target.number })?; + // The stage answers to the header, while downloaded ranges answered to the attempt. + if header.state_root() != attempt.state_root() { + return Err(ProviderError::StateRootMismatch(Box::new(RootMismatch { + root: GotExpected { got: attempt.state_root(), expected: header.state_root() }, + block_number: target.number, + block_hash: target.hash, + })) + .into()) + } + // Until the stage reaches the pivot after this state's hand-off, the trie holds no state + // for it: an earlier attempt's rebuild can end at the same block. + let handed_off = StoredRebuild::read(self)?.is_some_and(|stored| stored.write == write); + let rebuilt = self.get_stage_checkpoint(StageId::MerkleExecute)?; + if !handed_off || rebuilt.map(|checkpoint| checkpoint.block_number) != Some(target.number) { + return Err(ProviderError::StateForNumberNotFound(target.number).into()) + } + self.verify_snap_attempt(write)?; + Ok(VerifiedSnapState { attempt: attempt.id(), target, state_root: header.state_root() }) + } +} + +// The trie rebuild hand-off as persisted, tied to the write it was made for. +#[derive(Serialize, Deserialize)] +struct StoredRebuild { + // Encoding version, checked before the rest is decoded. + version: u32, + // Attempt and pivot the state was handed off at. + write: SnapWrite, +} + +impl SnapRecord for StoredRebuild { + const KEY: &'static str = "snap_trie_rebuild"; + const VERSION: u32 = 1; +} + +impl StoredRebuild { + // The hand-off of `write`'s state at this build's version. + const fn new(write: SnapWrite) -> Self { + Self { version: Self::VERSION, write } + } +} + +// Refuses the first account whose code is not stored, checking for cancellation every `chunk` +// accounts. +fn ensure_code_present( + tx: &impl DbTx, + chunk: u64, + cancel: &CancellationToken, +) -> Result<(), SnapSyncError> { + let chunk = chunk.max(1); + let mut cursor = tx.cursor_read::()?; + for (scanned, entry) in cursor.walk(None)?.enumerate() { + if (scanned as u64).is_multiple_of(chunk) && cancel.is_cancelled() { + return Err(SnapSyncError::Cancelled) + } + let (_, account) = entry?; + // Only presence matters, so stored code is not decoded. + if let Some(hash) = account.bytecode_hash.filter(|hash| *hash != KECCAK256_EMPTY) && + tx.get::>(RawKey::new(hash))?.is_none() + { + return Err(SnapSyncError::MissingCode { hash }) + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + test_utils::{account, hashed_factory, header, key, state_root}, + SnapGeneration, SnapStorageStore, StorageChunk, + }; + use alloy_primitives::{map::B256Map, Bytes, U256}; + use reth_db_api::transaction::DbTxMut; + use reth_primitives_traits::SealedHeader; + use reth_provider::{ + test_utils::{insert_headers, MockNodeTypesWithDB}, + DatabaseProviderFactory, ProviderFactory, + }; + use reth_stages::stages::MerkleStage; + use reth_stages_api::{ExecInput, Stage, StageError}; + use reth_trie_common::{root::storage_root_unsorted, HashedStorage, TrieAccount}; + use revm::bytecode::Bytecode; + + type Factory = ProviderFactory; + type Provider = ::ProviderRW; + + const CONTRACT: B256 = B256::repeat_byte(0xaa); + const SLOT: B256 = B256::repeat_byte(0x55); + + fn code() -> Bytecode { + Bytecode::new_raw(Bytes::from_static(&[0x60, 0x00])) + } + + fn storage() -> HashedStorage { + HashedStorage::from_iter([(SLOT, U256::from(7))]) + } + + // Two plain accounts and a contract with one slot and code. + fn accounts() -> Vec<(B256, TrieAccount)> { + let mut contract = account(3); + contract.storage_root = storage_root_unsorted([(SLOT, U256::from(7))]); + contract.code_hash = code().hash_slow(); + vec![(key(1), account(1)), (key(2), account(2)), (CONTRACT, contract)] + } + + // Blocks 0 through 2, each header committing to `header_root`. + fn insert_chain(factory: &Factory, header_root: B256) -> Vec { + let mut parent = B256::ZERO; + let headers: Vec<_> = (0..=2) + .map(|number| { + let mut header = header(number, parent, None); + header.state_root = header_root; + let sealed = SealedHeader::seal_slow(header); + parent = sealed.hash(); + sealed + }) + .collect(); + insert_headers(factory, &headers); + headers.iter().map(|header| BlockNumHash::new(header.number, header.hash())).collect() + } + + // An attempt pivoted at block 1 of a chain committing to `header_root`, with `served` of the + // accounts downloaded. + fn downloaded(header_root: B256, served: usize) -> (Factory, SnapWrite, Vec) { + let accounts = accounts(); + let factory = hashed_factory(); + let blocks = insert_chain(&factory, header_root); + let provider = factory.database_provider_rw().unwrap(); + let write = provider + .start_snap_attempt(SnapGeneration::new(blocks[1], state_root(&accounts))) + .unwrap(); + provider.start_account_coverage(write).unwrap(); + let targets = (served < accounts.len()).then(|| accounts[served - 1].0); + let range = + crate::test_utils::verified_range(&accounts, 0..served, B256::ZERO, targets.as_slice()); + let complete = served == accounts.len(); + let (storages, bytecodes) = if complete { + (B256Map::from_iter([(CONTRACT, storage())]), vec![(code().hash_slow(), code())]) + } else { + Default::default() + }; + provider.commit_account_range(write, &range, storages, bytecodes).unwrap(); + provider.commit().unwrap(); + (factory, write, blocks) + } + + fn start(provider: &Provider, write: SnapWrite) -> Result<(), SnapSyncError> { + provider.start_trie_rebuild(write, 1, &CancellationToken::new()) + } + + // Runs the merkle stage from its checkpoint to `target`, as the pipeline would. + fn run_merkle(provider: &Provider, target: u64) -> Result<(), StageError> { + let mut stage = MerkleStage::default_execution(); + loop { + let checkpoint = provider.get_stage_checkpoint(StageId::MerkleExecute).unwrap(); + let output = stage.execute(provider, ExecInput { target: Some(target), checkpoint })?; + provider.save_stage_checkpoint(StageId::MerkleExecute, output.checkpoint).unwrap(); + if output.done { + return Ok(()) + } + } + } + + fn merkle_checkpoint(provider: &Provider) -> Option { + provider.get_stage_checkpoint(StageId::MerkleExecute).unwrap().map(|c| c.block_number) + } + + #[test] + fn complete_state_is_verified_once_the_merkle_stage_reaches_the_pivot() { + let root = state_root(&accounts()); + let (factory, write, blocks) = downloaded(root, accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + + start(&provider, write).unwrap(); + run_merkle(&provider, 1).unwrap(); + let verified = provider.verify_state_root(write).unwrap(); + provider.commit().unwrap(); + + assert_eq!((verified.target(), verified.state_root()), (blocks[1], root)); + let provider = factory.database_provider_rw().unwrap(); + assert!(provider.snap_attempt().unwrap().unwrap().is_verified()); + // Verified state accepts no further writes, including a second verification. + assert!(matches!(provider.verify_state_root(write), Err(SnapSyncError::StaleWrite { .. }))); + } + + #[test] + fn unfinished_accounts_prevent_the_hand_off() { + let (factory, write, _) = downloaded(state_root(&accounts()), 1); + let provider = factory.database_provider_rw().unwrap(); + + assert!(matches!( + start(&provider, write), + Err(SnapSyncError::IncompleteAccounts { next }) if next == key(2) + )); + assert_eq!(merkle_checkpoint(&provider), None); + } + + #[test] + fn storage_persisted_ahead_of_its_range_prevents_the_hand_off() { + let (factory, write, _) = downloaded(state_root(&accounts()), 2); + let provider = factory.database_provider_rw().unwrap(); + let origin = provider.account_coverage(write).unwrap().unwrap().next().unwrap(); + let root = accounts()[2].1.storage_root; + let chunk = + StorageChunk::new(CONTRACT, root, B256::ZERO, vec![(SLOT, U256::from(7))], None); + provider.commit_storage_chunk(write, origin, chunk).unwrap(); + + assert!(matches!(start(&provider, write), Err(SnapSyncError::IncompleteAccounts { .. }))); + } + + #[test] + fn missing_code_prevents_the_hand_off() { + let (factory, write, _) = downloaded(state_root(&accounts()), accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + provider.tx_ref().delete::(code().hash_slow(), None).unwrap(); + + assert!(matches!( + start(&provider, write), + Err(SnapSyncError::MissingCode { hash }) if hash == code().hash_slow() + )); + } + + #[test] + fn catch_up_short_of_the_pivot_prevents_the_hand_off() { + let root = state_root(&accounts()); + let (factory, write, blocks) = downloaded(root, accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + let write = + provider.advance_snap_pivot(write, SnapGeneration::new(blocks[2], root)).unwrap(); + + assert!(matches!( + start(&provider, write), + Err(SnapSyncError::CatchUpBehindPivot { applied: 1, pivot: 2 }) + )); + } + + #[test] + fn a_cancelled_session_stops_the_completeness_scan() { + let (factory, write, _) = downloaded(state_root(&accounts()), accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + let cancel = CancellationToken::new(); + cancel.cancel(); + + assert!(matches!( + provider.start_trie_rebuild(write, 1, &cancel), + Err(SnapSyncError::Cancelled) + )); + assert_eq!(merkle_checkpoint(&provider), None); + } + + #[test] + fn corrupted_state_never_reaches_the_pivot() { + let (factory, write, _) = downloaded(state_root(&accounts()), accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + start(&provider, write).unwrap(); + provider.tx_ref().delete::(CONTRACT, None).unwrap(); + + // The stage refuses a root other than the header's. + assert!(matches!(run_merkle(&provider, 1), Err(StageError::Block { .. }))); + assert!(matches!( + provider.verify_state_root(write), + Err(SnapSyncError::Provider(ProviderError::StateForNumberNotFound(1))) + )); + assert!(provider.snap_attempt().unwrap().unwrap().is_unfinished()); + } + + #[test] + fn an_interrupted_rebuild_is_not_verified() { + let (factory, write, _) = downloaded(state_root(&accounts()), accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + // Progress recorded before the hand-off describes other state. + provider.save_stage_checkpoint(StageId::MerkleExecute, StageCheckpoint::new(1)).unwrap(); + + start(&provider, write).unwrap(); + + assert!(matches!( + provider.verify_state_root(write), + Err(SnapSyncError::Provider(ProviderError::StateForNumberNotFound(1))) + )); + run_merkle(&provider, 1).unwrap(); + provider.verify_state_root(write).unwrap(); + } + + #[test] + fn a_header_committing_to_another_root_is_refused() { + // Ranges authenticate against the attempt's root, but completion answers to the header. + let header_root = B256::repeat_byte(0xcc); + let (factory, write, _) = downloaded(header_root, accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + start(&provider, write).unwrap(); + + assert!(matches!(run_merkle(&provider, 1), Err(StageError::Block { .. }))); + match provider.verify_state_root(write) { + Err(SnapSyncError::Provider(ProviderError::StateRootMismatch(mismatch))) => { + assert_eq!( + mismatch.root, + GotExpected { got: state_root(&accounts()), expected: header_root } + ); + } + other => panic!("expected a state root mismatch, got {other:?}"), + } + } + + #[test] + fn a_rebuild_for_an_earlier_attempt_is_not_trusted() { + let root = state_root(&accounts()); + let (factory, write, blocks) = downloaded(root, accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + start(&provider, write).unwrap(); + run_merkle(&provider, 1).unwrap(); + + // A new attempt at the same pivot, with nothing downloaded yet. + let restarted = provider.start_snap_attempt(SnapGeneration::new(blocks[1], root)).unwrap(); + + assert_eq!(merkle_checkpoint(&provider), Some(1)); + assert!(matches!( + provider.verify_state_root(restarted), + Err(SnapSyncError::Provider(ProviderError::StateForNumberNotFound(1))) + )); + } + + #[test] + fn moving_the_pivot_after_the_hand_off_requires_a_new_one() { + let root = state_root(&accounts()); + let (factory, write, blocks) = downloaded(root, accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + start(&provider, write).unwrap(); + let advanced = + provider.advance_snap_pivot(write, SnapGeneration::new(blocks[2], root)).unwrap(); + + run_merkle(&provider, 2).unwrap(); + + assert!(matches!( + provider.verify_state_root(advanced), + Err(SnapSyncError::Provider(ProviderError::StateForNumberNotFound(2))) + )); + } + + #[test] + fn a_genesis_pivot_is_not_handed_off() { + let root = state_root(&accounts()); + let (factory, _, blocks) = downloaded(root, accounts().len()); + let provider = factory.database_provider_rw().unwrap(); + let write = provider.start_snap_attempt(SnapGeneration::new(blocks[0], root)).unwrap(); + + assert!(matches!(start(&provider, write), Err(SnapSyncError::GenesisPivot))); + assert_eq!(merkle_checkpoint(&provider), None); + } +} From 1136e1140023de2468e395a2633cf90cd3d03780 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Sat, 19 Sep 2026 16:31:53 +0000 Subject: [PATCH 022/133] fix(chain-state): preserve account reverts after empty test blocks (#27317) Co-authored-by: Amp --- crates/chain-state/src/test_utils.rs | 44 +++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/crates/chain-state/src/test_utils.rs b/crates/chain-state/src/test_utils.rs index 18fb6b5d43f..527e53091fc 100644 --- a/crates/chain-state/src/test_utils.rs +++ b/crates/chain-state/src/test_utils.rs @@ -251,11 +251,8 @@ impl TestBlockBuilder { let single_cost = Self::single_tx_cost(); // Look up parent's post-block state for correct revert construction. - let (pre_info, old_slot_value) = self - .post_block_state - .get(&parent_hash) - .cloned() - .unwrap_or_else(|| (initial_info.clone(), U256::ZERO)); + let (pre_info, old_slot_value) = + self.post_block_state.get(&parent_hash).cloned().unwrap_or((initial_info, U256::ZERO)); let mut final_balance = pre_info.balance; for _ in 0..num_txs { @@ -265,10 +262,11 @@ impl TestBlockBuilder { let post_info = AccountInfo { nonce: final_nonce, balance: final_balance, ..Default::default() }; - let account_revert = if pre_info.balance == initial_info.balance && pre_info.nonce == 0 { - Some(None) - } else { + // An empty parent creates the signer without changing its balance or nonce. + let account_revert = if self.post_block_state.contains_key(&parent_hash) { Some(Some(pre_info)) + } else { + Some(None) }; let new_slot_value = U256::from(block_number).wrapping_add(U256::from(1)); @@ -458,3 +456,33 @@ impl CanonStateSubscriptions for TestCanonStateSubscriptions { canon_notif_rx } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_revert_after_empty_parent() { + let mut builder = TestBlockBuilder::eth().with_state(); + let parent_hash = B256::repeat_byte(1); + let initial_info = AccountInfo::from_balance(U256::from(10).pow(U256::from(18))); + + // An empty parent still creates the signer with its initial balance and zero nonce. + // Seed that state directly so the test doesn't depend on random transaction counts. + builder.post_block_state.insert(parent_hash, (initial_info.clone(), U256::from(2))); + + for (parent_hash, expected_account) in + [(parent_hash, Some(initial_info)), (B256::repeat_byte(2), None)] + { + let block = builder.get_executed_block_with_number(2, parent_hash); + let mut state = block.execution_outcome().state.clone(); + state.revert(1); + + assert_eq!( + state.state.get(&builder.signer).and_then(|account| account.info.clone()), + expected_account, + "reverting must restore the signer's existence in the specified parent" + ); + } + } +} From 0d672728e1fe10df4530ec927ef65778d806b75f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 20 Sep 2026 07:39:44 +0200 Subject: [PATCH 023/133] feat(txpool): track common validation rejection reasons (#27321) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/transaction-pool/src/metrics.rs | 22 ++++++++++++++++++++- crates/transaction-pool/src/validate/eth.rs | 17 +++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/metrics.rs b/crates/transaction-pool/src/metrics.rs index b683ccf672a..0ba69ac2f4b 100644 --- a/crates/transaction-pool/src/metrics.rs +++ b/crates/transaction-pool/src/metrics.rs @@ -139,12 +139,32 @@ pub struct AllTransactionsMetrics { pub base_fee: Gauge, } -/// Transaction pool validation metrics +/// Transaction pool validation metrics. +/// +/// Rejection counters cover selected validation checks and count repeated attempts for the same +/// transaction separately. #[derive(Metrics)] #[metrics(scope = "transaction_pool")] pub struct TxPoolValidationMetrics { /// How long to successfully validate a blob pub blob_validation_duration: Histogram, + /// Number of stateless validation attempts rejected due to transaction data exceeding the size + /// limit. + pub rejected_oversized_data: Counter, + /// Number of stateless validation attempts rejected due to a priority fee exceeding the max + /// fee per gas. + pub rejected_tip_above_fee_cap: Counter, + /// Number of stateless validation attempts rejected due to a local transaction fee exceeding + /// the configured fee cap. + pub rejected_exceeds_fee_cap: Counter, + /// Number of stateless validation attempts rejected due to a priority fee below the configured + /// minimum. + pub rejected_priority_fee_below_minimum: Counter, + /// Number of stateless validation attempts rejected due to a gas limit below the intrinsic or + /// floor gas cost. + pub rejected_intrinsic_gas_too_low: Counter, + /// Number of validation attempts rejected by blob count or blob sidecar checks. + pub invalid_4844: Counter, } /// Transaction pool validator task metrics diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index 25339f13b48..ecea4e38a59 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -504,6 +504,7 @@ where .unwrap_or_default(), ); if tx_size > self.max_tx_input_bytes { + self.validation_metrics.rejected_oversized_data.increment(1); return Err(InvalidPoolTransactionError::OversizedData { size: tx_size, limit: self.max_tx_input_bytes, @@ -513,6 +514,7 @@ where // ensure the size of the non-blob transaction let tx_size = transaction.encoded_length(); if tx_size > self.max_tx_input_bytes { + self.validation_metrics.rejected_oversized_data.increment(1); return Err(InvalidPoolTransactionError::OversizedData { size: tx_size, limit: self.max_tx_input_bytes, @@ -549,6 +551,7 @@ where // Ensure max_priority_fee_per_gas (if EIP1559) is less than max_fee_per_gas if any. if transaction.max_priority_fee_per_gas() > Some(transaction.max_fee_per_gas()) { + self.validation_metrics.rejected_tip_above_fee_cap.increment(1); return Err(InvalidTransactionError::TipAboveFeeCap.into()) } @@ -563,6 +566,7 @@ where Some(tx_fee_cap_wei) => { let max_tx_fee_wei = transaction.cost().saturating_sub(transaction.value()); if max_tx_fee_wei > tx_fee_cap_wei { + self.validation_metrics.rejected_exceeds_fee_cap.increment(1); return Err(InvalidPoolTransactionError::ExceedsFeeCap { max_tx_fee_wei: max_tx_fee_wei.saturating_to(), tx_fee_cap_wei, @@ -577,6 +581,7 @@ where if transaction.is_dynamic_fee() && transaction.max_priority_fee_per_gas() < self.minimum_priority_fee { + self.validation_metrics.rejected_priority_fee_below_minimum.increment(1); return Err(InvalidPoolTransactionError::PriorityFeeBelowMinimum { minimum_priority_fee: self .minimum_priority_fee @@ -602,7 +607,9 @@ where } } - ensure_intrinsic_gas(transaction, &self.fork_tracker)?; + ensure_intrinsic_gas(transaction, &self.fork_tracker).inspect_err(|_| { + self.validation_metrics.rejected_intrinsic_gas_too_low.increment(1); + })?; // light blob tx pre-checks if transaction.is_eip4844() { @@ -614,6 +621,7 @@ where let blob_count = transaction.blob_count().unwrap_or(0); if blob_count == 0 { // no blobs + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidPoolTransactionError::Eip4844( Eip4844PoolTransactionError::NoEip4844Blobs, )) @@ -621,6 +629,7 @@ where let max_blob_count = self.fork_tracker.max_blob_count(); if blob_count > max_blob_count { + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidPoolTransactionError::Eip4844( Eip4844PoolTransactionError::TooManyEip4844Blobs { have: blob_count, @@ -803,6 +812,7 @@ where match transaction.take_blob() { EthBlobTransactionSidecar::None => { // this should not happen + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidTransactionError::TxTypeNotSupported.into()) } EthBlobTransactionSidecar::Missing => { @@ -813,6 +823,7 @@ where if self.blob_store.contains(*transaction.hash()).is_ok_and(|c| c) { // validated transaction is already in the store } else { + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidPoolTransactionError::Eip4844( Eip4844PoolTransactionError::MissingEip4844BlobSidecar, )) @@ -826,11 +837,13 @@ where // Standard Ethereum behavior if self.fork_tracker.is_osaka_activated() { if sidecar.is_eip4844() { + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidPoolTransactionError::Eip4844( Eip4844PoolTransactionError::UnexpectedEip4844SidecarAfterOsaka, )) } } else if sidecar.is_eip7594() && !self.allow_7594_sidecars() { + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidPoolTransactionError::Eip4844( Eip4844PoolTransactionError::UnexpectedEip7594SidecarBeforeOsaka, )) @@ -838,6 +851,7 @@ where } else { // EIP-7594 disabled: always reject v1 sidecars, accept v0 if sidecar.is_eip7594() { + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidPoolTransactionError::Eip4844( Eip4844PoolTransactionError::Eip7594SidecarDisallowed, )) @@ -846,6 +860,7 @@ where // validate the blob if let Err(err) = transaction.validate_blob(&sidecar, self.kzg_settings.get()) { + self.validation_metrics.invalid_4844.increment(1); return Err(InvalidPoolTransactionError::Eip4844( Eip4844PoolTransactionError::InvalidEip4844Blob(err), )) From 13074957e00cb5e65c984de1853e80305d4eaa7e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 20 Sep 2026 07:55:47 +0200 Subject: [PATCH 024/133] chore(ci): use shared Cyclops audit comment action (#27326) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .github/workflows/cyclops-audit.yml | 307 ++-------------------------- 1 file changed, 13 insertions(+), 294 deletions(-) diff --git a/.github/workflows/cyclops-audit.yml b/.github/workflows/cyclops-audit.yml index d4e3eaef337..909e73dcbf4 100644 --- a/.github/workflows/cyclops-audit.yml +++ b/.github/workflows/cyclops-audit.yml @@ -48,306 +48,25 @@ jobs: github.event.issue.pull_request && ( startsWith(github.event.comment.body, 'cyclops audit') || + startsWith(github.event.comment.body, 'cyclops private audit') || startsWith(github.event.comment.body, '@decofe cyclops audit') || + startsWith(github.event.comment.body, '@decofe cyclops private audit') || startsWith(github.event.comment.body, 'derek audit') ) runs-on: ubuntu-latest permissions: + contents: read issues: write - pull-requests: write + pull-requests: read steps: - - name: Check commenter permission - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + - name: Handle audit command + uses: tempoxyz/gh-actions/actions/pr-audit-comment@7741a8658354abd422dd1b941d07f809682ca96d with: + command-regex: '^(?:@decofe\s+)?(?:cyclops\s+(?:private\s+)?audit|derek\s+audit)\b' + permission-check-mode: association + allowed-associations: OWNER,MEMBER,COLLABORATOR + organization: paradigmxyz + events-key: ${{ secrets.EVENTS_KEY }} + events-cert: ${{ secrets.EVENTS_CERT }} + events-args: ${{ secrets.EVENTS_ARGS }} github-token: ${{ github.token }} - script: | - const allowed = new Set(['OWNER', 'MEMBER', 'COLLABORATOR']); - const commenterAssociation = context.payload.comment.author_association; - if (!allowed.has(commenterAssociation)) { - core.setFailed(`@${context.payload.comment.user.login} is not allowed to trigger Cyclops audits (${commenterAssociation})`); - return; - } - - const trustedPermissions = new Set(['admin', 'maintain', 'write']); - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - if (!allowed.has(pr.author_association)) { - const { data: authorPermission } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner: context.repo.owner, - repo: context.repo.repo, - username: pr.user.login, - }); - if (!trustedPermissions.has(authorPermission.permission)) { - core.setFailed(`PR author @${pr.user.login} is not allowed to trigger Cyclops audits (${pr.author_association}, ${authorPermission.permission})`); - return; - } - } - - - name: Parse arguments - id: args - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ github.token }} - script: | - const usage = [ - '**Usage:** `cyclops audit [fast] [perf] [iterations=N] [hours=N] [config=pr-review.yaml] ', - '[models="anthropic/claude-opus-4-7,openai/gpt-5.5"] [run-label=LABEL] ', - '[dry-run] [note="per-run audit guidance"]`', - ].join(''); - const body = context.payload.comment.body.trim(); - const prefix = /^(?:@decofe\s+)?(?:cyclops\s+audit|derek\s+audit)\b/i; - const args = body.replace(prefix, '').trim(); - const parts = []; - const argRegex = /(\S+?[=:]"[^"]*"|\S+?[=:]'[^']*'|\S+?[=:]\S+|\S+)/g; - let match; - while ((match = argRegex.exec(args)) !== null) parts.push(match[1]); - - const defaults = { - config: '', - iterations: '', - hours: '', - models: '', - 'run-label': '', - 'dry-run': 'false', - perf: 'false', - note: '', - }; - const intArgs = new Set(['iterations', 'hours']); - const stringArgs = new Set(['config', 'models', 'run-label', 'note']); - const boolArgs = new Set(['dry-run', 'perf']); - const unknown = []; - const invalid = []; - - for (const part of parts) { - if (part === 'fast') { - defaults.iterations = '1'; - continue; - } - - const eq = part.indexOf('='); - const colon = part.indexOf(':'); - const sep = eq === -1 ? colon : colon === -1 ? eq : Math.min(eq, colon); - if (sep === -1) { - if (boolArgs.has(part)) { - defaults[part] = 'true'; - } else { - unknown.push(part); - } - continue; - } - - const key = part.slice(0, sep); - let value = part.slice(sep + 1); - if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { - value = value.slice(1, -1); - } - - if (intArgs.has(key)) { - if (!/^[1-9]\d*$/.test(value)) { - invalid.push(`\`${key}=${value}\` (must be a positive integer)`); - } else { - defaults[key] = value; - } - } else if (boolArgs.has(key)) { - if (value === 'true' || value === 'false') { - defaults[key] = value; - } else { - invalid.push(`\`${key}=${value}\` (must be true or false)`); - } - } else if (stringArgs.has(key)) { - if (!value) { - invalid.push(`\`${key}=\` (must not be empty)`); - } else { - defaults[key] = value; - } - } else { - unknown.push(key); - } - } - - const errors = []; - if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``); - if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`); - if (errors.length) { - const msg = `Invalid cyclops audit command\n\n${errors.join('\n')}\n\n${usage}`; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: msg, - }); - core.setFailed(msg); - return; - } - - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - - const data = { - pr_number: context.issue.number, - sha: pr.head.sha, - source: 'comment', - actor: context.payload.comment.user.login, - comment_id: context.payload.comment.id, - dry_run: defaults['dry-run'] === 'true', - }; - if (defaults.config) data.config = defaults.config; - if (defaults.iterations) data.max_iterations = Number(defaults.iterations); - if (defaults.hours) data.max_hours = Number(defaults.hours); - if (defaults.models) data.models = defaults.models; - if (defaults['run-label']) data.run_label = defaults['run-label']; - if (defaults.note) data.audit_note_b64 = Buffer.from(defaults.note, 'utf8').toString('base64'); - if (defaults.perf === 'true') data.perf = true; - - const payload = { - repository: `${context.repo.owner}/${context.repo.repo}`, - event: 'pr_audit', - data, - }; - - const summaryParts = [ - defaults.config ? `config: \`${defaults.config}\`` : 'config: `default`', - defaults.iterations ? `iterations: \`${defaults.iterations}\`` : 'iterations: `default`', - defaults.hours ? `hours: \`${defaults.hours}\`` : 'hours: `default`', - ]; - if (defaults.models) summaryParts.push(`models: \`${defaults.models}\``); - if (defaults['run-label']) summaryParts.push(`run-label: \`${defaults['run-label']}\``); - if (defaults['dry-run'] === 'true') summaryParts.push('dry-run: `true`'); - if (defaults.perf === 'true') summaryParts.push('perf: `true`'); - if (defaults.note) { - const note = defaults.note.replace(/`/g, "'").slice(0, 160); - summaryParts.push(`note: \`${note}${defaults.note.length > 160 ? '...' : ''}\``); - } - - core.setOutput('actor', context.payload.comment.user.login); - core.setOutput('payload-b64', Buffer.from(JSON.stringify(payload), 'utf8').toString('base64')); - core.setOutput('summary', `**Config:** ${summaryParts.join(', ')}`); - - - name: Check publisher configuration - id: publisher-config - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - ACTOR: ${{ steps.args.outputs.actor }} - HAS_EVENTS_ARGS: ${{ secrets.EVENTS_ARGS != '' }} - HAS_EVENTS_KEY: ${{ secrets.EVENTS_KEY != '' }} - HAS_EVENTS_CERT: ${{ secrets.EVENTS_CERT != '' }} - with: - github-token: ${{ github.token }} - script: | - const required = { - EVENTS_ARGS: process.env.HAS_EVENTS_ARGS, - EVENTS_KEY: process.env.HAS_EVENTS_KEY, - EVENTS_CERT: process.env.HAS_EVENTS_CERT, - }; - const missing = Object.entries(required).filter(([, present]) => present !== 'true').map(([name]) => name); - if (!missing.length) return; - - const msg = `Cyclops audit is not configured: missing ${missing.map((name) => `\`${name}\``).join(', ')} secret${missing.length === 1 ? '' : 's'}.`; - try { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: `cc @${process.env.ACTOR}\n\n${msg}`, - }); - } catch (error) { - core.warning(`Could not create configuration failure comment: ${error.message}`); - } - core.setFailed(msg); - - - name: Acknowledge request - id: ack - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - ACTOR: ${{ steps.args.outputs.actor }} - SUMMARY: ${{ steps.args.outputs.summary }} - with: - github-token: ${{ github.token }} - script: | - try { - await github.rest.reactions.createForIssueComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: context.payload.comment.id, - content: 'eyes', - }); - } catch (error) { - core.warning(`Could not add acknowledgement reaction: ${error.message}`); - } - - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - try { - const { data: comment } = await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: `cc @${process.env.ACTOR}\n\nCyclops audit event queued. [View workflow run](${runUrl})\n\n${process.env.SUMMARY}`, - }); - core.setOutput('comment-id', String(comment.id)); - } catch (error) { - core.warning(`Could not create acknowledgement comment: ${error.message}`); - core.setOutput('comment-id', ''); - } - - - name: Publish event - id: publish - continue-on-error: true - env: - PAYLOAD_B64: ${{ steps.args.outputs.payload-b64 }} - HAS_EVENTS_ARGS: ${{ secrets.EVENTS_ARGS != '' }} - HAS_EVENTS_KEY: ${{ secrets.EVENTS_KEY != '' }} - HAS_EVENTS_CERT: ${{ secrets.EVENTS_CERT != '' }} - run: | - set -euo pipefail - - [[ "$HAS_EVENTS_ARGS" == "true" ]] || { echo "::error::Missing EVENTS_ARGS secret"; exit 1; } - [[ "$HAS_EVENTS_KEY" == "true" ]] || { echo "::error::Missing EVENTS_KEY secret"; exit 1; } - [[ "$HAS_EVENTS_CERT" == "true" ]] || { echo "::error::Missing EVENTS_CERT secret"; exit 1; } - - printf '%s' '${{ secrets.EVENTS_KEY }}' > "${RUNNER_TEMP}/key" - printf '%s' '${{ secrets.EVENTS_CERT }}' > "${RUNNER_TEMP}/cert" - printf '%s' "$PAYLOAD_B64" | base64 --decode > "${RUNNER_TEMP}/pr-audit-event.json" - - # EVENTS_ARGS may contain additional curl arguments, not just a URL. - curl -sf -o /dev/null -X POST ${{ secrets.EVENTS_ARGS }} \ - -H "Content-Type: application/json" \ - --key "${RUNNER_TEMP}/key" \ - --cert "${RUNNER_TEMP}/cert" \ - -d @"${RUNNER_TEMP}/pr-audit-event.json" - - - name: Update status - if: ${{ always() && steps.args.outcome == 'success' && steps.publisher-config.outcome == 'success' }} - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - COMMENT_ID: ${{ steps.ack.outputs['comment-id'] }} - PUBLISH_OUTCOME: ${{ steps.publish.outcome }} - ACTOR: ${{ steps.args.outputs.actor }} - SUMMARY: ${{ steps.args.outputs.summary }} - with: - github-token: ${{ github.token }} - script: | - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const success = process.env.PUBLISH_OUTCOME === 'success'; - const body = success - ? `cc @${process.env.ACTOR}\n\nCyclops audit event published. [View workflow run](${runUrl})\n\n${process.env.SUMMARY}` - : `cc @${process.env.ACTOR}\n\nCyclops audit event failed to publish. [View workflow run](${runUrl})\n\n${process.env.SUMMARY}`; - - if (process.env.COMMENT_ID) { - try { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: Number(process.env.COMMENT_ID), - body, - }); - } catch (error) { - core.warning(`Could not update acknowledgement comment: ${error.message}`); - } - } - if (!success) core.setFailed('Failed to publish pr_audit event'); From 4175485c7c65a6041e2eb2cbe746442d1232f60c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 20 Sep 2026 08:18:25 +0200 Subject: [PATCH 025/133] fix(ci): restore pull-requests write for Cyclops audit comments (#27327) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .github/workflows/cyclops-audit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cyclops-audit.yml b/.github/workflows/cyclops-audit.yml index 909e73dcbf4..52e7db317b9 100644 --- a/.github/workflows/cyclops-audit.yml +++ b/.github/workflows/cyclops-audit.yml @@ -57,7 +57,7 @@ jobs: permissions: contents: read issues: write - pull-requests: read + pull-requests: write steps: - name: Handle audit command uses: tempoxyz/gh-actions/actions/pr-audit-comment@7741a8658354abd422dd1b941d07f809682ca96d From 8bc57dec147ee7d6dcba7968ce065f51e7624836 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sun, 20 Sep 2026 06:15:21 +0000 Subject: [PATCH 026/133] perf(net): avoid boxed FetchClient request futures (#27323) --- crates/net/downloaders/src/file_client.rs | 22 ++++++------- crates/net/network/src/fetch/client.rs | 40 ++++++++++------------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/crates/net/downloaders/src/file_client.rs b/crates/net/downloaders/src/file_client.rs index 3e3348b1a8e..a969c17b69d 100644 --- a/crates/net/downloaders/src/file_client.rs +++ b/crates/net/downloaders/src/file_client.rs @@ -2,14 +2,14 @@ use alloy_consensus::BlockHeader; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{BlockHash, BlockNumber, Sealable, B256}; use async_compression::tokio::bufread::GzipDecoder; -use futures::Future; +use futures::{future, Future}; use itertools::{Either, Itertools}; use reth_consensus::{Consensus, ConsensusError}; use reth_network_p2p::{ - bodies::client::{BodiesClient, BodiesFut}, + bodies::client::BodiesClient, download::DownloadClient, - error::RequestError, - headers::client::{HeadersClient, HeadersDirection, HeadersFut, HeadersRequest}, + error::{PeerRequestResult, RequestError}, + headers::client::{HeadersClient, HeadersDirection, HeadersRequest}, priority::Priority, BlockClient, }; @@ -338,7 +338,7 @@ impl> FromReader impl HeadersClient for FileClient { type Header = B::Header; - type Output = HeadersFut; + type Output = future::Ready>>; fn get_headers_with_priority( &self, @@ -354,7 +354,7 @@ impl HeadersClient for FileClient { Some(num) => *num, None => { warn!(%hash, "Could not find starting block number for requested header hash"); - return Box::pin(async move { Err(RequestError::BadResponse) }) + return future::ready(Err(RequestError::BadResponse)) } }, BlockHashOrNumber::Number(num) => num, @@ -378,18 +378,18 @@ impl HeadersClient for FileClient { Some(header) => headers.push(header), None => { warn!(number=%block_number, "Could not find header"); - return Box::pin(async move { Err(RequestError::BadResponse) }) + return future::ready(Err(RequestError::BadResponse)) } } } - Box::pin(async move { Ok((PeerId::default(), headers).into()) }) + future::ready(Ok((PeerId::default(), headers).into())) } } impl BodiesClient for FileClient { type Body = B::Body; - type Output = BodiesFut; + type Output = future::Ready>>; fn get_block_bodies_with_priority_and_range_hint( &self, @@ -405,11 +405,11 @@ impl BodiesClient for FileClient { for hash in hashes { match self.bodies.get(&hash).cloned() { Some(body) => bodies.push(body), - None => return Box::pin(async move { Err(RequestError::BadResponse) }), + None => return future::ready(Err(RequestError::BadResponse)), } } - Box::pin(async move { Ok((PeerId::default(), bodies).into()) }) + future::ready(Ok((PeerId::default(), bodies).into())) } } diff --git a/crates/net/network/src/fetch/client.rs b/crates/net/network/src/fetch/client.rs index 345d07c7bdc..4f99f817377 100644 --- a/crates/net/network/src/fetch/client.rs +++ b/crates/net/network/src/fetch/client.rs @@ -11,12 +11,12 @@ use reth_eth_wire_types::snap::{ use reth_network_api::test_utils::PeersHandle; use reth_network_p2p::{ block_access_lists::client::{BalRequirement, BlockAccessListsClient}, - bodies::client::{BodiesClient, BodiesFut}, + bodies::client::BodiesClient, download::DownloadClient, error::{PeerRequestResult, RequestError}, headers::client::{HeadersClient, HeadersRequest}, priority::Priority, - receipts::client::{ReceiptsClient, ReceiptsFut}, + receipts::client::{ReceiptsClient, ReceiptsResponse}, snap::client::{SnapClient, SnapResponse}, BlockClient, }; @@ -64,24 +64,22 @@ impl FetchClient { &self, request: SnapProtocolMessage, priority: Priority, - ) -> std::pin::Pin> + Send + Sync>> - { + ) -> FetchClientFuture> { let (response, rx) = oneshot::channel(); if self.request_tx.send(DownloadRequest::GetSnap { request, response, priority }).is_ok() { - Box::pin(FlattenedResponse::from(rx)) + Either::Left(FlattenedResponse::from(rx)) } else { - Box::pin(future::err(RequestError::ChannelClosed)) + Either::Right(future::err(RequestError::ChannelClosed)) } } } -// The `Output` future of the [HeadersClient] impl of [FetchClient] that either returns a response -// or an error. -type HeadersClientFuture = Either, future::Ready>; +/// A request future that either waits for a network response or reports a closed request channel. +type FetchClientFuture = Either, future::Ready>; impl HeadersClient for FetchClient { type Header = N::BlockHeader; - type Output = HeadersClientFuture>>; + type Output = FetchClientFuture>>; /// Sends a `GetBlockHeaders` request to an available peer. fn get_headers_with_priority( @@ -104,7 +102,7 @@ impl HeadersClient for FetchClient { impl BodiesClient for FetchClient { type Body = N::BlockBody; - type Output = BodiesFut; + type Output = FetchClientFuture>>; /// Sends a `GetBlockBodies` request to an available peer. fn get_block_bodies_with_priority_and_range_hint( @@ -119,16 +117,16 @@ impl BodiesClient for FetchClient { .send(DownloadRequest::GetBlockBodies { request, response, priority, range_hint }) .is_ok() { - Box::pin(FlattenedResponse::from(rx)) + Either::Left(FlattenedResponse::from(rx)) } else { - Box::pin(future::err(RequestError::ChannelClosed)) + Either::Right(future::err(RequestError::ChannelClosed)) } } } impl ReceiptsClient for FetchClient { type Receipt = N::Receipt; - type Output = ReceiptsFut; + type Output = FetchClientFuture>>; fn get_receipts_with_priority(&self, request: Vec, priority: Priority) -> Self::Output { let (response, rx) = oneshot::channel(); @@ -137,9 +135,9 @@ impl ReceiptsClient for FetchClient { .send(DownloadRequest::GetReceipts { request, response, priority }) .is_ok() { - Box::pin(FlattenedResponse::from(rx)) + Either::Left(FlattenedResponse::from(rx)) } else { - Box::pin(future::err(RequestError::ChannelClosed)) + Either::Right(future::err(RequestError::ChannelClosed)) } } } @@ -149,8 +147,7 @@ impl BlockClient for FetchClient { } impl BlockAccessListsClient for FetchClient { - type Output = - std::pin::Pin> + Send + Sync>>; + type Output = FetchClientFuture>; fn get_block_access_lists_with_priority_and_requirement( &self, @@ -169,16 +166,15 @@ impl BlockAccessListsClient for FetchClient { }) .is_ok() { - Box::pin(FlattenedResponse::from(rx)) + Either::Left(FlattenedResponse::from(rx)) } else { - Box::pin(future::err(RequestError::ChannelClosed)) + Either::Right(future::err(RequestError::ChannelClosed)) } } } impl SnapClient for FetchClient { - type Output = - std::pin::Pin> + Send + Sync>>; + type Output = FetchClientFuture>; /// Sends a `GetAccountRange` (`snap/2`) request to an available peer. fn get_account_range_with_priority( From 5729f0e69090102a4ed8e51d3962ab66e23fcc46 Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Sun, 20 Sep 2026 07:02:56 +0000 Subject: [PATCH 027/133] fix(rpc): derive the chain config from the chain spec hardforks (#27304) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc/src/admin.rs | 54 +-------- crates/rpc/rpc/src/chain_config.rs | 169 +++++++++++++++++++++++++++++ crates/rpc/rpc/src/debug.rs | 4 +- crates/rpc/rpc/src/lib.rs | 1 + 4 files changed, 174 insertions(+), 54 deletions(-) create mode 100644 crates/rpc/rpc/src/chain_config.rs diff --git a/crates/rpc/rpc/src/admin.rs b/crates/rpc/rpc/src/admin.rs index a798a941dcb..14fbd3cf287 100644 --- a/crates/rpc/rpc/src/admin.rs +++ b/crates/rpc/rpc/src/admin.rs @@ -1,6 +1,5 @@ use std::sync::Arc; -use alloy_genesis::ChainConfig; use alloy_primitives::keccak256; use alloy_rpc_types_admin::{ EthInfo, EthPeerInfo, EthProtocolInfo, NodeInfo, PeerInfo, PeerNetworkInfo, PeerProtocolInfo, @@ -8,7 +7,7 @@ use alloy_rpc_types_admin::{ }; use async_trait::async_trait; use jsonrpsee::core::RpcResult; -use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks, ForkCondition}; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_network_api::{NetworkInfo, Peers}; use reth_network_peers::{AnyNode, NodeRecord}; use reth_network_types::PeerKind; @@ -123,56 +122,7 @@ where async fn node_info(&self) -> RpcResult { let enode = self.network.local_node_record(); let status = self.network.network_status().await.to_rpc_result()?; - let mut config = ChainConfig { - chain_id: self.chain_spec.chain().id(), - terminal_total_difficulty_passed: self - .chain_spec - .final_paris_total_difficulty() - .is_some(), - terminal_total_difficulty: self - .chain_spec - .ethereum_fork_activation(EthereumHardfork::Paris) - .ttd(), - deposit_contract_address: self.chain_spec.deposit_contract().map(|dc| dc.address), - ..self.chain_spec.genesis().config.clone() - }; - - // helper macro to set the block or time for a hardfork if known - macro_rules! set_block_or_time { - ($config:expr, [$( $field:ident => $fork:ident,)*]) => { - $( - // don't overwrite if already set - if $config.$field.is_none() { - $config.$field = match self.chain_spec.ethereum_fork_activation(EthereumHardfork::$fork) { - ForkCondition::Block(block) => Some(block), - ForkCondition::TTD { fork_block, .. } => fork_block, - ForkCondition::Timestamp(ts) => Some(ts), - ForkCondition::Never => None, - }; - } - )* - }; - } - - set_block_or_time!(config, [ - homestead_block => Homestead, - dao_fork_block => Dao, - eip150_block => Tangerine, - eip155_block => SpuriousDragon, - eip158_block => SpuriousDragon, - byzantium_block => Byzantium, - constantinople_block => Constantinople, - petersburg_block => Petersburg, - istanbul_block => Istanbul, - muir_glacier_block => MuirGlacier, - berlin_block => Berlin, - london_block => London, - arrow_glacier_block => ArrowGlacier, - gray_glacier_block => GrayGlacier, - shanghai_time => Shanghai, - cancun_time => Cancun, - prague_time => Prague, - ]); + let config = crate::chain_config::chain_config(&*self.chain_spec); Ok(NodeInfo { id: alloy_primitives::hex::encode(keccak256(enode.id.as_slice())), diff --git a/crates/rpc/rpc/src/chain_config.rs b/crates/rpc/rpc/src/chain_config.rs new file mode 100644 index 00000000000..8fadb1a9275 --- /dev/null +++ b/crates/rpc/rpc/src/chain_config.rs @@ -0,0 +1,169 @@ +//! Builds the [`ChainConfig`] reported by `debug_chainConfig` and `admin_nodeInfo`. + +use alloy_genesis::ChainConfig; +use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks, ForkCondition}; + +/// Returns the chain config of the given spec. +/// +/// The config starts from the genesis file, which is empty for the bundled specs and can be +/// outdated for custom ones, and is completed with the activations and blob params of the spec's +/// hardforks. +pub(crate) fn chain_config(spec: &C) -> ChainConfig +where + C: EthChainSpec + EthereumHardforks + ?Sized, +{ + let mut config = ChainConfig { + chain_id: spec.chain().id(), + terminal_total_difficulty_passed: spec.final_paris_total_difficulty().is_some(), + terminal_total_difficulty: spec.ethereum_fork_activation(EthereumHardfork::Paris).ttd(), + deposit_contract_address: spec.deposit_contract().map(|dc| dc.address), + ..spec.genesis().config.clone() + }; + + let activation = |fork: EthereumHardfork| match spec.ethereum_fork_activation(fork) { + ForkCondition::Block(block) => Some(block), + ForkCondition::TTD { fork_block, .. } => fork_block, + ForkCondition::Timestamp(ts) => Some(ts), + ForkCondition::Never => None, + }; + + macro_rules! set_block_or_time { + ($( $field:ident => $fork:ident,)*) => { + $( + // the hardforks are the source of truth, but keep the genesis value if the fork + // is unknown to the spec + config.$field = activation(EthereumHardfork::$fork).or(config.$field); + )* + }; + } + + set_block_or_time!( + homestead_block => Homestead, + dao_fork_block => Dao, + eip150_block => Tangerine, + eip155_block => SpuriousDragon, + eip158_block => SpuriousDragon, + byzantium_block => Byzantium, + constantinople_block => Constantinople, + petersburg_block => Petersburg, + istanbul_block => Istanbul, + muir_glacier_block => MuirGlacier, + berlin_block => Berlin, + london_block => London, + arrow_glacier_block => ArrowGlacier, + gray_glacier_block => GrayGlacier, + shanghai_time => Shanghai, + cancun_time => Cancun, + prague_time => Prague, + osaka_time => Osaka, + amsterdam_time => Amsterdam, + bogota_time => Bogota, + bpo1_time => Bpo1, + bpo2_time => Bpo2, + bpo3_time => Bpo3, + bpo4_time => Bpo4, + bpo5_time => Bpo5, + ); + + if config.dao_fork_block.is_some() { + config.dao_fork_support = true; + } + + let blob_forks = [EthereumHardfork::Cancun, EthereumHardfork::Prague, EthereumHardfork::Osaka] + .iter() + .chain(EthereumHardfork::bpo_variants()); + for fork in blob_forks { + if let ForkCondition::Timestamp(ts) = spec.ethereum_fork_activation(*fork) && + let Some(params) = spec.blob_params_at_timestamp(ts) + { + // Forks can share a timestamp, so preserve explicitly configured per-fork params. + config.blob_schedule.entry(fork.name().to_lowercase()).or_insert(params); + } + } + + config +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_eips::eip7840::BlobParams; + use reth_chainspec::{ChainSpec, HOODI, MAINNET, SEPOLIA}; + + #[test] + fn mainnet_config_is_derived_from_hardforks() { + let config = chain_config(&*MAINNET); + + assert_eq!(config.chain_id, 1); + assert_eq!(config.homestead_block, Some(1_150_000)); + assert_eq!(config.dao_fork_block, Some(1_920_000)); + assert!(config.dao_fork_support); + assert_eq!(config.london_block, Some(12_965_000)); + assert_eq!(config.shanghai_time, Some(1_681_338_455)); + assert_eq!(config.cancun_time, Some(1_710_338_135)); + assert_eq!(config.prague_time, Some(1_746_612_311)); + assert_eq!( + config.osaka_time, + MAINNET.ethereum_fork_activation(EthereumHardfork::Osaka).as_timestamp() + ); + assert_eq!( + config.bpo1_time, + MAINNET.ethereum_fork_activation(EthereumHardfork::Bpo1).as_timestamp() + ); + assert!(config.terminal_total_difficulty_passed); + assert!(config.terminal_total_difficulty.is_some()); + assert_eq!( + config.deposit_contract_address, + MAINNET.deposit_contract().map(|dc| dc.address) + ); + assert_eq!( + config.blob_schedule.get("cancun").map(|p| p.max_blob_count), + MAINNET.blob_params_at_timestamp(config.cancun_time.unwrap()).map(|p| p.max_blob_count) + ); + assert!(config.blob_schedule.contains_key("prague")); + assert!(config.blob_schedule.contains_key("osaka")); + assert_eq!(config.blob_schedule.contains_key("bpo1"), config.bpo1_time.is_some()); + } + + #[test] + fn sepolia_config_reports_its_chain_id() { + let config = chain_config(&*SEPOLIA); + assert_eq!(config.chain_id, SEPOLIA.chain().id()); + assert_eq!(config.shanghai_time, Some(1_677_557_088)); + assert!(config.blob_schedule.contains_key("cancun")); + } + + #[test] + fn hoodi_config_includes_forks_missing_from_genesis() { + let config = chain_config(&*HOODI); + let osaka = HOODI.ethereum_fork_activation(EthereumHardfork::Osaka).as_timestamp(); + assert!(osaka.is_some()); + assert_eq!(config.osaka_time, osaka); + assert!(config.blob_schedule.contains_key("osaka")); + } + + #[test] + fn genesis_config_fields_are_kept() { + let mut genesis = MAINNET.genesis().clone(); + genesis.config.chain_id = 1337; + genesis.config.extra_fields.insert("custom".to_string(), serde_json::json!(true)); + let spec = ChainSpec::from(genesis); + + let config = chain_config(&spec); + assert_eq!(config.chain_id, 1337); + assert_eq!(config.extra_fields.get("custom"), Some(&serde_json::json!(true))); + } + + #[test] + fn genesis_blob_schedule_is_kept_when_forks_share_a_timestamp() { + let mut genesis = MAINNET.genesis().clone(); + genesis.config.cancun_time = Some(0); + genesis.config.prague_time = Some(0); + genesis.config.blob_schedule.insert("cancun".to_string(), BlobParams::cancun()); + genesis.config.blob_schedule.insert("prague".to_string(), BlobParams::prague()); + let expected = genesis.config.blob_schedule.clone(); + let spec = ChainSpec::from(genesis); + + assert_eq!(chain_config(&spec).blob_schedule, expected); + } +} diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 1154f793839..f10ee838152 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -17,7 +17,7 @@ use async_trait::async_trait; use futures::Stream; use jsonrpsee::{core::RpcResult, PendingSubscriptionSink, SubscriptionMessage}; use parking_lot::RwLock; -use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; +use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; use reth_engine_primitives::ConsensusEngineEvent; use reth_errors::RethError; use reth_evm::{block::BlockExecutor, execute::Executor, ConfigureEvm, EvmEnvFor}; @@ -1231,7 +1231,7 @@ where } async fn debug_chain_config(&self) -> RpcResult { - Ok(self.provider().chain_spec().genesis().config.clone()) + Ok(crate::chain_config::chain_config(&*self.provider().chain_spec())) } async fn debug_chaindb_property(&self, _property: String) -> RpcResult<()> { diff --git a/crates/rpc/rpc/src/lib.rs b/crates/rpc/rpc/src/lib.rs index f38465e3559..ff932160c21 100644 --- a/crates/rpc/rpc/src/lib.rs +++ b/crates/rpc/rpc/src/lib.rs @@ -27,6 +27,7 @@ mod admin; mod aliases; +mod chain_config; mod debug; mod engine; pub mod eth; From a58c6abc0ca0069910fe3731a14cd0c9a58e708f Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sun, 20 Sep 2026 07:11:16 +0000 Subject: [PATCH 028/133] fix(rpc): reject zero-valued concurrency limits (#26846) Co-authored-by: Matthias Seitz --- crates/node/core/src/args/rpc_server.rs | 30 ++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 08eda4a6e47..46aa59c3557 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -613,7 +613,7 @@ pub struct RpcServerArgs { /// Tracing requests are generally CPU bound. /// Choosing a value that is higher than the available CPU cores can have a negative impact on /// the performance of the node and affect the node's ability to maintain sync. - #[arg(long = "rpc.max-tracing-requests", alias = "rpc-max-tracing-requests", value_name = "COUNT", default_value_t = DefaultRpcServerArgs::get_global().rpc_max_tracing_requests)] + #[arg(long = "rpc.max-tracing-requests", alias = "rpc-max-tracing-requests", value_name = "COUNT", value_parser = RangedU64ValueParser::::new().range(1..), default_value_t = DefaultRpcServerArgs::get_global().rpc_max_tracing_requests)] pub rpc_max_tracing_requests: usize, /// Maximum number of concurrent blocking IO requests. @@ -621,7 +621,7 @@ pub struct RpcServerArgs { /// Blocking IO requests include `eth_call`, `eth_estimateGas`, and similar methods that /// require EVM execution. These are spawned as blocking tasks to avoid blocking the async /// runtime. - #[arg(long = "rpc.max-blocking-io-requests", alias = "rpc-max-blocking-io-requests", value_name = "COUNT", default_value_t = DefaultRpcServerArgs::get_global().rpc_max_blocking_io_requests)] + #[arg(long = "rpc.max-blocking-io-requests", alias = "rpc-max-blocking-io-requests", value_name = "COUNT", value_parser = RangedU64ValueParser::::new().range(1..), default_value_t = DefaultRpcServerArgs::get_global().rpc_max_blocking_io_requests)] pub rpc_max_blocking_io_requests: usize, /// Maximum number of blocks for `trace_filter` requests. @@ -693,7 +693,7 @@ pub struct RpcServerArgs { pub rpc_eth_proof_window: u64, /// Maximum number of concurrent getproof requests. - #[arg(long = "rpc.proof-permits", alias = "rpc-proof-permits", value_name = "COUNT", default_value_t = DefaultRpcServerArgs::get_global().rpc_proof_permits)] + #[arg(long = "rpc.proof-permits", alias = "rpc-proof-permits", value_name = "COUNT", value_parser = RangedU64ValueParser::::new().range(1..), default_value_t = DefaultRpcServerArgs::get_global().rpc_proof_permits)] pub rpc_proof_permits: usize, /// Configures the pending block behavior for RPC responses. @@ -1083,6 +1083,30 @@ mod tests { assert_eq!(apis, expected); } + #[test] + fn rpc_concurrency_limits_reject_zero() { + for flag in + ["--rpc.max-tracing-requests", "--rpc.max-blocking-io-requests", "--rpc.proof-permits"] + { + let result = CommandParser::::try_parse_from(["reth", flag, "0"]); + assert!(result.is_err(), "{flag} should reject zero"); + } + + let args = CommandParser::::parse_from([ + "reth", + "--rpc.max-tracing-requests", + "1", + "--rpc.max-blocking-io-requests", + "2", + "--rpc.proof-permits", + "3", + ]) + .args; + assert_eq!(args.rpc_max_tracing_requests, 1); + assert_eq!(args.rpc_max_blocking_io_requests, 2); + assert_eq!(args.rpc_proof_permits, 3); + } + #[test] fn http_compression_algorithms_are_optional() { let args = CommandParser::::parse_from(["reth"]).args; From e2d464bd3e8f871fe9c613827148bdf46ce9cc52 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sun, 20 Sep 2026 08:33:07 +0000 Subject: [PATCH 029/133] refactor(rpc): centralize state cache miss scheduling (#27324) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-eth-types/src/cache/mod.rs | 239 +++++++++++++++++----- 1 file changed, 190 insertions(+), 49 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index 3d07789b7b9..67d5b28bd9d 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -23,6 +23,7 @@ use reth_storage_api::{BalProvider, BlockReader, TransactionVariant}; use reth_tasks::Runtime; use schnellru::{ByLength, Limiter, LruMap}; use std::{ + collections::VecDeque, future::Future, pin::Pin, sync::Arc, @@ -30,7 +31,7 @@ use std::{ }; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedSender}, - oneshot, Semaphore, + oneshot, OwnedSemaphorePermit, Semaphore, }; use tokio_stream::wrappers::UnboundedReceiverStream; @@ -112,6 +113,7 @@ impl EthStateCache { action_rx: UnboundedReceiverStream::new(rx), action_task_spawner, rate_limiter: Arc::new(Semaphore::new(max_concurrent_db_requests)), + pending_fetches: VecDeque::new(), tx_hash_index: LruMap::new(ByLength::new(max_cached_tx_hashes)), }; let cache = Self { to_service }; @@ -325,6 +327,11 @@ pub(crate) struct EthStateCacheService< /// /// This restricts the max concurrent fetch tasks at the same time. rate_limiter: Arc, + /// Cache misses waiting for a database request slot. + /// + /// A request is only moved to the blocking pool after it acquires a permit. This prevents + /// cache misses that are waiting for a slot from occupying Tokio blocking threads. + pending_fetches: VecDeque, /// LRU index mapping transaction hashes to their block hash and index within the block. tx_hash_index: LruMap, } @@ -333,6 +340,71 @@ impl EthStateCacheService where Provider: BlockReader + BalProvider + Clone + Unpin + 'static, { + fn queue_fetch(&mut self, fetch: CacheFetch) { + if self.pending_fetches.is_empty() && + let Ok(permit) = self.rate_limiter.clone().try_acquire_owned() + { + self.spawn_fetch(fetch, permit); + return + } + + self.pending_fetches.push_back(fetch); + self.spawn_pending_fetches(); + } + + fn spawn_pending_fetches(&mut self) { + while let Some(fetch) = self.pending_fetches.pop_front() { + let Ok(permit) = self.rate_limiter.clone().try_acquire_owned() else { + self.pending_fetches.push_front(fetch); + return + }; + + self.spawn_fetch(fetch, permit); + } + } + + fn spawn_fetch(&self, fetch: CacheFetch, permit: OwnedSemaphorePermit) { + let provider = self.provider.clone(); + let action_tx = self.action_tx.clone(); + + match fetch { + CacheFetch::Block(block_hash) => { + let mut action_sender = ActionSender::new(CacheKind::Block, block_hash, action_tx); + self.action_task_spawner.spawn_blocking_task(async move { + let block_sender = provider + .sealed_block_with_senders( + BlockHashOrNumber::Hash(block_hash), + TransactionVariant::WithHash, + ) + .map(|maybe_block| maybe_block.map(Arc::new)); + drop(permit); + action_sender.send_block(block_sender); + }); + } + CacheFetch::Receipts(block_hash) => { + let mut action_sender = + ActionSender::new(CacheKind::Receipt, block_hash, action_tx); + self.action_task_spawner.spawn_blocking_task(async move { + let res = provider + .receipts_by_block(block_hash.into()) + .map(|maybe_receipts| maybe_receipts.map(Arc::new)); + drop(permit); + action_sender.send_receipts(res); + }); + } + CacheFetch::Bal(block_hash) => { + let mut action_sender = ActionSender::new(CacheKind::Bal, block_hash, action_tx); + self.action_task_spawner.spawn_blocking_task(async move { + let res = provider.get_bal_by_hash(block_hash).and_then(|maybe_bal| { + maybe_bal.map(CachedRevmBal::try_from_raw).transpose() + }); + drop(permit); + action_sender.send_bal(res); + }); + } + } + } + /// Indexes all transactions in a block by transaction hash. fn index_block_transactions(&mut self, block: &RecoveredBlock) { let block_hash = block.hash(); @@ -495,24 +567,7 @@ where // block is not in the cache, request it if this is the first consumer if this.full_block_cache.queue(block_hash, response_tx) { - let provider = this.provider.clone(); - let action_tx = this.action_tx.clone(); - let rate_limiter = this.rate_limiter.clone(); - let mut action_sender = - ActionSender::new(CacheKind::Block, block_hash, action_tx); - this.action_task_spawner.spawn_blocking_task(async move { - // Acquire permit - let _permit = rate_limiter.acquire().await; - // Only look in the database to prevent situations where we - // looking up the tree is blocking - let block_sender = provider - .sealed_block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - ) - .map(|maybe_block| maybe_block.map(Arc::new)); - action_sender.send_block(block_sender); - }); + this.queue_fetch(CacheFetch::Block(block_hash)); } } CacheAction::GetReceipts { block_hash, response_tx } => { @@ -524,20 +579,7 @@ where // block is not in the cache, request it if this is the first consumer if this.receipts_cache.queue(block_hash, response_tx) { - let provider = this.provider.clone(); - let action_tx = this.action_tx.clone(); - let rate_limiter = this.rate_limiter.clone(); - let mut action_sender = - ActionSender::new(CacheKind::Receipt, block_hash, action_tx); - this.action_task_spawner.spawn_blocking_task(async move { - // Acquire permit - let _permit = rate_limiter.acquire().await; - let res = provider - .receipts_by_block(block_hash.into()) - .map(|maybe_receipts| maybe_receipts.map(Arc::new)); - - action_sender.send_receipts(res); - }); + this.queue_fetch(CacheFetch::Receipts(block_hash)); } } CacheAction::GetBal { block_hash, response_tx } => { @@ -547,20 +589,7 @@ where } if this.bal_cache.queue(block_hash, response_tx) { - let provider = this.provider.clone(); - let action_tx = this.action_tx.clone(); - let rate_limiter = this.rate_limiter.clone(); - let mut action_sender = - ActionSender::new(CacheKind::Bal, block_hash, action_tx); - this.action_task_spawner.spawn_blocking_task(async move { - let _permit = rate_limiter.acquire().await; - let res = provider.get_bal_by_hash(block_hash).and_then( - |maybe_bal| { - maybe_bal.map(CachedRevmBal::try_from_raw).transpose() - }, - ); - action_sender.send_bal(res); - }); + this.queue_fetch(CacheFetch::Bal(block_hash)); } } CacheAction::ReceiptsResult { block_hash, res } => { @@ -630,6 +659,7 @@ where let _ = response_tx.send(result); } }; + this.spawn_pending_fetches(); } } } @@ -734,6 +764,13 @@ enum CacheKind { Bal, } +#[derive(Copy, Clone, Debug)] +enum CacheFetch { + Block(B256), + Receipts(B256), + Bal(B256), +} + /// Drop aware sender struct that ensures a response is always emitted even if the db task panics /// before a result could be sent. /// @@ -918,7 +955,15 @@ mod tests { BlockHashReader, BlockNumReader, BlockReader, BlockSource, HeaderProvider, ReceiptProvider, TransactionVariant, TransactionsProvider, }; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc::{self, Receiver, SyncSender}, + Mutex, + }, + thread, + time::Duration, + }; fn test_service() -> EthStateCacheService { let (_cache, service) = EthStateCache::::create( @@ -995,6 +1040,78 @@ mod tests { assert!(service.bal_cache.contains_key(&hash)); } + #[tokio::test] + async fn cache_misses_wait_in_service_queue_for_db_slot() { + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (release_tx, release_rx) = mpsc::sync_channel(1); + let (_cache, mut service) = EthStateCache::::create( + TestBalProvider::new_blocking(started_tx, release_rx), + Runtime::test(), + EthStateCacheConfig { max_concurrent_db_requests: 1, ..Default::default() }, + ); + let permit = service.rate_limiter.clone().try_acquire_owned().unwrap(); + + service.queue_fetch(CacheFetch::Bal(B256::repeat_byte(0x01))); + service.queue_fetch(CacheFetch::Bal(B256::repeat_byte(0x02))); + + assert_eq!(service.pending_fetches.len(), 2); + + drop(permit); + service.spawn_pending_fetches(); + + started_rx.recv_timeout(Duration::from_secs(1)).expect("first fetch started"); + assert_eq!(service.pending_fetches.len(), 1); + release_tx.send(()).unwrap(); + } + + #[test] + fn pending_cache_miss_does_not_block_unrelated_blocking_work() { + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (release_tx, release_rx) = mpsc::sync_channel(1); + let (unrelated_tx, unrelated_rx) = mpsc::sync_channel(1); + + let test = thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async move { + let provider = TestBalProvider::new_blocking(started_tx, release_rx); + let (_cache, mut service) = EthStateCache::::create( + provider, + Runtime::test(), + EthStateCacheConfig { + max_blocks: 0, + max_receipts: 0, + max_bals: 0, + cache_computed_bals: false, + prewarm_bals: None, + max_concurrent_db_requests: 1, + max_cached_tx_hashes: 0, + }, + ); + + service.queue_fetch(CacheFetch::Bal(B256::repeat_byte(0x01))); + started_rx.recv_timeout(Duration::from_secs(1)).expect("first fetch started"); + + service.queue_fetch(CacheFetch::Bal(B256::repeat_byte(0x02))); + assert_eq!(service.pending_fetches.len(), 1); + + tokio::task::spawn_blocking(move || unrelated_tx.send(()).unwrap()); + release_tx.send(()).unwrap(); + + unrelated_rx + .recv_timeout(Duration::from_secs(1)) + .expect("unrelated blocking work was not starved"); + }); + }); + + test.join().unwrap(); + } + #[test] fn reorg_forwards_bal_to_queued_requests() { let mut service = test_service(); @@ -1184,13 +1301,26 @@ mod tests { impl TestBalProvider { fn new(fetches: Arc) -> Self { - Self { bal_store: BalStoreHandle::new(TestBalStore { fetches }), block: None } + Self { + bal_store: BalStoreHandle::new(TestBalStore { fetches, blocking_fetch: None }), + block: None, + } } fn with_block(mut self, block: RecoveredBlock) -> Self { self.block = Some(block); self } + + fn new_blocking(started: SyncSender<()>, release: Receiver<()>) -> Self { + Self { + bal_store: BalStoreHandle::new(TestBalStore { + fetches: Arc::new(AtomicUsize::default()), + blocking_fetch: Some(BlockingFetch { started, release: Mutex::new(release) }), + }), + block: None, + } + } } impl BalProvider for TestBalProvider { @@ -1202,6 +1332,13 @@ mod tests { #[derive(Debug)] struct TestBalStore { fetches: Arc, + blocking_fetch: Option, + } + + #[derive(Debug)] + struct BlockingFetch { + started: SyncSender<()>, + release: Mutex>, } impl BalStore for TestBalStore { @@ -1215,6 +1352,10 @@ mod tests { fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> ProviderResult>> { self.fetches.fetch_add(1, Ordering::SeqCst); + if let Some(blocking_fetch) = &self.blocking_fetch { + blocking_fetch.started.send(()).unwrap(); + blocking_fetch.release.lock().unwrap().recv().unwrap(); + } Ok(block_hashes.iter().map(|_| Some(Bytes::from_static(&[0xc0]))).collect()) } } From a2aff3428bd08e4ad0b8b8bdb34e2426991076a5 Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Sun, 20 Sep 2026 08:35:35 +0000 Subject: [PATCH 030/133] fix(provider): reject expired history in the block range readers (#27337) --- .../provider/src/providers/database/mod.rs | 66 +++++++++++++++++++ .../src/providers/database/provider.rs | 10 +++ 2 files changed, 76 insertions(+) diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 25cdf18884e..f4cae91d56b 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -1070,6 +1070,72 @@ mod tests { provider.block_hash(0).unwrap(); } + #[test] + fn block_range_readers_reject_expired_history() { + use crate::BlockReader; + use reth_static_file_types::{SegmentHeader, SegmentRangeInclusive, StaticFileSegment}; + + let factory = create_test_provider_factory(); + let mut rng = generators::rng(); + let provider_rw = factory.provider_rw().unwrap(); + let mut parent = None; + for number in 0..6 { + let block = random_block( + &mut rng, + number, + BlockParams { parent, tx_count: Some(1), ..Default::default() }, + ); + parent = Some(block.hash()); + provider_rw.insert_block(&block.try_recover().unwrap()).unwrap(); + } + provider_rw.commit().unwrap(); + + // history below block 3 has been expired + let static_provider = factory.static_file_provider(); + { + let mut writer = + static_provider.latest_writer(StaticFileSegment::Transactions).unwrap(); + let header = writer.user_header().clone(); + *writer.user_header_mut() = SegmentHeader::new( + header.expected_block_range(), + Some(SegmentRangeInclusive::new(3, 5)), + header.tx_range(), + StaticFileSegment::Transactions, + ); + writer.inner().set_dirty(); + writer.commit().unwrap(); + } + static_provider.initialize_index().unwrap(); + assert_eq!(static_provider.earliest_history_height(), 3); + + let provider = factory.provider().unwrap(); + // single block lookups already reject expired blocks + assert_matches!( + provider.block(1.into()), + Err(ProviderError::BlockExpired { requested: 1, earliest_available: 3 }) + ); + assert_matches!( + provider.recovered_block(1.into(), Default::default()), + Err(ProviderError::BlockExpired { requested: 1, earliest_available: 3 }) + ); + // and so must the range readers instead of returning blocks with stripped bodies + assert_matches!( + provider.block_range(1..=4), + Err(ProviderError::BlockExpired { requested: 1, earliest_available: 3 }) + ); + assert_matches!( + provider.block_with_senders_range(1..=4), + Err(ProviderError::BlockExpired { requested: 1, earliest_available: 3 }) + ); + assert_matches!( + provider.recovered_block_range(1..=4), + Err(ProviderError::BlockExpired { requested: 1, earliest_available: 3 }) + ); + // ranges within the available history keep working + assert_eq!(provider.block_range(3..=5).unwrap().len(), 3); + assert_eq!(provider.recovered_block_range(3..=5).unwrap().len(), 3); + } + #[test] fn insert_block_with_prune_modes() { let block = TEST_BLOCK.clone(); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index f6e9ef97034..53169e8932a 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1158,6 +1158,16 @@ impl DatabaseProvider { return Ok(Vec::new()) } + // like the single block lookups, reject ranges that reach into expired history instead + // of assembling blocks whose bodies are no longer available + let earliest_available = self.static_file_provider.earliest_history_height(); + if *range.start() < earliest_available { + return Err(ProviderError::BlockExpired { + requested: *range.start(), + earliest_available, + }) + } + let len = range.end().saturating_sub(*range.start()) as usize + 1; let mut blocks = Vec::with_capacity(len); From 03cb186c1d36eebbacc7bda08f36e25711d0804e Mon Sep 17 00:00:00 2001 From: Bruce0X <109469941+Bruce039@users.noreply.github.com> Date: Sun, 20 Sep 2026 08:57:29 +0000 Subject: [PATCH 031/133] docs(rpc): document the remaining reth_ and debug_ methods (#27340) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- docs/vocs/docs/pages/jsonrpc/debug.mdx | 32 +++++++++++ docs/vocs/docs/pages/jsonrpc/reth.mdx | 80 ++++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/docs/vocs/docs/pages/jsonrpc/debug.mdx b/docs/vocs/docs/pages/jsonrpc/debug.mdx index 299b0ec52b2..d64a9e06c6c 100644 --- a/docs/vocs/docs/pages/jsonrpc/debug.mdx +++ b/docs/vocs/docs/pages/jsonrpc/debug.mdx @@ -45,6 +45,14 @@ Returns an array of EIP-2718 binary-encoded receipts. | ------ | ------------------------------------------------------- | | RPC | `{"method": "debug_getRawReceipts", "params": [block]}` | +## `debug_getRawBlockAccessList` + +Returns the RLP-encoded EIP-7928 block access list of the given block. Returns an error if the block is unknown. + +| Client | Method invocation | +| ------ | ---------------------------------------------------------------- | +| RPC | `{"method": "debug_getRawBlockAccessList", "params": [block]}` | + ## `debug_getBadBlocks` Returns an array of recent bad blocks that the client has seen on the network. @@ -53,6 +61,22 @@ Returns an array of recent bad blocks that the client has seen on the network. | ------ | ------------------------------------------------ | | RPC | `{"method": "debug_getBadBlocks", "params": []}` | +## `debug_traceBadBlock` + +Re-executes a block from the bad block cache (see `debug_getBadBlocks`) and returns the structured logs of every transaction, using the same tracer options as `debug_traceBlock`. + +| Client | Method invocation | +| ------ | ------------------------------------------------------------------- | +| RPC | `{"method": "debug_traceBadBlock", "params": [block_hash, opts]}` | + +## `debug_clearTxpool` + +Removes all transactions from the transaction pool. Equivalent to `admin_clearTxpool`, but does not report the number of removed transactions. + +| Client | Method invocation | +| ------ | ------------------------------------------------ | +| RPC | `{"method": "debug_clearTxpool", "params": []}` | + ## `debug_subscribe`, `debug_unsubscribe` Subscribes to structured logs created during EVM execution between two blocks. The start block is @@ -185,6 +209,14 @@ Retrieves a raw value from the database. | ------ | -------------------------------------------------- | | RPC | `{"method": "debug_dbGet", "params": [key]}` | +## `debug_stateRootWithUpdates` + +Computes the state root that results from applying the given `HashedPostState` on top of the state of the specified block and returns it together with the trie updates. If no block identifier is provided it defaults to `latest`. + +| Client | Method invocation | +| ------ | -------------------------------------------------------------------------- | +| RPC | `{"method": "debug_stateRootWithUpdates", "params": [hashed_state, block]}` | + ## `debug_storageRangeAt` Returns the storage at the given block height and transaction index. The result can be paged by providing a `maxResult` to cap the number of storage slots returned as well as specifying the offset via `keyStart`. diff --git a/docs/vocs/docs/pages/jsonrpc/reth.mdx b/docs/vocs/docs/pages/jsonrpc/reth.mdx index 80e49d33ab6..7af7faa4617 100644 --- a/docs/vocs/docs/pages/jsonrpc/reth.mdx +++ b/docs/vocs/docs/pages/jsonrpc/reth.mdx @@ -27,6 +27,44 @@ The method accepts a block identifier (number, hash, or tag like `latest`) and r The result is a mapping of addresses to their new balance after the block was executed. Only addresses whose balance changed during block execution are included. +## `reth_getBlockExecutionOutcome` + +Re-executes a block and returns its execution outcome: the bundle of state changes, the receipts and the EIP-7685 requests. + +If `count` is provided, up to `count` consecutive available blocks starting at the given block are re-executed and the merged execution outcome is returned. `count` must be between 1 and 128. + +Returns `null` if the block is unknown. The genesis block yields an empty outcome because it has no parent state to execute against. + +| Client | Method invocation | +| ------ | ---------------------------------------------------------------------------- | +| RPC | `{"method": "reth_getBlockExecutionOutcome", "params": [block, count]}` | + +### Example + +```js +// > {"jsonrpc":"2.0","id":1,"method":"reth_getBlockExecutionOutcome","params":["latest"]} +{"jsonrpc":"2.0","id":1,"result":{"bundle":{"state":{"0x95222290dd7278aa3ddd389cc1e1d165cc4bafe5":{"info":{"balance":"0x1bc16d674ec80000","nonce":0,"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470","code":null},"original_info":{"balance":"0x0","nonce":0,"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470","code":null},"storage":{},"status":"Changed"}},"contracts":{},"reverts":[[["0x95222290dd7278aa3ddd389cc1e1d165cc4bafe5",{"account":{"RevertTo":{"balance":"0x0","nonce":0,"code_hash":"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470","code":null}},"storage":{},"previous_status":"Loaded","wipe_storage":false}]]],"state_size":1,"reverts_size":1},"receipts":[[]],"first_block":21000000,"requests":[[]]}} +``` + +The result contains the `bundle` of account and storage changes (including the reverts required to undo them), one list of `receipts` per executed block, the number of the `first_block` of the executed range and the `requests` (EIP-7685) of each block. + +## `reth_jit` + +Controls the JIT compilation backend of the EVM, if the node was built with one. The method is a no-op when no JIT backend is configured. + +The action is one of `enable`, `disable`, `pause`, `unpause` or `clear`. The `clear` action clears resident and persisted JIT artifacts. + +| Client | Method invocation | +| ------ | ---------------------------------------------------- | +| RPC | `{"method": "reth_jit", "params": [action]}` | + +### Example + +```js +// > {"jsonrpc":"2.0","id":1,"method":"reth_jit","params":["pause"]} +{"jsonrpc":"2.0","id":1,"result":null} +``` + ## `reth_subscribeChainNotifications`, `reth_unsubscribeChainNotifications` Subscribe to canonical chain state notifications. This creates a subscription that emits notifications whenever the canonical chain state changes. @@ -47,7 +85,7 @@ The subscription emits events with the following structure: ```json { "jsonrpc": "2.0", - "method": "reth_subscription", + "method": "reth_subscribeChainNotifications", "params": { "subscription": "0xcd0c3e8af590364c09d0fa6a1210faf5", "result": { @@ -76,8 +114,8 @@ This is particularly useful for applications that need to react immediately to c // responds with subscription ID {"jsonrpc":"2.0","id":1,"result":"0xcd0c3e8af590364c09d0fa6a1210faf5"} -// Example notification when new blocks are committed -{"jsonrpc":"2.0","method":"reth_subscription","params":{"subscription":"0xcd0c3e8af590364c09d0fa6a1210faf5","result":{"Commit":{"new":{"blocks":[...],"receipts":[...],"first_block":1000,"last_block":1000}}}}} +// Example notification when new blocks are committed (nested values abbreviated) +{"jsonrpc":"2.0","method":"reth_subscribeChainNotifications","params":{"subscription":"0xcd0c3e8af590364c09d0fa6a1210faf5","result":{"Commit":{"new":{"blocks":{"1000":...},"execution_outcome":{"bundle":...,"receipts":[...],"first_block":1000,"requests":[...]},"trie_data":{}}}}}} // Unsubscribe // > {"jsonrpc":"2.0","id":2,"method":"reth_unsubscribeChainNotifications","params":["0xcd0c3e8af590364c09d0fa6a1210faf5"]} @@ -109,7 +147,7 @@ To unsubscribe from persisted block notifications, call `reth_unsubscribePersist {"jsonrpc":"2.0","id":1,"result":"0xab1c2d3e4f590364c09d0fa6a1210faf5"} // Example notification when a block is persisted -{"jsonrpc":"2.0","method":"reth_subscription","params":{"subscription":"0xab1c2d3e4f590364c09d0fa6a1210faf5","result":{"number":"0x1a2b3c","hash":"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"}}} +{"jsonrpc":"2.0","method":"reth_subscribePersistedBlock","params":{"subscription":"0xab1c2d3e4f590364c09d0fa6a1210faf5","result":{"number":"0x1a2b3c","hash":"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"}}} // Unsubscribe // > {"jsonrpc":"2.0","id":2,"method":"reth_unsubscribePersistedBlock","params":["0xab1c2d3e4f590364c09d0fa6a1210faf5"]} @@ -119,3 +157,37 @@ To unsubscribe from persisted block notifications, call `reth_unsubscribePersist :::note This subscription is only available over WebSocket and IPC transports, as HTTP does not support server-initiated messages. ::: + +## `reth_subscribeFinalizedChainNotifications`, `reth_unsubscribeFinalizedChainNotifications` + +Subscribe to finalized chain notifications. Committed chain notifications are buffered and emitted once a new finalized block is received; each notification contains a list of buffered chain segments whose last block is at or below the finalized block. Each segment uses the same format as `reth_subscribeChainNotifications`. Segments that extend beyond the finalized block remain buffered until their last block is finalized. + +Blocks that were reverted by a reorg before they were finalized are dropped from the buffer, so subscribers only ever see the finalized chain. + +Like other subscription methods, this returns the ID of the subscription, which is then used in all events subsequently. + +To unsubscribe from finalized chain notifications, call `reth_unsubscribeFinalizedChainNotifications` with the subscription ID. + +| Client | Method invocation | +| ------ | ------------------------------------------------------------------------------------- | +| RPC | `{"method": "reth_subscribeFinalizedChainNotifications", "params": []}` | +| RPC | `{"method": "reth_unsubscribeFinalizedChainNotifications", "params": [subscription_id]}` | + +### Example + +```js +// > {"jsonrpc":"2.0","id":1,"method":"reth_subscribeFinalizedChainNotifications","params":[]} +// responds with subscription ID +{"jsonrpc":"2.0","id":1,"result":"0xcd2e3f4a5b6c7d8e9f0a1b2c3d4e5f60"} + +// Example notification once a new finalized block is known (nested values abbreviated) +{"jsonrpc":"2.0","method":"reth_subscribeFinalizedChainNotifications","params":{"subscription":"0xcd2e3f4a5b6c7d8e9f0a1b2c3d4e5f60","result":[{"Commit":{"new":{"blocks":{"1000":...,"1001":...,"1002":...},"execution_outcome":{"bundle":...,"receipts":[...],"first_block":1000,"requests":[...]},"trie_data":{}}}}]}} + +// Unsubscribe +// > {"jsonrpc":"2.0","id":2,"method":"reth_unsubscribeFinalizedChainNotifications","params":["0xcd2e3f4a5b6c7d8e9f0a1b2c3d4e5f60"]} +{"jsonrpc":"2.0","id":2,"result":true} +``` + +:::note +This subscription is only available over WebSocket and IPC transports, as HTTP does not support server-initiated messages. +::: From 62ff56be8625b9fc6cd1fc30c9bd242a6a3cd2b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 09:32:30 +0000 Subject: [PATCH 032/133] chore(deps): bump the ci-weekly group with 4 updates (#27347) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bench-scheduled.yml | 4 ++-- .github/workflows/bench.yml | 4 ++-- .github/workflows/check-alloy.yml | 2 +- .github/workflows/compact.yml | 2 +- .github/workflows/e2e.yml | 8 +++---- .github/workflows/integration.yml | 8 +++---- .github/workflows/lint.yml | 32 +++++++++++++-------------- .github/workflows/release-dist.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/stage.yml | 2 +- .github/workflows/sync-era.yml | 2 +- .github/workflows/sync.yml | 2 +- .github/workflows/unit.yml | 10 ++++----- 13 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 5e08ac69436..10507d5689e 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -347,11 +347,11 @@ jobs: const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; core.exportVariable('BENCH_JOB_URL', jobUrl); - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install dependencies env: diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 897514909ff..b88fe00d3fb 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1226,11 +1226,11 @@ jobs: }); } - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Mask OTLP endpoints uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/check-alloy.yml b/.github/workflows/check-alloy.yml index 93df79f1dbb..c9dc0d51197 100644 --- a/.github/workflows/check-alloy.yml +++ b/.github/workflows/check-alloy.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 52ad1ff8374..494a0aac872 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6031cd278a5..8d539fdc195 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,11 +32,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: nextest - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -68,11 +68,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: nextest - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3d3ff5abf6d..e61f5985d2d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -39,12 +39,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -83,10 +83,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5f30c2d041a..9972347999f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -34,7 +34,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@clippy with: @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly with: @@ -80,11 +80,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -106,11 +106,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@stable with: target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -135,10 +135,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -157,7 +157,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@master with: @@ -180,7 +180,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -203,7 +203,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt @@ -221,14 +221,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: cargo-udeps - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked @@ -243,7 +243,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -306,7 +306,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@stable - name: Ensure no arbitrary or proptest dependency on default build run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0 @@ -324,7 +324,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: zepter diff --git a/.github/workflows/release-dist.yml b/.github/workflows/release-dist.yml index e9594d9fff5..5a5a1e12b12 100644 --- a/.github/workflows/release-dist.yml +++ b/.github/workflows/release-dist.yml @@ -15,7 +15,7 @@ jobs: permissions: {} steps: - name: Update Homebrew formula - uses: dawidd6/action-homebrew-bump-formula@a0e064e08103c01870c6d1b05168d1b726aab119 # v8 + uses: dawidd6/action-homebrew-bump-formula@b12d2bc99c46444125e606142f28200ec71a1214 # v10 with: token: ${{ secrets.HOMEBREW }} no_fork: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5dc76c32844..390593f482c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,7 +112,7 @@ jobs: with: persist-credentials: false ref: ${{ env.RELEASE_REF }} - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@stable with: target: ${{ matrix.configs.target }} diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index bb5c2422482..54847480ca6 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index 87774748908..04b90978b77 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -41,7 +41,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index dd1f3ef826f..55cc74c7931 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -41,7 +41,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 9e59205b8d8..96aa4d68afd 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -48,7 +48,7 @@ jobs: prefix-key: v1-rust-unit-targets cache-bin: false cache-on-failure: true - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: nextest - name: Run tests @@ -91,10 +91,10 @@ jobs: mkdir -p testing/ef-tests/execution-spec-tests URL="https://github.com/ethereum/execution-spec-tests/releases/download/${EEST_TESTS_TAG}/fixtures_stable.tar.gz" curl -L "$URL" | tar -xz --strip-components=1 -C testing/ef-tests/execution-spec-tests - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 with: tool: nextest - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -115,7 +115,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 From 609cba205b2c143c71c4bde035ba64eebbec8f5a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 21 Sep 2026 10:20:03 +0000 Subject: [PATCH 033/133] feat: schedule Sepolia Amsterdam activation (#27350) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 12 ++-- Cargo.toml | 2 +- crates/chainspec/src/spec.rs | 83 ++++++++++++++++++++++++++- crates/ethereum/node/tests/e2e/rpc.rs | 50 ++++++++++++++++ 4 files changed, 138 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 795aa758aaf..49b05e4a2bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -299,7 +299,7 @@ checksum = "7b6bd5f7e5ce9858bac99275e3f548c3b63bb3e5806da0142dc0b2f518d3ec4a" dependencies = [ "alloy-consensus", "alloy-eips", - "alloy-hardforks 0.4.8", + "alloy-hardforks 0.4.9", "alloy-primitives", "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "alloy-hardforks" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865371fb677c005f7cb0ea9c2847a1ba4fdb042caf6428b1769fee20368bbfc7" +checksum = "09d5ff0394d9547798fa4b666c46e356d09a113f33d45fb3d75175681c1e2652" dependencies = [ "alloy-chains", "alloy-eip2124", @@ -7876,7 +7876,7 @@ dependencies = [ "alloy-consensus", "alloy-eips", "alloy-evm", - "alloy-hardforks 0.4.8", + "alloy-hardforks 0.4.9", "alloy-primitives", "alloy-rlp", "alloy-rpc-types", @@ -8795,7 +8795,7 @@ dependencies = [ "alloy-eip7928", "alloy-eips", "alloy-genesis", - "alloy-hardforks 0.4.8", + "alloy-hardforks 0.4.9", "alloy-primitives", "alloy-rlp", "alloy-trie", @@ -8914,7 +8914,7 @@ name = "reth-ethereum-forks" version = "2.5.2" dependencies = [ "alloy-eip2124", - "alloy-hardforks 0.4.8", + "alloy-hardforks 0.4.9", "alloy-primitives", "arbitrary", "auto_impl", diff --git a/Cargo.toml b/Cargo.toml index 53965b57963..f15b9929862 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -443,7 +443,7 @@ alloy-evm = { version = "0.39.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } -alloy-hardforks = "0.4.8" +alloy-hardforks = "0.4.9" alloy-consensus = { version = "2.4.2", default-features = false } alloy-contract = { version = "2.4.2", default-features = false } diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index ac0a247c37b..58c772bf592 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -1688,6 +1688,31 @@ Post-merge hard forks (timestamp based): next: sepolia::SEPOLIA_OSAKA_TIMESTAMP, }, ), + ( + EthereumHardfork::Osaka, + ForkId { + hash: ForkHash(hex!("0xe2ae4999")), + next: sepolia::SEPOLIA_BPO1_TIMESTAMP, + }, + ), + ( + EthereumHardfork::Bpo1, + ForkId { + hash: ForkHash(hex!("0x56078a1e")), + next: sepolia::SEPOLIA_BPO2_TIMESTAMP, + }, + ), + ( + EthereumHardfork::Bpo2, + ForkId { + hash: ForkHash(hex!("0x268956b6")), + next: sepolia::SEPOLIA_AMSTERDAM_TIMESTAMP, + }, + ), + ( + EthereumHardfork::Amsterdam, + ForkId { hash: ForkHash(hex!("0x6c1d9423")), next: 0 }, + ), ], ); } @@ -1940,6 +1965,60 @@ Post-merge hard forks (timestamp based): next: sepolia::SEPOLIA_BPO1_TIMESTAMP, }, ), + // First BPO1 block + ( + Head { + number: 1735377, + timestamp: sepolia::SEPOLIA_BPO1_TIMESTAMP, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0x56078a1e")), + next: sepolia::SEPOLIA_BPO2_TIMESTAMP, + }, + ), + // First BPO2 block + ( + Head { + number: 1735377, + timestamp: sepolia::SEPOLIA_BPO2_TIMESTAMP, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0x268956b6")), + next: sepolia::SEPOLIA_AMSTERDAM_TIMESTAMP, + }, + ), + // Last block before Amsterdam + ( + Head { + number: 1735377, + timestamp: sepolia::SEPOLIA_AMSTERDAM_TIMESTAMP - 1, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0x268956b6")), + next: sepolia::SEPOLIA_AMSTERDAM_TIMESTAMP, + }, + ), + // First Amsterdam block + ( + Head { + number: 1735377, + timestamp: sepolia::SEPOLIA_AMSTERDAM_TIMESTAMP, + ..Default::default() + }, + ForkId { hash: ForkHash(hex!("0x6c1d9423")), next: 0 }, + ), + // After Amsterdam + ( + Head { + number: 1735377, + timestamp: sepolia::SEPOLIA_AMSTERDAM_TIMESTAMP + 1, + ..Default::default() + }, + ForkId { hash: ForkHash(hex!("0x6c1d9423")), next: 0 }, + ), ], ); } @@ -2798,8 +2877,8 @@ Post-merge hard forks (timestamp based): #[test] fn latest_sepolia_mainnet_fork_id() { - // BPO2 - assert_eq!(ForkId { hash: ForkHash(hex!("0x268956b6")), next: 0 }, SEPOLIA.latest_fork_id()) + // Amsterdam + assert_eq!(ForkId { hash: ForkHash(hex!("0x6c1d9423")), next: 0 }, SEPOLIA.latest_fork_id()) } #[test] diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 12b12593357..4df2ccb6d1b 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -1003,6 +1003,56 @@ async fn test_eth_config() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_sepolia_amsterdam_eth_config() -> eyre::Result<()> { + use alloy_consensus::Header; + use alloy_primitives::hex; + use reth_chainspec::{ + sepolia::{SEPOLIA_AMSTERDAM_TIMESTAMP, SEPOLIA_BPO2_TIMESTAMP}, + SEPOLIA, + }; + use reth_primitives_traits::SealedHeader; + use reth_provider::CanonChainTracker; + + let node_config = NodeConfig::test() + .with_chain(SEPOLIA.clone()) + .with_unused_ports() + .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); + let NodeHandle { node, .. } = NodeBuilder::new(node_config) + .testing_node(Runtime::test()) + .node(EthereumNode::default()) + .launch() + .await?; + let provider = + ProviderBuilder::new().connect_http(node.rpc_server_handle().http_url().unwrap().parse()?); + + // eth_config reads the canonical header; move it across the fork without syncing the testnet. + for timestamp in [SEPOLIA_AMSTERDAM_TIMESTAMP - 1, SEPOLIA_AMSTERDAM_TIMESTAMP] { + node.provider.set_canonical_head(SealedHeader::seal_slow(Header { + number: 10_000_000, + timestamp, + base_fee_per_gas: Some(1_000_000_000), + excess_blob_gas: Some(0), + ..Default::default() + })); + let config = provider.client().request_noparams::("eth_config").await?; + if timestamp < SEPOLIA_AMSTERDAM_TIMESTAMP { + assert_eq!(config.current.activation_time, SEPOLIA_BPO2_TIMESTAMP); + assert_eq!(config.current.fork_id, Bytes::from_static(&hex!("268956b6"))); + let next = config.next.unwrap(); + assert_eq!(next.activation_time, SEPOLIA_AMSTERDAM_TIMESTAMP); + assert_eq!(next.fork_id, Bytes::from_static(&hex!("6c1d9423"))); + assert_eq!(config.last.unwrap(), next); + } else { + assert_eq!(config.current.activation_time, SEPOLIA_AMSTERDAM_TIMESTAMP); + assert_eq!(config.current.fork_id, Bytes::from_static(&hex!("6c1d9423"))); + assert!(config.next.is_none()); + assert!(config.last.is_none()); + } + } + Ok(()) +} + // #[tokio::test] async fn test_admin_external_ip() -> eyre::Result<()> { From 3fa8556452cb814287e345f1b8166f89d8862f94 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 21 Sep 2026 16:55:07 +0200 Subject: [PATCH 034/133] fix(test): stabilize hash builder and chain import tests (#27354) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../tests/e2e-testsuite/main.rs | 64 ++++++++----------- crates/trie/common/src/hash_builder/state.rs | 28 +++++--- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index 01367f0f7ce..f3ac2f95892 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -1,16 +1,16 @@ //! Example tests using the test suite framework. use alloy_primitives::{Address, B256}; -use alloy_rpc_types_engine::PayloadAttributes; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes, PayloadStatusEnum}; use eyre::Result; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::{ test_rlp_utils::{generate_test_blocks, write_blocks_to_rlp}, testsuite::{ actions::{ - Action, AssertChainTip, AssertMineBlock, BlockReference, CaptureBlock, - CaptureBlockOnNode, CompareNodeChainTips, CreateFork, FinalizeBlock, MakeCanonical, - ProduceBlocks, ReorgTo, SelectActiveNode, UpdateBlockInfo, + expect_fcu_valid, Action, AssertChainTip, AssertMineBlock, BlockReference, + CaptureBlock, CaptureBlockOnNode, CompareNodeChainTips, CreateFork, FinalizeBlock, + MakeCanonical, ProduceBlocks, ReorgTo, SelectActiveNode, UpdateBlockInfo, }, setup::{NetworkSetup, Setup}, Environment, TestBuilder, @@ -71,39 +71,31 @@ async fn test_apply_with_import() -> Result<()> { let mut make_canonical = MakeCanonical::new(); make_canonical.execute(&mut env).await?; - // Wait for the pipeline to finish processing all stages - debug!("Waiting for pipeline to finish processing imported blocks..."); - let start = std::time::Instant::now(); - loop { - // Check if we can get the block from RPC (indicates pipeline finished) - let client = &env.node_clients[0]; - let block_result = reth_rpc_api::clients::EthApiClient::< - alloy_rpc_types_eth::TransactionRequest, - alloy_rpc_types_eth::Transaction, - alloy_rpc_types_eth::Block, - alloy_rpc_types_eth::Receipt, - alloy_rpc_types_eth::Header, - reth_ethereum_primitives::TransactionSigned, - >::block_by_number( - &client.rpc, - alloy_eips::BlockNumberOrTag::Number(10), - true, // Include full transaction details - ) - .await; - - if let Ok(Some(block)) = block_result && - block.header.number == 10 - { - debug!("Pipeline finished, block 10 is fully available"); - break; - } - - if start.elapsed() > std::time::Duration::from_secs(10) { - return Err(eyre::eyre!("Timeout waiting for pipeline to finish")); + // Imported blocks are already readable from the database while the engine may still be + // syncing. Wait for the engine to accept the imported head before building on top of it. + let head = test_blocks.last().expect("imported blocks").hash(); + let engine = env.node_clients[0].engine.http_client(); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let response = + reth_rpc_api::clients::EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState { + head_block_hash: head, + safe_block_hash: head, + finalized_block_hash: B256::ZERO, + }, + None, + ) + .await?; + if response.payload_status.status != PayloadStatusEnum::Syncing { + return expect_fcu_valid(&response, "imported head"); + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } - - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } + }) + .await + .map_err(|_| eyre::eyre!("Timeout waiting for engine to accept imported head"))??; // Update block info again after making canonical let mut update_block_info_2 = UpdateBlockInfo::default(); diff --git a/crates/trie/common/src/hash_builder/state.rs b/crates/trie/common/src/hash_builder/state.rs index 0df582f8f5c..dc35f7150e4 100644 --- a/crates/trie/common/src/hash_builder/state.rs +++ b/crates/trie/common/src/hash_builder/state.rs @@ -7,11 +7,7 @@ use nybbles::Nibbles; /// Check the `reth-trie` crate for more info on hash builder. #[derive(Debug, Clone, PartialEq, Eq, Default)] #[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr( - feature = "arbitrary", - derive(arbitrary::Arbitrary), - reth_codecs::add_arbitrary_tests(compact) -)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct HashBuilderState { /// The current key. pub key: Vec, @@ -152,6 +148,8 @@ impl reth_codecs::Compact for HashBuilderState { #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "arbitrary")] + use proptest::{collection::vec, prelude::*}; use reth_codecs::Compact; #[test] @@ -164,14 +162,28 @@ mod tests { assert_eq!(state, decoded); } + // Generate fields directly: the arbitrary interop's fixed byte buffer can run out while + // generating variable-length fields, aborting the test before checking the codec. #[cfg(feature = "arbitrary")] proptest::proptest! { #[test] - fn hash_builder_state_roundtrip(state in proptest_arbitrary_interop::arb::()) { + fn hash_builder_state_roundtrip( + key in vec(0u8..16, 0..=64), + value in any::(), + stack in vec(any::(), 0..=64), + groups in vec(any::(), 0..=64), + tree_masks in vec(any::(), 0..=64), + hash_masks in vec(any::(), 0..=64), + stored_in_database in any::(), + ) { + let state = HashBuilderState { + key, value, stack, groups, tree_masks, hash_masks, stored_in_database, + }; let mut buf = vec![]; let len = state.to_compact(&mut buf); - let (decoded, _) = HashBuilderState::from_compact(&buf, len); - assert_eq!(state, decoded); + let (decoded, rest) = HashBuilderState::from_compact(&buf, len); + prop_assert_eq!(state, decoded); + prop_assert!(rest.is_empty()); } } } From 5b0120dbb373b12adce0a169c16ed0accd614a38 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:18:37 +0000 Subject: [PATCH 035/133] fix(rpc): lower default block and receipt cache limits (#27310) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-types/src/cache/config.rs | 6 +++--- crates/rpc/rpc-server-types/src/constants.rs | 8 ++++---- docs/vocs/docs/pages/cli/reth/node.mdx | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/cache/config.rs b/crates/rpc/rpc-eth-types/src/cache/config.rs index ac534abb39e..5fa675b276f 100644 --- a/crates/rpc/rpc-eth-types/src/cache/config.rs +++ b/crates/rpc/rpc-eth-types/src/cache/config.rs @@ -13,11 +13,11 @@ use reth_rpc_server_types::constants::cache::{ pub struct EthStateCacheConfig { /// Max number of blocks in cache. /// - /// Default is 5000. + /// Default is 1000. pub max_blocks: u32, - /// Max number receipts in cache. + /// Max number of blocks' receipts in cache. /// - /// Default is 2000. + /// Default is 500. pub max_receipts: u32, /// Max number of EVM BALs in cache. /// diff --git a/crates/rpc/rpc-server-types/src/constants.rs b/crates/rpc/rpc-server-types/src/constants.rs index e2ad2e6671d..1e58d4eeb57 100644 --- a/crates/rpc/rpc-server-types/src/constants.rs +++ b/crates/rpc/rpc-server-types/src/constants.rs @@ -114,11 +114,11 @@ pub mod gas_oracle { /// Cache specific constants pub mod cache { - /// Default cache size for the block cache: 5000 blocks. - pub const DEFAULT_BLOCK_CACHE_MAX_LEN: u32 = 5000; + /// Default cache size for the block cache: 1000 blocks. + pub const DEFAULT_BLOCK_CACHE_MAX_LEN: u32 = 1000; - /// Default cache size for the receipts cache: 2000 receipts. - pub const DEFAULT_RECEIPT_CACHE_MAX_LEN: u32 = 2000; + /// Default cache size for the receipts cache: 500 blocks' receipts. + pub const DEFAULT_RECEIPT_CACHE_MAX_LEN: u32 = 500; /// Legacy default for the no-op RPC header cache option. pub const DEFAULT_HEADER_CACHE_MAX_LEN: u32 = 1000; diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 867d8f62b5d..e1f1e8f4ee6 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -537,12 +537,12 @@ RPC State Cache: --rpc-cache.max-blocks Max number of blocks in cache - [default: 5000] + [default: 1000] --rpc-cache.max-receipts Max number receipts in cache - [default: 2000] + [default: 500] --rpc-cache.max-bals Max number of block access lists in cache From f79901e0022e08b4701426e6e8b084247344c1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:32:46 +0000 Subject: [PATCH 036/133] feat(snap-sync): run snap sync as one resumable session (#27345) --- Cargo.lock | 1 + crates/snap-sync/Cargo.toml | 1 + crates/snap-sync/src/account/mod.rs | 1 + crates/snap-sync/src/account/store.rs | 2 +- crates/snap-sync/src/attempt.rs | 34 +- crates/snap-sync/src/bootstrap.rs | 780 ++++++++++++++++++++++++++ crates/snap-sync/src/common/record.rs | 6 + crates/snap-sync/src/error.rs | 20 + crates/snap-sync/src/lib.rs | 6 +- crates/snap-sync/src/pivot.rs | 10 +- crates/snap-sync/src/session.rs | 30 + crates/snap-sync/src/storage/mod.rs | 2 +- crates/snap-sync/src/storage/store.rs | 2 +- crates/snap-sync/src/test_utils.rs | 11 + crates/snap-sync/src/verify.rs | 12 +- 15 files changed, 908 insertions(+), 10 deletions(-) create mode 100644 crates/snap-sync/src/bootstrap.rs diff --git a/Cargo.lock b/Cargo.lock index 49b05e4a2bf..b1ee9ee83da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10431,6 +10431,7 @@ dependencies = [ "thiserror 2.0.20", "tokio", "tokio-util", + "tracing", ] [[package]] diff --git a/crates/snap-sync/Cargo.toml b/crates/snap-sync/Cargo.toml index 6d1470960fe..afd5eaf5a91 100644 --- a/crates/snap-sync/Cargo.toml +++ b/crates/snap-sync/Cargo.toml @@ -38,6 +38,7 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true thiserror.workspace = true tokio-util.workspace = true +tracing.workspace = true [dev-dependencies] alloy-consensus.workspace = true diff --git a/crates/snap-sync/src/account/mod.rs b/crates/snap-sync/src/account/mod.rs index 765ab7cba08..c347c535c36 100644 --- a/crates/snap-sync/src/account/mod.rs +++ b/crates/snap-sync/src/account/mod.rs @@ -4,4 +4,5 @@ mod download; mod store; pub use download::{AccountRangeDownload, AccountRangeStep, VerifiedRange}; +pub(crate) use store::StoredCoverage; pub use store::{AccountCoverage, SnapAccountStore}; diff --git a/crates/snap-sync/src/account/store.rs b/crates/snap-sync/src/account/store.rs index c52a5cca7bb..d4a8ed971f7 100644 --- a/crates/snap-sync/src/account/store.rs +++ b/crates/snap-sync/src/account/store.rs @@ -113,7 +113,7 @@ impl AccountCoverage { // The coverage record as persisted, tied to the attempt that recorded it. #[derive(Serialize, Deserialize)] -struct StoredCoverage { +pub(crate) struct StoredCoverage { // Encoding version, checked before the rest is decoded. version: u32, // Attempt the coverage belongs to. diff --git a/crates/snap-sync/src/attempt.rs b/crates/snap-sync/src/attempt.rs index c26fff0901b..1d0cbfd38e0 100644 --- a/crates/snap-sync/src/attempt.rs +++ b/crates/snap-sync/src/attempt.rs @@ -4,7 +4,10 @@ //! attempt produced it. Every write presents a [`SnapWrite`]; ones that no longer match are //! refused. -use crate::{CatchUpProgress, SnapGeneration, SnapSyncError}; +use crate::{ + account::StoredCoverage, common::SnapRecord, storage::StoredProgress, verify::StoredRebuild, + CatchUpProgress, SnapGeneration, SnapSyncError, +}; use reth_storage_api::{ BlockHashReader, MetadataProvider, MetadataWriter, SnapAttempt, SnapAttemptId, StorageSettings, }; @@ -15,8 +18,8 @@ use serde::{Deserialize, Serialize}; /// Blanket-implemented over node metadata access, so these writes join the caller's transaction: /// state, bytecode and the attempt record commit together or not at all. pub trait SnapAttemptStore { - /// Starts an attempt anchored to `generation`, superseding any already recorded, with its - /// catch-up progress at that pivot. + /// Starts an attempt anchored to `generation`, superseding any already recorded and the + /// progress it kept, with its catch-up progress at that pivot. fn start_snap_attempt(&self, generation: SnapGeneration) -> Result where Self: MetadataWriter; @@ -102,6 +105,11 @@ impl SnapAttemptStore for T { self.write_snap_attempt(&attempt)?; // Ranges committed at this pivot need every list after it, however far the pivot moves. CatchUpProgress::at_pivot(attempt.pivot()).write(self, attempt.id())?; + // Earlier attempts' progress never applies to this one, and a record another build wrote + // would fail every read of it. + StoredCoverage::clear(self)?; + StoredProgress::clear(self)?; + StoredRebuild::clear(self)?; Ok(SnapWrite::of(&attempt)) } @@ -271,6 +279,26 @@ mod tests { assert_eq!(provider.snap_attempt().unwrap(), None); } + #[test] + fn a_new_attempt_drops_progress_another_build_wrote() { + let factory = factory(); + let provider = factory.database_provider_rw().unwrap(); + let unreadable = br#"{"version":999}"#.to_vec(); + for key in [StoredCoverage::KEY, StoredProgress::KEY, StoredRebuild::KEY] { + provider.write_metadata(key, unreadable.clone()).unwrap(); + } + + let write = provider.start_snap_attempt(generation(1)).unwrap(); + + assert_eq!( + crate::SnapAccountStore::start_account_coverage(&provider, write).unwrap(), + crate::AccountCoverage::START + ); + for key in [StoredProgress::KEY, StoredRebuild::KEY] { + assert_eq!(provider.get_metadata(key).unwrap(), None); + } + } + #[test] fn restarting_at_the_same_pivot_takes_a_new_identity() { let factory = factory(); diff --git a/crates/snap-sync/src/bootstrap.rs b/crates/snap-sync/src/bootstrap.rs new file mode 100644 index 00000000000..a549d05ff93 --- /dev/null +++ b/crates/snap-sync/src/bootstrap.rs @@ -0,0 +1,780 @@ +//! Runs snap synchronization from pivot selection until its state is handed to the trie rebuild. +//! +//! Every step commits before the next one starts and the attempt record is re-read on each pass, +//! so a run stopped anywhere resumes from what the database holds. + +use crate::{ + AccountRangeDownload, AccountRangeStep, BlockAccessListCatchUp, BytecodeDownload, BytecodeStep, + CatchUpStep, SnapAccountStore, SnapAttemptStore, SnapCatchUpStore, SnapGeneration, + SnapPivotPolicy, SnapStateVerifier, SnapSyncError, SnapSyncSession, SnapWrite, + StorageRangeDownload, StorageRangeStep, VerifiedRange, DEFAULT_SCAN_CHUNK, +}; +use alloy_eips::BlockNumHash; +use reth_db_api::transaction::DbTxMut; +use reth_network_p2p::snap::client::SnapClient; +use reth_storage_api::{ + BlockHashReader, DBProvider, DatabaseProviderFactory, HeaderProvider, MetadataProvider, + MetadataWriter, StageCheckpointWriter, StateWriter, +}; +use reth_storage_errors::provider::ProviderError; +use reth_tasks::Runtime; +use std::{fmt, future::Future}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info}; + +/// Default number of account ranges committed between pivot checks. +/// +/// Bounds how long a pivot can lag unnoticed while ranges download. +pub const DEFAULT_RANGES_PER_CHECK: usize = 64; + +/// Drives snap synchronization until its downloaded state is ready for the trie rebuild. +/// +/// Each pass moves a lagging pivot forward, applies the block access lists that carry the +/// downloaded state to it, then downloads account ranges with their storage and code. An attempt +/// whose pivot leaves the canonical chain or outlives the served lists is abandoned and started +/// over. +pub struct SnapBootstrap { + factory: F, + // Blocking database work runs off the async worker. + runtime: Runtime, + // Head, finality and peer progress come from the node running the sync. + context: X, + policy: SnapPivotPolicy, + // Target of the attempt being driven, rebuilt from the attempt record on every pass. + session: SnapSyncSession, + ranges_per_check: usize, + // Stops the run at the next step boundary, leaving committed progress in place. + cancel: CancellationToken, + accounts: AccountRangeDownload, + storage: StorageRangeDownload, + bytecode: BytecodeDownload, + catch_up: BlockAccessListCatchUp, +} + +impl SnapBootstrap { + /// Creates a run that has not touched the network or the database yet. + pub fn new(client: C, factory: F, runtime: Runtime, context: X) -> Self { + let policy = SnapPivotPolicy::default(); + Self { + accounts: AccountRangeDownload::new(client.clone(), factory.clone(), runtime.clone()), + storage: StorageRangeDownload::new(client.clone(), factory.clone(), runtime.clone()), + bytecode: BytecodeDownload::new(client.clone(), factory.clone(), runtime.clone()), + catch_up: BlockAccessListCatchUp::new(client, factory.clone(), runtime.clone()), + factory, + runtime, + context, + policy, + session: SnapSyncSession::new(policy), + ranges_per_check: DEFAULT_RANGES_PER_CHECK, + cancel: CancellationToken::new(), + } + } +} + +impl SnapBootstrap { + /// Returns this run anchoring and re-anchoring pivots with `policy`. + pub fn with_policy(mut self, policy: SnapPivotPolicy) -> Self { + self.policy = policy; + self.session = SnapSyncSession::new(policy); + self + } + + /// Returns this run committing at most `ranges` account ranges between pivot checks, at + /// least one. + pub const fn with_ranges_per_check(mut self, ranges: usize) -> Self { + self.ranges_per_check = if ranges == 0 { 1 } else { ranges }; + self + } + + /// Returns this run stopping once `cancel` fires. + pub fn with_cancellation(mut self, cancel: CancellationToken) -> Self { + self.cancel = cancel; + self + } +} + +impl SnapBootstrap +where + C: SnapClient + Clone + Unpin, + F: DatabaseProviderFactory + Clone + 'static, + F::Provider: HeaderProvider + MetadataProvider + BlockHashReader, + F::ProviderRW: HeaderProvider + + BlockHashReader + + MetadataProvider + + MetadataWriter + + StageCheckpointWriter + + StateWriter + + DBProvider, + X: SnapSyncContext, +{ + /// Runs until the downloaded state is handed to the merkle stage, or the run stops. + /// + /// Peers that do not serve the state, or headers not downloaded yet, wait for the context to + /// report progress instead of failing the run. + pub async fn run(&mut self) -> Result { + loop { + if self.cancel.is_cancelled() { + return Ok(SnapBootstrapOutcome::Stopped) + } + let head = self.context.head()?; + let step = match self.resolve(head)? { + Resolved::Verified(pivot) => return Ok(SnapBootstrapOutcome::Verified { pivot }), + Resolved::Waiting => { + debug!(target: "sync::snap", head, "No eligible snap pivot"); + Step::Wait + } + Resolved::Active(write) => match self.drive(write, head).await { + Ok(step) => step, + Err(error) if error.is_transient() => { + debug!(target: "sync::snap", %error, "Waiting for peers or headers"); + Step::Wait + } + Err(error) if error.is_reorg() => { + info!(target: "sync::snap", %error, "Snap pivot was reorged, restarting"); + Step::Restart + } + // Progress this build cannot read is unusable, so the attempt starts over. + Err(error @ SnapSyncError::UnsupportedRecord { .. }) => { + info!(target: "sync::snap", %error, "Snap progress is unreadable, restarting"); + Step::Restart + } + Err(SnapSyncError::Cancelled) => Step::Stop, + Err(error) => return Err(error), + }, + }; + match step { + Step::HandedOff(write) => { + let pivot = self.pivot()?; + info!(target: "sync::snap", ?pivot, "Snap state handed to the trie rebuild"); + return Ok(SnapBootstrapOutcome::TrieRebuild { write, pivot }) + } + Step::Continue => {} + Step::Wait => { + if !self.wait(head).await { + return Ok(SnapBootstrapOutcome::Stopped) + } + } + Step::Restart => { + let provider = self.factory.database_provider_rw()?; + provider.abandon_snap_attempt()?; + provider.commit()?; + } + Step::Stop => return Ok(SnapBootstrapOutcome::Stopped), + } + } + } + + // Resumes the recorded attempt while its pivot is canonical, otherwise starts one at the + // pivot under `head`. These are single-record writes, cheap enough for the async worker. + fn resolve(&mut self, head: u64) -> Result { + let provider = self.factory.database_provider_rw()?; + let mut session = SnapSyncSession::new(self.policy); + if let Some(attempt) = provider.snap_attempt()? { + if attempt.is_verified() { + return Ok(Resolved::Verified(attempt.pivot())) + } + if let Some(write) = provider.active_snap_write()? { + let generation = SnapGeneration::new(attempt.pivot(), attempt.state_root()); + if generation.is_canonical(&provider)? { + session.resume(generation); + self.session = session; + return Ok(Resolved::Active(write)) + } + info!(target: "sync::snap", pivot = ?attempt.pivot(), "Snap pivot was reorged, restarting"); + } + } + + session.select(&provider, head, self.context.finalized())?; + let Some((generation, _)) = session.start() else { return Ok(Resolved::Waiting) }; + let write = provider.start_snap_attempt(generation)?; + provider.start_account_coverage(write)?; + provider.commit()?; + info!( + target: "sync::snap", + pivot = ?generation.target(), + state_root = %generation.state_root(), + "Started snap attempt" + ); + self.session = session; + Ok(Resolved::Active(write)) + } + + // One pass over the attempt: pivot, catch-up, then up to `ranges_per_check` account ranges. + async fn drive(&mut self, write: SnapWrite, head: u64) -> Result { + let (applied, covered) = { + let provider = self.factory.database_provider_ro()?; + if provider.is_trie_rebuild_started(write)? { + return Ok(Step::HandedOff(write)) + } + let applied = provider + .catch_up_progress(write)? + .ok_or(SnapSyncError::NoCatchUpProgress)? + .applied(); + let covered = provider.account_coverage(write)?.is_some_and(|c| c.is_complete()); + (applied, covered) + }; + // Complete state carried to its pivot needs no further lists, only its trie rebuilt, so + // neither their retention nor a newer pivot applies to it. + if covered && applied == self.pivot()? { + return self.hand_off(write).await + } + // Catch-up continues from the last applied block, not the pivot, so once that block's + // successor is no longer served the state cannot be carried to any newer pivot. + if !self.policy.is_catchable_from(applied.number, head) { + info!(target: "sync::snap", ?applied, head, "Snap catch-up outlived the served block access lists, restarting"); + return Ok(Step::Restart) + } + let write = self.advance_pivot(write, head)?; + // Lists only update accounts already downloaded, so coverage grows once they reach the + // pivot. + if let Some(step) = self.catch_up(write).await? { + return Ok(step) + } + self.download_ranges(write).await + } + + // Moves a lagging pivot forward, since a few lists cost less than downloading state peers no + // longer serve. Returns the write the attempt accepts afterwards. + fn advance_pivot(&mut self, write: SnapWrite, head: u64) -> Result { + let provider = self.factory.database_provider_rw()?; + let from = self.pivot()?; + let Some(advanced) = + self.session.advance(&provider, write, head, self.context.finalized())? + else { + return Ok(write) + }; + // The database takes one writer at a time, so this commits before any download does. + provider.commit()?; + info!(target: "sync::snap", ?from, to = ?self.pivot()?, "Advanced snap pivot"); + Ok(advanced) + } + + // Applies lists until the downloaded state reaches the pivot. `Some` ends the pass. + async fn catch_up(&mut self, write: SnapWrite) -> Result, SnapSyncError> { + let pivot = self.pivot()?.number; + loop { + match self.catch_up.next(write, pivot).await? { + CatchUpStep::Complete => return Ok(None), + CatchUpStep::Applied { progress, .. } => { + debug!(target: "sync::snap", applied = ?progress.applied(), pivot, "Applied block access lists"); + } + CatchUpStep::Unavailable { .. } => return Ok(Some(Step::Wait)), + } + if self.cancel.is_cancelled() { + return Ok(Some(Step::Stop)) + } + } + } + + // Commits up to `ranges_per_check` account ranges, handing the state off once none remain. + async fn download_ranges(&mut self, write: SnapWrite) -> Result { + for _ in 0..self.ranges_per_check { + if self.cancel.is_cancelled() { + return Ok(Step::Stop) + } + let range = match self.accounts.next().await? { + Some(AccountRangeStep::Verified(range)) => range, + Some(AccountRangeStep::Unavailable { .. }) => return Ok(Step::Wait), + None => return self.hand_off(write).await, + }; + if let Some(step) = self.download_storage_and_code(&range).await? { + return Ok(step) + } + self.accounts.commit(range, Default::default(), Vec::new()).await?; + } + Ok(Step::Continue) + } + + // Persists the storage and code `range` needs. `Some` ends the pass. Both commit as they + // arrive, so a range dropped here is fetched again without repeating them. + async fn download_storage_and_code( + &mut self, + range: &VerifiedRange, + ) -> Result, SnapSyncError> { + loop { + match self.storage.next(range).await? { + StorageRangeStep::Complete => break, + StorageRangeStep::Committed(_) => {} + StorageRangeStep::Unavailable { .. } => return Ok(Some(Step::Wait)), + } + // A large contract takes many responses, each committed, so any of them is a + // resumable place to stop. + if self.cancel.is_cancelled() { + return Ok(Some(Step::Stop)) + } + } + loop { + match self.bytecode.next(range).await? { + BytecodeStep::Complete => return Ok(None), + BytecodeStep::Committed { .. } => {} + BytecodeStep::Unavailable { .. } => return Ok(Some(Step::Wait)), + } + if self.cancel.is_cancelled() { + return Ok(Some(Step::Stop)) + } + } + } + + // Checks the downloaded state is complete and hands it to the merkle stage. The scan reads + // every account, so it runs on the blocking pool. + async fn hand_off(&self, write: SnapWrite) -> Result { + let factory = self.factory.clone(); + let cancel = self.cancel.clone(); + self.runtime + .spawn_blocking(move || -> Result<(), SnapSyncError> { + let provider = factory.database_provider_rw()?; + provider.start_trie_rebuild(write, DEFAULT_SCAN_CHUNK, &cancel)?; + provider.commit()?; + Ok(()) + }) + .await + .map_err(|error| SnapSyncError::Provider(ProviderError::other(error)))??; + Ok(Step::HandedOff(write)) + } + + // Pivot of the attempt being driven. + fn pivot(&self) -> Result { + self.session.target().map(SnapGeneration::target).ok_or(SnapSyncError::NoAttempt) + } + + // Whether the context reported progress before the run was cancelled. + async fn wait(&mut self, head: u64) -> bool { + let Self { cancel, context, .. } = self; + cancel.run_until_cancelled(context.wait_for_progress(head)).await.unwrap_or(false) + } +} + +impl fmt::Debug for SnapBootstrap { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SnapBootstrap") + .field("policy", &self.policy) + .field("session", &self.session) + .field("ranges_per_check", &self.ranges_per_check) + .finish_non_exhaustive() + } +} + +/// How a [`SnapBootstrap`] run ended. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SnapBootstrapOutcome { + /// The downloaded state is complete and handed to the merkle stage; once the stage reaches + /// `pivot`, [`SnapStateVerifier::verify_state_root`] accepts it under `write`. + TrieRebuild { + /// Write the state was downloaded under. + write: SnapWrite, + /// Block the state is anchored to. + pivot: BlockNumHash, + }, + /// An earlier run's state was already verified, so there is nothing to download. + Verified { + /// Block the verified state is anchored to. + pivot: BlockNumHash, + }, + /// The run was cancelled or the context reported no further progress. Committed progress is + /// kept for the next run. + Stopped, +} + +/// What a [`SnapBootstrap`] needs to know about the chain and peers it synchronizes from. +pub trait SnapSyncContext: Send { + /// Highest block whose canonical header is downloaded. + fn head(&self) -> Result; + + /// Latest finalized block, preferred as the pivot when recent enough. + fn finalized(&self) -> Option { + None + } + + /// Waits until the head moves past `head` or new peers connect, returning `false` once no + /// further progress will come. + fn wait_for_progress(&mut self, head: u64) -> impl Future + Send; +} + +// Whether the attempt being driven can continue, and how. +enum Resolved { + // An unfinished attempt, resumed or just started, accepting this write. + Active(SnapWrite), + // An earlier run's state was verified at this pivot, so nothing is left to download. + Verified(BlockNumHash), + // No block is eligible as a pivot yet. + Waiting, +} + +// What one pass over the attempt left to do. +enum Step { + HandedOff(SnapWrite), + // The range budget ran out; the pivot is checked again before more ranges. + Continue, + // Peers or headers are missing until the node progresses. + Wait, + // The attempt's state can no longer be carried forward. + Restart, + // The run was cancelled; committed progress stays for the next one. + Stop, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{ + account, account_range, hashed_factory, header, key, policy, state_root, storage_ranges, + storage_root_of, verified_range, ScriptedSnapClient, + }; + use alloy_eip7928::{compute_block_access_list_hash, AccountChanges}; + use alloy_eips::eip7928::bal::Bal; + use alloy_primitives::{Bytes, B256, U256}; + use reth_eth_wire_types::{snap::BlockAccessListsMessage, BlockAccessLists}; + use reth_network_p2p::{ + error::{PeerRequestResult, RequestError}, + snap::client::SnapResponse, + }; + use reth_network_peers::{PeerId, WithPeerId}; + use reth_primitives_traits::SealedHeader; + use reth_provider::{ + test_utils::{insert_headers, MockNodeTypesWithDB}, + ProviderFactory, + }; + use reth_stages::stages::MerkleStage; + use reth_stages_api::{ExecInput, Stage}; + use reth_stages_types::StageId; + use reth_storage_api::{SnapAttemptId, StageCheckpointReader}; + use reth_trie_common::TrieAccount; + use std::{cell::RefCell, collections::VecDeque, sync::Arc}; + + type Factory = ProviderFactory; + type Bootstrap = SnapBootstrap, Factory, TestContext>; + + const FAR: B256 = B256::repeat_byte(0xaa); + + fn accounts() -> Vec<(B256, TrieAccount)> { + vec![(key(1), account(1)), (key(2), account(2)), (FAR, account(3))] + } + + // Blocks `0..=tip`, each committing to an empty list and to `root`, so any of them anchors + // the same state. + fn insert_chain(factory: &Factory, tip: u64, root: B256) { + let commitment = compute_block_access_list_hash(&Vec::::new()); + let mut parent = B256::ZERO; + let headers: Vec<_> = (0..=tip) + .map(|number| { + let mut header = header(number, parent, Some(commitment)); + header.state_root = root; + let sealed = SealedHeader::seal_slow(header); + parent = sealed.hash(); + sealed + }) + .collect(); + insert_headers(factory, &headers); + } + + // A peer serving `blocks` empty lists. + fn empty_lists(request_id: u64, blocks: usize) -> PeerRequestResult { + let list: Bytes = alloy_rlp::encode(Bal::from(Vec::::new())).into(); + let message = BlockAccessListsMessage { + request_id, + block_access_lists: BlockAccessLists(vec![Some(list); blocks]), + }; + Ok(WithPeerId::new(PeerId::random(), SnapResponse::BlockAccessLists(message))) + } + + // A peer holding no list for the first block it is asked for. + fn no_lists(request_id: u64) -> PeerRequestResult { + let message = BlockAccessListsMessage { + request_id, + block_access_lists: BlockAccessLists(vec![None]), + }; + Ok(WithPeerId::new(PeerId::random(), SnapResponse::BlockAccessLists(message))) + } + + fn scripted( + factory: &Factory, + responses: impl IntoIterator>, + heads: impl IntoIterator, + ) -> (Arc, Bootstrap) { + let client = Arc::new(ScriptedSnapClient::new(responses)); + let context = TestContext { heads: RefCell::new(heads.into_iter().collect()), waits: 0 }; + let bootstrap = + SnapBootstrap::new(Arc::clone(&client), factory.clone(), Runtime::test(), context) + .with_policy(policy()); + (client, bootstrap) + } + + fn attempt_id(factory: &Factory) -> SnapAttemptId { + factory.database_provider_ro().unwrap().snap_attempt().unwrap().unwrap().id() + } + + #[tokio::test] + async fn downloads_the_state_and_hands_it_to_the_trie_rebuild() { + let accounts = accounts(); + let factory = hashed_factory(); + insert_chain(&factory, 3, state_root(&accounts)); + let (client, mut bootstrap) = + scripted(&factory, [account_range(1, &accounts, 0..3, &[])], [3]); + + let outcome = bootstrap.run().await.unwrap(); + + let SnapBootstrapOutcome::TrieRebuild { write, pivot } = outcome else { + panic!("the state is complete: {outcome:?}") + }; + assert_eq!(pivot.number, 2); + assert_eq!(*client.origins(), [B256::ZERO]); + let provider = factory.database_provider_ro().unwrap(); + assert!(provider.is_trie_rebuild_started(write).unwrap()); + } + + #[tokio::test] + async fn a_lagging_pivot_advances_before_more_ranges_download() { + let accounts = accounts(); + let root = state_root(&accounts); + let factory = hashed_factory(); + insert_chain(&factory, 8, root); + let responses = [ + account_range(1, &accounts, 0..1, &[key(1)]), + // Blocks 3 through 7 carry the first account from pivot 2 to pivot 7. + empty_lists(1, 5), + account_range(2, &accounts, 1..3, &[key(2), FAR]), + ]; + // The head moves past the advance window after the first range. + let (client, bootstrap) = scripted(&factory, responses, [3, 8]); + let mut bootstrap = bootstrap.with_ranges_per_check(1); + + let outcome = bootstrap.run().await.unwrap(); + + let SnapBootstrapOutcome::TrieRebuild { write, pivot } = outcome else { + panic!("the state is complete: {outcome:?}") + }; + assert_eq!(pivot.number, 7); + assert_eq!(*client.origins(), [B256::ZERO, key(2)]); + assert_eq!(client.block_requests().len(), 1); + + // The merkle stage rebuilds the trie, which the pivot's header then accepts. + let provider = factory.database_provider_rw().unwrap(); + let mut stage = MerkleStage::default_execution(); + loop { + let checkpoint = provider.get_stage_checkpoint(StageId::MerkleExecute).unwrap(); + let output = + stage.execute(&provider, ExecInput { target: Some(7), checkpoint }).unwrap(); + provider.save_stage_checkpoint(StageId::MerkleExecute, output.checkpoint).unwrap(); + if output.done { + break + } + } + let verified = provider.verify_state_root(write).unwrap(); + assert_eq!((verified.target(), verified.state_root()), (pivot, root)); + } + + #[tokio::test] + async fn peers_without_the_state_wait_and_the_next_run_resumes() { + let accounts = accounts(); + let factory = hashed_factory(); + insert_chain(&factory, 3, state_root(&accounts)); + let responses = [ + account_range(1, &accounts, 0..1, &[key(1)]), + Err(RequestError::UnsupportedCapability), + ]; + let (_, mut bootstrap) = scripted(&factory, responses, [3]); + + assert_eq!(bootstrap.run().await.unwrap(), SnapBootstrapOutcome::Stopped); + assert_eq!(bootstrap.context.waits, 1); + let attempt = attempt_id(&factory); + + let (client, mut resumed) = + scripted(&factory, [account_range(1, &accounts, 1..3, &[key(2), FAR])], [3]); + let outcome = resumed.run().await.unwrap(); + + assert!(matches!(outcome, SnapBootstrapOutcome::TrieRebuild { .. })); + assert_eq!(attempt_id(&factory), attempt); + assert_eq!(*client.origins(), [key(2)]); + } + + #[tokio::test] + async fn no_eligible_pivot_waits_without_starting_an_attempt() { + let factory = hashed_factory(); + insert_chain(&factory, 0, state_root(&accounts())); + let (client, mut bootstrap) = scripted(&factory, [], [0]); + + assert_eq!(bootstrap.run().await.unwrap(), SnapBootstrapOutcome::Stopped); + + assert_eq!(bootstrap.context.waits, 1); + assert!(client.origins().is_empty()); + assert!(factory.database_provider_ro().unwrap().snap_attempt().unwrap().is_none()); + } + + #[tokio::test] + async fn a_reorged_pivot_restarts_the_attempt() { + let accounts = accounts(); + let factory = hashed_factory(); + insert_chain(&factory, 3, state_root(&accounts)); + // A previous run anchored to a block the canonical chain no longer holds. + let provider = factory.database_provider_rw().unwrap(); + let orphaned = SnapGeneration::new( + BlockNumHash::new(2, B256::repeat_byte(0xee)), + state_root(&accounts), + ); + provider.start_snap_attempt(orphaned).unwrap(); + provider.commit().unwrap(); + let orphaned = attempt_id(&factory); + let (_, mut bootstrap) = scripted(&factory, [account_range(1, &accounts, 0..3, &[])], [3]); + + let outcome = bootstrap.run().await.unwrap(); + + let SnapBootstrapOutcome::TrieRebuild { pivot, .. } = outcome else { + panic!("the state is complete: {outcome:?}") + }; + assert_ne!(pivot.hash, B256::repeat_byte(0xee)); + assert_ne!(attempt_id(&factory), orphaned); + } + + #[tokio::test] + async fn a_handed_off_attempt_is_not_downloaded_again() { + let accounts = accounts(); + let factory = hashed_factory(); + insert_chain(&factory, 3, state_root(&accounts)); + let (_, mut bootstrap) = scripted(&factory, [account_range(1, &accounts, 0..3, &[])], [3]); + let first = bootstrap.run().await.unwrap(); + + let (client, mut resumed) = scripted(&factory, [], [3]); + + assert_eq!(resumed.run().await.unwrap(), first); + assert!(client.origins().is_empty()); + } + + #[tokio::test] + async fn a_cancelled_run_stops_before_any_work() { + let factory = hashed_factory(); + insert_chain(&factory, 3, state_root(&accounts())); + let cancel = CancellationToken::new(); + let (client, bootstrap) = scripted(&factory, [], [3]); + let mut bootstrap = bootstrap.with_cancellation(cancel.clone()); + cancel.cancel(); + + assert_eq!(bootstrap.run().await.unwrap(), SnapBootstrapOutcome::Stopped); + assert!(client.origins().is_empty()); + } + + #[tokio::test] + async fn a_catch_up_stalled_past_the_served_lists_restarts() { + let accounts = accounts(); + let factory = hashed_factory(); + insert_chain(&factory, 11, state_root(&accounts)); + // The pivot moves from 2 to 7, but no peer serves block 3's list. + let responses = [account_range(1, &accounts, 0..1, &[key(1)]), no_lists(1)]; + let (_, stalled) = scripted(&factory, responses, [3, 8]); + let mut stalled = stalled.with_ranges_per_check(1); + assert_eq!(stalled.run().await.unwrap(), SnapBootstrapOutcome::Stopped); + let attempt = attempt_id(&factory); + + // Pivot 7 is still recent under head 11, but block 3's list is past the served history. + let (client, mut restarted) = + scripted(&factory, [account_range(1, &accounts, 0..3, &[])], [11]); + let outcome = restarted.run().await.unwrap(); + + let SnapBootstrapOutcome::TrieRebuild { pivot, .. } = outcome else { + panic!("the state is complete: {outcome:?}") + }; + assert_eq!(pivot.number, 10); + assert_ne!(attempt_id(&factory), attempt); + assert!(client.block_requests().is_empty()); + } + + #[tokio::test] + async fn complete_state_is_handed_off_past_the_served_lists() { + let accounts = accounts(); + let factory = hashed_factory(); + insert_chain(&factory, 11, state_root(&accounts)); + // A run committed every range at pivot 2, then stopped before the hand-off. + let provider = factory.database_provider_rw().unwrap(); + let pivot = provider.sealed_header(2).unwrap().unwrap(); + let generation = SnapGeneration::new(pivot.num_hash(), pivot.state_root); + let downloaded = provider.start_snap_attempt(generation).unwrap(); + provider.start_account_coverage(downloaded).unwrap(); + let range = verified_range(&accounts, 0..3, B256::ZERO, &[]); + provider.commit_account_range(downloaded, &range, Default::default(), Vec::new()).unwrap(); + provider.commit().unwrap(); + + // Block 3's list is past the served history under head 11, but nothing needs it. + let (client, mut resumed) = scripted(&factory, [], [11]); + let outcome = resumed.run().await.unwrap(); + + assert_eq!( + outcome, + SnapBootstrapOutcome::TrieRebuild { write: downloaded, pivot: pivot.num_hash() } + ); + assert!(client.origins().is_empty()); + assert!(client.block_requests().is_empty()); + } + + #[tokio::test] + async fn unreadable_progress_restarts_the_attempt() { + let accounts = accounts(); + let factory = hashed_factory(); + insert_chain(&factory, 3, state_root(&accounts)); + let (_, mut stopped) = scripted(&factory, [Err(RequestError::UnsupportedCapability)], [3]); + assert_eq!(stopped.run().await.unwrap(), SnapBootstrapOutcome::Stopped); + // Another build rewrote the coverage the attempt resumes from. + let provider = factory.database_provider_rw().unwrap(); + let stale = provider.active_snap_write().unwrap().unwrap(); + provider.write_metadata("snap_account_coverage", br#"{"version":999}"#.to_vec()).unwrap(); + provider.commit().unwrap(); + + let (_, mut restarted) = scripted(&factory, [account_range(1, &accounts, 0..3, &[])], [3]); + let outcome = restarted.run().await.unwrap(); + + let SnapBootstrapOutcome::TrieRebuild { write, .. } = outcome else { + panic!("the state is complete: {outcome:?}") + }; + assert_ne!(write.attempt(), stale.attempt()); + let provider = factory.database_provider_ro().unwrap(); + assert!(matches!( + provider.authorize_snap_write(stale), + Err(SnapSyncError::StaleWrite { .. }) + )); + } + + #[tokio::test] + async fn cancellation_between_storage_responses_stops_the_run() { + let slots = vec![(key(1), U256::from(11)), (key(2), U256::from(12))]; + let mut contract = account(1); + contract.storage_root = storage_root_of(&slots); + let accounts = vec![(key(1), contract)]; + let factory = hashed_factory(); + insert_chain(&factory, 3, state_root(&accounts)); + let cancel = CancellationToken::new(); + let on_request = cancel.clone(); + // The contract's storage takes two responses, and shutdown fires during the first. + let client = Arc::new( + ScriptedSnapClient::new([ + account_range(1, &accounts, 0..1, &[]), + storage_ranges(1, &[&slots[..1]], &slots, &[B256::ZERO, key(1)]), + ]) + .on_storage_request(move || on_request.cancel()), + ); + let context = TestContext { heads: RefCell::new(VecDeque::from([3])), waits: 0 }; + let mut bootstrap = + SnapBootstrap::new(Arc::clone(&client), factory.clone(), Runtime::test(), context) + .with_policy(policy()) + .with_cancellation(cancel); + + assert_eq!(bootstrap.run().await.unwrap(), SnapBootstrapOutcome::Stopped); + assert_eq!(client.storage_requests().len(), 1); + } + + // Serves heads in order, repeating the last, and ends the run at the first wait. + struct TestContext { + heads: RefCell>, + waits: usize, + } + + impl SnapSyncContext for TestContext { + fn head(&self) -> Result { + let mut heads = self.heads.borrow_mut(); + let head = if heads.len() > 1 { heads.pop_front() } else { heads.front().copied() }; + Ok(head.expect("the context is given at least one head")) + } + + fn wait_for_progress(&mut self, _head: u64) -> impl Future + Send { + self.waits += 1; + std::future::ready(false) + } + } +} diff --git a/crates/snap-sync/src/common/record.rs b/crates/snap-sync/src/common/record.rs index b34744395b2..a39120ad25a 100644 --- a/crates/snap-sync/src/common/record.rs +++ b/crates/snap-sync/src/common/record.rs @@ -30,6 +30,12 @@ pub(crate) trait SnapRecord: Serialize + DeserializeOwned { Ok(Some(serde_json::from_value(value).map_err(ProviderError::other)?)) } + /// Removes the record, whichever version wrote it. + fn clear(provider: &impl MetadataWriter) -> Result<(), SnapSyncError> { + provider.delete_metadata(Self::KEY)?; + Ok(()) + } + /// Writes this record under its key. fn write(&self, provider: &impl MetadataWriter) -> Result<(), SnapSyncError> { provider diff --git a/crates/snap-sync/src/error.rs b/crates/snap-sync/src/error.rs index 2e48563d655..7a6fc4fe450 100644 --- a/crates/snap-sync/src/error.rs +++ b/crates/snap-sync/src/error.rs @@ -184,6 +184,26 @@ pub enum SnapSyncError { Cancelled, } +impl SnapSyncError { + /// Whether the node's progress resolves this error, as new peers serve the state or missing + /// headers are downloaded. + /// + /// A closed channel means the network is gone, not that peers lack the state. + pub const fn is_transient(&self) -> bool { + match self { + Self::Request(error) => !error.is_channel_closed(), + Self::MissingHeader { .. } => true, + _ => false, + } + } + + /// Whether a block the attempt builds on left the canonical chain, so its downloaded state + /// belongs to another fork. + pub const fn is_reorg(&self) -> bool { + matches!(self, Self::NonCanonicalBlock { .. } | Self::ForkedBlock { .. }) + } +} + impl From for SnapSyncError { fn from(error: DatabaseError) -> Self { Self::Provider(error.into()) diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index c3eb8ec83b3..352c2f97ec8 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -7,7 +7,7 @@ //! //! This crate owns download progress only. Authenticated downloads come from //! `reth-downloaders`, and verified state is handed back to node integration once its trie root -//! matches the target header. +//! matches the target header. [`SnapBootstrap`] runs these steps as one resumable sync. //! //! Downloaded state goes into the hashed state tables, owned by an attempt record that commits //! with it, along with how far the account key space has been downloaded. An account range only @@ -38,6 +38,7 @@ mod account; mod attempt; +mod bootstrap; mod bytecode; mod catch_up; mod common; @@ -55,6 +56,9 @@ pub use account::{ AccountCoverage, AccountRangeDownload, AccountRangeStep, SnapAccountStore, VerifiedRange, }; pub use attempt::{SnapAttemptStore, SnapWrite}; +pub use bootstrap::{ + SnapBootstrap, SnapBootstrapOutcome, SnapSyncContext, DEFAULT_RANGES_PER_CHECK, +}; pub use bytecode::{BytecodeDownload, BytecodeStep, SnapBytecodeStore, DEFAULT_CODE_HASHES}; pub use catch_up::{ BalStateUpdate, BlockAccessListCatchUp, CatchUpProgress, CatchUpStep, DownloadedAccount, diff --git a/crates/snap-sync/src/pivot.rs b/crates/snap-sync/src/pivot.rs index 8409a498fdf..bfa4ed267a5 100644 --- a/crates/snap-sync/src/pivot.rs +++ b/crates/snap-sync/src/pivot.rs @@ -94,7 +94,15 @@ impl SnapPivotPolicy { /// /// Once they are not, its state cannot be carried forward and the attempt has to restart. pub const fn is_catchable(&self, generation: SnapGeneration, head: u64) -> bool { - generation.lag(head) <= self.history + self.is_catchable_from(generation.target().number, head) + } + + /// Returns whether the list of the block after `applied` remains servable under `head`. + /// + /// Catch-up continues from the last applied block, which trails the pivot while lists are + /// missing, so an attempt whose pivot is recent can still need an expired list. + pub const fn is_catchable_from(&self, applied: u64, head: u64) -> bool { + head.saturating_sub(applied) <= self.history } /// Returns a fresh generation for the canonical pivot under `head`. diff --git a/crates/snap-sync/src/session.rs b/crates/snap-sync/src/session.rs index 7a34404564e..b29b5d0bdd1 100644 --- a/crates/snap-sync/src/session.rs +++ b/crates/snap-sync/src/session.rs @@ -73,6 +73,19 @@ impl SnapSyncSession { Some((generation, self.cancellation.clone())) } + /// Takes over the target of an attempt a previous run recorded, as [`Self::start`] does for a + /// selected one. + /// + /// Only a session that has not handed out a target can take one over. + pub fn resume(&mut self, generation: SnapGeneration) -> Option { + if !matches!(self.state, SnapSyncSessionState::Waiting | SnapSyncSessionState::Selected(_)) + { + return None + } + self.state = SnapSyncSessionState::Downloading(generation); + Some(self.cancellation.clone()) + } + /// Re-anchors the downloaded target once it lags too far behind `head`, returning the new /// write. /// @@ -240,6 +253,23 @@ mod tests { assert!(session.start().is_none()); } + #[test] + fn a_recorded_target_is_resumed_once() { + let (factory, _, write) = downloading(); + let provider = factory.database_provider_rw().unwrap(); + let attempt = provider.authorize_snap_write(write).unwrap(); + let recorded = SnapGeneration::new(attempt.pivot(), attempt.state_root()); + let mut session = SnapSyncSession::new(policy().with_advance_after(1)); + + assert!(session.resume(recorded).is_some()); + assert!(session.resume(recorded).is_none()); + assert!(session.start().is_none()); + assert_eq!(session.state(), &SnapSyncSessionState::Downloading(recorded)); + // A resumed target advances like a started one. + assert!(session.advance(&provider, write, 3, None).unwrap().is_some()); + assert_eq!(session.target().unwrap().target().number, 2); + } + #[test] fn cancellation_stops_outstanding_work() { let provider = provider_with(chain(Some(0))); diff --git a/crates/snap-sync/src/storage/mod.rs b/crates/snap-sync/src/storage/mod.rs index 976641b934f..d6433124c36 100644 --- a/crates/snap-sync/src/storage/mod.rs +++ b/crates/snap-sync/src/storage/mod.rs @@ -4,5 +4,5 @@ mod download; mod store; pub use download::{StorageRangeDownload, StorageRangeStep, DEFAULT_STORAGE_ACCOUNTS}; -pub(crate) use store::persisted_storage_root; +pub(crate) use store::{persisted_storage_root, StoredProgress}; pub use store::{SnapStorageStore, StorageChunk, StorageProgress}; diff --git a/crates/snap-sync/src/storage/store.rs b/crates/snap-sync/src/storage/store.rs index d0e5b24abba..e3373bbf3af 100644 --- a/crates/snap-sync/src/storage/store.rs +++ b/crates/snap-sync/src/storage/store.rs @@ -203,7 +203,7 @@ struct PartialStorage { // The progress record as persisted, tied to the write and range that recorded it. #[derive(Serialize, Deserialize)] -struct StoredProgress { +pub(crate) struct StoredProgress { // Encoding version, checked before the rest is decoded. version: u32, // Attempt the slots belong to. diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs index 27b468e53bb..3b53edfc689 100644 --- a/crates/snap-sync/src/test_utils.rs +++ b/crates/snap-sync/src/test_utils.rs @@ -371,6 +371,7 @@ pub(crate) struct ScriptedSnapClient { code_requests: Mutex>>, block_requests: Mutex>>, on_block_request: Mutex>>, + on_storage_request: Mutex>>, } impl ScriptedSnapClient { @@ -384,6 +385,7 @@ impl ScriptedSnapClient { code_requests: Mutex::new(Vec::new()), block_requests: Mutex::new(Vec::new()), on_block_request: Mutex::new(None), + on_storage_request: Mutex::new(None), } } @@ -393,6 +395,12 @@ impl ScriptedSnapClient { self } + /// Runs `hook` after the next storage request is recorded, before returning its response. + pub(crate) fn on_storage_request(self, hook: impl FnOnce() + Send + 'static) -> Self { + *self.on_storage_request.lock().unwrap() = Some(Box::new(hook)); + self + } + /// Origins of the account range requests sent so far. pub(crate) fn origins(&self) -> MutexGuard<'_, Vec> { self.origins.lock().unwrap() @@ -456,6 +464,9 @@ impl SnapClient for ScriptedSnapClient { ) -> Self::Output { let from = request.starting_hash.unwrap_or(B256::ZERO); self.storage_requests.lock().unwrap().push((request.account_hashes, from)); + if let Some(hook) = self.on_storage_request.lock().unwrap().take() { + hook(); + } self.next_response() } diff --git a/crates/snap-sync/src/verify.rs b/crates/snap-sync/src/verify.rs index 4e9938700c8..75b5277051a 100644 --- a/crates/snap-sync/src/verify.rs +++ b/crates/snap-sync/src/verify.rs @@ -52,6 +52,10 @@ pub trait SnapStateVerifier { where Self: DBProvider; + /// Returns whether `write`'s state was handed to the merkle stage, so a resumed sync does not + /// reset its rebuild. + fn is_trie_rebuild_started(&self, write: SnapWrite) -> Result; + /// Accepts the state once the merkle stage has rebuilt its trie up to the pivot. /// /// The stage checks the root against the pivot header, which must commit to the root the @@ -138,6 +142,10 @@ impl SnapStateVerifier for T { ensure_code_present(self.tx_ref(), chunk, cancel) } + fn is_trie_rebuild_started(&self, write: SnapWrite) -> Result { + Ok(StoredRebuild::read(self)?.is_some_and(|stored| stored.write == write)) + } + fn verify_state_root(&self, write: SnapWrite) -> Result where Self: BlockHashReader + HeaderProvider + MetadataWriter + StageCheckpointReader, @@ -159,7 +167,7 @@ impl SnapStateVerifier for T { } // Until the stage reaches the pivot after this state's hand-off, the trie holds no state // for it: an earlier attempt's rebuild can end at the same block. - let handed_off = StoredRebuild::read(self)?.is_some_and(|stored| stored.write == write); + let handed_off = self.is_trie_rebuild_started(write)?; let rebuilt = self.get_stage_checkpoint(StageId::MerkleExecute)?; if !handed_off || rebuilt.map(|checkpoint| checkpoint.block_number) != Some(target.number) { return Err(ProviderError::StateForNumberNotFound(target.number).into()) @@ -171,7 +179,7 @@ impl SnapStateVerifier for T { // The trie rebuild hand-off as persisted, tied to the write it was made for. #[derive(Serialize, Deserialize)] -struct StoredRebuild { +pub(crate) struct StoredRebuild { // Encoding version, checked before the rest is decoded. version: u32, // Attempt and pivot the state was handed off at. From 778a9e16823a087249575c68dc1861cae32f9add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:19:12 +0200 Subject: [PATCH 037/133] chore(engine): replace `build_engine_orchestrator` args with a builder struct (#27359) --- crates/engine/tree/src/launch.rs | 150 +++++++++++++++-------- crates/node/builder/src/launch/engine.rs | 31 ++--- 2 files changed, 115 insertions(+), 66 deletions(-) diff --git a/crates/engine/tree/src/launch.rs b/crates/engine/tree/src/launch.rs index 9d524c8c35e..24297608bb1 100644 --- a/crates/engine/tree/src/launch.rs +++ b/crates/engine/tree/src/launch.rs @@ -1,7 +1,7 @@ //! Engine orchestrator launch helper. //! -//! Provides [`build_engine_orchestrator`](crate::launch::build_engine_orchestrator) which wires -//! together all engine components and returns a +//! Provides [`EngineOrchestratorBuilder`](crate::launch::EngineOrchestratorBuilder) which wires +//! together all engine components and builds a //! [`ChainOrchestrator`](crate::chain::ChainOrchestrator) ready to be polled as a `Stream`. use crate::{ @@ -29,9 +29,19 @@ use reth_storage_overlay::OverlayManager; use reth_tasks::Runtime; use std::sync::Arc; -/// Builds the engine [`ChainOrchestrator`] that drives the chain forward. +/// The [`ChainOrchestrator`] built by [`EngineOrchestratorBuilder`]. +pub type EngineOrchestrator = ChainOrchestrator< + EngineHandler< + EngineApiRequestHandler, N>, + S, + BasicBlockDownloader::Block>, + >, + B, +>; + +/// Components needed to build the engine [`ChainOrchestrator`] that drives the chain forward. /// -/// This spawns and wires together the following components: +/// [`build`](Self::build) spawns and wires together the following components: /// /// - **[`BasicBlockDownloader`]** — downloads blocks on demand from the network during live sync. /// - **[`PersistenceHandle`]** — spawns the persistence service on a background thread for writing @@ -47,32 +57,46 @@ use std::sync::Arc; /// [`ChainEvent`]s. /// /// [`ChainEvent`]: crate::chain::ChainEvent -#[expect(clippy::too_many_arguments, clippy::type_complexity)] -pub fn build_engine_orchestrator( - engine_kind: EngineApiKind, - consensus: Arc>, - client: Client, - incoming_requests: S, - pipeline: Pipeline, - pipeline_task_spawner: Runtime, - provider: ProviderFactory, - blockchain_db: BlockchainProvider, - pruner: PrunerWithFactory>, - payload_builder: PayloadBuilderHandle, - payload_validator: V, - overlay_manager: OverlayManager, - tree_config: TreeConfig, - sync_metrics_tx: MetricEventsSender, - evm_config: C, - runtime: Runtime, -) -> ChainOrchestrator< - EngineHandler< - EngineApiRequestHandler, N::Primitives>, - S, - BasicBlockDownloader::Block>, - >, - PipelineSync, -> +#[derive(Debug)] +pub struct EngineOrchestratorBuilder +where + N: ProviderNodeTypes, +{ + /// The engine API flavor to run. + pub engine_kind: EngineApiKind, + /// Consensus used to validate downloaded and incoming blocks. + pub consensus: Arc>, + /// Network client used to download blocks during live sync. + pub client: Client, + /// Incoming consensus layer messages. + pub incoming_requests: S, + /// Staged sync pipeline used for backfill. + pub pipeline: Pipeline, + /// Runtime the pipeline runs on. + pub pipeline_task_spawner: Runtime, + /// Provider factory handed to the persistence service. + pub provider: ProviderFactory, + /// Blockchain provider backing the engine tree. + pub blockchain_db: BlockchainProvider, + /// Pruner run by the persistence service. + pub pruner: PrunerWithFactory>, + /// Handle to the payload builder service. + pub payload_builder: PayloadBuilderHandle, + /// Validator for incoming payloads. + pub payload_validator: V, + /// Overlay manager for state on top of the database. + pub overlay_manager: OverlayManager, + /// Engine tree configuration. + pub tree_config: TreeConfig, + /// Sender for sync metric events. + pub sync_metrics_tx: MetricEventsSender, + /// EVM configuration used to execute payloads. + pub evm_config: C, + /// Runtime used to spawn engine tree tasks. + pub runtime: Runtime, +} + +impl EngineOrchestratorBuilder where N: ProviderNodeTypes, Client: BlockClient::Block> @@ -82,31 +106,55 @@ where V: EngineValidator + WaitForCaches, C: ConfigureEvm + 'static, { - let downloader = BasicBlockDownloader::new(client, consensus.clone()); + /// Spawns the engine services and returns the [`ChainOrchestrator`] driving them. + pub fn build( + self, + ) -> EngineOrchestrator> { + let Self { + engine_kind, + consensus, + client, + incoming_requests, + pipeline, + pipeline_task_spawner, + provider, + blockchain_db, + pruner, + payload_builder, + payload_validator, + overlay_manager, + tree_config, + sync_metrics_tx, + evm_config, + runtime, + } = self; + + let downloader = BasicBlockDownloader::new(client, consensus.clone()); - let persistence_handle = - PersistenceHandle::::spawn_service(provider, pruner, sync_metrics_tx); + let persistence_handle = + PersistenceHandle::::spawn_service(provider, pruner, sync_metrics_tx); - let canonical_in_memory_state = blockchain_db.canonical_in_memory_state(); + let canonical_in_memory_state = blockchain_db.canonical_in_memory_state(); - let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new( - blockchain_db, - consensus, - payload_validator, - persistence_handle, - payload_builder, - canonical_in_memory_state, - overlay_manager, - tree_config, - engine_kind, - evm_config, - runtime, - ); + let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new( + blockchain_db, + consensus, + payload_validator, + persistence_handle, + payload_builder, + canonical_in_memory_state, + overlay_manager, + tree_config, + engine_kind, + evm_config, + runtime, + ); - let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree); - let handler = EngineHandler::new(engine_handler, downloader, incoming_requests); + let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree); + let handler = EngineHandler::new(engine_handler, downloader, incoming_requests); - let backfill_sync = PipelineSync::new(pipeline, pipeline_task_spawner); + let backfill_sync = PipelineSync::new(pipeline, pipeline_task_spawner); - ChainOrchestrator::new(handler, backfill_sync) + ChainOrchestrator::new(handler, backfill_sync) + } } diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index b4dbf9111d2..4bc37c3de4f 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -16,7 +16,7 @@ use reth_db::{database_metrics::DatabaseMetrics, Database}; use reth_engine_tree::{ chain::{ChainEvent, FromOrchestrator}, engine::{EngineApiKind, EngineApiRequest, EngineRequestHandler}, - launch::build_engine_orchestrator, + launch::EngineOrchestratorBuilder, tree::TreeConfig, }; use reth_engine_util::EngineMessageStreamExt; @@ -244,24 +244,25 @@ impl EngineNodeLauncher { EngineApiKind::Ethereum }; - let mut orchestrator = build_engine_orchestrator( + let mut orchestrator = EngineOrchestratorBuilder { engine_kind, - consensus.clone(), - network_client.clone(), - Box::pin(consensus_engine_stream), + consensus, + client: network_client, + incoming_requests: Box::pin(consensus_engine_stream), pipeline, - ctx.task_executor().clone(), - ctx.provider_factory().clone(), - ctx.blockchain_db().clone(), + pipeline_task_spawner: ctx.task_executor().clone(), + provider: ctx.provider_factory().clone(), + blockchain_db: ctx.blockchain_db().clone(), pruner, - ctx.components().payload_builder_handle().clone(), - engine_validator, + payload_builder: ctx.components().payload_builder_handle().clone(), + payload_validator: engine_validator, overlay_manager, - engine_tree_config, - ctx.sync_metrics_tx(), - ctx.components().evm_config().clone(), - ctx.task_executor().clone(), - ); + tree_config: engine_tree_config, + sync_metrics_tx: ctx.sync_metrics_tx(), + evm_config: ctx.components().evm_config().clone(), + runtime: ctx.task_executor().clone(), + } + .build(); info!(target: "reth::cli", "Consensus engine initialized"); From 3dd806a43352cdcf975f26e3399aa21f09e6c784 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Sep 2026 13:37:15 +0000 Subject: [PATCH 038/133] fix(rpc): include transaction hash in individual replays (#27365) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-api/src/trace.rs | 2 +- crates/rpc/rpc/src/trace.rs | 90 +++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 1c4b148a098..4e5a404efd5 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -58,7 +58,7 @@ pub trait TraceApi { &self, transaction: B256, trace_types: HashSet, - ) -> RpcResult; + ) -> RpcResult; /// Returns traces created at given block. #[method(name = "block")] diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index c79b087b169..ad8976b1ded 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -196,7 +196,7 @@ where &self, hash: B256, trace_types: HashSet, - ) -> Result { + ) -> Result { let config = TracingInspectorConfig::from_parity_config(&trace_types); self.eth_api() .spawn_trace_transaction_in_block(hash, config, move |_, inspector, res, db| { @@ -204,7 +204,10 @@ where .into_parity_builder() .into_trace_results_with_state(&res, &trace_types, &db) .map_err(Eth::Error::from_eth_err)?; - Ok(trace_res) + Ok(TraceResultsWithTransactionHash { + transaction_hash: hash, + full_trace: trace_res, + }) }) .await .transpose() @@ -753,7 +756,7 @@ where &self, transaction: B256, trace_types: HashSet, - ) -> RpcResult { + ) -> RpcResult { let _permit = self.acquire_trace_permit().await; Ok(Self::replay_transaction(self, transaction, trace_types).await.map_err(Into::into)?) } @@ -888,6 +891,87 @@ fn reward_trace( mod tests { use super::*; + #[tokio::test] + async fn replay_transaction_includes_transaction_hash() { + use crate::EthApiBuilder; + use alloy_consensus::{Header, TxLegacy}; + use alloy_primitives::{Signature, TxKind}; + use reth_chain_state::CanonStateNotification; + use reth_ethereum_primitives::{Block, BlockBody, TransactionSigned}; + use reth_evm_ethereum::EthEvmConfig; + use reth_execution_types::{Chain, ExecutionOutcome}; + use reth_network_api::noop::NoopNetwork; + use reth_primitives_traits::{RecoveredBlock, SignerRecoverable}; + use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; + use reth_rpc_eth_types::cache::cache_new_blocks_task; + use reth_transaction_pool::test_utils::testing_pool; + + let provider = MockEthProvider::default(); + let tx = TransactionSigned::new_unhashed( + TxLegacy { + gas_limit: 21_000, + to: TxKind::Call(Address::with_last_byte(0x42)), + value: U256::from(1), + ..Default::default() + } + .into(), + Signature::test_signature(), + ); + let hash = *tx.hash(); + let sender = tx.recover_signer().unwrap(); + provider.add_account(sender, ExtendedAccount::new(0, U256::from(1_000_000))); + let parent = Header { gas_limit: 30_000_000, ..Default::default() }; + let parent_hash = parent.hash_slow(); + provider.add_header(parent_hash, parent); + let block = Block { + header: Header { parent_hash, number: 1, gas_limit: 30_000_000, ..Default::default() }, + body: BlockBody { transactions: vec![tx], ..Default::default() }, + }; + let block_hash = block.header.hash_slow(); + provider.add_block(block_hash, block.clone()); + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + // MockEthProvider does not implement recovered_block, so seed the RPC cache. + cache_new_blocks_task( + eth_api.cache().clone(), + futures::stream::iter([CanonStateNotification::Commit { + new: Arc::new(Chain::new( + [RecoveredBlock::new_unhashed(block, vec![sender])], + ExecutionOutcome { + receipts: vec![vec![]], + first_block: 1, + ..Default::default() + }, + Default::default(), + )), + }]), + ) + .await; + let api = TraceApi::new(eth_api, BlockingTaskGuard::new(1), EthConfig::default()); + let module = api.clone().into_rpc(); + for types in + [HashSet::default(), HashSet::from_iter([TraceType::Trace, TraceType::StateDiff])] + { + let request = serde_json::json!({ + "jsonrpc": "2.0", "id": 1, "method": "trace_replayTransaction", + "params": [hash, types], + }); + let (response, _) = module.raw_json_request(&request.to_string(), 1).await.unwrap(); + let response: serde_json::Value = serde_json::from_str(response.get()).unwrap(); + assert!(response.get("error").is_none(), "{response}"); + assert_eq!(response["result"]["transactionHash"], serde_json::json!(hash)); + + let block_replay = + api.replay_block_transactions(block_hash.into(), types).await.unwrap().unwrap(); + assert_eq!(response["result"], serde_json::to_value(&block_replay[0]).unwrap()); + } + } + fn localized_transaction_trace( block_number: u64, transaction_position: u64, From 534c60db9d4be32d349e5b31456416ecb4b4dbee Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Sep 2026 13:55:05 +0000 Subject: [PATCH 039/133] fix(rpc): classify pruned changeset errors as unavailable history (#27367) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-eth-types/src/error/mod.rs | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/rpc/rpc-eth-types/src/error/mod.rs b/crates/rpc/rpc-eth-types/src/error/mod.rs index 7374de36678..2049db7cb68 100644 --- a/crates/rpc/rpc-eth-types/src/error/mod.rs +++ b/crates/rpc/rpc-eth-types/src/error/mod.rs @@ -545,6 +545,11 @@ impl From for EthApiError { ProviderError::BlockExpired { requested, earliest_available } => { Self::PrunedHistoryUnavailable { requested, earliest_available } } + ProviderError::InsufficientChangesets { requested, available } + if requested < *available.start() => + { + Self::PrunedHistoryUnavailable { requested, earliest_available: *available.start() } + } err => Self::Internal(err.into()), } } @@ -1255,6 +1260,28 @@ mod tests { ); } + #[test] + fn pruned_state_errors_use_history_unavailable_code() { + use reth_errors::ProviderError; + + let err = EthApiError::from(ProviderError::InsufficientChangesets { + requested: 1, + available: 43..=48, + }) + .into_rpc_err(); + assert_eq!(err.code(), 4444); + assert_eq!(err.message(), "pruned history unavailable: requested 1, earliest available 43"); + + for requested in [43, 49] { + let err = EthApiError::from(ProviderError::InsufficientChangesets { + requested, + available: 43..=48, + }) + .into_rpc_err(); + assert_eq!(err.code(), -32603); + } + } + #[test] fn receipts_not_found_message() { let err: jsonrpsee_types::error::ErrorObject<'static> = From e3d7f0952dc298ff94fc31ece9a59eb25ebcf340 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Sep 2026 14:21:19 +0000 Subject: [PATCH 040/133] fix(rpc): return null for missing transaction replays (#27364) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-api/src/trace.rs | 4 +-- crates/rpc/rpc-builder/tests/it/http.rs | 6 ++-- crates/rpc/rpc/src/trace.rs | 39 +++++++++++++++++++++---- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 4e5a404efd5..7ba12e17b42 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -52,13 +52,13 @@ pub trait TraceApi { trace_types: HashSet, ) -> RpcResult>>; - /// Replays a transaction, returning the traces. + /// Replays a transaction, returning the traces or `None` if the transaction does not exist. #[method(name = "replayTransaction")] async fn replay_transaction( &self, transaction: B256, trace_types: HashSet, - ) -> RpcResult; + ) -> RpcResult>; /// Returns traces created at given block. #[method(name = "block")] diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 5160ce3020f..3025b61c3f8 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -561,14 +561,14 @@ where ) .await .unwrap_err(); - TraceApiClient::::replay_transaction( + assert!(TraceApiClient::::replay_transaction( client, B256::default(), HashSet::default(), ) .await - .err() - .unwrap(); + .unwrap() + .is_none()); TraceApiClient::::trace_block(client, block_id).await.unwrap_err(); assert!(TraceApiClient::::replay_block_transactions( client, diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index ad8976b1ded..ea52320a49b 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -191,12 +191,12 @@ where .await } - /// Replays a transaction, returning the traces. + /// Replays a transaction, returning the traces or `None` if the transaction does not exist. pub async fn replay_transaction( &self, hash: B256, trace_types: HashSet, - ) -> Result { + ) -> Result, Eth::Error> { let config = TracingInspectorConfig::from_parity_config(&trace_types); self.eth_api() .spawn_trace_transaction_in_block(hash, config, move |_, inspector, res, db| { @@ -210,8 +210,6 @@ where }) }) .await - .transpose() - .ok_or(EthApiError::TransactionNotFound)? } /// Returns transaction trace objects at the given index @@ -756,7 +754,7 @@ where &self, transaction: B256, trace_types: HashSet, - ) -> RpcResult { + ) -> RpcResult> { let _permit = self.acquire_trace_permit().await; Ok(Self::replay_transaction(self, transaction, trace_types).await.map_err(Into::into)?) } @@ -891,6 +889,37 @@ fn reward_trace( mod tests { use super::*; + #[tokio::test] + async fn replay_missing_transaction_returns_null() { + use crate::EthApiBuilder; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_provider::test_utils::MockEthProvider; + use reth_transaction_pool::test_utils::testing_pool; + + let provider = MockEthProvider::default(); + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + let api = TraceApi::new(eth_api, BlockingTaskGuard::new(1), EthConfig::default()); + let module = api.into_rpc(); + for types in [serde_json::json!([]), serde_json::json!(["trace", "stateDiff", "vmTrace"])] { + let request = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "trace_replayTransaction", + "params": [B256::with_last_byte(1), types], + }); + let (response, _) = module.raw_json_request(&request.to_string(), 1).await.unwrap(); + let response: serde_json::Value = serde_json::from_str(response.get()).unwrap(); + assert_eq!(response, serde_json::json!({"jsonrpc": "2.0", "id": 1, "result": null})); + } + } + #[tokio::test] async fn replay_transaction_includes_transaction_hash() { use crate::EthApiBuilder; From 5dfd334ba469bebc32f9f49a9a6617822f142283 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:07:24 +0000 Subject: [PATCH 041/133] fix(tasks): avoid starving the OpenTelemetry span exporter (#27370) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Co-authored-by: Brian Picciano --- crates/tasks/src/utils.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/tasks/src/utils.rs b/crates/tasks/src/utils.rs index f535922bc27..aac5fbcd219 100644 --- a/crates/tasks/src/utils.rs +++ b/crates/tasks/src/utils.rs @@ -34,9 +34,9 @@ pub fn increase_thread_priority() { } } -/// Deprioritizes known background threads spawned by third-party libraries (`OpenTelemetry`, -/// `tracing-appender`, `reqwest`) by scanning `/proc//task/` for matching thread names and -/// setting `SCHED_IDLE` scheduling policy + maximum niceness on them. +/// Deprioritizes known background threads spawned by third-party libraries (`tracing-appender`, +/// `reqwest`) by scanning `/proc//task/` for matching thread names and setting `SCHED_IDLE` +/// scheduling policy on them. /// /// This is a hack: these threads are spawned by libraries that do not expose a way to hook into /// thread initialization or expose the TIDs, so we have to discover them after the fact by @@ -52,9 +52,11 @@ pub fn deprioritize_background_threads() { } /// Thread name prefixes to deprioritize. +/// +/// Do not re-add the OTLP span exporter (`OpenTelemetry.T`) to avoid CPU spinning. +/// See . #[cfg(target_os = "linux")] -const DEPRIORITIZE_THREAD_PREFIXES: &[&str] = - &["OpenTelemetry.T", "tracing-appende", "reqwest-interna"]; +const DEPRIORITIZE_THREAD_PREFIXES: &[&str] = &["tracing-appende", "reqwest-interna"]; #[cfg(target_os = "linux")] fn _deprioritize_background_threads() { From 8760eb0008f8f99d345d62039552622ebc057dda Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 22 Sep 2026 16:03:56 +0000 Subject: [PATCH 042/133] fix(rpc): select trace_get results by tree path (#27366) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-api/src/trace.rs | 9 +-- crates/rpc/rpc/src/trace.rs | 127 +++++++++++++++++++++++++++++--- 2 files changed, 120 insertions(+), 16 deletions(-) diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 7ba12e17b42..56f824700ef 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -73,12 +73,11 @@ pub trait TraceApi { #[method(name = "filter")] async fn trace_filter(&self, filter: TraceFilter) -> RpcResult>; - /// Returns transaction trace at given index. + /// Returns the transaction trace at the given `traceAddress` path. /// - /// `indices` represent the index positions of the traces. - /// - /// Note: This expects a list of indices but only one is supported since this function returns a - /// single [`LocalizedTransactionTrace`]. + /// An empty path selects the root, `[0]` selects its first child, and `[0, 1]` selects that + /// child's second child. Returns `None` if the transaction or path does not exist. + /// Callers requiring a flat index can index the result of `trace_transaction` instead. #[method(name = "get")] async fn trace_get( &self, diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index ea52320a49b..3185a3c9aab 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -212,25 +212,31 @@ where .await } - /// Returns transaction trace objects at the given index + /// Returns the transaction trace at the given `traceAddress` path. /// - /// Note: For compatibility reasons this only supports 1 single index, since this method is - /// supposed to return a single trace. See also: - /// - /// This returns `None` if `indices` is empty + /// An empty path selects the root. Returns `None` if the transaction or path does not exist. pub async fn trace_get( &self, hash: B256, indices: Vec, ) -> Result, Eth::Error> { - if indices.len() != 1 { - // The OG impl failed if it gets more than a single index - return Ok(None) - } - self.trace_get_index(hash, indices[0]).await + self.eth_api() + .spawn_trace_transaction_in_block( + hash, + TracingInspectorConfig::default_parity(), + move |tx_info, inspector, _, _| { + Ok(inspector + .into_parity_builder() + .into_localized_transaction_traces_iter(tx_info) + .find(|trace| trace.trace.trace_address == indices)) + }, + ) + .await + .map(Option::flatten) } - /// Returns transaction trace object at the given index. + /// Returns the transaction trace at the given flat index, unlike [`Self::trace_get`], which + /// selects a `traceAddress` path. /// /// Returns `None` if the trace object at that index does not exist pub async fn trace_get_index( @@ -889,6 +895,105 @@ fn reward_trace( mod tests { use super::*; + #[tokio::test] + async fn trace_get_selects_tree_paths() { + use crate::EthApiBuilder; + use alloy_consensus::{Header, TxLegacy}; + use alloy_primitives::{Signature, TxKind}; + use reth_chain_state::CanonStateNotification; + use reth_ethereum_primitives::{Block, BlockBody, TransactionSigned}; + use reth_evm_ethereum::EthEvmConfig; + use reth_execution_types::{Chain, ExecutionOutcome}; + use reth_network_api::noop::NoopNetwork; + use reth_primitives_traits::{RecoveredBlock, SignerRecoverable}; + use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; + use reth_rpc_eth_types::cache::cache_new_blocks_task; + use reth_transaction_pool::test_utils::testing_pool; + + let provider = MockEthProvider::default(); + // The root calls 0x43 (which calls 0x44), then calls 0x45. + for (address, code) in [ + (0x42, "60006000600060006000604361fffff15060006000600060006000604561fffff15000"), + (0x43, "600060006000600060006044611000f15000"), + ] { + provider.add_account( + Address::with_last_byte(address), + ExtendedAccount::new(0, U256::ZERO).with_bytecode(code.parse().unwrap()), + ); + } + let tx = TransactionSigned::new_unhashed( + TxLegacy { + gas_limit: 500_000, + to: TxKind::Call(Address::with_last_byte(0x42)), + value: U256::from(1), + ..Default::default() + } + .into(), + Signature::test_signature(), + ); + let hash = *tx.hash(); + let sender = tx.recover_signer().unwrap(); + provider.add_account(sender, ExtendedAccount::new(0, U256::from(1_000_000))); + let parent = Header { gas_limit: 30_000_000, ..Default::default() }; + let parent_hash = parent.hash_slow(); + provider.add_header(parent_hash, parent); + let block = Block { + header: Header { parent_hash, number: 1, gas_limit: 30_000_000, ..Default::default() }, + body: BlockBody { transactions: vec![tx], ..Default::default() }, + }; + let block_hash = block.header.hash_slow(); + provider.add_block(block_hash, block.clone()); + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + // MockEthProvider does not implement recovered_block, so seed the RPC cache. + cache_new_blocks_task( + eth_api.cache().clone(), + futures::stream::iter([CanonStateNotification::Commit { + new: Arc::new(Chain::new( + [RecoveredBlock::new_unhashed(block, vec![sender])], + ExecutionOutcome { + receipts: vec![vec![]], + first_block: 1, + ..Default::default() + }, + Default::default(), + )), + }]), + ) + .await; + let api = TraceApi::new(eth_api, BlockingTaskGuard::new(1), EthConfig::default()); + let module = api.clone().into_rpc(); + let traces = api.trace_transaction(hash).await.unwrap().unwrap(); + let paths: Vec<_> = traces.iter().map(|trace| trace.trace.trace_address.clone()).collect(); + assert_eq!(paths, [vec![], vec![0], vec![0, 0], vec![1]]); + for (path, expected) in [ + (vec![], Some(&traces[0])), + (vec![0], Some(&traces[1])), + (vec![0, 0], Some(&traces[2])), + (vec![1], Some(&traces[3])), + (vec![2], None), + (vec![0, 1], None), + (vec![1, 0], None), + ] { + let indices: Vec<_> = path.iter().map(|index| format!("0x{index:x}")).collect(); + let request = serde_json::json!({ + "jsonrpc": "2.0", "id": 1, "method": "trace_get", "params": [hash, indices], + }); + let (response, _) = module.raw_json_request(&request.to_string(), 1).await.unwrap(); + let response: serde_json::Value = serde_json::from_str(response.get()).unwrap(); + assert!(response.get("error").is_none(), "{response}"); + assert_eq!(response["result"], serde_json::to_value(expected).unwrap()); + } + assert!(api.trace_get(B256::ZERO, vec![]).await.unwrap().is_none()); + assert!(api.trace_get(B256::ZERO, vec![0, 0]).await.unwrap().is_none()); + assert_eq!(api.trace_get_index(hash, 2).await.unwrap().as_ref(), Some(&traces[2])); + } + #[tokio::test] async fn replay_missing_transaction_returns_null() { use crate::EthApiBuilder; From b9b8485e65614b3a372ead2a2235c7498228fb5f Mon Sep 17 00:00:00 2001 From: christn Date: Tue, 22 Sep 2026 16:18:45 +0000 Subject: [PATCH 043/133] fix(rpc): skip the payment check for a zero-value builder submission (#27371) --- crates/rpc/rpc/src/validation.rs | 154 ++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index 7468a994011..b7b74488b35 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -289,6 +289,11 @@ where output: &BlockExecutionOutput<::Receipt>, message: &BidTrace, ) -> Result<(), ValidationApiError> { + // A zero-value bid promises the proposer nothing, so the payload owes nothing. + if message.value.is_zero() { + return Ok(()) + } + let (mut balance_before, balance_after) = if let Some(acc) = output.state.state.get(&message.proposer_fee_recipient) { @@ -799,11 +804,26 @@ pub(crate) struct ValidationMetrics { #[cfg(test)] mod tests { use super::{ - hash_disallow_list, validate_message_against_payload, AddressSet, ValidationApiError, + hash_disallow_list, validate_message_against_payload, AddressSet, ValidationApi, + ValidationApiConfig, ValidationApiError, }; - use alloy_primitives::{Address, B256}; + use alloy_consensus::{BlockHeader, Header}; + use alloy_primitives::{Address, B256, U256}; use alloy_rpc_types_beacon::relay::BidTrace; - use alloy_rpc_types_engine::{ExecutionPayload, ExecutionPayloadV1}; + use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload, ExecutionPayloadV1}; + use reth_consensus::noop::NoopConsensus; + use reth_engine_primitives::PayloadValidator; + use reth_ethereum_engine_primitives::EthPayloadTypes; + use reth_ethereum_primitives::Block; + use reth_evm_ethereum::EthEvmConfig; + use reth_execution_types::BlockExecutionOutput; + use reth_node_api::NewPayloadError; + use reth_primitives_traits::{RecoveredBlock, SealedBlock, SealedHeader}; + use reth_provider::test_utils::MockEthProvider; + use reth_revm::db::{states::bundle_state::BundleState, AccountStatus, BundleAccount}; + use reth_tasks::Runtime; + use revm::state::AccountInfo; + use std::sync::Arc; fn test_execution_payload() -> ExecutionPayload { ExecutionPayload::V1(ExecutionPayloadV1 { @@ -943,4 +963,132 @@ mod tests { let hash = hash_disallow_list(&blocklist); assert_eq!(expected_hash, hash); } + + /// Only [`ValidationApi::validate_message_against_block`] is exercised below, which never + /// converts a payload. + #[derive(Debug)] + struct UnusedPayloadValidator; + + impl PayloadValidator for UnusedPayloadValidator { + type Block = Block; + + fn convert_payload_to_block( + &self, + _payload: ExecutionData, + ) -> Result, NewPayloadError> { + unimplemented!() + } + } + + fn test_validation_api( + provider: MockEthProvider, + ) -> ValidationApi { + ValidationApi::new( + provider, + NoopConsensus::arc(), + EthEvmConfig::mainnet(), + ValidationApiConfig::default(), + Runtime::test(), + Arc::new(UnusedPayloadValidator), + ) + } + + /// A submission whose payload pays the proposer nothing, like a trustless ePBS bid: there the + /// payment settles from the builder's stake on the consensus layer. + fn payment_free_submission() -> (MockEthProvider, RecoveredBlock, BidTrace) { + let provider = MockEthProvider::default(); + + let parent = Header { gas_limit: 30_000_000, ..Default::default() }; + let parent = SealedHeader::seal_slow(parent); + provider.add_block( + parent.hash(), + Block { header: parent.clone_header(), body: Default::default() }, + ); + + let header = Header { + parent_hash: parent.hash(), + number: parent.number() + 1, + gas_limit: parent.gas_limit(), + timestamp: parent.timestamp() + 12, + ..Default::default() + }; + let block = SealedBlock::seal_slow(Block { header, body: Default::default() }) + .try_recover() + .unwrap(); + provider.state_roots.lock().push(block.state_root()); + + let message = BidTrace { + parent_hash: block.parent_hash(), + block_hash: block.hash(), + gas_limit: block.gas_limit(), + gas_used: block.gas_used(), + proposer_fee_recipient: Address::repeat_byte(0x42), + value: U256::from(1_000_000_000_000_000_000u64), + ..Default::default() + }; + + (provider, block, message) + } + + #[tokio::test] + async fn test_payment_check_runs_for_a_nonzero_bid() { + let (provider, block, message) = payment_free_submission(); + let registered_gas_limit = block.gas_limit(); + + let err = test_validation_api(provider) + .validate_message_against_block(block, message, registered_gas_limit, None) + .await + .unwrap_err(); + + assert!(matches!(err, ValidationApiError::ProposerPayment), "unexpected error: {err}"); + } + + #[tokio::test] + async fn test_zero_value_bid_skips_the_payment_check() { + let (provider, block, mut message) = payment_free_submission(); + let registered_gas_limit = block.gas_limit(); + message.value = U256::ZERO; + + test_validation_api(provider) + .validate_message_against_block(block, message, registered_gas_limit, None) + .await + .unwrap(); + } + + /// A zero-value bid promises the proposer nothing, so there is nothing to verify. The + /// balance-delta branch already lets one through whenever the fee recipient's balance does + /// not fall -- but a fee recipient that merely sends a transaction of its own in the same + /// block ends it poorer, drops through to the last-transaction fallback and is rejected. + #[test] + fn test_zero_value_bid_is_accepted_when_the_fee_recipient_spends() { + let (provider, block, mut message) = payment_free_submission(); + message.value = U256::ZERO; + + let output = BlockExecutionOutput { + result: Default::default(), + state: fee_recipient_spent(message.proposer_fee_recipient), + }; + + test_validation_api(provider) + .ensure_payment(block.sealed_block(), &output, &message) + .unwrap(); + } + + /// Execution state for a block in which `address` ended up poorer than it started. + fn fee_recipient_spent(address: Address) -> BundleState { + let balance = + |wei: u64| Some(AccountInfo { balance: U256::from(wei), ..Default::default() }); + + let mut state = BundleState::default(); + state.state.insert( + address, + BundleAccount { + original_info: balance(1_000), + info: balance(999), + storage: Default::default(), + status: AccountStatus::Changed, + }, + ); + state + } } From 7719d7df22ec3eef46b2da0347bbfa49be4cd1fb Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:22:40 +0200 Subject: [PATCH 044/133] docs: streamline agent guidance (#27381) --- AGENTS.md | 615 +++++++----------------------------------------------- 1 file changed, 76 insertions(+), 539 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4d4d767b1c1..3ed9e42e7fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,549 +1,86 @@ -# Reth Development Guide for AI Agents +# AGENTS.md -This guide provides comprehensive instructions for AI agents working on the Reth codebase. It covers the architecture, development workflows, and critical guidelines for effective contributions. +Guidance for AI coding agents working in this repository. ## Project Overview -Reth is a high-performance Ethereum execution client written in Rust, focusing on modularity, performance, and contributor-friendliness. The codebase is organized into well-defined crates with clear boundaries and responsibilities. +Reth is an Ethereum execution client written in Rust. The workspace contains the +node binary and libraries for execution, storage, networking, and RPC. -## Architecture Overview - -### Core Components - -1. **Consensus (`crates/consensus/`)**: Validates blocks according to Ethereum consensus rules -2. **Storage (`crates/storage/`)**: Hybrid database using MDBX + static files for optimal performance -3. **Networking (`crates/net/`)**: P2P networking stack with discovery, sync, and transaction propagation -4. **RPC (`crates/rpc/`)**: JSON-RPC server supporting all standard Ethereum APIs -5. **Execution (`crates/evm/`, `crates/ethereum/`)**: Transaction execution and state transitions -6. **Pipeline (`crates/stages/`)**: Staged sync architecture for blockchain synchronization -7. **Trie (`crates/trie/`)**: Merkle Patricia Trie implementation with a sparse trie state root task and parallel proof computation -8. **Node Builder (`crates/node/`)**: High-level node orchestration and configuration -9. **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated) - -### Key Design Principles - -- **Modularity**: Each crate can be used as a standalone library -- **Performance**: Extensive use of parallelism, memory-mapped I/O, and optimized data structures -- **Extensibility**: Traits and generic types allow for different chain implementations -- **Type Safety**: Strong typing throughout with minimal use of dynamic dispatch - -## Development Workflow - -### Code Style and Standards - -1. **Formatting**: Always use nightly rustfmt - ```bash - cargo +nightly fmt --all - ``` - -2. **Linting**: Run clippy with all features - ```bash - cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features - ``` - -3. **Testing**: Use nextest for faster test execution - ```bash - cargo nextest run --workspace - ``` - -### Common Contribution Types - -Based on actual recent PRs, here are typical contribution patterns: - -#### 1. Small Bug Fixes (1-10 lines) -Real example: Fixing beacon block root handling ([#16767](https://github.com/paradigmxyz/reth/pull/16767)) -```rust -// Changed a single line to fix logic error -- parent_beacon_block_root: parent.parent_beacon_block_root(), -+ parent_beacon_block_root: parent.parent_beacon_block_root().map(|_| B256::ZERO), -``` - -#### 2. Integration with Upstream Changes -Real example: Integrating revm updates ([#16752](https://github.com/paradigmxyz/reth/pull/16752)) -```rust -// Update code to use new APIs from dependencies -- if self.fork_tracker.is_shanghai_activated() { -- if let Err(err) = transaction.ensure_max_init_code_size(MAX_INIT_CODE_BYTE_SIZE) { -+ if let Some(init_code_size_limit) = self.fork_tracker.max_initcode_size() { -+ if let Err(err) = transaction.ensure_max_init_code_size(init_code_size_limit) { -``` - -#### 3. Adding Comprehensive Tests -Real example: ETH69 protocol tests ([#16759](https://github.com/paradigmxyz/reth/pull/16759)) -```rust -#[tokio::test(flavor = "multi_thread")] -async fn test_eth69_peers_can_connect() { - // Create test network with specific protocol versions - let p0 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth69.into())); - // Test connection and version negotiation -} -``` - -#### 4. Making Components Generic -Real example: Making EthEvmConfig generic over chainspec ([#16758](https://github.com/paradigmxyz/reth/pull/16758)) -```rust -// Before: Hardcoded to ChainSpec -- pub struct EthEvmConfig { -- pub executor_factory: EthBlockExecutorFactory, EvmFactory>, - -// After: Generic over any chain spec type -+ pub struct EthEvmConfig -+ where -+ C: EthereumHardforks, -+ { -+ pub executor_factory: EthBlockExecutorFactory, EvmFactory>, -``` - -#### 5. Resource Management Improvements -Real example: ETL directory cleanup ([#16770](https://github.com/paradigmxyz/reth/pull/16770)) -```rust -// Add cleanup logic on startup -+ if let Err(err) = fs::remove_dir_all(&etl_path) { -+ warn!(target: "reth::cli", ?etl_path, %err, "Failed to remove ETL path on launch"); -+ } -``` - -#### 6. Feature Additions -Real example: Sharded mempool support ([#16756](https://github.com/paradigmxyz/reth/pull/16756)) -```rust -// Add new filtering policies for transaction announcements -pub struct ShardedMempoolAnnouncementFilter { - pub inner: T, - pub shard_bits: u8, - pub node_id: Option, -} -``` - -### Testing Guidelines - -1. **Unit Tests**: Test individual functions and components -2. **Integration Tests**: Test interactions between components -3. **Benchmarks**: For performance-critical code -4. **Fuzz Tests**: For parsing and serialization code -5. **Property Tests**: For checking component correctness on a wide variety of inputs - -Example test structure: -```rust -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_component_behavior() { - // Arrange - let component = Component::new(); - - // Act - let result = component.operation(); - - // Assert - assert_eq!(result, expected); - } -} -``` - -### Performance Considerations - -1. **Avoid Allocations in Hot Paths**: Use references and borrowing -2. **Parallel Processing**: Use rayon for CPU-bound parallel work -3. **Async/Await**: Use tokio for I/O-bound operations -4. **File Operations**: Use `reth_fs_util` instead of `std::fs` for better error handling - -### Common Pitfalls - -1. **Don't Block Async Tasks**: Use `spawn_blocking` for CPU-intensive work or work with lots of blocking I/O -2. **Handle Errors Properly**: Use `?` operator and proper error types - -### What to Avoid - -Based on PR patterns, avoid: - -1. **Large, sweeping changes**: Keep PRs focused and reviewable -2. **Mixing unrelated changes**: One logical change per PR -3. **Ignoring CI failures**: All checks must pass -4. **Incomplete implementations**: Finish features before submitting -5. **Modifying libmdbx sources**: Never modify files in `crates/storage/libmdbx-rs/mdbx-sys/libmdbx/` - this is vendored third-party code - -### CI Requirements - -Before submitting changes, ensure: - -1. **Format Check**: `cargo +nightly fmt --all --check` -2. **Clippy**: No warnings -3. **Tests Pass**: All unit and integration tests -4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items` -5. **CLI Docs** (if CLI changed): Run `make update-book-cli` (see below) -6. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.) - -### CLI Reference Docs (`book` CI Job) - -The CLI reference pages under `docs/vocs/docs/pages/cli/` are **auto-generated** from the `reth` binary's `--help` output. **Do not edit these files manually** — any hand edits will be overwritten and CI will fail regardless. - -When you add, remove, or modify CLI commands, subcommands, or flags, regenerate the CLI docs by running: - -```bash -make update-book-cli -``` - -This builds `reth` in debug mode and runs `docs/cli/update.sh` to regenerate all CLI pages. Commit the resulting changes. - -The `book` CI job (`.github/workflows/lint.yml`) enforces this by regenerating the docs and running `git diff --exit-code`. If the committed docs don't match the generated output, CI fails. Manually editing these pages is never productive — always use `make update-book-cli`. - -### Opening PRs against - -#### Titles - -Use [Conventional Commits](https://www.conventionalcommits.org/) with an optional scope: - -``` -(): -``` - -**Types**: `feat`, `fix`, `perf`, `refactor`, `docs`, `test`, `chore` - -**Scope** (optional): crate or area, e.g. `evm`, `trie`, `rpc`, `engine`, `net` - -Examples: -- `fix(rpc): correct gas estimation for ERC-20 transfers` -- `perf: batch trie updates to reduce cursor overhead` -- `feat(engine): add new_payload_interval metric` - -#### Descriptions - -Keep it short. Say what changed and why — nothing more. - -**Do:** -- Write 1–3 sentences summarizing the change -- Explain _why_ if the diff doesn't make it obvious -- Link related issues or EIPs -- Include benchmark numbers for perf changes - -**Don't:** -- List every file changed — that's what the diff is for -- Repeat the title in the body -- Add "Files changed" or "Changes" sections -- Write walls of text that go stale when the diff is updated -- Use filler like "This PR introduces...", "comprehensive", "robust", "enhance", "leverage" - -**Template:** - -``` -Closes # - - - - -``` - -**Good example:** - -``` -Closes #16800 - -Adds fallback for external IP resolution so node startup doesn't fail -when STUN is unreachable. Falls back to the configured default. -``` - -**Bad example:** - -``` -## Summary -This PR introduces comprehensive improvements to the IP resolution system. - -## Changes -- Modified `crates/net/discv4/src/lib.rs` to add fallback -- Modified `crates/net/discv4/src/config.rs` to add default IP -- Added tests in `crates/net/discv4/src/tests/ip.rs` - -## Files Changed -- crates/net/discv4/src/lib.rs -- crates/net/discv4/src/config.rs -- crates/net/discv4/src/tests/ip.rs -``` - -#### Labels and CI - -Label PRs appropriately, first check the available labels and then apply the relevant ones: -* when changes are RPC related, add A-rpc label -* when changes are docs related, add C-docs label -* ... and so on, check the available labels for more options. -* if being tasked to open a pr, ensure that all changes are properly formatted: `cargo +nightly fmt --all` - -If changes in reth include changes to dependencies, run commands `zepter` and `make lint-toml` before finalizing the pr. Assume `zepter` binary is installed. - -### Debugging Tips - -1. **Logging**: Use `tracing` crate with appropriate levels - ```rust - tracing::debug!(target: "reth::component", ?value, "description"); - ``` - -2. **Metrics**: Add metrics for monitoring - ```rust - metrics::counter!("reth_component_operations").increment(1); - ``` - -3. **Test Isolation**: Use separate test databases/directories - -### Finding Where to Contribute - -1. **Check Issues**: Look for issues labeled `good-first-issue` or `help-wanted` -2. **Review TODOs**: Search for `TODO` comments in the codebase -3. **Improve Tests**: Areas with low test coverage are good targets -4. **Documentation**: Improve code comments and documentation -5. **Performance**: Profile and optimize hot paths (with benchmarks) - -### Common PR Patterns - -#### Small, Focused Changes -Most PRs change only 1-5 files. Examples: -- Single-line bug fixes -- Adding a missing trait implementation -- Updating error messages -- Adding test cases for edge conditions - -#### Integration Work -When dependencies update (especially revm), code needs updating: -- Check for breaking API changes -- Update to use new features (like EIP implementations) -- Ensure compatibility with new versions - -#### Test Improvements -Tests often need expansion for: -- New protocol versions (ETH68, ETH69) -- Edge cases in state transitions -- Network behavior under specific conditions -- Concurrent operations - -#### Making Code More Generic -Common refactoring pattern: -- Replace concrete types with generics -- Add trait bounds for flexibility -- Enable reuse across different chain types - -#### When to Comment - -Write comments that remain valuable after the PR is merged. Future readers won't have PR context - they only see the current code. - -##### ✅ DO: Add Value - -**Explain WHY and non-obvious behavior:** -```rust -// Process must handle allocations atomically to prevent race conditions -// between dealloc on drop and concurrent limit checks -unsafe impl GlobalAlloc for LimitedAllocator { ... } - -// Binary search requires sorted input. Panics on unsorted slices. -fn find_index(items: &[Item], target: &Item) -> Option - -// Timeout set to 5s to match EVM block processing limits -const TRACER_TIMEOUT: Duration = Duration::from_secs(5); -``` - -**Document constraints and assumptions:** -```rust -/// Returns heap size estimate. -/// -/// Note: May undercount shared references (Rc/Arc). For precise -/// accounting, combine with an allocator-based approach. -fn deep_size_of(&self) -> usize -``` - -**Explain complex logic:** -```rust -// We reset limits at task start because tokio reuses threads in -// spawn_blocking pool. Without reset, second task inherits first -// task's allocation count and immediately hits limit. -THREAD_ALLOCATED.with(|allocated| allocated.set(0)); -``` - -##### ❌ DON'T: Describe Changes -```rust -// ❌ BAD - Describes the change, not the code -// Changed from Vec to HashMap for O(1) lookups - -// ✅ GOOD - Explains the decision -// HashMap provides O(1) symbol lookups during trace replay -``` -```rust -// ❌ BAD - PR-specific context -// Fix for issue #234 where memory wasn't freed - -// ✅ GOOD - Documents the actual behavior -// Explicitly drop allocations before limit check to ensure -// accurate accounting -``` -```rust -// ❌ BAD - States the obvious -// Increment counter -counter += 1; - -// ✅ GOOD - Explains non-obvious purpose -// Track allocations across all threads for global limit enforcement -GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst); -``` - -✅ **Comment when:** -- Non-obvious behavior or edge cases -- Performance trade-offs -- Safety requirements (unsafe blocks must always be documented) -- Limitations or gotchas -- Why simpler alternatives don't work - -❌ **Don't comment when:** -- Code is self-explanatory -- Just restating the code in English -- Describing what changed in this PR - -##### The Test: "Will this make sense in 6 months?" - -Before adding a comment, ask: Would someone reading just the current code (no PR, no history) find this helpful? - - -#### Rust Style Guides - -##### Type Ordering in Files - -When defining structs, traits, and functions in a file, follow this ordering convention. The file's primary type (matching the file name) comes first, followed by supporting public types, then private types and helpers. - -```rust -use ...; - -/// The primary type of this file (matches filename). -pub struct PayloadProcessor { ... } - -impl PayloadProcessor { ... } - -// Followed by public auxiliary types that support the primary type - -/// Configuration for the processor. -pub struct PayloadProcessorConfig { ... } - -/// Result type returned by processor operations. -pub struct ProcessorResult { ... } - -// Followed by public traits related to the primary type - -pub trait ProcessorExt { ... } - -// Followed by private helper types - -struct InternalState { ... } - -// Followed by private helper functions - -fn validate_input() { ... } -``` - -❌ **Bad**: Adding new traits and auxiliary types **above** the file's primary type (see [#22133](https://github.com/paradigmxyz/reth/pull/22133)): - -```rust -use ...; - -// ❌ BAD - new auxiliary struct added before the file's main type -pub struct CacheWaitDurations { ... } - -// ❌ BAD - new trait added before the file's main type -pub trait WaitForCaches { ... } - -// The file's primary type is buried below unrelated additions -pub struct PayloadProcessor { ... } -``` - -✅ **Good**: New types go **after** the primary type: - -```rust -use ...; - -// ✅ The file's primary type stays at the top -pub struct PayloadProcessor { ... } - -impl PayloadProcessor { ... } - -// ✅ Auxiliary types follow the primary type -pub struct CacheWaitDurations { ... } - -pub trait WaitForCaches { ... } - -impl WaitForCaches for PayloadProcessor { ... } -``` - -### Example Contribution Workflow - -Let's say you want to fix a bug where external IP resolution fails on startup: - -1. **Create a branch**: - ```bash - git checkout -b fix-external-ip-resolution - ``` - -2. **Find the relevant code**: - ```bash - # Search for IP resolution code - rg "external.*ip" --type rust - ``` - -3. **Reason about the problem, when the problem is identified, make the fix**: - ```rust - // In crates/net/discv4/src/lib.rs - pub fn resolve_external_ip() -> Option { - // Add fallback mechanism - nat::external_ip() - .or_else(|| nat::external_ip_from_stun()) - .or_else(|| Some(DEFAULT_IP)) - } - ``` - -4. **Add a test**: - ```rust - #[test] - fn test_external_ip_fallback() { - // Test that resolution has proper fallbacks - } - ``` - -5. **Run checks** (IMPORTANT!): - ```bash - cargo +nightly fmt --all - cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles! - cargo nextest run -p reth-discv4 - ``` - -6. **Commit with clear message**: - ```bash - git commit -m "fix: add fallback for external IP resolution - - Previously, node startup could fail if external IP resolution - failed. This adds fallback mechanisms to ensure the node can - always start with a reasonable default." - ``` - -## Quick Reference - -### Essential Commands +## Commands ```bash -# Format code -cargo +nightly fmt --all - -# Run lints -cargo +nightly clippy --workspace --all-features - -# Run tests +cargo build -p reth +cargo fmt --all +cargo clippy --workspace --all-targets --all-features cargo nextest run --workspace - -# Run specific benchmark -cargo bench --bench bench_name - -# Build optimized binary -cargo build --release - -# Check compilation for all features -cargo check --workspace --all-features - -# Check documentation cargo docs --document-private-items - -# Regenerate CLI reference docs (after CLI changes) -make update-book-cli ``` + +## Architecture + +- `bin/reth/`: node binary. +- `crates/node/`, `crates/engine/`: node setup and consensus engine. +- `crates/evm/`, `crates/ethereum/`: execution and Ethereum-specific components. +- `crates/storage/`, `crates/trie/`, `crates/stages/`: persistence, state roots, and sync. +- `crates/net/`, `crates/rpc/`: P2P networking and JSON-RPC. + +## Testing + +- Add regression coverage in existing unit or integration tests. +- Reuse test helpers and isolate test databases and directories. +- Run affected packages first; broaden checks for changes across crates. +- Use existing benchmarks for performance changes and report measured results. + +## Commit and PR Style + +Use conventional commits and PR titles: `type: description`, with an optional +scope. Keep subjects under 50 characters where practical. Explain what changed +and why in one short paragraph. Link related issues and include real measurements +for performance claims; omit templates and validation boilerplate. + +## Notes + +- **Testing**: Use fuzz tests for parsing and serialization, and property tests for invariants. +- **Hot paths**: Avoid unnecessary allocations; reuse buffers and borrow where practical. +- **Logging and metrics**: Follow the component's existing `tracing` targets and metrics conventions. +- **Filesystem**: Use `reth_fs_util` for filesystem operations and its error context. +- **Async work**: Keep blocking work off executor threads; follow the component's task model. +- **Vendored code**: Do not edit `crates/storage/libmdbx-rs/mdbx-sys/libmdbx/`. +- **Type order**: Keep the file's primary type before supporting types and private helpers. +- **Documentation**: Update relevant docs when behavior or public APIs change. Regenerate CLI + reference pages with `make update-book-cli` after changing commands or flags; do not edit + generated pages by hand. +- **Dependencies**: Run `zepter run check` and `make lint-toml` for dependency changes. +- **PR labels**: Check available labels and apply those relevant to the change. + +## Code Style + +- Follow existing patterns and keep changes focused. +- Explain non-obvious behavior and safety requirements in comments, not PR history. +- Comments end with periods, except URLs. Document safety requirements of unsafe code. +- Files use LF and end with a newline. +- Never expose secrets. + +### Rust + +- Generally add new Rust functions, methods, `impl` blocks, modules, imports, Cargo dependencies, and other items at the bottom of the relevant scope, section, or group. Constructors usually go at the top of an `impl` block. First check where similar items sit in the file and match the existing order, grouping, and style, including alphabetical order where used. +- Put doc comments before attributes, always: `/// ...` comes before `#[derive]`, `#[inline]`, `#[cfg]`, and every other attribute. +- Put module documentation at the top of the module file with inner doc comments (`//! ...`), not on the `mod` item in the parent module. +- NEVER put imports inside functions unless required for `#[cfg(...)]` gating. All imports go at the top of the file. +- Group all `use` imports together. Keep `pub use` imports in a separate group. For local module re-exports, write `mod x;` before `pub use x;`; for re-exporting another module or external crate, use `use x;`, then a blank line, then `pub use y;`, then a blank line before local `mod my_mod; pub use my_mod::*;`. +- Put imports used only by tests inside the relevant `#[cfg(test)]` module, merging them into its ordinary imports instead of adding `#[cfg(test)] use` items to the parent. Keep any additional feature or platform conditions on those imports. Retain parent-level test-gated imports or re-exports only when test-only helpers or multiple test modules need them there. +- Keep crate-level dependency anchors such as `#[cfg(test)] use cc as _;` at crate scope. +- Put conditional imports after unconditional imports, separated by a blank line. Group imports with the same `#[cfg(...)]` condition together, with a blank line between different conditions. Apply this to every feature, platform, and test gate, including imports in nested modules. Within each group, merge imports from the same crate when their conditions and other attributes match. Keep the full condition on each `use`; do not introduce import-only modules or macros to avoid repeated attributes. Apply the same ordering within the separate `pub use` group, keeping local re-exports after their module declarations. +- In test modules, always import the parent module with `use super::*`. +- In `Cargo.toml`, generally group optional dependencies for a feature together. Put a comment immediately above the group containing only the feature name, for example `# jit`. +- Prefer `let Some(x) = x else { return };` / `let Ok(x) = x else { return };` over `match x { Some(x) => x, _ => return }`. +- Use `let ... else` only for a single early-exit guard. When multiple conditions or patterns gate the same block, prefer a combined `if let` / `let` chain instead of several sequential `let ... else` statements. +- Use combined `if let` chains (`if let Some(x) = x && let Some(y) = y { ... }`) instead of nesting (`if let Some(x) = x { if let Some(y) = y { ... } }`). +- In loops, prefer an `if let` chain around the loop body over multiple `let ... else { continue };` statements when the body only runs if all patterns match. +- NEVER use `ref` / `ref mut` in patterns as the first resort. Always prefer borrowing the expression with `&` / `&mut` instead. +- Prefer map entry APIs such as `entry`, `or_insert`, and `or_insert_with` when multiple consecutive operations would otherwise look up and then insert or update the same key. +- Avoid specifying type hints in variables unless absolutely necessary (e.g. `HashMap<_, Vec<_>>` for `x.entry(y).or_default().push(z)` where type inference won't work). Rely on the compiler. +- When type hints are needed, prefer turbofish (`let x = Type::::new()`) over annotation (`let x: Type = Type::new()`). +- In tests, avoid `.contains` assertions for error/output strings when the project has snapshot testing support such as `snapbox`. Prefer exact snapshot assertions (`stderr_eq`, `stdout_eq`, `assert_data_eq!`, etc.) and use redactions only for genuinely variable parts. +- Always leave a blank line in between module doc-comments, items or item categories, unless in rare exceptions: it's a one-shot struct with one single impl block, or it's a list of impls that are all very similar. But in general blank line in between items is the norm but it's just unenforced. Items includes imports (together) too. The previous rules apply first. From 47c2166bc48bba25b95787c1f0640ec42bb9dee0 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 22 Sep 2026 20:20:50 +0000 Subject: [PATCH 045/133] fix(storage): evict ready sibling overlays (#27380) --- crates/storage/storage-overlay/src/manager.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 8eb63374718..43f9cac41af 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -514,6 +514,15 @@ impl OverlayManager { Compute(Arc>), } + let parent_hash = parent_state.block_ref().recovered_block().parent_hash(); + cache.retain(|sibling_key, entry| { + sibling_key.tip_hash == tip_hash || + !matches!(entry, OverlayCacheEntry::Ready(_)) || + self.blocks + .get(&sibling_key.tip_hash) + .is_none_or(|block| block.recovered_block().parent_hash() != parent_hash) + }); + let action = match cache.entries.entry(key) { Entry::Occupied(entry) => { let entry = entry.get().clone(); @@ -1130,6 +1139,37 @@ mod tests { assert_eq!(short.accounts().len(), 1); } + #[test] + fn computing_sibling_evicts_ready_cached_overlays() { + let manager = OverlayManager::default(); + let mut builder = TestBlockBuilder::eth(); + let anchor_hash = B256::random(); + let parent = builder.get_executed_block_with_number(1, anchor_hash); + let parent_hash = parent.recovered_block().hash(); + let first = builder.get_executed_block_with_number(2, parent_hash); + let sibling = builder.get_executed_block_with_number(2, parent_hash); + let sibling_hash = sibling.recovered_block().hash(); + let first_key = OverlayCacheKey { anchor_hash, tip_hash: first.recovered_block().hash() }; + + manager.insert_block(parent); + manager.insert_block(first); + manager.insert_block(sibling); + manager + .state_trie_overlays + .entries + .insert(first_key, OverlayCacheEntry::Ready(Arc::new(TrieInputSorted::default()))); + manager + .execution_overlays + .entries + .insert(first_key, OverlayCacheEntry::Ready(Arc::new(ExecutionOverlay::default()))); + + overlay_for_parent(&manager, sibling_hash, anchor_hash).unwrap(); + manager.execution_overlay_for_parent(sibling_hash, anchor_hash).unwrap(); + + assert!(!manager.state_trie_overlays.entries.contains_key(&first_key)); + assert!(!manager.execution_overlays.entries.contains_key(&first_key)); + } + #[test] fn execution_overlay_for_parent_at_anchor_is_empty() { let manager = OverlayManager::::default(); From 81a13259d0bfa62138d47fa1316893e22a2dffe8 Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Wed, 23 Sep 2026 01:59:41 +0400 Subject: [PATCH 046/133] fix(rpc): fill missing fee fields in eth_sendTransaction (#27341) --- .../rpc-eth-api/src/helpers/transaction.rs | 19 ++++++++++++- crates/rpc/rpc/src/eth/helpers/transaction.rs | 27 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 32f6f044974..1222b5ffbb8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -493,7 +493,7 @@ pub trait EthTransactions: LoadTransaction { mut request: RpcTxReq, ) -> impl Future> + Send where - Self: EthApiSpec + LoadBlock + EstimateCall, + Self: EthApiSpec + LoadBlock + EstimateCall + LoadFee, { async move { let from = match request.as_ref().from() { @@ -526,6 +526,23 @@ pub trait EthTransactions: LoadTransaction { } request.as_mut().set_chain_id(chain_id.to()); + // set fee defaults if not already set before, same as `fill_transaction` + if request.as_ref().gas_price().is_none() { + let tip = if let Some(tip) = request.as_ref().max_priority_fee_per_gas() { + tip + } else { + let tip = self.suggested_priority_fee().await?.to::(); + request.as_mut().set_max_priority_fee_per_gas(tip); + tip + }; + if request.as_ref().max_fee_per_gas().is_none() { + let header = + self.provider().latest_header().map_err(Self::Error::from_eth_err)?; + let base_fee = header.and_then(|h| h.base_fee_per_gas()).unwrap_or_default(); + request.as_mut().set_max_fee_per_gas(base_fee as u128 * 2 + tip); + } + } + if request.as_ref().gas_limit().is_none() { let estimated_gas = self .estimate_gas_at(request.clone(), BlockId::pending(), EvmOverrides::default()) diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 52473d687b5..247f814f5dd 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -313,6 +313,33 @@ mod tests { assert_eq!(pooled.transaction.gas_limit(), provided_gas_limit); } + #[tokio::test] + async fn send_transaction_fills_missing_fee_fields() { + let signers = DevSigner::random_signers(1); + let address = signers[0].accounts()[0]; + let accounts = AddressMap::from_iter([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), + )]); + let eth_api = mock_eth_api(accounts); + eth_api.signers().write().extend(signers); + + // no gas, gasPrice or 1559 fee fields: the node is expected to fill them + let tx_req = TransactionRequest { + from: Some(address), + to: Some(address.into()), + value: Some(U256::from(1)), + ..Default::default() + }; + + let hash = eth_api + .send_transaction_request(tx_req) + .await + .expect("send_transaction should fill the missing fee fields"); + let pooled = eth_api.pool().get(&hash).expect("transaction should be in the pool"); + assert!(pooled.transaction.max_fee_per_gas() > 0); + } + #[tokio::test] async fn send_transaction_rejects_mismatched_chain_id() { let signers = DevSigner::random_signers(1); From 0a1d3e761230b88271a481b94b6957a3f0f50bfc Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:31:37 +0000 Subject: [PATCH 047/133] fix(txpool): scale validation channel capacity with worker count (#27362) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com> --- crates/transaction-pool/src/validate/task.rs | 25 +++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/validate/task.rs b/crates/transaction-pool/src/validate/task.rs index a3718cdce91..c71c2554bc4 100644 --- a/crates/transaction-pool/src/validate/task.rs +++ b/crates/transaction-pool/src/validate/task.rs @@ -216,7 +216,10 @@ impl TransactionValidationTaskExecutor { /// This spawns `additional_tasks` extra blocking tasks plus one critical blocking task /// for the validation service. pub fn spawn(validator: V, tasks: &Runtime, additional_tasks: usize) -> Self { - let (tx, task) = ValidationTask::new(); + // Buffer two jobs per worker so workers can dequeue without waiting for + // a producer to refill the channel after every handoff. + let (tx, task) = + ValidationTask::with_capacity(additional_tasks.saturating_add(1).saturating_mul(2)); for _ in 0..additional_tasks { let task = task.clone(); @@ -442,6 +445,26 @@ mod tests { assert!(out.iter().all(|o| matches!(o, TransactionValidationOutcome::Valid { .. }))); } + #[tokio::test] + async fn configured_workers_have_bounded_handoff_capacity() { + let runtime = Runtime::test(); + for (additional_tasks, expected_capacity) in [(0, 2), (1, 4), (8, 18)] { + let executor = + TransactionValidationTaskExecutor::spawn(NoopValidator, &runtime, additional_tasks); + assert_eq!( + executor.to_validation_task.lock().await.tx.max_capacity(), + expected_capacity + ); + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(5), + executor.validate_transaction(TransactionOrigin::Local, MockTransaction::legacy()), + ) + .await + .expect("configured validation workers must process jobs"); + assert!(matches!(outcome, TransactionValidationOutcome::Valid { .. })); + } + } + #[derive(Debug)] struct SameOriginBatchValidator; From fe01db183447231759d2de98307c52d842692883 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:21:01 +0000 Subject: [PATCH 048/133] fix(txpool): remove validation producer mutex (#27361) Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com> --- crates/transaction-pool/src/validate/task.rs | 136 +++++++++++++++---- 1 file changed, 108 insertions(+), 28 deletions(-) diff --git a/crates/transaction-pool/src/validate/task.rs b/crates/transaction-pool/src/validate/task.rs index c71c2554bc4..d71150b8549 100644 --- a/crates/transaction-pool/src/validate/task.rs +++ b/crates/transaction-pool/src/validate/task.rs @@ -14,10 +14,7 @@ use reth_primitives_traits::{HeaderTy, SealedBlock}; use reth_storage_api::BlockReaderIdExt; use reth_tasks::Runtime; use std::{future::Future, pin::Pin, sync::Arc}; -use tokio::{ - sync, - sync::{mpsc, oneshot}, -}; +use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; /// Represents a future outputting unit type and is sendable. @@ -90,13 +87,8 @@ impl ValidationJobSender { job: Pin + Send>>, ) -> Result<(), TransactionValidatorError> { self.metrics.inflight_validation_jobs.increment(1); - let res = self - .tx - .send(job) - .await - .map_err(|_| TransactionValidatorError::ValidationServiceUnreachable); - self.metrics.inflight_validation_jobs.decrement(1); - res + let _guard = DecrementPendingOnDrop(&self.metrics.inflight_validation_jobs); + self.tx.send(job).await.map_err(|_| TransactionValidatorError::ValidationServiceUnreachable) } } @@ -106,8 +98,9 @@ impl ValidationJobSender { pub struct TransactionValidationTaskExecutor { /// The validator that will validate transactions on a separate task. pub validator: Arc, - /// The sender half to validation tasks that perform the actual validation. - pub to_validation_task: Arc>, + /// The bounded multi-producer queue supplies backpressure without serializing + /// producers behind an additional lock. + pub to_validation_task: Arc, } impl Clone for TransactionValidationTaskExecutor { @@ -202,13 +195,7 @@ impl TransactionValidationTaskExecutor { /// validation tasks. pub fn new(validator: V) -> (Self, ValidationTask) { let (tx, task) = ValidationTask::new(); - ( - Self { - validator: Arc::new(validator), - to_validation_task: Arc::new(sync::Mutex::new(tx)), - }, - task, - ) + (Self { validator: Arc::new(validator), to_validation_task: Arc::new(tx) }, task) } /// Creates a new executor and spawns the validation tasks on the given runtime. @@ -232,7 +219,7 @@ impl TransactionValidationTaskExecutor { task.run().await; }); - Self { validator: Arc::new(validator), to_validation_task: Arc::new(sync::Mutex::new(tx)) } + Self { validator: Arc::new(validator), to_validation_task: Arc::new(tx) } } } @@ -257,7 +244,7 @@ where let res = validator.validate_transaction(origin, transaction).await; let _ = tx.send(res); }); - self.to_validation_task.lock().await.send(fut).await + self.to_validation_task.send(fut).await }; if res.is_err() { return TransactionValidationOutcome::Error( @@ -291,7 +278,7 @@ where let res = validator.validate_transactions(transactions).await; let _ = tx.send(res); }); - self.to_validation_task.lock().await.send(fut).await + self.to_validation_task.send(fut).await }; if res.is_err() { return validation_service_error_outcomes(hashes) @@ -317,7 +304,7 @@ where let _ = tx.send(res); }); - if self.to_validation_task.lock().await.send(fut).await.is_err() { + if self.to_validation_task.send(fut).await.is_err() { return validation_service_error_outcomes(hashes) } @@ -332,6 +319,15 @@ where } } +/// Decrements the pending-send count even if the send future is cancelled. +struct DecrementPendingOnDrop<'a>(&'a metrics::Gauge); + +impl Drop for DecrementPendingOnDrop<'_> { + fn drop(&mut self) { + self.0.decrement(1); + } +} + #[inline] fn validation_service_error_outcomes( hashes: Vec, @@ -356,6 +352,42 @@ mod tests { TransactionOrigin, }; use alloy_primitives::{Address, U256}; + use metrics::atomics::AtomicU64; + use std::sync::atomic::Ordering; + + #[tokio::test] + async fn validation_send_metrics_track_cancellation_and_completion() { + for close_channel in [false, true] { + let (mut sender, task) = ValidationTask::new(); + let gauge = Arc::new(AtomicU64::new(0)); + sender.metrics.inflight_validation_jobs = metrics::Gauge::from_arc(gauge.clone()); + let pending_sends = || f64::from_bits(gauge.load(Ordering::Relaxed)); + + sender.send(Box::pin(async {})).await.unwrap(); + assert_eq!(pending_sends(), 0.0); + + // Keep the channel full so every producer waits for capacity. + let mut producers = + (0..8).map(|_| Box::pin(sender.send(Box::pin(async {})))).collect::>(); + for producer in &mut producers { + assert!(futures_util::poll!(producer.as_mut()).is_pending()); + } + assert_eq!(pending_sends(), 8.0); + + let survivor = producers.remove(0); + drop(producers); + assert_eq!(pending_sends(), 1.0); + + if close_channel { + drop(task); + } else { + // Free a slot for the remaining producer. + drop(task.validation_jobs.lock().await.next().await.unwrap()); + } + assert_eq!(survivor.await.is_err(), close_channel); + assert_eq!(pending_sends(), 0.0); + } + } #[tokio::test] async fn cloned_workers_validate_while_another_job_is_blocked() { @@ -445,16 +477,64 @@ mod tests { assert!(out.iter().all(|o| matches!(o, TransactionValidationOutcome::Valid { .. }))); } + #[tokio::test] + async fn cloned_executors_share_bounded_queue() { + let (executor, task) = TransactionValidationTaskExecutor::new(NoopValidator); + let mut submissions = tokio::task::JoinSet::new(); + for index in 0..64 { + let executor = executor.clone(); + submissions.spawn(async move { + let transaction = MockTransaction::legacy(); + let outcomes = match index % 3 { + 0 => vec![ + executor.validate_transaction(TransactionOrigin::Local, transaction).await, + ], + 1 => { + executor + .validate_transactions(vec![(TransactionOrigin::Local, transaction)]) + .await + } + _ => { + executor + .validate_transactions_with_origin( + TransactionOrigin::Local, + vec![transaction], + ) + .await + } + }; + assert_eq!(outcomes.len(), 1); + assert!(matches!(outcomes[0], TransactionValidationOutcome::Valid { .. })); + }); + } + + // No worker has started: one job can be queued and the other producers + // must remain pending on the same bounded channel. + tokio::task::yield_now().await; + assert_eq!(executor.to_validation_task.tx.capacity(), 0); + assert!(submissions.try_join_next().is_none()); + + let first_worker = tokio::spawn(task.clone().run()); + let second_worker = tokio::spawn(task.run()); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while let Some(result) = submissions.join_next().await { + result.unwrap(); + } + drop(executor); + first_worker.await.unwrap(); + second_worker.await.unwrap(); + }) + .await + .expect("concurrent producers and workers must drain and shut down"); + } + #[tokio::test] async fn configured_workers_have_bounded_handoff_capacity() { let runtime = Runtime::test(); for (additional_tasks, expected_capacity) in [(0, 2), (1, 4), (8, 18)] { let executor = TransactionValidationTaskExecutor::spawn(NoopValidator, &runtime, additional_tasks); - assert_eq!( - executor.to_validation_task.lock().await.tx.max_capacity(), - expected_capacity - ); + assert_eq!(executor.to_validation_task.tx.max_capacity(), expected_capacity); let outcome = tokio::time::timeout( std::time::Duration::from_secs(5), executor.validate_transaction(TransactionOrigin::Local, MockTransaction::legacy()), From 695b4f174b495e5a2bbc8779165422a678858bd7 Mon Sep 17 00:00:00 2001 From: Christopher Campbell Date: Wed, 23 Sep 2026 09:49:23 +0000 Subject: [PATCH 049/133] fix(trie): state root task persists EIP-161-empty accounts into HashedAccounts (#27252) Co-authored-by: Claude Co-authored-by: Brian Picciano --- crates/trie/parallel/src/state_root_task.rs | 200 +++++++++++++++++++- 1 file changed, 196 insertions(+), 4 deletions(-) diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index 5eb920317a9..382124aa63a 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -497,9 +497,19 @@ pub fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostState { trace!(target: "trie::parallel::sparse", ?address, ?hashed_address, "Adding account to state update"); let destroyed = account.is_selfdestructed(); - if account.info != account.original_info() { - let info = if destroyed { None } else { Some(account.info.into()) }; - hashed_state.accounts.insert(hashed_address, info); + // EIP-161: a touched account that ends up empty is deleted, so it must be emitted + // as a removal rather than as an all-zero account. This mirrors what revm does in + // the bundle path (`CacheAccount::touch_empty_eip161`) and what the sibling + // producer for this consumer already does in `send_bal_hashed_state`. + // An address that never existed and still does not exist is not a deletion: revm + // emits no transition for `LoadedNotExisting`. Skipping it matches the bundle + // producer; every `None` here becomes a storage-trie cursor walk in `StateRoot`. + let deleted = destroyed || (account.is_empty() && !account.is_loaded_as_not_existing()); + if deleted { + hashed_state.accounts.insert(hashed_address, None); + } else if account.info != account.original_info() { + // A touched but unchanged account produces no bundle transition either. + hashed_state.accounts.insert(hashed_address, Some(account.info.into())); } let mut changed_storage_iter = account @@ -533,7 +543,7 @@ mod tests { #[test] fn created_selfdestruct_does_not_emit_storage() { let address = Address::repeat_byte(0x01); - let mut account = Account::default(); + let mut account = Account::new_not_existing(TransactionId::ZERO); account.mark_touch(); assert!(account.mark_created_locally()); assert!(account.mark_selfdestructed_locally()); @@ -573,6 +583,188 @@ mod tests { assert!(!hashed_state.storages.contains_key(&hashed_address)); } + /// An account drained to zero balance, with nonce 0 and no code, is EIP-161-empty and + /// canonical execution deletes it. Asserts the converter reports the deletion as `None`, + /// matching `HashedPostState::from_bundle_state`, so `write_hashed_state` removes the + /// `HashedAccounts` row instead of upserting an all-zero one. + #[test] + fn emptied_account_is_deleted() { + let address = Address::repeat_byte(0x05); + let mut account = Account::default(); + // Pre-state: the account exists and holds a balance. + account.info.balance = U256::from(1); + account.set_current_info_as_original(); + // This block drains it. Not selfdestructed: an ordinary value transfer out. + account.mark_touch(); + account.info.balance = U256::ZERO; + assert!(account.is_empty(), "the drained account must be EIP-161-empty"); + assert!(!account.is_selfdestructed()); + + let hashed_state = + evm_state_to_hashed_post_state(EvmState::from_iter([(address, account)])); + + assert_eq!(hashed_state.accounts.get(&keccak256(address)), Some(&None)); + } + + /// A pre-existing EIP-161-empty account that is merely touched must be deleted. revm marks it + /// for removal (`touch_empty_eip161`) and the `BundleState` path reports `None`. Asserts the + /// converter emits the deletion even though `info == original_info()`, so an existing all-zero + /// `HashedAccounts` row can be cleared by a touch. + #[test] + fn touched_preexisting_empty_account_is_deleted() { + let address = Address::repeat_byte(0x06); + // Empty in the pre-state too: nonce 0, balance 0, no code. + let mut account = Account::default(); + account.set_current_info_as_original(); + account.mark_touch(); + assert!(account.is_empty()); + + let hashed_state = + evm_state_to_hashed_post_state(EvmState::from_iter([(address, account)])); + + assert_eq!(hashed_state.accounts.get(&keccak256(address)), Some(&None)); + } + + /// The two `HashedPostState` producers that feed the same consumer must agree. + /// + /// `evm_state_to_hashed_post_state` converts the raw `EvmState` handed to the state hook; + /// `HashedPostState::from_bundle_state` converts the `BundleState` revm produces from the + /// same execution. Since "perf: avoid hashing the state twice" the engine persists whichever + /// one it gets, so a disagreement between them is a disagreement about durable state. + /// + /// Asserts both producers agree for a touched account drained to EIP-161-empty, which revm + /// marks for removal in the bundle path (`CacheAccount::touch_empty_eip161`). + #[test] + fn matches_bundle_state_for_emptied_account() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + state::AccountInfo, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x07); + let pre = AccountInfo { balance: U256::from(1), ..Default::default() }; + + // The EvmState the state hook observes: a funded account drained to empty. + let mut account = Account::from(pre.clone()); + account.mark_touch(); + account.info.balance = U256::ZERO; + let evm_state = EvmState::from_iter([(address, account)]); + + // Same execution, through revm's own bundle machinery. + let mut db = State::builder().with_bundle_update().build(); + db.insert_account(address, pre); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts, from_bundle.accounts, + "state-hook and bundle producers disagree about durable account state" + ); + } + + /// An account created during the block whose final info is empty. EIP-161 deletes it, so + /// the bundle producer reports a removal. + #[test] + fn created_empty_account_matches_bundle_state() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x08); + let mut account = Account::default(); + account.mark_touch(); + assert!(account.mark_created_locally()); + assert!(account.is_empty()); + let evm_state = EvmState::from_iter([(address, account)]); + + let mut db = State::builder().with_bundle_update().build(); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts.get(&keccak256(address)).copied().flatten(), + from_bundle.accounts.get(&keccak256(address)).copied().flatten(), + "state-hook and bundle producers disagree about a created-empty account" + ); + } + + /// A zero-value call to an address that never existed leaves it non-existent. revm's bundle + /// path emits no transition for it (`CacheAccount::touch_empty_eip161` returns `None` for + /// `LoadedNotExisting`). Asserts the converter also emits nothing, rather than a deletion + /// that would queue a `HashedAccounts` delete and a storage-trie wipe for an unused address. + #[test] + fn touched_never_existing_account_matches_bundle_state() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x09); + let mut account = Account::new_not_existing(TransactionId::default()); + account.mark_touch(); + let evm_state = EvmState::from_iter([(address, account)]); + + let mut db = State::builder().with_bundle_update().build(); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts, from_bundle.accounts, + "state-hook and bundle producers disagree about a never-existing touched account" + ); + } + + /// An account touched without being changed. revm's bundle producer reports nothing for it. + /// Asserts the converter agrees, so a block full of zero-value calls does not rewrite + /// unchanged `HashedAccounts` rows. + #[test] + fn touched_unchanged_account_matches_bundle_state() { + use revm::{ + database::{states::bundle_state::BundleRetention, State}, + state::AccountInfo, + DatabaseCommit, + }; + + let address = Address::repeat_byte(0x0a); + let pre = AccountInfo { balance: U256::from(7), nonce: 1, ..Default::default() }; + let mut account = Account::from(pre.clone()); + account.mark_touch(); + assert!(!account.is_empty()); + let evm_state = EvmState::from_iter([(address, account)]); + + let mut db = State::builder().with_bundle_update().build(); + db.insert_account(address, pre); + db.commit(evm_state.clone()); + db.merge_transitions(BundleRetention::PlainState); + let bundle = db.take_bundle(); + + let from_bundle = + HashedPostState::from_bundle_state::(bundle.state.iter()); + let from_hook = evm_state_to_hashed_post_state(evm_state); + + assert_eq!( + from_hook.accounts, from_bundle.accounts, + "state-hook and bundle producers disagree about a touched but unchanged account" + ); + } + #[derive(Default)] struct CountingSink { access_hints: AtomicUsize, From 458d609fb61d47eabc0ec1c027f1514ec405813c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Sep 2026 12:28:23 +0200 Subject: [PATCH 050/133] chore(deps): bump alloy to 2.5.0 (#27387) --- Cargo.lock | 158 +++++++++++++++++++++++++++++------------------------ Cargo.toml | 46 ++++++++-------- 2 files changed, 110 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1ee9ee83da..0717cd855de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b836b372d3b742b7c083d2617d6e68563cd5dadbd8752c461a935a756551e69" +checksum = "727016fed834c2392d014dd697ea473f3e47bcec31ba31ccffa9c1d18ed904e5" dependencies = [ "alloy-eips", "alloy-primitives", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1befb37485ea71e8599e32ffd76c98af7c3720a34741d30abd87f41c3148407" +checksum = "6ea5ae60bbb3ce9dc52685a70a3755ad4432045fd38079b763cfe5127c706134" dependencies = [ "alloy-consensus", "alloy-eips", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b432cf09e307820b39ec5b856b25f396d8fde6089358f0c1d96d9654ad1e0b7d" +checksum = "8c807c974ae5ac07fa98431310b498d85f029da0d2f9a088b3ea673e5e33bb81" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -265,16 +265,31 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "alloy-eip8141" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36422c14bd57dcd638e276c8c9d7ac09eb16598e3a9dbd34e4b55267d4565b5e" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "serde", + "thiserror 2.0.20", +] + [[package]] name = "alloy-eips" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3e76d47e8de277cb55e7e77fa38adde835162fc01cddbe1b258b5e0a86035b" +checksum = "e01905444bd2fc330673f11d2466c0f54e8488f7ce3132b6a5e6497916fca765" dependencies = [ "alloy-eip2124", "alloy-eip2930", "alloy-eip7702", "alloy-eip7928", + "alloy-eip8141", "alloy-primitives", "alloy-rlp", "alloy-serde", @@ -313,9 +328,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0680a2f7d4bbc6f48c0cb43368fa544918a1fa291e772bdb47c834ade338f363" +checksum = "53cfa407c1d874cb306187996e30eb551bd98025e60e913e69e298ed2c8ea045" dependencies = [ "alloy-eips", "alloy-primitives", @@ -367,9 +382,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a523ca087dda941e53f6ecde5bf5df78b9dbe760f3f77488e13dbf355de4a84" +checksum = "fff9cb760601d352d1e59aab5803c3fb6dc99824b6883f5c7a781aacd4b54c66" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -382,9 +397,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f263c8d752b15e19b7c2d3cbad00338ec2d7cc923a46cd1ff98aea2fa333166b" +checksum = "e11dfba13dac9e54576bf41e1b7461491367e0bce6a2a4293b9e40ffa3ebd5e9" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -408,9 +423,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8795eecf50374a0dd942caff57922c59333c9991ead9eccbbc1709f55d73c7c2" +checksum = "3819bf6794d25f679d42c3f2b168e12cae08c56bb41d393cb6249b55e6d5933e" dependencies = [ "alloy-consensus", "alloy-eips", @@ -421,9 +436,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a58b30703aa2afe3b3ba66b0e9422393a9e869c6b84bff3b0a2f7e23f6fe5d1" +checksum = "75442e1238881ace33211a369d0cf614507cafb0211af3d0c37c81210ffb6589" dependencies = [ "alloy-genesis", "alloy-hardforks 0.2.13", @@ -475,9 +490,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef487e44dfae9c87cb732989d3f43682f65aefb11a0f56045b26be357f671fcf" +checksum = "9f8250dbbc93fab5b00fe0d3c1db3ede67d2a5fdbae8255de70b34ae39da244b" dependencies = [ "alloy-chains", "alloy-consensus", @@ -520,9 +535,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18662bde9be7d5218bed6e6fcb472caac3a65ea31730654678b6bba7c1b0553a" +checksum = "fcf12a49e488baefa21619f3ec2ce39f20857153aa1e91619fb5c1ce4d69d893" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -564,9 +579,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98267c2d47f9c185b575bac3f772a8ca66bcbd712693535e50980c12eab80a42" +checksum = "1f73519dc8f4ab73dafe90ac47aec4b9055c421c762d770d911eed83fcbef28d" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -590,9 +605,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "637fed69eda26e19fb42314006570519003c1bfc338c54165e610a657e3b46ff" +checksum = "e30da7fc40ff208becb4ebe462a149045c8613f60ee84f65731a522c67c351fd" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -603,9 +618,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91f1a1fa4d3c95ea5dba30c02d8987f0c052137dd7f1d55086cf36270517cbc" +checksum = "8d217ca58fed01c4c96d66767af04af39d57ee0d3af016aaf686a2d600aa66d0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -615,9 +630,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848bfe2315338ec605863073aae54a90fcea9cec7e6855a9e93d988bf42e9740" +checksum = "ad4774d898cc10846f65c426a385071f3e2872876d7f0e7836f1f75ffa54d448" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -627,9 +642,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1176430031ba27abfb51b95516cf71e8ce3ff4172c72f3136627f0f89813a739" +checksum = "9c5d320312e4a9481b18185429e57fa161dbf9468414ba5667aff4ebe7bd37e7" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", @@ -642,9 +657,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20f226662436bbdc221c9156a1f445c257da5e695b39a26878aa8b2644f710e" +checksum = "01f0837510a7e7789f99cf86bc0d0e3b54cfa663f25de803dad613364eef9fd3" dependencies = [ "alloy-eips", "alloy-primitives", @@ -662,9 +677,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a97960aa11b9984a18580b8dfbfc3eacfce1befa0d588b03ccc0bc5563d01f8" +checksum = "ce17ebd75d90c490897ca320e34bd5c95e0d8620d35ba5842aee9e7b5a90ea21" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -675,9 +690,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a43d7b6b86be4885c7a76f1dfcde9651af2d419ff571e8ee7257de47101203b" +checksum = "77b967576e97165fed3fae1cf6adad964e192293e4ee61d9187afe3ecaa12323" dependencies = [ "alloy-consensus", "alloy-eips", @@ -696,9 +711,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756eb5803e70985be4066b636194ccb5f6629e5d141817c1f0c50cea8350bf5c" +checksum = "32c475e21fe8a8caa02166b326ed22207a1a0597fc9b0dcdfb46e2fe6f80dadf" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -708,6 +723,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", + "alloy-trie", "arbitrary", "itertools 0.14.0", "serde", @@ -718,9 +734,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c833a9b946d827d2b2aa8cad8e92279a4340ec1b7cf666f3e1ee6ef4684e0159" +checksum = "8c230cc2e3f19a76889664522ffc10d746292f43fa1f57a3a9e20fe011346203" dependencies = [ "alloy-consensus", "alloy-eips", @@ -733,9 +749,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1933d60c2f51b8906103484ea2ca2218357264dbf16f948f7c2dd4ac5818e1d8" +checksum = "e3d8edefcef7d45a3da4bb869bbee1ee4c71b7437b22b87794f0912c89b5702c" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -747,9 +763,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb5d708e45799bd963d7e3bd8d04110e90983833ed02f91a9099811ed39ce6d" +checksum = "2fcacd9928d9788790afcd1d3fd8d0206329d2ef2391070867bafcc20ecf278f" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -759,9 +775,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc142e40df0c3eb5983b1637a68ab33471d8d22b8f402c9ed8260411a4e89d86" +checksum = "4f44f297d3b9e3ca2c3039e3c0ed69acde9b35edda15381cf78f725b7263fbe2" dependencies = [ "alloy-primitives", "arbitrary", @@ -771,9 +787,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c84f88f8a29d2607f98993687ed55a5943ba06819bc2126bb078dca54be943" +checksum = "e7de6fd34b5ba1789d482fa800dd97219b34765d6b07633a989a24d2dace770e" dependencies = [ "alloy-primitives", "async-trait", @@ -786,9 +802,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58c81472fb0b73cbb18106e323e33e3f249f56318a330a8e0f6aa6e913e3519a" +checksum = "d6909c09d586c7b55eced688754d57dde8a6bcbae2b83e69a9b9250e74cfd79e" dependencies = [ "alloy-consensus", "alloy-network", @@ -875,9 +891,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6576826236d84590b037e333bfc3643ae0a90cb325ca9da565419102047b3b81" +checksum = "e10be13ab13ddadcfaec3320cc6cee83548a1ae32ada214ff864604cbf5c6f77" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -898,9 +914,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e38c841f63141cad3f1a4378cedf121914b32932cf09b66c6006d4a906112136" +checksum = "df948f2f9530ed0325d789ad98aa08403aaeaf55ddd6eeaf0cda143e02afab39" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -914,9 +930,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03044aa44ec1da9468adf6a6e8c27d2bbaafd7a854ae14ac3e093b214139816d" +checksum = "a757d266eb7927d59a04878c99b450df0da49125e61d641f48fa7c6a82dd4418" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -935,9 +951,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55691c488ab3c88f664b6ffcaf9027f106b4642e6ff220502b6f645543a9efe8" +checksum = "8c799e5804cadb4a09e0ff21be230cc1025e5a21e02a41aa9c0022fdb41f830f" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -954,9 +970,9 @@ dependencies = [ [[package]] name = "alloy-trie" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f14b5d9b2c2173980202c6ff470d96e7c5e202c65a9f67884ad565226df7fbb" +checksum = "7895d0d2c946c0ae9253c227db77b9fc707383831058e6a5232fdfc774b28259" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -974,9 +990,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25484345e4d89c259235e200d4f895eba33e70f125f4e8bb37a7aa69a97bd656" +checksum = "60fa90c57738ace5b38106c9f743f3d6d47de09a82e82ecfc42702c1acf116ee" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -3496,7 +3512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5239,7 +5255,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6369,7 +6385,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11554,7 +11570,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11634,7 +11650,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12527,7 +12543,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -13791,7 +13807,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f15b9929862..79892fa6e46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -445,29 +445,29 @@ alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.9" -alloy-consensus = { version = "2.4.2", default-features = false } -alloy-contract = { version = "2.4.2", default-features = false } -alloy-eips = { version = "2.4.2", default-features = false } -alloy-genesis = { version = "2.4.2", default-features = false } -alloy-json-rpc = { version = "2.4.2", default-features = false } -alloy-network = { version = "2.4.2", default-features = false } -alloy-node-bindings = "2.4.2" -alloy-provider = { version = "2.4.2", features = ["reqwest", "debug-api"], default-features = false } -alloy-rpc-client = { version = "2.4.2", default-features = false } -alloy-rpc-types = { version = "2.4.2", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "2.4.2", default-features = false } -alloy-rpc-types-anvil = { version = "2.4.2", default-features = false } -alloy-rpc-types-beacon = { version = "2.4.2", default-features = false } -alloy-rpc-types-debug = { version = "2.4.2", default-features = false } -alloy-rpc-types-engine = { version = "2.4.2", default-features = false } -alloy-rpc-types-eth = { version = "2.4.2", default-features = false } -alloy-rpc-types-mev = { version = "2.4.2", default-features = false } -alloy-rpc-types-trace = { version = "2.4.2", default-features = false } -alloy-rpc-types-txpool = { version = "2.4.2", default-features = false } -alloy-serde = { version = "2.4.2", default-features = false } -alloy-signer = { version = "2.4.2", default-features = false } -alloy-signer-local = { version = "2.4.2", default-features = false } -alloy-transport = { version = "2.4.2" } +alloy-consensus = { version = "2.5.0", default-features = false } +alloy-contract = { version = "2.5.0", default-features = false } +alloy-eips = { version = "2.5.0", default-features = false } +alloy-genesis = { version = "2.5.0", default-features = false } +alloy-json-rpc = { version = "2.5.0", default-features = false } +alloy-network = { version = "2.5.0", default-features = false } +alloy-node-bindings = "2.5.0" +alloy-provider = { version = "2.5.0", features = ["reqwest", "debug-api"], default-features = false } +alloy-rpc-client = { version = "2.5.0", default-features = false } +alloy-rpc-types = { version = "2.5.0", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "2.5.0", default-features = false } +alloy-rpc-types-anvil = { version = "2.5.0", default-features = false } +alloy-rpc-types-beacon = { version = "2.5.0", default-features = false } +alloy-rpc-types-debug = { version = "2.5.0", default-features = false } +alloy-rpc-types-engine = { version = "2.5.0", default-features = false } +alloy-rpc-types-eth = { version = "2.5.0", default-features = false } +alloy-rpc-types-mev = { version = "2.5.0", default-features = false } +alloy-rpc-types-trace = { version = "2.5.0", default-features = false } +alloy-rpc-types-txpool = { version = "2.5.0", default-features = false } +alloy-serde = { version = "2.5.0", default-features = false } +alloy-signer = { version = "2.5.0", default-features = false } +alloy-signer-local = { version = "2.5.0", default-features = false } +alloy-transport = { version = "2.5.0" } # misc either = { version = "1.16.0", default-features = false } From 8ae06eca59c85d35c58ba10aff111b0e7e11f177 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 23 Sep 2026 12:45:58 +0200 Subject: [PATCH 051/133] perf(storage): reuse canonical BlockState chain when building overlay providers (#27319) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/engine/tree/src/tree/mod.rs | 6 +- .../src/providers/blockchain_provider.rs | 108 ++++++++++++++++-- crates/storage/storage-overlay/src/builder.rs | 97 +++++++++++++++- crates/storage/storage-overlay/src/manager.rs | 14 ++- 4 files changed, 207 insertions(+), 18 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index df80919699f..4a39a4b76fd 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2320,11 +2320,15 @@ where } let finalized = self.state.forkchoice_state_tracker.last_valid_finalized(); - self.remove_before(in_memory_persisted_block, finalized)?; + // Trim the canonical in-memory state first: state providers build their overlays from the + // canonical chain, so it must never reference blocks whose overlays the manager has + // already pruned. `remove_before` does not read the canonical in-memory state, so the + // order between the two trims is free to choose. self.canonical_in_memory_state.remove_persisted_blocks_until( self.persistence_state.last_persisted_block, in_memory_persisted_block.number, ); + self.remove_before(in_memory_persisted_block, finalized)?; // Persistence changes the overlay anchor. Prepare the remaining canonical range before // the next payload needs to read execution state against the new durable frontier. self.state.tree_state.overlay_manager.precompute_execution_overlay( diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 318a374421a..54ccd5b26f9 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -16,7 +16,7 @@ use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes, TxHash, TxNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ - CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, + BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, PersistedBlockNotifications, PersistedBlockSubscriptions, }; use reth_chainspec::ChainInfo; @@ -149,11 +149,17 @@ impl BlockchainProvider { ConsistentProvider::new(self.database.clone(), self.canonical_in_memory_state()) } - /// Returns a state provider for the post-state of `block_hash`. - fn state_provider_at_block_hash(&self, block_hash: B256) -> ProviderResult { + /// Returns a state provider for the post-state of an already resolved in-memory chain. + /// + /// This is the post-state of `state.hash()`. Because the caller already holds the in-memory + /// chain, the overlay manager does not have to rebuild an identical one from its block graph. + fn state_provider_for_state( + &self, + state: Arc>, + ) -> ProviderResult { let state_provider_factory = OverlayStateProviderFactory::new( self.database.clone(), - self.database.overlay_manager().overlay_builder(block_hash), + self.database.overlay_manager().overlay_builder_for_state(state), ); Ok(Box::new(state_provider_factory.database_provider_ro()?)) } @@ -185,7 +191,7 @@ impl BlockchainProvider { let state_provider_factory = OverlayStateProviderFactory::new( self.database.clone(), - self.database.overlay_manager().overlay_builder(matched.hash()), + self.database.overlay_manager().overlay_builder_for_state(matched), ); state_provider_factory.database_provider_ro().map(Some) } @@ -729,7 +735,7 @@ impl StateProviderFactory for BlockchainProvider { // use latest state provider if the head state exists if let Some(state) = self.canonical_in_memory_state.head_state() { trace!(target: "providers::blockchain", "Using head state for latest state provider"); - self.state_provider_at_block_hash(state.hash()) + self.state_provider_for_state(state) } else { trace!(target: "providers::blockchain", "Using database state for latest state provider"); self.database.latest() @@ -789,7 +795,7 @@ impl StateProviderFactory for BlockchainProvider { fn state_by_block_hash(&self, hash: BlockHash) -> ProviderResult { trace!(target: "providers::blockchain", ?hash, "Getting state by block hash"); if let Some(state) = self.canonical_in_memory_state.state_by_hash(hash) { - self.state_provider_at_block_hash(state.hash()) + self.state_provider_for_state(state) } else if let Ok(state) = self.history_by_block_hash(hash) { // This could be tracked by a historical block Ok(state) @@ -811,7 +817,7 @@ impl StateProviderFactory for BlockchainProvider { if let Some(pending) = self.canonical_in_memory_state.pending_state() { // we have a pending block - return self.state_provider_at_block_hash(pending.hash()); + return self.state_provider_for_state(Arc::new(pending)); } // fallback to latest state if the pending block is not available @@ -822,14 +828,14 @@ impl StateProviderFactory for BlockchainProvider { if let Some(pending) = self.canonical_in_memory_state.pending_state() && pending.hash() == block_hash { - return self.state_provider_at_block_hash(pending.hash()).map(Some); + return self.state_provider_for_state(Arc::new(pending)).map(Some); } Ok(None) } fn maybe_pending(&self) -> ProviderResult> { if let Some(pending) = self.canonical_in_memory_state.pending_state() { - return self.state_provider_at_block_hash(pending.hash()).map(Some) + return self.state_provider_for_state(Arc::new(pending)).map(Some) } Ok(None) @@ -3232,6 +3238,88 @@ mod tests { Ok(()) } + #[test] + fn latest_reads_in_memory_state_like_the_hash_lookup() -> eyre::Result<()> { + use crate::{AccountReader, StateProvider, StateProviderBox}; + use reth_storage_api::DatabaseProviderROFactory; + use reth_storage_overlay::OverlayStateProviderFactory; + + let mut rng = generators::rng(); + let factory = test_provider_factory_with_genesis()?; + let provider = BlockchainProvider::new(factory)?; + let genesis = provider.canonical_in_memory_state.get_canonical_head(); + + let (address, account) = random_account(1); + let hashed_address = keccak256(address); + let slot_key = B256::with_last_byte(1); + let slot = U256::from_be_bytes(slot_key.0); + let value = U256::from(42); + + let mut hashed_state = HashedPostState::default(); + hashed_state.accounts.insert(hashed_address, Some(account)); + hashed_state + .storages + .insert(hashed_address, HashedStorage::from_iter([(keccak256(slot_key), value)])); + + let mut storage = HashMap::default(); + storage.insert(slot, (U256::ZERO, value)); + + let block = random_block( + &mut rng, + genesis.number + 1, + BlockParams { parent: Some(genesis.hash()), tx_count: Some(0), ..Default::default() }, + ) + .try_recover() + .expect("failed to seal block with senders"); + let block_hash = block.hash(); + + let execution_output = BlockExecutionOutput { + result: BlockExecutionResult { + receipts: Default::default(), + requests: Default::default(), + gas_used: 0, + blob_gas_used: 0, + }, + state: BundleState::new( + [(address, None, Some(account.into()), storage)], + [[(address, Some(None), [])]], + [], + ), + }; + let executed = ExecutedBlock::new( + Arc::new(block), + Arc::new(execution_output), + ComputedTrieData::new( + Arc::new(hashed_state.into_sorted()), + Arc::new(TrieUpdates::default().into_sorted()), + ), + ); + provider.database.overlay_manager().insert_block(executed.clone()); + provider + .canonical_in_memory_state + .update_chain(NewCanonicalChain::Commit { new: vec![executed] }); + + // `latest` reuses the canonical `Arc` chain, while the hash-based lookup + // rebuilds an equivalent chain from the overlay manager's block graph. Both must observe + // the same in-memory account and storage values. + let rebuilt_from_hash = OverlayStateProviderFactory::new( + provider.database.clone(), + provider.database.overlay_manager().overlay_builder(block_hash), + ) + .database_provider_ro()?; + + for state in [ + Box::new(rebuilt_from_hash) as StateProviderBox, + provider.latest()?, + provider.state_by_block_hash(block_hash)?, + ] { + assert_eq!(state.basic_account(&address)?, Some(account)); + assert_eq!(state.storage(address, slot_key)?, Some(value)); + } + + Ok(()) + } + #[test] fn state_range_provider_resolves_root_from_in_memory_block() -> eyre::Result<()> { let mut rng = generators::rng(); diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index b78df88c9d3..fde2d0f5c1b 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -218,7 +218,10 @@ pub struct OverlayBuilder { /// Manager used for cached changesets and overlays. overlay_manager: OverlayManager, /// Snapshot of the in-memory chain ending at the requested parent. - parent_state: Option>, + /// + /// This is shared with the caller so that a chain that is already maintained elsewhere (for + /// example the canonical in-memory chain) can be reused instead of rebuilt. + parent_state: Option>>, /// Anchor hash of the reused sparse trie, if this task reused one. reused_sparse_trie_anchor_hash: Option, /// Whether building the overlay may query revert changesets. @@ -233,7 +236,7 @@ impl OverlayBuilder { /// Create a new manager-backed overlay builder. pub(crate) fn new( parent_hash: B256, - parent_state: Option>, + parent_state: Option>>, overlay_manager: OverlayManager, ) -> Self { Self { @@ -268,7 +271,7 @@ impl OverlayBuilder { debug_assert_eq!(block.recovered_block().parent_hash(), self.parent_hash); self.parent_hash = block.recovered_block().hash(); self.parent_state = - Some(BlockState::with_parent(block, self.parent_state.take().map(Arc::new))); + Some(Arc::new(BlockState::with_parent(block, self.parent_state.take()))); self.reused_sparse_trie_anchor_hash = None; self.overlay_cache_config.write_to_cache = false; self @@ -668,7 +671,7 @@ impl OverlayBuilder { if anchor_hash == self.parent_hash { Ok((Arc::new(TrieUpdatesSorted::default()), Arc::new(HashedPostStateSorted::default()))) } else { - let parent_state = self.parent_state.as_ref().ok_or_else(|| { + let parent_state = self.parent_state.as_deref().ok_or_else(|| { ProviderError::other(std::io::Error::other( "state trie overlay cannot be anchored without in-memory parent state", )) @@ -687,7 +690,7 @@ impl OverlayBuilder { if anchor_hash == self.parent_hash { Ok(Arc::new(ExecutionOverlay::default())) } else { - let parent_state = self.parent_state.as_ref().ok_or_else(|| { + let parent_state = self.parent_state.as_deref().ok_or_else(|| { ProviderError::other(std::io::Error::other("missing in-memory parent state")) })?; self.overlay_manager @@ -839,7 +842,9 @@ enum AnchorForParent { mod tests { use super::*; use alloy_primitives::{map::HashMap, Address, U256}; - use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; + use reth_chain_state::{ + test_utils::TestBlockBuilder, CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain, + }; use reth_db::{ models::{AccountBeforeTx, BlockNumberAddress}, tables, @@ -937,6 +942,23 @@ mod tests { (factory, blocks) } + /// Tracks `blocks` in a [`CanonicalInMemoryState`] the way the engine does, so the resulting + /// `Arc` chain matches what state providers hold. + fn canonical_in_memory_state( + blocks: &[ExecutedBlock], + ) -> CanonicalInMemoryState { + let state = CanonicalInMemoryState::empty(); + state.update_chain(NewCanonicalChain::Commit { new: blocks.to_vec() }); + state + } + + const fn anchor_num_hash(anchor: &AnchorForParent) -> BlockNumHash { + match anchor { + AnchorForParent::NoReverts { anchor } | + AnchorForParent::RevertsRequired { anchor, .. } => *anchor, + } + } + fn account_keys(overlay: &StateTrieOverlay) -> Vec { overlay.input().state.accounts.iter().map(|(key, _)| *key).collect() } @@ -1047,6 +1069,69 @@ mod tests { assert!(overlay.code_hashes.contains_key(&later_code_hash)); } + #[test] + fn overlay_builder_for_state_matches_hash_lookup() { + // The state trie frontier sits at block 1 while Finish is at block 3, so the in-memory + // chain straddles the state-masking frontier. + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + for block in &blocks[2..=4] { + manager.insert_block(block.clone()); + } + let canonical = canonical_in_memory_state(&blocks[2..=4]); + let provider = factory.provider().unwrap(); + + // The head, a block below the head, and the oldest in-memory block, whose chain no longer + // covers the Finish frontier and therefore anchors below it. + for index in [4usize, 3, 2] { + let hash = blocks[index].recovered_block().hash(); + let state = canonical.state_by_hash(hash).expect("canonical state for in-memory block"); + assert_eq!(state.hash(), hash); + + let from_hash = manager.overlay_builder(hash); + let from_state = manager.overlay_builder_for_state(state); + + let anchor = anchor_num_hash(&from_hash.anchor_at_parent(&provider).unwrap()); + assert_eq!( + anchor_num_hash(&from_state.anchor_at_parent(&provider).unwrap()), + anchor, + "block {index} must resolve the same anchor from both builders" + ); + + let (hash_overlay, hash_fallback) = from_hash.execution_overlay(&provider).unwrap(); + let (state_overlay, state_fallback) = from_state.execution_overlay(&provider).unwrap(); + assert_eq!(state_fallback, hash_fallback); + assert!( + Arc::ptr_eq(&hash_overlay, &state_overlay), + "block {index} must resolve the cached execution overlay" + ); + + let (hash_nodes, hash_state) = + from_hash.resolve_state_trie_overlays(anchor.hash).unwrap(); + let (state_nodes, state_state) = + from_state.resolve_state_trie_overlays(anchor.hash).unwrap(); + assert!( + Arc::ptr_eq(&hash_nodes, &state_nodes), + "block {index} must resolve the cached trie overlay nodes" + ); + assert!( + Arc::ptr_eq(&hash_state, &state_state), + "block {index} must resolve the cached trie overlay state" + ); + + // The fully built overlay folds in database reverts for anchors below Finish, so + // compare it by value rather than by pointer. + let hash_trie = from_hash.build_state_trie_overlay(&provider, false).unwrap(); + let state_trie = from_state.build_state_trie_overlay(&provider, false).unwrap(); + assert_eq!(account_keys(&state_trie), account_keys(&hash_trie), "block {index}"); + assert_eq!( + account_node_paths(&state_trie), + account_node_paths(&hash_trie), + "block {index}" + ); + } + } + #[test] fn managed_overlay_starts_at_state_trie_frontier() { let (factory, blocks) = setup_frontiers(1, 3); diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 43f9cac41af..c195531a03f 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -95,8 +95,20 @@ impl OverlayManager { } /// Creates an overlay builder for `parent_hash`. + /// + /// This rebuilds the in-memory chain ending at `parent_hash` from the manager's block graph. + /// Prefer [`Self::overlay_builder_for_state`] whenever the caller already holds the chain. pub fn overlay_builder(&self, parent_hash: B256) -> OverlayBuilder { - OverlayBuilder::new(parent_hash, self.block_state(parent_hash), self.clone()) + OverlayBuilder::new(parent_hash, self.block_state(parent_hash).map(Arc::new), self.clone()) + } + + /// Creates an overlay builder for an already materialized in-memory chain. + /// + /// The chain tip is used as the parent hash, so no block graph lookup or chain rebuild is + /// performed. The builder only reads `state`, which makes it safe to share the same + /// [`BlockState`] with other holders. + pub fn overlay_builder_for_state(&self, state: Arc>) -> OverlayBuilder { + OverlayBuilder::new(state.hash(), Some(state), self.clone()) } pub(crate) fn block_state(&self, parent_hash: B256) -> Option> { From 9e6823bed8c77d4c5a35fbe639723a06afd5bd80 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 23 Sep 2026 11:03:33 +0000 Subject: [PATCH 052/133] refactor(provider): remove memory overlay provider (#26923) --- Cargo.lock | 5 +- crates/chain-state/Cargo.toml | 9 +- crates/chain-state/src/in_memory.rs | 229 +----------- crates/chain-state/src/lib.rs | 5 - crates/chain-state/src/memory_overlay.rs | 341 ------------------ crates/chain-state/src/noop.rs | 36 -- crates/chain-state/src/notifications.rs | 8 +- crates/chain-state/src/test_utils.rs | 5 +- crates/node/ethstats/src/ethstats.rs | 4 +- crates/rpc/rpc-builder/src/lib.rs | 3 +- crates/rpc/rpc-eth-api/Cargo.toml | 2 +- .../rpc-eth-api/src/helpers/pending_block.rs | 18 +- crates/rpc/rpc-eth-api/src/node.rs | 10 +- crates/rpc/rpc/Cargo.toml | 2 +- crates/rpc/rpc/src/eth/core.rs | 7 +- crates/rpc/rpc/src/reth.rs | 11 +- crates/storage/provider/Cargo.toml | 2 +- .../src/providers/blockchain_provider.rs | 20 +- .../storage/provider/src/test_utils/mock.rs | 14 +- crates/storage/provider/src/traits/full.rs | 8 +- crates/storage/rpc-provider/Cargo.toml | 1 + crates/storage/rpc-provider/src/lib.rs | 23 ++ crates/storage/storage-api/Cargo.toml | 8 + crates/storage/storage-api/src/noop.rs | 48 +++ crates/storage/storage-api/src/state.rs | 15 +- crates/storage/storage-overlay/src/builder.rs | 26 ++ crates/storage/storage-overlay/src/manager.rs | 68 +++- 27 files changed, 271 insertions(+), 657 deletions(-) delete mode 100644 crates/chain-state/src/memory_overlay.rs delete mode 100644 crates/chain-state/src/noop.rs diff --git a/Cargo.lock b/Cargo.lock index 0717cd855de..d316bcdf0e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7947,12 +7947,10 @@ dependencies = [ "pin-project", "rand 0.9.4", "reth-chainspec", - "reth-errors", "reth-ethereum-primitives", "reth-execution-types", "reth-metrics", "reth-primitives-traits", - "reth-storage-api", "reth-testing-utils", "reth-trie", "reth-trie-sparse", @@ -10614,6 +10612,7 @@ dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", "auto_impl", + "reth-chain-state", "reth-chainspec", "reth-db-api", "reth-db-models", @@ -10626,6 +10625,7 @@ dependencies = [ "reth-trie-common", "revm", "serde_json", + "tokio", ] [[package]] @@ -10686,6 +10686,7 @@ dependencies = [ "alloy-rpc-types", "alloy-rpc-types-engine", "dashmap", + "reth-chain-state", "reth-chainspec", "reth-db-api", "reth-errors", diff --git a/crates/chain-state/Cargo.toml b/crates/chain-state/Cargo.toml index 4a4b494821f..2d96c857761 100644 --- a/crates/chain-state/Cargo.toml +++ b/crates/chain-state/Cargo.toml @@ -14,12 +14,10 @@ workspace = true [dependencies] # reth reth-chainspec.workspace = true -reth-errors.workspace = true reth-execution-types.workspace = true reth-metrics.workspace = true reth-ethereum-primitives.workspace = true reth-primitives-traits.workspace = true -reth-storage-api.workspace = true reth-trie-sparse = { workspace = true, features = ["std"] } reth-trie.workspace = true @@ -27,7 +25,7 @@ reth-trie.workspace = true alloy-eips.workspace = true alloy-primitives = { workspace = true, features = ["std"] } alloy-consensus.workspace = true -revm.workspace = true +revm = { workspace = true, optional = true } # async tokio = { workspace = true, default-features = false, features = ["sync", "macros"] } @@ -54,6 +52,7 @@ reth-testing-utils.workspace = true alloy-signer.workspace = true alloy-signer-local.workspace = true rand.workspace = true +revm.workspace = true [features] serde = [ @@ -67,8 +66,7 @@ serde = [ "reth-execution-types/serde", "reth-primitives-traits/serde", "reth-trie/serde", - "revm/serde", - "reth-storage-api/serde", + "revm?/serde", ] test-utils = [ "alloy-primitives/getrandom", @@ -80,4 +78,5 @@ test-utils = [ "reth-trie/test-utils", "reth-ethereum-primitives/test-utils", "reth-trie-sparse/test-utils", + "dep:revm", ] diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index 29ca16a9682..6a095c7eccb 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -1,8 +1,7 @@ //! Types for tracking the canonical chain state in memory. use crate::{ - CanonStateNotification, CanonStateNotificationSender, CanonStateNotifications, - ChainInfoTracker, MemoryOverlayStateProvider, + CanonStateNotification, CanonStateNotificationSender, CanonStateNotifications, ChainInfoTracker, }; use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; use alloy_eips::{BlockHashOrNumber, BlockNumHash}; @@ -18,7 +17,6 @@ use reth_primitives_traits::{ BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, SignedTransaction, }; -use reth_storage_api::StateProviderBox; use reth_trie::{ updates::TrieUpdatesSorted, ComputedTrieData, HashedPostStateSorted, LazyTrieData, }; @@ -545,24 +543,6 @@ impl CanonicalInMemoryState { self.inner.canon_state_notification_sender.send(event).ok(); } - /// Return state provider with reference to in-memory blocks that overlay database state. - /// - /// This merges the state of all blocks that are part of the chain that the requested block is - /// the head of. This includes all blocks that connect back to the canonical block on disk. - pub fn state_provider( - &self, - hash: B256, - historical: StateProviderBox, - ) -> MemoryOverlayStateProvider { - let in_memory = if let Some(state) = self.state_by_hash(hash) { - state.chain().map(|block_state| block_state.block()).collect() - } else { - Vec::new() - }; - - MemoryOverlayStateProvider::new(historical, in_memory) - } - /// Returns an iterator over all __canonical blocks__ in the in-memory state, from newest to /// oldest (highest to lowest). /// @@ -717,16 +697,6 @@ impl BlockState { std::iter::successors(Some(self), |state| state.parent.clone()) } - /// Return state provider with reference to in-memory blocks that overlay database state. - /// - /// This merges the state of all blocks that are part of the chain that the this block is - /// the head of. This includes all blocks that connect back to the canonical block on disk. - pub fn state_provider(&self, historical: StateProviderBox) -> MemoryOverlayStateProvider { - let in_memory = self.chain().map(|block_state| block_state.block()).collect(); - - MemoryOverlayStateProvider::new(historical, in_memory) - } - /// Tries to find a block by [`BlockHashOrNumber`] in the chain ending at this block. pub fn block_on_chain(&self, hash_or_num: BlockHashOrNumber) -> Option<&Self> { self.chain().find(|block| match hash_or_num { @@ -1055,19 +1025,9 @@ mod tests { use super::*; use crate::test_utils::TestBlockBuilder; use alloy_eips::eip7685::Requests; - use alloy_primitives::{Address, BlockNumber, Bytes, StorageKey, StorageValue}; + use alloy_primitives::Bytes; use rand::Rng; - use reth_errors::ProviderResult; use reth_ethereum_primitives::{EthPrimitives, Receipt}; - use reth_primitives_traits::{Account, Bytecode}; - use reth_storage_api::{ - AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, - StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, - }; - use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, - MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, - }; fn create_mock_state( test_block_builder: &mut TestBlockBuilder, @@ -1100,141 +1060,6 @@ mod tests { chain } - struct MockStateProvider; - - impl StateProvider for MockStateProvider { - fn storage( - &self, - _address: Address, - _storage_key: StorageKey, - ) -> ProviderResult> { - Ok(None) - } - } - - impl BytecodeReader for MockStateProvider { - fn bytecode_by_hash(&self, _code_hash: &B256) -> ProviderResult> { - Ok(None) - } - } - - impl BlockHashReader for MockStateProvider { - fn block_hash(&self, _number: BlockNumber) -> ProviderResult> { - Ok(None) - } - - fn canonical_hashes_range( - &self, - _start: BlockNumber, - _end: BlockNumber, - ) -> ProviderResult> { - Ok(vec![]) - } - } - - impl AccountReader for MockStateProvider { - fn basic_account(&self, _address: &Address) -> ProviderResult> { - Ok(None) - } - } - - impl StateRootProvider for MockStateProvider { - fn state_root(&self, _hashed_state: HashedPostState) -> ProviderResult { - Ok(B256::random()) - } - - fn state_root_from_nodes(&self, _input: TrieInput) -> ProviderResult { - Ok(B256::random()) - } - - fn state_root_with_updates( - &self, - _hashed_state: HashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - Ok((B256::random(), TrieUpdates::default())) - } - - fn state_root_from_nodes_with_updates( - &self, - _input: TrieInput, - ) -> ProviderResult<(B256, TrieUpdates)> { - Ok((B256::random(), TrieUpdates::default())) - } - } - - impl HashedPostStateProvider for MockStateProvider { - fn hashed_post_state( - &self, - _bundle_state: &revm::database::BundleState, - ) -> ProviderResult { - Ok(HashedPostState::default()) - } - } - - impl StorageRootProvider for MockStateProvider { - fn storage_root( - &self, - _address: Address, - _hashed_storage: HashedStorage, - ) -> ProviderResult { - Ok(B256::random()) - } - - fn storage_proof( - &self, - _address: Address, - slot: B256, - _hashed_storage: HashedStorage, - ) -> ProviderResult { - Ok(StorageProof::new(slot)) - } - - fn storage_multiproof( - &self, - _address: Address, - _slots: &[B256], - _hashed_storage: HashedStorage, - ) -> ProviderResult { - Ok(StorageMultiProof::empty()) - } - } - - impl StateProofProvider for MockStateProvider { - fn proof( - &self, - _input: TrieInput, - _address: Address, - _slots: &[B256], - ) -> ProviderResult { - Ok(AccountProof::new(Address::random())) - } - - fn multiproof( - &self, - _input: TrieInput, - _targets: MultiProofTargets, - ) -> ProviderResult { - Ok(MultiProof::default()) - } - - fn multiproof_v2( - &self, - _input: TrieInput, - _targets: reth_trie::MultiProofTargetsV2, - ) -> ProviderResult { - Ok(reth_trie::DecodedMultiProofV2::default()) - } - - fn witness( - &self, - _input: TrieInput, - _target: HashedPostState, - _mode: reth_trie::ExecutionWitnessMode, - ) -> ProviderResult> { - Ok(Vec::default()) - } - } - #[test] fn test_in_memory_state_impl_state_by_hash() { let mut state_by_hash = B256Map::default(); @@ -1428,56 +1253,6 @@ mod tests { ); } - #[test] - fn test_canonical_in_memory_state_state_provider() { - let mut test_block_builder: TestBlockBuilder = TestBlockBuilder::default(); - let block1 = test_block_builder.get_executed_block_with_number(1, B256::random()); - let block2 = - test_block_builder.get_executed_block_with_number(2, block1.recovered_block().hash()); - let block3 = - test_block_builder.get_executed_block_with_number(3, block2.recovered_block().hash()); - - let state1 = Arc::new(BlockState::new(block1.clone())); - let state2 = Arc::new(BlockState::with_parent(block2.clone(), Some(state1.clone()))); - let state3 = Arc::new(BlockState::with_parent(block3.clone(), Some(state2.clone()))); - - let mut blocks = B256Map::default(); - blocks.insert(block1.recovered_block().hash(), state1); - blocks.insert(block2.recovered_block().hash(), state2); - blocks.insert(block3.recovered_block().hash(), state3); - - let mut numbers = BTreeMap::new(); - numbers.insert(1, block1.recovered_block().hash()); - numbers.insert(2, block2.recovered_block().hash()); - numbers.insert(3, block3.recovered_block().hash()); - - let canonical_state = CanonicalInMemoryState::new(blocks, numbers, None, None, None); - - let historical: StateProviderBox = Box::new(MockStateProvider); - - let overlay_provider = - canonical_state.state_provider(block3.recovered_block().hash(), historical); - - assert_eq!(overlay_provider.in_memory.len(), 3); - assert_eq!(overlay_provider.in_memory[0].recovered_block().number, 3); - assert_eq!(overlay_provider.in_memory[1].recovered_block().number, 2); - assert_eq!(overlay_provider.in_memory[2].recovered_block().number, 1); - - assert_eq!( - overlay_provider.in_memory[0].recovered_block().parent_hash, - overlay_provider.in_memory[1].recovered_block().hash() - ); - assert_eq!( - overlay_provider.in_memory[1].recovered_block().parent_hash, - overlay_provider.in_memory[2].recovered_block().hash() - ); - - let unknown_hash = B256::random(); - let empty_overlay_provider = - canonical_state.state_provider(unknown_hash, Box::new(MockStateProvider)); - assert_eq!(empty_overlay_provider.in_memory.len(), 0); - } - #[test] fn test_canonical_in_memory_state_canonical_chain_empty() { let state: CanonicalInMemoryState = CanonicalInMemoryState::empty(); diff --git a/crates/chain-state/src/lib.rs b/crates/chain-state/src/lib.rs index c13232504a4..5c171e2f22d 100644 --- a/crates/chain-state/src/lib.rs +++ b/crates/chain-state/src/lib.rs @@ -17,8 +17,6 @@ pub use in_memory::*; mod preserved_sparse_trie; pub use preserved_sparse_trie::*; -mod noop; - mod chain_info; pub use chain_info::ChainInfoTracker; @@ -30,9 +28,6 @@ pub use notifications::{ WatchValueStream, }; -mod memory_overlay; -pub use memory_overlay::{MemoryOverlayStateProvider, MemoryOverlayStateProviderRef}; - #[cfg(any(test, feature = "test-utils"))] /// Common test helpers pub mod test_utils; diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs deleted file mode 100644 index 5173a16b06b..00000000000 --- a/crates/chain-state/src/memory_overlay.rs +++ /dev/null @@ -1,341 +0,0 @@ -use super::ExecutedBlock; -use alloy_consensus::BlockHeader; -use alloy_primitives::{ - keccak256, Address, BlockNumber, Bytes, StorageKey, StorageValue, B256, U256, -}; -use reth_errors::ProviderResult; -use reth_primitives_traits::{Account, Bytecode, NodePrimitives}; -use reth_storage_api::{ - AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider, - StateProvider, StateProviderBox, StateRootProvider, StorageRootProvider, -}; -use reth_trie::{ - updates::TrieUpdates, AccountProof, DecodedMultiProofV2, HashedPostState, HashedStorage, - MultiProof, MultiProofTargets, MultiProofTargetsV2, StorageMultiProof, TrieInput, -}; -use revm::database::BundleState; -use std::{borrow::Cow, sync::OnceLock}; - -/// A state provider that stores references to in-memory blocks along with their state as well as a -/// reference of the historical state provider for fallback lookups. -#[expect(missing_debug_implementations)] -pub struct MemoryOverlayStateProviderRef< - 'a, - N: NodePrimitives = reth_ethereum_primitives::EthPrimitives, -> { - /// Historical state provider for state lookups that are not found in memory blocks. - pub(crate) historical: Box, - /// The collection of executed parent blocks. Expected order is newest to oldest. - pub(crate) in_memory: Cow<'a, [ExecutedBlock]>, - /// Lazy-loaded in-memory trie data. - pub(crate) trie_input: OnceLock, -} - -impl<'a, N: NodePrimitives> MemoryOverlayStateProviderRef<'a, N> { - /// Create new memory overlay state provider. - /// - /// ## Arguments - /// - /// - `in_memory` - the collection of executed ancestor blocks in reverse. - /// - `historical` - a historical state provider for the latest ancestor block stored in the - /// database. - pub fn new(historical: Box, in_memory: Vec>) -> Self { - Self { historical, in_memory: Cow::Owned(in_memory), trie_input: OnceLock::new() } - } - - /// Turn this state provider into a state provider - pub fn boxed(self) -> Box { - Box::new(self) - } - - /// Return lazy-loaded trie state aggregated from in-memory blocks. - fn trie_input(&self) -> &TrieInput { - self.trie_input.get_or_init(|| { - let mut input = TrieInput::default(); - // Iterate from oldest to newest - for block in self.in_memory.iter().rev() { - let data = block.trie_data(); - input.nodes.extend_from_sorted(&data.sorted.trie_updates); - input.state.extend_from_sorted(&data.sorted.hashed_state); - } - input - }) - } - - fn merged_hashed_storage(&self, address: Address, storage: HashedStorage) -> HashedStorage { - let state = &self.trie_input().state; - let mut hashed = state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); - hashed.extend(&storage); - hashed - } -} - -impl BlockHashReader for MemoryOverlayStateProviderRef<'_, N> { - fn block_hash(&self, number: BlockNumber) -> ProviderResult> { - for block in self.in_memory.iter() { - if block.recovered_block().number() == number { - return Ok(Some(block.recovered_block().hash())); - } - } - - self.historical.block_hash(number) - } - - fn canonical_hashes_range( - &self, - start: BlockNumber, - end: BlockNumber, - ) -> ProviderResult> { - let range = start..end; - let mut earliest_block_number = None; - let mut in_memory_hashes = Vec::with_capacity(range.size_hint().0); - - // iterate in ascending order (oldest to newest = low to high) - for block in self.in_memory.iter() { - let block_num = block.recovered_block().number(); - if range.contains(&block_num) { - in_memory_hashes.push(block.recovered_block().hash()); - earliest_block_number = Some(block_num); - } - } - - // `self.in_memory` stores executed blocks in ascending order (oldest to newest). - // However, `in_memory_hashes` should be constructed in descending order (newest to oldest), - // so we reverse the vector after collecting the hashes. - in_memory_hashes.reverse(); - - let mut hashes = - self.historical.canonical_hashes_range(start, earliest_block_number.unwrap_or(end))?; - hashes.append(&mut in_memory_hashes); - Ok(hashes) - } -} - -impl AccountReader for MemoryOverlayStateProviderRef<'_, N> { - fn basic_account(&self, address: &Address) -> ProviderResult> { - for block in self.in_memory.iter() { - if let Some(account) = block.execution_output.account(address) { - return Ok(account); - } - } - - self.historical.basic_account(address) - } -} - -impl StateRootProvider for MemoryOverlayStateProviderRef<'_, N> { - fn state_root(&self, state: HashedPostState) -> ProviderResult { - self.state_root_from_nodes(TrieInput::from_state(state)) - } - - fn state_root_from_nodes(&self, mut input: TrieInput) -> ProviderResult { - input.prepend_self(self.trie_input().clone()); - self.historical.state_root_from_nodes(input) - } - - fn state_root_with_updates( - &self, - state: HashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - self.state_root_from_nodes_with_updates(TrieInput::from_state(state)) - } - - fn state_root_from_nodes_with_updates( - &self, - mut input: TrieInput, - ) -> ProviderResult<(B256, TrieUpdates)> { - input.prepend_self(self.trie_input().clone()); - self.historical.state_root_from_nodes_with_updates(input) - } -} - -impl StorageRootProvider for MemoryOverlayStateProviderRef<'_, N> { - // TODO: Currently this does not reuse available in-memory trie nodes. - fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { - let merged = self.merged_hashed_storage(address, storage); - self.historical.storage_root(address, merged) - } - - // TODO: Currently this does not reuse available in-memory trie nodes. - fn storage_proof( - &self, - address: Address, - slot: B256, - storage: HashedStorage, - ) -> ProviderResult { - let merged = self.merged_hashed_storage(address, storage); - self.historical.storage_proof(address, slot, merged) - } - - // TODO: Currently this does not reuse available in-memory trie nodes. - fn storage_multiproof( - &self, - address: Address, - slots: &[B256], - storage: HashedStorage, - ) -> ProviderResult { - let merged = self.merged_hashed_storage(address, storage); - self.historical.storage_multiproof(address, slots, merged) - } -} - -impl StateProofProvider for MemoryOverlayStateProviderRef<'_, N> { - fn proof( - &self, - mut input: TrieInput, - address: Address, - slots: &[B256], - ) -> ProviderResult { - input.prepend_self(self.trie_input().clone()); - self.historical.proof(input, address, slots) - } - - fn multiproof( - &self, - mut input: TrieInput, - targets: MultiProofTargets, - ) -> ProviderResult { - input.prepend_self(self.trie_input().clone()); - self.historical.multiproof(input, targets) - } - - fn multiproof_v2( - &self, - mut input: TrieInput, - targets: MultiProofTargetsV2, - ) -> ProviderResult { - input.prepend_self(self.trie_input().clone()); - self.historical.multiproof_v2(input, targets) - } - - fn witness( - &self, - mut input: TrieInput, - target: HashedPostState, - mode: reth_trie::ExecutionWitnessMode, - ) -> ProviderResult> { - input.prepend_self(self.trie_input().clone()); - self.historical.witness(input, target, mode) - } -} - -impl HashedPostStateProvider for MemoryOverlayStateProviderRef<'_, N> { - fn hashed_post_state(&self, bundle_state: &BundleState) -> ProviderResult { - let mut hashed_state = self.historical.hashed_post_state(bundle_state)?; - - for (address, account) in bundle_state.state() { - // Accounts created in this bundle cannot have parent storage to zero. - if !account.was_destroyed() || account.original_info.is_none() { - continue - } - - let hashed_address = keccak256(address); - let Some(parent_storage) = self.trie_input().state.storages.get(&hashed_address) else { - continue - }; - let storage = &mut hashed_state.storages.entry(hashed_address).or_default().storage; - for hashed_slot in parent_storage.storage.keys() { - storage.entry(*hashed_slot).or_insert(U256::ZERO); - } - } - - Ok(hashed_state) - } -} - -impl StateProvider for MemoryOverlayStateProviderRef<'_, N> { - fn storage( - &self, - address: Address, - storage_key: StorageKey, - ) -> ProviderResult> { - for block in self.in_memory.iter() { - if let Some(value) = block.execution_output.storage(&address, storage_key.into()) { - return Ok(Some(value)); - } - } - - self.historical.storage(address, storage_key) - } -} - -impl BytecodeReader for MemoryOverlayStateProviderRef<'_, N> { - fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { - for block in self.in_memory.iter() { - if let Some(contract) = block.execution_output.bytecode(code_hash) { - return Ok(Some(contract)); - } - } - - self.historical.bytecode_by_hash(code_hash) - } -} - -/// An owned state provider that stores references to in-memory blocks along with their state as -/// well as a reference of the historical state provider for fallback lookups. -#[expect(missing_debug_implementations)] -pub struct MemoryOverlayStateProvider { - /// Historical state provider for state lookups that are not found in memory blocks. - pub(crate) historical: StateProviderBox, - /// The collection of executed parent blocks. Expected order is newest to oldest. - pub(crate) in_memory: Vec>, - /// Lazy-loaded in-memory trie data. - pub(crate) trie_input: OnceLock, -} - -impl MemoryOverlayStateProvider { - /// Create new memory overlay state provider. - /// - /// ## Arguments - /// - /// - `in_memory` - the collection of executed ancestor blocks in reverse. - /// - `historical` - a historical state provider for the latest ancestor block stored in the - /// database. - pub fn new(historical: StateProviderBox, in_memory: Vec>) -> Self { - Self { historical, in_memory, trie_input: OnceLock::new() } - } - - /// Returns a new provider that takes the `TX` as reference - #[inline(always)] - fn as_ref(&self) -> MemoryOverlayStateProviderRef<'_, N> { - MemoryOverlayStateProviderRef { - historical: Box::new(self.historical.as_ref()), - in_memory: Cow::Borrowed(&self.in_memory), - trie_input: self.trie_input.clone(), - } - } - - /// Wraps the [`Self`] in a `Box`. - pub fn boxed(self) -> StateProviderBox { - Box::new(self) - } -} - -// Delegates all provider impls to [`MemoryOverlayStateProviderRef`] -reth_storage_api::macros::delegate_provider_impls!(MemoryOverlayStateProvider where [N: NodePrimitives]); - -#[cfg(test)] -mod tests { - use super::*; - use reth_ethereum_primitives::EthPrimitives; - use reth_storage_api::noop::NoopProvider; - use revm::database::{AccountStatus, BundleAccount}; - - #[test] - fn created_and_destroyed_account_skips_in_memory_trie_aggregation() { - let address = Address::with_last_byte(1); - let provider = MemoryOverlayStateProviderRef::::new( - Box::new(NoopProvider::default()), - Vec::new(), - ); - let mut bundle_state = BundleState::default(); - bundle_state.state.insert( - address, - BundleAccount::new(None, None, Default::default(), AccountStatus::Destroyed), - ); - - provider.hashed_post_state(&bundle_state).unwrap(); - - assert!(provider.trie_input.get().is_none()); - } -} diff --git a/crates/chain-state/src/noop.rs b/crates/chain-state/src/noop.rs deleted file mode 100644 index c6862b945ad..00000000000 --- a/crates/chain-state/src/noop.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! Noop impls for testing. - -use crate::{ - CanonStateNotifications, CanonStateSubscriptions, ForkChoiceNotifications, - ForkChoiceSubscriptions, PersistedBlockNotifications, PersistedBlockSubscriptions, -}; -use reth_primitives_traits::NodePrimitives; -use reth_storage_api::noop::NoopProvider; -use tokio::sync::{broadcast, watch}; - -impl CanonStateSubscriptions for NoopProvider { - fn subscribe_to_canonical_state(&self) -> CanonStateNotifications { - broadcast::channel(1).1 - } -} - -impl ForkChoiceSubscriptions for NoopProvider { - type Header = N::BlockHeader; - - fn subscribe_safe_block(&self) -> ForkChoiceNotifications { - let (_, rx) = watch::channel(None); - ForkChoiceNotifications(rx) - } - - fn subscribe_finalized_block(&self) -> ForkChoiceNotifications { - let (_, rx) = watch::channel(None); - ForkChoiceNotifications(rx) - } -} - -impl PersistedBlockSubscriptions for NoopProvider { - fn subscribe_persisted_block(&self) -> PersistedBlockNotifications { - let (_, rx) = watch::channel(None); - PersistedBlockNotifications(rx) - } -} diff --git a/crates/chain-state/src/notifications.rs b/crates/chain-state/src/notifications.rs index 3132715c014..8ff25845d44 100644 --- a/crates/chain-state/src/notifications.rs +++ b/crates/chain-state/src/notifications.rs @@ -4,7 +4,6 @@ use alloy_eips::BlockNumHash; use derive_more::{Deref, DerefMut}; use reth_execution_types::{BlockReceipts, Chain}; use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedHeader}; -use reth_storage_api::NodePrimitivesProvider; use std::{ pin::Pin, sync::Arc, @@ -26,7 +25,10 @@ pub type CanonStateNotificationSender>; /// A type that allows to register chain related event subscriptions. -pub trait CanonStateSubscriptions: NodePrimitivesProvider + Send + Sync { +pub trait CanonStateSubscriptions: Send + Sync { + /// The node primitive types. + type Primitives: NodePrimitives; + /// Get notified when a new canonical chain was imported. /// /// A canonical chain be one or more blocks, a reorg or a revert. @@ -41,6 +43,8 @@ pub trait CanonStateSubscriptions: NodePrimitivesProvider + Send + Sync { } impl CanonStateSubscriptions for &T { + type Primitives = T::Primitives; + fn subscribe_to_canonical_state(&self) -> CanonStateNotifications { (*self).subscribe_to_canonical_state() } diff --git a/crates/chain-state/src/test_utils.rs b/crates/chain-state/src/test_utils.rs index 527e53091fc..e2688390019 100644 --- a/crates/chain-state/src/test_utils.rs +++ b/crates/chain-state/src/test_utils.rs @@ -19,7 +19,6 @@ use reth_primitives_traits::{ Account, NodePrimitives, Recovered, RecoveredBlock, SealedBlock, SealedHeader, SignedTransaction, }; -use reth_storage_api::NodePrimitivesProvider; use reth_trie::{root::state_root_unhashed, ComputedTrieData, SortedTrieData}; use revm::{database::BundleState, state::AccountInfo}; use std::{ @@ -443,11 +442,9 @@ impl TestCanonStateSubscriptions { } } -impl NodePrimitivesProvider for TestCanonStateSubscriptions { +impl CanonStateSubscriptions for TestCanonStateSubscriptions { type Primitives = EthPrimitives; -} -impl CanonStateSubscriptions for TestCanonStateSubscriptions { /// Sets up a broadcast channel with a buffer size of 100. fn subscribe_to_canonical_state(&self) -> CanonStateNotifications { let (canon_notif_tx, canon_notif_rx) = broadcast::channel(100); diff --git a/crates/node/ethstats/src/ethstats.rs b/crates/node/ethstats/src/ethstats.rs index 82c2974655f..e77badda816 100644 --- a/crates/node/ethstats/src/ethstats.rs +++ b/crates/node/ethstats/src/ethstats.rs @@ -12,7 +12,7 @@ use alloy_primitives::U256; use reth_chain_state::{CanonStateNotification, CanonStateSubscriptions}; use reth_network_api::{NetworkInfo, Peers}; use reth_primitives_traits::{Block, BlockBody}; -use reth_storage_api::{BlockReader, BlockReaderIdExt, NodePrimitivesProvider}; +use reth_storage_api::{BlockReader, BlockReaderIdExt}; use reth_transaction_pool::TransactionPool; use chrono::Local; @@ -320,7 +320,7 @@ where /// * `head` - Optional canonical state notification containing new block info async fn report_block( &self, - head: Option::Primitives>>, + head: Option::Primitives>>, ) -> Result<(), EthStatsError> { let conn = self.conn.read().await; let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index b2eeab22815..d903d60ea26 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -317,7 +317,8 @@ where + CanonStateSubscriptions + ForkChoiceSubscriptions

+ PersistedBlockSubscriptions - + ChangeSetReader, + + ChangeSetReader + + NodePrimitivesProvider, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, EvmConfig: ConfigureEvm + 'static, diff --git a/crates/rpc/rpc-eth-api/Cargo.toml b/crates/rpc/rpc-eth-api/Cargo.toml index f3b12654670..1a10bd76071 100644 --- a/crates/rpc/rpc-eth-api/Cargo.toml +++ b/crates/rpc/rpc-eth-api/Cargo.toml @@ -19,7 +19,7 @@ revm-inspectors.workspace = true reth-primitives-traits = { workspace = true, features = ["rpc-compat"] } reth-errors.workspace = true reth-evm.workspace = true -reth-storage-api.workspace = true +reth-storage-api = { workspace = true, features = ["chain-state"] } reth-revm.workspace = true reth-rpc-convert.workspace = true reth-tasks = { workspace = true, features = ["rayon"] } diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 4642172c646..9fc4f3558fd 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -8,7 +8,7 @@ use alloy_eips::eip7840::BlobParams; use alloy_primitives::{B256, U256}; use alloy_rpc_types_eth::{BlockNumberOrTag, BlockOverrides}; use futures::Future; -use reth_chain_state::{BlockState, ExecutedBlock}; +use reth_chain_state::ExecutedBlock; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_errors::{BlockExecutionError, BlockValidationError, ProviderError, RethError}; use reth_evm::{ @@ -121,14 +121,14 @@ pub trait LoadPendingBlock: return Ok(None); }; - let latest_historical = self - .provider() - .history_by_block_hash(pending_block.block().parent_hash()) - .map_err(Self::Error::from_eth_err)?; - - let state = BlockState::from(pending_block); - - Ok(Some(Box::new(state.state_provider(latest_historical)) as StateProviderBox)) + Ok(Some( + self.provider() + .state_with_block_appended( + pending_block.block().parent_hash(), + pending_block.executed_block, + ) + .map_err(Self::Error::from_eth_err)?, + )) } } diff --git a/crates/rpc/rpc-eth-api/src/node.rs b/crates/rpc/rpc-eth-api/src/node.rs index 1c26cba7998..1ddb901f09d 100644 --- a/crates/rpc/rpc-eth-api/src/node.rs +++ b/crates/rpc/rpc-eth-api/src/node.rs @@ -8,8 +8,8 @@ use reth_node_api::{FullNodeComponents, NodePrimitives, PrimitivesTy}; use reth_primitives_traits::{BlockTy, HeaderTy, ReceiptTy, TxTy}; use reth_rpc_eth_types::EthStateCache; use reth_storage_api::{ - BalProvider, BlockReader, BlockReaderIdExt, PruneCheckpointReader, StageCheckpointReader, - StateProviderFactory, + BalProvider, BlockReader, BlockReaderIdExt, NodePrimitivesProvider, PruneCheckpointReader, + StageCheckpointReader, StateProviderFactory, }; use reth_transaction_pool::{PoolTransaction, TransactionPool}; @@ -35,7 +35,8 @@ pub trait RpcNodeCore: Clone + Send + Sync + Unpin + 'static { ChainSpec: EthChainSpec
> + Hardforks + EthereumHardforks, - > + StateProviderFactory + > + StateProviderFactory + + NodePrimitivesProvider + CanonStateSubscriptions + StageCheckpointReader + PruneCheckpointReader @@ -130,7 +131,8 @@ where ChainSpec: EthChainSpec
> + Hardforks + EthereumHardforks, - > + StateProviderFactory + > + StateProviderFactory + + NodePrimitivesProvider + CanonStateSubscriptions + StageCheckpointReader + PruneCheckpointReader diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 9af5fee3909..da210b42a8e 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -20,7 +20,7 @@ reth-rpc-eth-api.workspace = true reth-engine-primitives.workspace = true reth-errors.workspace = true reth-metrics.workspace = true -reth-storage-api.workspace = true +reth-storage-api = { workspace = true, features = ["chain-state"] } reth-execution-types = { workspace = true, features = ["serde"] } reth-chain-state.workspace = true reth-transaction-pool.workspace = true diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index a721dc3c621..fc9005d877e 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -566,7 +566,9 @@ mod tests { PruneCheckpointReader, StageCheckpointReader, }; use reth_rpc_eth_api::{node::RpcNodeCoreAdapter, EthApiServer}; - use reth_storage_api::{BalProvider, BlockReader, BlockReaderIdExt, StateProviderFactory}; + use reth_storage_api::{ + BalProvider, BlockReader, BlockReaderIdExt, NodePrimitivesProvider, StateProviderFactory, + }; use reth_testing_utils::generators; use reth_transaction_pool::test_utils::{testing_pool, TestPool}; @@ -583,7 +585,8 @@ mod tests { Transaction = reth_ethereum_primitives::TransactionSigned, > + BlockReader + ChainSpecProvider - + StateProviderFactory + + StateProviderFactory + + NodePrimitivesProvider + CanonStateSubscriptions + StageCheckpointReader + PruneCheckpointReader diff --git a/crates/rpc/rpc/src/reth.rs b/crates/rpc/rpc/src/reth.rs index 2b9b1889f6e..ab03b422e6a 100644 --- a/crates/rpc/rpc/src/reth.rs +++ b/crates/rpc/rpc/src/reth.rs @@ -191,12 +191,15 @@ where Provider: BlockReaderIdExt + ChangeSetReader + StateProviderFactory - + BlockReader::Block> - + CanonStateSubscriptions - + ForkChoiceSubscriptions
::BlockHeader> + + BlockReader< + Block = <::Primitives as NodePrimitives>::Block, + > + CanonStateSubscriptions + + ForkChoiceSubscriptions< + Header = <::Primitives as NodePrimitives>::BlockHeader, + > + PersistedBlockSubscriptions + 'static, - EvmConfig: ConfigureEvm + 'static, + EvmConfig: ConfigureEvm::Primitives> + 'static, { /// Handler for `reth_getBalanceChangesInBlock` async fn reth_get_balance_changes_in_block( diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index c31ecfaa331..b2d8be7e29c 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -19,7 +19,7 @@ reth-ethereum-primitives = { workspace = true, features = ["reth-codec"] } reth-primitives-traits = { workspace = true, features = ["reth-codec", "secp256k1", "dashmap"] } reth-errors.workspace = true reth-storage-errors.workspace = true -reth-storage-api = { workspace = true, features = ["std", "db-api"] } +reth-storage-api = { workspace = true, features = ["std", "db-api", "chain-state"] } reth-storage-overlay.workspace = true reth-db = { workspace = true, features = ["mdbx"] } reth-db-api.workspace = true diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 54ccd5b26f9..eb8ca1ca2e4 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -16,8 +16,8 @@ use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes, TxHash, TxNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ - BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, - PersistedBlockNotifications, PersistedBlockSubscriptions, + BlockState, CanonicalInMemoryState, ExecutedBlock, ForkChoiceNotifications, + ForkChoiceSubscriptions, PersistedBlockNotifications, PersistedBlockSubscriptions, }; use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; @@ -729,6 +729,8 @@ impl ChainSpecProvider for BlockchainProvider { } impl StateProviderFactory for BlockchainProvider { + type Primitives = N::Primitives; + /// Storage provider for latest block fn latest(&self) -> ProviderResult { trace!(target: "providers::blockchain", "Getting latest block state provider"); @@ -742,6 +744,18 @@ impl StateProviderFactory for BlockchainProvider { } } + fn state_with_block_appended( + &self, + parent_hash: BlockHash, + block: ExecutedBlock, + ) -> ProviderResult { + let state_provider_factory = OverlayStateProviderFactory::new( + self.database.clone(), + self.database.overlay_manager().overlay_builder(parent_hash).with_appended_block(block), + ); + Ok(Box::new(state_provider_factory.database_provider_ro()?)) + } + /// Returns a [`StateProviderBox`] indexed by the given block number or tag. fn state_by_block_number_or_tag( &self, @@ -902,6 +916,8 @@ where } impl CanonStateSubscriptions for BlockchainProvider { + type Primitives = N::Primitives; + fn subscribe_to_canonical_state(&self) -> CanonStateNotifications { self.canonical_in_memory_state.subscribe_canon_state() } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 55d54e91954..75860cf7334 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -19,7 +19,7 @@ use alloy_primitives::{ Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, TxHash, TxNumber, B256, U256, }; use parking_lot::Mutex; -use reth_chain_state::{CanonStateNotifications, CanonStateSubscriptions}; +use reth_chain_state::{CanonStateNotifications, CanonStateSubscriptions, ExecutedBlock}; use reth_chainspec::{ChainInfo, EthChainSpec}; use reth_db::transaction::DbTx; use reth_db_api::{ @@ -1223,11 +1223,21 @@ impl StorageSettingsCache impl StateProviderFactory for MockEthProvider { + type Primitives = T; + fn latest(&self) -> ProviderResult { self.ensure_snap_state_reads_succeed()?; Ok(Box::new(self.clone())) } + fn state_with_block_appended( + &self, + _parent_hash: BlockHash, + _block: ExecutedBlock, + ) -> ProviderResult { + Err(ProviderError::UnsupportedProvider) + } + fn state_by_block_number_or_tag( &self, number_or_tag: BlockNumberOrTag, @@ -1360,6 +1370,8 @@ impl StateReader for MockEthProvider< impl CanonStateSubscriptions for MockEthProvider { + type Primitives = T; + fn subscribe_to_canonical_state(&self) -> CanonStateNotifications { broadcast::channel(1).1 } diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 04af272ce5b..8277d54bd86 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -35,13 +35,13 @@ pub trait FullProvider: Receipt = ReceiptTy, Header = HeaderTy, > + BalProvider - + StateProviderFactory + + StateProviderFactory + StateRangeProviderFactory + StateReader + ChainSpecProvider + ChangeSetReader + StorageChangeSetReader - + CanonStateSubscriptions + + CanonStateSubscriptions + ForkChoiceSubscriptions
> + PersistedBlockSubscriptions + StageCheckpointReader @@ -73,13 +73,13 @@ impl FullProvider for T where Receipt = ReceiptTy, Header = HeaderTy, > + BalProvider - + StateProviderFactory + + StateProviderFactory + StateRangeProviderFactory + StateReader + ChainSpecProvider + ChangeSetReader + StorageChangeSetReader - + CanonStateSubscriptions + + CanonStateSubscriptions + ForkChoiceSubscriptions
> + PersistedBlockSubscriptions + StageCheckpointReader diff --git a/crates/storage/rpc-provider/Cargo.toml b/crates/storage/rpc-provider/Cargo.toml index 7717aea07c9..540a56f521a 100644 --- a/crates/storage/rpc-provider/Cargo.toml +++ b/crates/storage/rpc-provider/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] # reth reth-storage-api.workspace = true +reth-chain-state.workspace = true reth-chainspec.workspace = true reth-ethereum-primitives.workspace = true reth-primitives-traits.workspace = true diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs index 60e9b0f3099..c87a87bd4aa 100644 --- a/crates/storage/rpc-provider/src/lib.rs +++ b/crates/storage/rpc-provider/src/lib.rs @@ -32,6 +32,7 @@ use alloy_provider::{ext::DebugApi, network::Network, Provider}; use alloy_rpc_types::{AccountInfo, BlockId}; use alloy_rpc_types_engine::ForkchoiceState; use dashmap::DashMap; +use reth_chain_state::ExecutedBlock; use reth_chainspec::{ChainInfo, ChainSpecProvider}; use reth_db_api::{ mock::{DatabaseMock, TxMock}, @@ -737,10 +738,20 @@ where N: Network, Node: NodeTypes, { + type Primitives = PrimitivesTy; + fn latest(&self) -> Result { Ok(Box::new(self.create_state_provider(self.best_block_number()?.into()))) } + fn state_with_block_appended( + &self, + _parent_hash: BlockHash, + _block: ExecutedBlock>, + ) -> ProviderResult { + Err(ProviderError::UnsupportedProvider) + } + fn state_by_block_id(&self, block_id: BlockId) -> Result { Ok(Box::new(self.create_state_provider(block_id))) } @@ -879,6 +890,8 @@ where N: Network, Node: NodeTypes, { + type Primitives = PrimitivesTy; + fn subscribe_to_canonical_state(&self) -> CanonStateNotifications> { trace!(target: "alloy-provider", "Subscribing to canonical state notifications"); self.canon_state_notification.subscribe() @@ -1794,10 +1807,20 @@ where N: Network, Self: Clone + 'static, { + type Primitives = PrimitivesTy; + fn latest(&self) -> Result { Ok(Box::new(self.with_block_id(self.best_block_number()?.into()))) } + fn state_with_block_appended( + &self, + _parent_hash: BlockHash, + _block: ExecutedBlock>, + ) -> ProviderResult { + Err(ProviderError::UnsupportedProvider) + } + fn state_by_block_id(&self, block_id: BlockId) -> Result { Ok(Box::new(self.with_block_id(block_id))) } diff --git a/crates/storage/storage-api/Cargo.toml b/crates/storage/storage-api/Cargo.toml index 5c28a941fc4..5f489c75bde 100644 --- a/crates/storage/storage-api/Cargo.toml +++ b/crates/storage/storage-api/Cargo.toml @@ -15,6 +15,7 @@ workspace = true # reth reth-db-models.workspace = true reth-chainspec.workspace = true +reth-chain-state = { workspace = true, optional = true } reth-db-api = { workspace = true, optional = true } reth-execution-types.workspace = true reth-primitives-traits.workspace = true @@ -34,6 +35,7 @@ alloy-rpc-types-engine.workspace = true auto_impl.workspace = true serde_json = { workspace = true, optional = true } +tokio = { workspace = true, features = ["sync"], optional = true } [dev-dependencies] alloy-primitives = { workspace = true, features = ["getrandom"] } @@ -64,7 +66,13 @@ db-api = [ "dep:serde_json", ] +chain-state = [ + "dep:reth-chain-state", + "dep:tokio", +] + serde = [ + "reth-chain-state?/serde", "reth-ethereum-primitives/serde", "reth-db-models/serde", "reth-execution-types/serde", diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 5eb5a01e3c7..a1779868cd9 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -28,6 +28,11 @@ use core::{ marker::PhantomData, ops::{RangeBounds, RangeInclusive}, }; +#[cfg(feature = "chain-state")] +use reth_chain_state::{ + CanonStateNotifications, CanonStateSubscriptions, ExecutedBlock, ForkChoiceNotifications, + ForkChoiceSubscriptions, PersistedBlockNotifications, PersistedBlockSubscriptions, +}; use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec, MAINNET}; #[cfg(feature = "db-api")] use reth_db_api::mock::{DatabaseMock, TxMock}; @@ -604,10 +609,21 @@ impl BytecodeReader for NoopProvider { } impl StateProviderFactory for NoopProvider { + type Primitives = N; + fn latest(&self) -> ProviderResult { Ok(Box::new(self.clone())) } + #[cfg(feature = "chain-state")] + fn state_with_block_appended( + &self, + _parent_hash: BlockHash, + _block: ExecutedBlock, + ) -> ProviderResult { + Ok(Box::new(self.clone())) + } + fn state_by_block_number_or_tag( &self, number_or_tag: BlockNumberOrTag, @@ -661,6 +677,38 @@ impl StateProviderFactory for NoopP } } +#[cfg(feature = "chain-state")] +impl CanonStateSubscriptions for NoopProvider { + type Primitives = N; + + fn subscribe_to_canonical_state(&self) -> CanonStateNotifications { + tokio::sync::broadcast::channel(1).1 + } +} + +#[cfg(feature = "chain-state")] +impl ForkChoiceSubscriptions for NoopProvider { + type Header = N::BlockHeader; + + fn subscribe_safe_block(&self) -> ForkChoiceNotifications { + let (_, rx) = tokio::sync::watch::channel(None); + ForkChoiceNotifications(rx) + } + + fn subscribe_finalized_block(&self) -> ForkChoiceNotifications { + let (_, rx) = tokio::sync::watch::channel(None); + ForkChoiceNotifications(rx) + } +} + +#[cfg(feature = "chain-state")] +impl PersistedBlockSubscriptions for NoopProvider { + fn subscribe_persisted_block(&self) -> PersistedBlockNotifications { + let (_, rx) = tokio::sync::watch::channel(None); + PersistedBlockNotifications(rx) + } +} + impl StageCheckpointReader for NoopProvider { fn get_stage_checkpoint(&self, _id: StageId) -> ProviderResult> { Ok(None) diff --git a/crates/storage/storage-api/src/state.rs b/crates/storage/storage-api/src/state.rs index d448671c8ea..200c4c3fd68 100644 --- a/crates/storage/storage-api/src/state.rs +++ b/crates/storage/storage-api/src/state.rs @@ -7,8 +7,10 @@ use alloy_consensus::constants::KECCAK_EMPTY; use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_primitives::{Address, BlockHash, BlockNumber, StorageKey, StorageValue, B256, U256}; use auto_impl::auto_impl; +#[cfg(feature = "chain-state")] +use reth_chain_state::ExecutedBlock; use reth_execution_types::ExecutionOutcome; -use reth_primitives_traits::Bytecode; +use reth_primitives_traits::{Bytecode, NodePrimitives}; use reth_storage_errors::provider::ProviderResult; use reth_trie_common::HashedPostState; use revm::database::BundleState; @@ -137,9 +139,20 @@ pub trait BytecodeReader { /// to be used, since block `n` was executed on its parent block's state. #[auto_impl(&, Box, Arc)] pub trait StateProviderFactory: BlockIdReader + Send { + /// The node primitive types. + type Primitives: NodePrimitives; + /// Storage provider for latest block. fn latest(&self) -> ProviderResult; + /// Returns a state provider after applying `block` to `parent_hash`. + #[cfg(feature = "chain-state")] + fn state_with_block_appended( + &self, + parent_hash: BlockHash, + block: ExecutedBlock, + ) -> ProviderResult; + /// Returns a [`StateProvider`] indexed by the given [`BlockId`]. /// /// Note: if a number or hash is provided this will __only__ look at historical(canonical) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index fde2d0f5c1b..c7234cacec6 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1427,6 +1427,32 @@ mod tests { } } + #[test] + fn builder_appends_block_to_parent_state() { + let manager = OverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks[2..=4] { + manager.insert_block(block.clone()); + } + + let block = TestBlockBuilder::eth().get_executed_block_with_number( + blocks[4].recovered_block().number() + 1, + blocks[4].recovered_block().hash(), + ); + let builder = manager + .overlay_builder(block.recovered_block().parent_hash()) + .with_appended_block(block.clone()); + + assert_eq!(builder.parent_hash, block.recovered_block().hash()); + assert_eq!( + builder.parent_state.unwrap().chain().map(BlockState::hash).collect::>(), + std::iter::once(&block) + .chain(blocks[2..=4].iter().rev()) + .map(|block| block.recovered_block().hash()) + .collect::>(), + ); + } + #[test] fn overlay_after_state_trie_frontier_requires_managed_coverage() { let (factory, blocks) = setup_frontiers(1, 3); diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index c195531a03f..4b1349fcfdf 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -399,7 +399,14 @@ impl OverlayManager { anchor_hash, parent_state, cache_config, - |input, span| self.compute_state_trie_overlay(input, anchor_hash, span), + |input, span| { + self.compute_state_trie_overlay( + input, + anchor_hash, + span, + cache_config.write_to_cache, + ) + }, )? .expect("required overlay lookup cannot skip an in-progress computation"); Ok((Arc::clone(&input.nodes), Arc::clone(&input.state))) @@ -457,7 +464,14 @@ impl OverlayManager { anchor_hash, parent_state, cache_config, - |input, span| self.compute_execution_overlay(input, anchor_hash, span), + |input, span| { + self.compute_execution_overlay( + input, + anchor_hash, + span, + cache_config.write_to_cache, + ) + }, ) } @@ -649,7 +663,12 @@ impl OverlayManager { compute_input: ComputeOverlayInput, anchor_hash: B256, _span: tracing::Span, + write_to_cache: bool, ) -> TrieInputSorted { + if !write_to_cache { + return compute_overlay(compute_input, anchor_hash, &self.metrics) + } + #[cfg(feature = "rayon")] { if let Some(worker_pool) = &self.worker_pool { @@ -670,7 +689,16 @@ impl OverlayManager { compute_input: ComputeOverlayInput, anchor_hash: B256, _span: tracing::Span, + write_to_cache: bool, ) -> ExecutionOverlay { + if !write_to_cache { + return compute_execution_overlay_inner( + compute_input, + anchor_hash, + &self.execution_metrics, + ) + } + #[cfg(feature = "rayon")] { if let Some(worker_pool) = &self.worker_pool { @@ -1298,6 +1326,42 @@ mod tests { assert!(!manager.execution_overlays.entries.contains_key(&child_key)); } + #[cfg(feature = "rayon")] + #[test] + fn uncached_overlays_do_not_use_worker_pool() { + let worker_pool = Arc::new(WorkerPool::new(1, "uncached-overlay-test")); + let manager = OverlayManager::new(Arc::clone(&worker_pool)); + let block = test_blocks().remove(0); + let anchor_hash = block.recovered_block().parent_hash(); + let parent_state = BlockState::new(block); + let cache_config = OverlayCacheConfig { precompute: false, write_to_cache: false }; + + let (started_tx, started_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + worker_pool.spawn(move || { + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }); + started_rx.recv().unwrap(); + + let (completed_tx, completed_rx) = mpsc::channel(); + let task = thread::spawn(move || { + let execution = + manager.execution_overlay_for_block_state(&parent_state, anchor_hash, cache_config); + let state = manager.overlay_for_parent(&parent_state, anchor_hash, cache_config); + completed_tx.send((execution, state)).unwrap(); + }); + + let completed = completed_rx.recv_timeout(Duration::from_millis(100)); + release_tx.send(()).unwrap(); + task.join().unwrap(); + + assert!(completed.is_ok(), "uncached overlay used the worker pool"); + let (execution, state) = completed.unwrap(); + assert!(execution.is_ok()); + assert!(state.is_ok()); + } + #[cfg(feature = "rayon")] #[test] fn precomputes_execution_overlay_for_cached_parent() { From 04c0a7f2284425e13d8b20bd2a695905ed3e12d0 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:44:26 +0200 Subject: [PATCH 053/133] chore(ci): protect dependency installs with Aegis (#27358) Co-authored-by: grandizzy <38490174+grandizzy@users.noreply.github.com> Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- .github/sts/cargo-update-pr.sts.yaml | 14 + .github/workflows/bench-benchmarkoor.yml | 49 ++- .github/workflows/bench-scheduled.yml | 72 +++-- .github/workflows/bench.yml | 348 ++++++++++++++------- .github/workflows/book.yml | 10 +- .github/workflows/check-alloy.yml | 19 +- .github/workflows/compact.yml | 7 +- .github/workflows/cyclops-audit.yml | 2 +- .github/workflows/dependencies.yml | 8 +- .github/workflows/docker-tag-latest.yml | 4 +- .github/workflows/docker-test.yml | 20 +- .github/workflows/docker.yml | 43 ++- .github/workflows/e2e.yml | 28 +- .github/workflows/hive.yml | 18 +- .github/workflows/integration.yml | 30 +- .github/workflows/kurtosis.yml | 6 +- .github/workflows/lint-actions.yml | 15 +- .github/workflows/lint.yml | 122 ++++++-- .github/workflows/release-dist.yml | 6 +- .github/workflows/release-reproducible.yml | 36 ++- .github/workflows/release.yml | 62 ++-- .github/workflows/reproducible-build.yml | 6 +- .github/workflows/scan-github-actions.yml | 19 ++ .github/workflows/stage.yml | 7 +- .github/workflows/sync-era.yml | 7 +- .github/workflows/sync.yml | 7 +- .github/workflows/unit.yml | 37 ++- Dockerfile.depot | 2 +- 28 files changed, 721 insertions(+), 283 deletions(-) create mode 100644 .github/sts/cargo-update-pr.sts.yaml create mode 100644 .github/workflows/scan-github-actions.yml diff --git a/.github/sts/cargo-update-pr.sts.yaml b/.github/sts/cargo-update-pr.sts.yaml new file mode 100644 index 00000000000..c61d5a403d9 --- /dev/null +++ b/.github/sts/cargo-update-pr.sts.yaml @@ -0,0 +1,14 @@ +# Only the pinned dependency updater on main may create dependency PRs. +subject: + - repo:paradigmxyz/reth:ref:refs/heads/main + - repo:paradigmxyz@97369466/reth@537233603:ref:refs/heads/main + +claim_pattern: + repository_id: '537233603' + event_name: '(schedule|workflow_dispatch)' + workflow_ref: 'paradigmxyz/reth/\.github/workflows/dependencies\.yml@refs/heads/main' + job_workflow_ref: 'tempoxyz/gh-actions/\.github/workflows/cargo-update-pr\.yml@117919c943b804057be733b0c4034c5542e58959' + +permissions: + contents: write + pull_requests: write diff --git a/.github/workflows/bench-benchmarkoor.yml b/.github/workflows/bench-benchmarkoor.yml index c9e69ef6bb3..77554b2ea02 100644 --- a/.github/workflows/bench-benchmarkoor.yml +++ b/.github/workflows/bench-benchmarkoor.yml @@ -153,6 +153,7 @@ jobs: name: bench-benchmarkoor runs-on: [self-hosted, Linux, X64, available] permissions: + id-token: write contents: read timeout-minutes: 180 env: @@ -175,6 +176,8 @@ jobs: BENCHMARKOOR_REPLAY_PATH: ${{ inputs.benchmarkoor_replay_path || '~/projects/benchmarkoor-replay' }} BENCHMARKOOR_CACHE: ${{ github.workspace }}/bench-work/benchmarkoor-cache steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - name: Clean up previous bench-work run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true @@ -196,7 +199,9 @@ jobs: submodules: true fetch-depth: 0 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true @@ -259,6 +264,8 @@ jobs: set -euo pipefail replay_path="$BENCHMARKOOR_REPLAY_PATH" + # Match a literal tilde before expanding it ourselves. + # shellcheck disable=SC2088 case "$replay_path" in "~") replay_path="$HOME" ;; "~/"*) replay_path="$HOME/${replay_path#"~/"}" ;; @@ -306,6 +313,8 @@ jobs: set -euo pipefail metadata_root="$BENCHMARKOOR_METADATA_ROOT" + # Match a literal tilde before expanding it ourselves. + # shellcheck disable=SC2088 case "$metadata_root" in "~") metadata_root="$HOME" ;; "~/"*) metadata_root="$HOME/${metadata_root#"~/"}" ;; @@ -377,6 +386,11 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Prepare source dirs + env: + INPUT_BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }} + INPUT_DOWNLOAD_NAME: ${{ steps.refs.outputs.download-name }} + INPUT_DOWNLOAD_REF: ${{ steps.refs.outputs.download-ref }} + INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }} run: | set -euo pipefail prepare_source_dir() { @@ -400,13 +414,16 @@ jobs: git -C "$dir" submodule update --init --recursive } - prepare_source_dir ../reth-baseline "${{ steps.refs.outputs.baseline-ref }}" - prepare_source_dir ../reth-feature "${{ steps.refs.outputs.feature-ref }}" - prepare_source_dir ../reth-download "${{ steps.refs.outputs.download-ref }}" "${{ steps.refs.outputs.download-name }}" + prepare_source_dir ../reth-baseline "${INPUT_BASELINE_REF}" + prepare_source_dir ../reth-feature "${INPUT_FEATURE_REF}" + prepare_source_dir ../reth-download "${INPUT_DOWNLOAD_REF}" "${INPUT_DOWNLOAD_NAME}" - name: Build binaries id: build env: + INPUT_BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }} + INPUT_DOWNLOAD_REF: ${{ steps.refs.outputs.download-ref }} + INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEREK_PAT: ${{ secrets.DEREK_PAT }} DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }} @@ -418,11 +435,11 @@ jobs: FEATURE_DIR="$(cd ../reth-feature && pwd)" DOWNLOAD_DIR="$(cd ../reth-download && pwd)" - .github/scripts/bench-txgen-build.sh baseline "$BASELINE_DIR" "${{ steps.refs.outputs.baseline-ref }}" & + .github/scripts/bench-txgen-build.sh baseline "$BASELINE_DIR" "${INPUT_BASELINE_REF}" & PID_BASELINE=$! - .github/scripts/bench-txgen-build.sh feature "$FEATURE_DIR" "${{ steps.refs.outputs.feature-ref }}" & + .github/scripts/bench-txgen-build.sh feature "$FEATURE_DIR" "${INPUT_FEATURE_REF}" & PID_FEATURE=$! - .github/scripts/bench-txgen-build.sh baseline "$DOWNLOAD_DIR" "${{ steps.refs.outputs.download-ref }}" & + .github/scripts/bench-txgen-build.sh baseline "$DOWNLOAD_DIR" "${INPUT_DOWNLOAD_REF}" & PID_DOWNLOAD=$! FAIL=0 @@ -566,10 +583,13 @@ jobs: - name: Import snapshot into schelk id: snapshot + env: + INPUT_DOWNLOAD_NAME: ${{ steps.refs.outputs.download-name }} + INPUT_DOWNLOAD_REF: ${{ steps.refs.outputs.download-ref }} run: | set -euo pipefail DOWNLOAD_RETH_BINARY="$(pwd)/../reth-download/target/profiling/reth" - echo "Using download reth from ${{ steps.refs.outputs.download-name }} (${{ steps.refs.outputs.download-ref }})" + echo "Using download reth from ${INPUT_DOWNLOAD_NAME} (${INPUT_DOWNLOAD_REF})" "$DOWNLOAD_RETH_BINARY" --version BENCH_RETH_BINARY="$DOWNLOAD_RETH_BINARY" \ .github/scripts/bench-benchmarkoor-snapshot.sh @@ -649,6 +669,11 @@ jobs: - name: Parse results id: results if: success() + env: + INPUT_BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }} + INPUT_BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }} + INPUT_FEATURE_NAME: ${{ steps.refs.outputs.feature-name }} + INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }} run: | set -euo pipefail RESULTS="$BENCH_WORK_DIR/results.jsonl" @@ -666,10 +691,10 @@ jobs: --output-json "$BENCH_WORK_DIR/summary.json" \ --output-markdown "$BENCH_WORK_DIR/summary.md" \ --repo "${{ github.repository }}" \ - --baseline-name "${{ steps.refs.outputs.baseline-name }}" \ - --baseline-ref "${{ steps.refs.outputs.baseline-ref }}" \ - --feature-name "${{ steps.refs.outputs.feature-name }}" \ - --feature-ref "${{ steps.refs.outputs.feature-ref }}" + --baseline-name "${INPUT_BASELINE_NAME}" \ + --baseline-ref "${INPUT_BASELINE_REF}" \ + --feature-name "${INPUT_FEATURE_NAME}" \ + --feature-ref "${INPUT_FEATURE_REF}" - name: Write job summary if: success() diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 10507d5689e..6c2936f0207 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -143,6 +143,7 @@ jobs: - name: Resolve refs id: refs env: + INPUT_MODE: ${{ steps.mode.outputs.mode }} GH_TOKEN: ${{ github.token }} DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -150,7 +151,7 @@ jobs: INPUT_FORCE: ${{ inputs.force || 'false' }} run: | FORCE="${INPUT_FORCE:-false}" - MODE="${{ steps.mode.outputs.mode }}" + MODE="${INPUT_MODE}" .github/scripts/bench-scheduled-refs.sh "$FORCE" "$MODE" - name: Alert on long-running hourly @@ -199,6 +200,9 @@ jobs: if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never') uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: + INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }} + INPUT_NIGHTLY_CREATED: ${{ steps.refs.outputs.nightly-created }} + INPUT_STALE_AGE_HOURS: ${{ steps.refs.outputs.stale-age-hours }} SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }} SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }} with: @@ -210,9 +214,9 @@ jobs: return; } - const ageHours = '${{ steps.refs.outputs.stale-age-hours }}'; - const created = '${{ steps.refs.outputs.nightly-created }}'; - const featureRef = '${{ steps.refs.outputs.feature-ref }}'; + const ageHours = process.env.INPUT_STALE_AGE_HOURS; + const created = process.env.INPUT_NIGHTLY_CREATED; + const featureRef = process.env.INPUT_FEATURE_REF; const shortSha = featureRef.slice(0, 8); const repo = '${{ github.repository }}'; const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`; @@ -286,6 +290,7 @@ jobs: name: bench-scheduled runs-on: [self-hosted, Linux, X64, available] permissions: + id-token: write contents: read actions: read timeout-minutes: 120 @@ -323,6 +328,8 @@ jobs: CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }} BENCH_VICTORIAMETRICS_URL: ${{ secrets.BENCH_VICTORIAMETRICS_URL }} steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - name: Clean up previous bench-work run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true @@ -348,7 +355,9 @@ jobs: core.exportVariable('BENCH_JOB_URL', jobUrl); - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -418,15 +427,21 @@ jobs: run: | FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8) if [ "$BENCH_MODE" = "release" ] && [ -n "$RELEASE_TAG" ]; then - echo "baseline-name=${RELEASE_TAG}" >> "$GITHUB_OUTPUT" - echo "baseline-ref=${RELEASE_TAG}" >> "$GITHUB_OUTPUT" + { + echo "baseline-name=${RELEASE_TAG}" + echo "baseline-ref=${RELEASE_TAG}" + } >> "$GITHUB_OUTPUT" else BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8) - echo "baseline-name=${BENCH_MODE}-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT" - echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT" + { + echo "baseline-name=${BENCH_MODE}-${BASELINE_SHORT}" + echo "baseline-ref=$BASELINE_REF" + } >> "$GITHUB_OUTPUT" fi - echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT" - echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT" + { + echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" + echo "feature-ref=$FEATURE_REF" + } >> "$GITHUB_OUTPUT" - name: Prepare source dirs run: | @@ -630,14 +645,18 @@ jobs: fi printf '
%s: %d panic(s), %d error(s)\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE" if [ "$panics" -gt 0 ]; then - printf '**Panics:**\n```\n' >> "$ERRORS_FILE" - grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE" - printf '```\n' >> "$ERRORS_FILE" + { + printf '**Panics:**\n```\n' + grep -E 'panicked at' "$LOG" | head -10 + printf '```\n' + } >> "$ERRORS_FILE" fi if [ "$errors" -gt 0 ]; then - printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE" - grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE" - printf '```\n' >> "$ERRORS_FILE" + { + printf '**Errors (first 20):**\n```\n' + grep ' ERROR ' "$LOG" | head -20 + printf '```\n' + } >> "$ERRORS_FILE" fi printf '\n
\n\n' >> "$ERRORS_FILE" fi @@ -647,6 +666,7 @@ jobs: id: results if: success() env: + INPUT_GRAFANA_URL: ${{ steps.metrics.outputs.grafana-url }} BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }} FEATURE_NAME: ${{ steps.refs.outputs.feature-name }} BASELINE_REF_DISPLAY: ${{ steps.refs.outputs.baseline-ref }} @@ -667,7 +687,7 @@ jobs: SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS" SUMMARY_ARGS="$SUMMARY_ARGS --run-pairs $BENCH_RUN_PAIRS" - GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}' + GRAFANA_URL="${INPUT_GRAFANA_URL}" if [ -n "$GRAFANA_URL" ]; then SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL" fi @@ -680,6 +700,7 @@ jobs: - name: Upload to ClickHouse if: success() env: + INPUT_GRAFANA_URL: ${{ steps.metrics.outputs.grafana-url }} CLICKHOUSE_HOST: ${{ secrets.CLICKHOUSE_HOST }} CLICKHOUSE_USER: ${{ secrets.CLICKHOUSE_USER }} CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }} @@ -690,7 +711,7 @@ jobs: WORKFLOW_NAME="workflows-nightly-regression-${{ github.run_id }}" fi DIFF_URL="https://github.com/${{ github.repository }}/compare/${BASELINE_REF}...${FEATURE_REF}" - GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}' + GRAFANA_URL="${INPUT_GRAFANA_URL}" JOB_URL="${BENCH_JOB_URL:-${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}}" python3 .github/scripts/bench-upload-clickhouse.py \ @@ -775,6 +796,9 @@ jobs: - name: Write job summary if: success() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_GRAFANA_URL: ${{ steps.metrics.outputs.grafana-url }} + INPUT_SHA: ${{ steps.push-charts.outputs.sha }} with: script: | const fs = require('fs'); @@ -822,7 +846,7 @@ jobs: } // Charts - const chartSha = '${{ steps.push-charts.outputs.sha }}'; + const chartSha = process.env.INPUT_SHA; if (chartSha) { const runId = '${{ github.run_id }}'; const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`; @@ -839,7 +863,7 @@ jobs: } } - const grafanaUrl = '${{ steps.metrics.outputs.grafana-url }}'; + const grafanaUrl = process.env.INPUT_GRAFANA_URL; if (grafanaUrl) { md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`; } @@ -1101,10 +1125,12 @@ jobs: - name: Push state to charts repo env: + INPUT_FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }} + INPUT_MODE: ${{ needs.resolve-refs.outputs.mode }} DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }} run: | - MODE="${{ needs.resolve-refs.outputs.mode }}" - FEATURE_REF="${{ needs.resolve-refs.outputs.feature-ref }}" + MODE="${INPUT_MODE}" + FEATURE_REF="${INPUT_FEATURE_REF}" CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git" TMP_DIR=$(mktemp -d) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b88fe00d3fb..9530bef873d 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -84,12 +84,12 @@ on: samply: description: "Enable samply profiling" required: false - default: "false" + default: false type: boolean tracing_chrome: description: "Enable Chrome trace recording" required: false - default: "false" + default: false type: boolean cores: description: "Limit reth to N CPU cores (0 = all available)" @@ -114,12 +114,12 @@ on: otlp: description: "Export OTLP traces and logs" required: false - default: "true" + default: true type: boolean metrics: description: "Upload txgen metrics to VictoriaMetrics" required: false - default: "false" + default: false type: boolean call_args: description: "Call mode arguments as key=value pairs (corpus, class, methods, rps, duration, passes, concurrency)" @@ -813,6 +813,39 @@ jobs: id: ack if: steps.command.outputs.command == 'bench' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_ACTOR: ${{ steps.args.outputs.actor }} + INPUT_BAL: ${{ steps.args.outputs.bal }} + INPUT_BASELINE_ARGS: ${{ steps.args.outputs.baseline-args }} + INPUT_BASELINE_NAME: ${{ steps.args.outputs.baseline-name }} + INPUT_BIG_BLOCKS: ${{ steps.args.outputs.big-blocks }} + INPUT_BLOCK_TIME: ${{ steps.args.outputs.block-time }} + INPUT_BLOCKS: ${{ steps.args.outputs.blocks }} + INPUT_CLASS: ${{ steps.args.outputs.class }} + INPUT_CONCURRENCY: ${{ steps.args.outputs.concurrency }} + INPUT_CORES: ${{ steps.args.outputs.cores }} + INPUT_CORPUS: ${{ steps.args.outputs.corpus }} + INPUT_DURATION: ${{ steps.args.outputs.duration }} + INPUT_FEATURE_ARGS: ${{ steps.args.outputs.feature-args }} + INPUT_FEATURE_NAME: ${{ steps.args.outputs.feature-name }} + INPUT_METHODS: ${{ steps.args.outputs.methods }} + INPUT_METRICS: ${{ steps.args.outputs.metrics }} + INPUT_MODE: ${{ steps.args.outputs.mode }} + INPUT_NAMESPACE: ${{ steps.args.outputs.namespace }} + INPUT_OTLP: ${{ steps.args.outputs.otlp }} + INPUT_PASSES: ${{ steps.args.outputs.passes }} + INPUT_PR: ${{ steps.args.outputs.pr }} + INPUT_REORG: ${{ steps.args.outputs.reorg }} + INPUT_RPS: ${{ steps.args.outputs.rps }} + INPUT_RUN_ORDER: ${{ steps.args.outputs.run-order }} + INPUT_RUN_PAIRS: ${{ steps.args.outputs.run-pairs }} + INPUT_SAMPLY: ${{ steps.args.outputs.samply }} + INPUT_SLACK: ${{ steps.args.outputs.slack }} + INPUT_TOP_GAS: ${{ steps.args.outputs.top-gas }} + INPUT_TRACER: ${{ steps.args.outputs.tracer }} + INPUT_TRACING_CHROME: ${{ steps.args.outputs.tracing-chrome }} + INPUT_WAIT_TIME: ${{ steps.args.outputs.wait-time }} + INPUT_WARMUP: ${{ steps.args.outputs.warmup }} with: github-token: ${{ secrets.DEREK_PAT }} script: | @@ -825,7 +858,7 @@ jobs: }); } - const pr = '${{ steps.args.outputs.pr }}'; + const pr = process.env.INPUT_PR; if (!pr) return; const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; @@ -860,34 +893,34 @@ jobs: core.info(`Skipping queue tracking: ${e.message}`); } - const actor = '${{ steps.args.outputs.actor }}'; - const blocks = '${{ steps.args.outputs.blocks }}'; - const warmup = '${{ steps.args.outputs.warmup }}'; - const baseline = '${{ steps.args.outputs.baseline-name }}'; - const feature = '${{ steps.args.outputs.feature-name }}'; - const samply = '${{ steps.args.outputs.samply }}' === 'true'; - const tracingChrome = '${{ steps.args.outputs.tracing-chrome }}' === 'true'; - const slack = '${{ steps.args.outputs.slack }}' || 'always'; - const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true'; - const mode = '${{ steps.args.outputs.mode }}' || 'engine'; - const reorg = '${{ steps.args.outputs.reorg }}'; - const bal = '${{ steps.args.outputs.bal }}' || 'false'; + const actor = process.env.INPUT_ACTOR; + const blocks = process.env.INPUT_BLOCKS; + const warmup = process.env.INPUT_WARMUP; + const baseline = process.env.INPUT_BASELINE_NAME; + const feature = process.env.INPUT_FEATURE_NAME; + const samply = process.env.INPUT_SAMPLY === 'true'; + const tracingChrome = process.env.INPUT_TRACING_CHROME === 'true'; + const slack = process.env.INPUT_SLACK || 'always'; + const bigBlocks = process.env.INPUT_BIG_BLOCKS === 'true'; + const mode = process.env.INPUT_MODE || 'engine'; + const reorg = process.env.INPUT_REORG; + const bal = process.env.INPUT_BAL || 'false'; const samplyNote = samply ? ', samply: `enabled`' : ''; const tracingChromeNote = tracingChrome ? ', tracing-chrome: `enabled`' : ''; const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : ''; const reorgNote = reorg ? `, reorg: \`${reorg}\`` : ''; const balNote = bal !== 'false' ? `, BAL: \`${bal}\`` : ''; const modeNote = mode !== 'engine' ? `, mode: \`${mode}\`` : ''; - const corpus = '${{ steps.args.outputs.corpus }}'; - const callClass = '${{ steps.args.outputs.class }}'; - const callMethods = '${{ steps.args.outputs.methods }}'; - const callRps = '${{ steps.args.outputs.rps }}'; - const callDuration = '${{ steps.args.outputs.duration }}'; - const callPasses = '${{ steps.args.outputs.passes }}'; - const callConcurrency = '${{ steps.args.outputs.concurrency }}'; - const callTopGas = '${{ steps.args.outputs.top-gas }}'; - const callNamespace = '${{ steps.args.outputs.namespace }}'; - const callTracer = '${{ steps.args.outputs.tracer }}'; + const corpus = process.env.INPUT_CORPUS; + const callClass = process.env.INPUT_CLASS; + const callMethods = process.env.INPUT_METHODS; + const callRps = process.env.INPUT_RPS; + const callDuration = process.env.INPUT_DURATION; + const callPasses = process.env.INPUT_PASSES; + const callConcurrency = process.env.INPUT_CONCURRENCY; + const callTopGas = process.env.INPUT_TOP_GAS; + const callNamespace = process.env.INPUT_NAMESPACE; + const callTracer = process.env.INPUT_TRACER; const callExtras = [ callTopGas ? `top-gas: \`${callTopGas}\`` : '', callNamespace ? `namespace: \`${callNamespace}\`` : '', @@ -896,22 +929,22 @@ jobs: const callNote = mode === 'call' ? `, corpus: \`${corpus}\`, class: \`${callClass}\`, methods: \`${callMethods}\`, rps: \`${callRps}\`, duration: \`${callDuration}\`, passes: \`${callPasses}\`, concurrency: \`${callConcurrency}\`${callExtras}` : ''; - const cores = '${{ steps.args.outputs.cores }}'; + const cores = process.env.INPUT_CORES; const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : ''; - const runPairs = '${{ steps.args.outputs.run-pairs }}' || '6'; - const runOrder = '${{ steps.args.outputs.run-order }}'; + const runPairs = process.env.INPUT_RUN_PAIRS || '6'; + const runOrder = process.env.INPUT_RUN_ORDER; const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`; - const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false'; + const otlpEnabled = process.env.INPUT_OTLP !== 'false'; const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : ''; - const metricsEnabled = '${{ steps.args.outputs.metrics }}' === 'true'; + const metricsEnabled = process.env.INPUT_METRICS === 'true'; const metricsNote = metricsEnabled ? ', metrics: `enabled`' : ''; - const waitTimeVal = '${{ steps.args.outputs.wait-time }}'; + const waitTimeVal = process.env.INPUT_WAIT_TIME; const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : ''; - const blockTimeVal = '${{ steps.args.outputs.block-time }}'; + const blockTimeVal = process.env.INPUT_BLOCK_TIME; const blockTimeNote = blockTimeVal ? `, block-time: \`${blockTimeVal}\`` : ''; - const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}'; + const baselineArgsVal = process.env.INPUT_BASELINE_ARGS; const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : ''; - const featureArgsVal = '${{ steps.args.outputs.feature-args }}'; + const featureArgsVal = process.env.INPUT_FEATURE_ARGS; const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : ''; const blocksDesc = mode === 'call' ? `${blocks} corpus blocks, ${warmup}s warmup` @@ -930,39 +963,74 @@ jobs: - name: Poll queue position if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_COMMENT_ID: ${{ steps.ack.outputs.comment-id }} + INPUT_QUEUE_POSITION: ${{ steps.ack.outputs.queue-position }} + INPUT_ACTOR: ${{ steps.args.outputs.actor }} + INPUT_BAL: ${{ steps.args.outputs.bal }} + INPUT_BASELINE_ARGS: ${{ steps.args.outputs.baseline-args }} + INPUT_BASELINE_NAME: ${{ steps.args.outputs.baseline-name }} + INPUT_BIG_BLOCKS: ${{ steps.args.outputs.big-blocks }} + INPUT_BLOCK_TIME: ${{ steps.args.outputs.block-time }} + INPUT_BLOCKS: ${{ steps.args.outputs.blocks }} + INPUT_CLASS: ${{ steps.args.outputs.class }} + INPUT_CONCURRENCY: ${{ steps.args.outputs.concurrency }} + INPUT_CORES: ${{ steps.args.outputs.cores }} + INPUT_CORPUS: ${{ steps.args.outputs.corpus }} + INPUT_DURATION: ${{ steps.args.outputs.duration }} + INPUT_FEATURE_ARGS: ${{ steps.args.outputs.feature-args }} + INPUT_FEATURE_NAME: ${{ steps.args.outputs.feature-name }} + INPUT_METHODS: ${{ steps.args.outputs.methods }} + INPUT_METRICS: ${{ steps.args.outputs.metrics }} + INPUT_MODE: ${{ steps.args.outputs.mode }} + INPUT_NAMESPACE: ${{ steps.args.outputs.namespace }} + INPUT_OTLP: ${{ steps.args.outputs.otlp }} + INPUT_PASSES: ${{ steps.args.outputs.passes }} + INPUT_PR: ${{ steps.args.outputs.pr }} + INPUT_REORG: ${{ steps.args.outputs.reorg }} + INPUT_RPS: ${{ steps.args.outputs.rps }} + INPUT_RUN_ORDER: ${{ steps.args.outputs.run-order }} + INPUT_RUN_PAIRS: ${{ steps.args.outputs.run-pairs }} + INPUT_SAMPLY: ${{ steps.args.outputs.samply }} + INPUT_SLACK: ${{ steps.args.outputs.slack }} + INPUT_TOP_GAS: ${{ steps.args.outputs.top-gas }} + INPUT_TRACER: ${{ steps.args.outputs.tracer }} + INPUT_TRACING_CHROME: ${{ steps.args.outputs.tracing-chrome }} + INPUT_WAIT_TIME: ${{ steps.args.outputs.wait-time }} + INPUT_WARMUP: ${{ steps.args.outputs.warmup }} with: github-token: ${{ secrets.DEREK_PAT }} script: | - const pr = '${{ steps.args.outputs.pr }}'; - const commentId = parseInt('${{ steps.ack.outputs.comment-id }}'); - const actor = '${{ steps.args.outputs.actor }}'; - const blocks = '${{ steps.args.outputs.blocks }}'; - const warmup = '${{ steps.args.outputs.warmup }}'; - const baseline = '${{ steps.args.outputs.baseline-name }}'; - const feature = '${{ steps.args.outputs.feature-name }}'; - const samply = '${{ steps.args.outputs.samply }}' === 'true'; - const tracingChrome = '${{ steps.args.outputs.tracing-chrome }}' === 'true'; - const slack = '${{ steps.args.outputs.slack }}' || 'always'; - const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true'; - const mode = '${{ steps.args.outputs.mode }}' || 'engine'; - const reorg = '${{ steps.args.outputs.reorg }}'; - const bal = '${{ steps.args.outputs.bal }}' || 'false'; + const pr = process.env.INPUT_PR; + const commentId = parseInt(process.env.INPUT_COMMENT_ID); + const actor = process.env.INPUT_ACTOR; + const blocks = process.env.INPUT_BLOCKS; + const warmup = process.env.INPUT_WARMUP; + const baseline = process.env.INPUT_BASELINE_NAME; + const feature = process.env.INPUT_FEATURE_NAME; + const samply = process.env.INPUT_SAMPLY === 'true'; + const tracingChrome = process.env.INPUT_TRACING_CHROME === 'true'; + const slack = process.env.INPUT_SLACK || 'always'; + const bigBlocks = process.env.INPUT_BIG_BLOCKS === 'true'; + const mode = process.env.INPUT_MODE || 'engine'; + const reorg = process.env.INPUT_REORG; + const bal = process.env.INPUT_BAL || 'false'; const samplyNote = samply ? ', samply: `enabled`' : ''; const tracingChromeNote = tracingChrome ? ', tracing-chrome: `enabled`' : ''; const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : ''; const reorgNote = reorg ? `, reorg: \`${reorg}\`` : ''; const balNote = bal !== 'false' ? `, BAL: \`${bal}\`` : ''; const modeNote = mode !== 'engine' ? `, mode: \`${mode}\`` : ''; - const corpus = '${{ steps.args.outputs.corpus }}'; - const callClass = '${{ steps.args.outputs.class }}'; - const callMethods = '${{ steps.args.outputs.methods }}'; - const callRps = '${{ steps.args.outputs.rps }}'; - const callDuration = '${{ steps.args.outputs.duration }}'; - const callPasses = '${{ steps.args.outputs.passes }}'; - const callConcurrency = '${{ steps.args.outputs.concurrency }}'; - const callTopGas = '${{ steps.args.outputs.top-gas }}'; - const callNamespace = '${{ steps.args.outputs.namespace }}'; - const callTracer = '${{ steps.args.outputs.tracer }}'; + const corpus = process.env.INPUT_CORPUS; + const callClass = process.env.INPUT_CLASS; + const callMethods = process.env.INPUT_METHODS; + const callRps = process.env.INPUT_RPS; + const callDuration = process.env.INPUT_DURATION; + const callPasses = process.env.INPUT_PASSES; + const callConcurrency = process.env.INPUT_CONCURRENCY; + const callTopGas = process.env.INPUT_TOP_GAS; + const callNamespace = process.env.INPUT_NAMESPACE; + const callTracer = process.env.INPUT_TRACER; const callExtras = [ callTopGas ? `top-gas: \`${callTopGas}\`` : '', callNamespace ? `namespace: \`${callNamespace}\`` : '', @@ -971,22 +1039,22 @@ jobs: const callNote = mode === 'call' ? `, corpus: \`${corpus}\`, class: \`${callClass}\`, methods: \`${callMethods}\`, rps: \`${callRps}\`, duration: \`${callDuration}\`, passes: \`${callPasses}\`, concurrency: \`${callConcurrency}\`${callExtras}` : ''; - const cores = '${{ steps.args.outputs.cores }}'; + const cores = process.env.INPUT_CORES; const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : ''; - const runPairs = '${{ steps.args.outputs.run-pairs }}' || '6'; - const runOrder = '${{ steps.args.outputs.run-order }}'; + const runPairs = process.env.INPUT_RUN_PAIRS || '6'; + const runOrder = process.env.INPUT_RUN_ORDER; const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`; - const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false'; + const otlpEnabled = process.env.INPUT_OTLP !== 'false'; const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : ''; - const metricsEnabled = '${{ steps.args.outputs.metrics }}' === 'true'; + const metricsEnabled = process.env.INPUT_METRICS === 'true'; const metricsNote = metricsEnabled ? ', metrics: `enabled`' : ''; - const waitTimeVal = '${{ steps.args.outputs.wait-time }}'; + const waitTimeVal = process.env.INPUT_WAIT_TIME; const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : ''; - const blockTimeVal = '${{ steps.args.outputs.block-time }}'; + const blockTimeVal = process.env.INPUT_BLOCK_TIME; const blockTimeNote = blockTimeVal ? `, block-time: \`${blockTimeVal}\`` : ''; - const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}'; + const baselineArgsVal = process.env.INPUT_BASELINE_ARGS; const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : ''; - const featureArgsVal = '${{ steps.args.outputs.feature-args }}'; + const featureArgsVal = process.env.INPUT_FEATURE_ARGS; const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : ''; const blocksDesc = mode === 'call' ? `${blocks} corpus blocks, ${warmup}s warmup` @@ -1015,7 +1083,7 @@ jobs: return { ahead: Math.max(0, totalAhead - numRunners + 1), numRunners }; } - let lastPosition = parseInt('${{ steps.ack.outputs.queue-position }}'); + let lastPosition = parseInt(process.env.INPUT_QUEUE_POSITION); const sleep = ms => new Promise(r => setTimeout(r, ms)); while (true) { @@ -1046,6 +1114,7 @@ jobs: name: bench-txgen runs-on: [self-hosted, Linux, X64, available] permissions: + id-token: write contents: read pull-requests: write timeout-minutes: 120 @@ -1101,17 +1170,21 @@ jobs: BENCH_OTLP_LOGS_ENDPOINT: ${{ needs.bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_LOGS_ENDPOINT || '' }} BENCH_VICTORIAMETRICS_URL: ${{ needs.bench-ack.outputs.metrics == 'true' && secrets.BENCH_VICTORIAMETRICS_URL || '' }} steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - name: Clean up previous bench-work run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true - name: Resolve checkout ref id: checkout-ref uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_REF: ${{ github.ref }} with: script: | if (!process.env.BENCH_PR) { core.setOutput('repository', '${{ github.repository }}'); - core.setOutput('ref', '${{ github.ref }}'); + core.setOutput('ref', process.env.INPUT_REF); return; } const sha = process.env.BENCH_PR_HEAD_SHA; @@ -1134,6 +1207,9 @@ jobs: - name: Initialize run summary and update status uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_BASELINE_NAME: ${{ needs.bench-ack.outputs.baseline-name }} + INPUT_FEATURE_NAME: ${{ needs.bench-ack.outputs.feature-name }} with: github-token: ${{ secrets.DEREK_PAT }} script: | @@ -1151,8 +1227,8 @@ jobs: const blocks = process.env.BENCH_BLOCKS; const warmup = process.env.BENCH_WARMUP_BLOCKS; - const baseline = '${{ needs.bench-ack.outputs.baseline-name }}'; - const feature = '${{ needs.bench-ack.outputs.feature-name }}'; + const baseline = process.env.INPUT_BASELINE_NAME; + const feature = process.env.INPUT_FEATURE_NAME; const samply = process.env.BENCH_SAMPLY === 'true'; const tracingChrome = process.env.BENCH_TRACING_CHROME === 'true'; const slack = process.env.BENCH_SLACK || 'always'; @@ -1227,7 +1303,9 @@ jobs: } - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 continue-on-error: true - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 @@ -1322,42 +1400,49 @@ jobs: - name: Resolve PR head branch id: pr-info uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_REF_NAME: ${{ github.ref_name }} with: script: | if (process.env.BENCH_PR) { - const ref = process.env.BENCH_PR_HEAD_REF || '${{ github.ref_name }}'; + const ref = process.env.BENCH_PR_HEAD_REF || process.env.INPUT_REF_NAME; const sha = process.env.BENCH_PR_HEAD_SHA || '${{ github.sha }}'; core.setOutput('head-ref', ref); core.setOutput('head-sha', sha); } else { - core.setOutput('head-ref', '${{ github.ref_name }}'); + core.setOutput('head-ref', process.env.INPUT_REF_NAME); core.setOutput('head-sha', '${{ github.sha }}'); } - name: Resolve refs id: refs uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_BASELINE: ${{ needs.bench-ack.outputs.baseline }} + INPUT_FEATURE: ${{ needs.bench-ack.outputs.feature }} + INPUT_HEAD_REF: ${{ steps.pr-info.outputs.head-ref }} + INPUT_HEAD_SHA: ${{ steps.pr-info.outputs.head-sha }} with: script: | - const { execSync } = require('child_process'); - const run = (cmd) => execSync(cmd, { encoding: 'utf8' }).trim(); + const { execFileSync } = require('child_process'); + const run = (...args) => execFileSync('git', args, { encoding: 'utf8' }).trim(); - const baselineArg = '${{ needs.bench-ack.outputs.baseline }}'; - const featureArg = '${{ needs.bench-ack.outputs.feature }}'; + const baselineArg = process.env.INPUT_BASELINE; + const featureArg = process.env.INPUT_FEATURE; let baselineRef, baselineName, featureRef, featureName; if (baselineArg) { - try { run(`git fetch origin "${baselineArg}" --quiet`); } catch {} + try { run('fetch', '--quiet', '--', 'origin', baselineArg); } catch {} try { - baselineRef = run(`git rev-parse "${baselineArg}"`); + baselineRef = run('rev-parse', '--verify', '--end-of-options', baselineArg); } catch { - baselineRef = run(`git rev-parse "origin/${baselineArg}"`); + baselineRef = run('rev-parse', '--verify', '--end-of-options', `origin/${baselineArg}`); } baselineName = baselineArg; } else { try { - baselineRef = run('git merge-base HEAD origin/main'); + baselineRef = run('merge-base', 'HEAD', 'origin/main'); } catch { baselineRef = '${{ github.sha }}'; } @@ -1365,16 +1450,16 @@ jobs: } if (featureArg) { - try { run(`git fetch origin "${featureArg}" --quiet`); } catch {} + try { run('fetch', '--quiet', '--', 'origin', featureArg); } catch {} try { - featureRef = run(`git rev-parse "${featureArg}"`); + featureRef = run('rev-parse', '--verify', '--end-of-options', featureArg); } catch { - featureRef = run(`git rev-parse "origin/${featureArg}"`); + featureRef = run('rev-parse', '--verify', '--end-of-options', `origin/${featureArg}`); } featureName = featureArg; } else { - featureRef = '${{ steps.pr-info.outputs.head-sha }}'; - featureName = '${{ steps.pr-info.outputs.head-ref }}'; + featureRef = process.env.INPUT_HEAD_SHA; + featureName = process.env.INPUT_HEAD_REF; } core.setOutput('baseline-ref', baselineRef); @@ -1383,6 +1468,9 @@ jobs: core.setOutput('feature-name', featureName); - name: Prepare source dirs + env: + INPUT_BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }} + INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }} run: | prepare_source_dir() { local dir="$1" @@ -1399,15 +1487,17 @@ jobs: git -C "$dir" checkout --force "$ref" } - BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}" + BASELINE_REF="${INPUT_BASELINE_REF}" prepare_source_dir ../reth-baseline "$BASELINE_REF" - FEATURE_REF="${{ steps.refs.outputs.feature-ref }}" + FEATURE_REF="${INPUT_FEATURE_REF}" prepare_source_dir ../reth-feature "$FEATURE_REF" - name: Build binaries id: build env: + INPUT_BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }} + INPUT_FEATURE_REF: ${{ steps.refs.outputs.feature-ref }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TXGEN_DEPLOY_KEY: ${{ secrets.TXGEN_DEPLOY_KEY }} TXGEN_TOKEN: ${{ secrets.TXGEN_TOKEN }} @@ -1422,9 +1512,9 @@ jobs: .github/scripts/bench-txgen-install.sh - .github/scripts/bench-txgen-build.sh baseline "${BASELINE_DIR}" "${{ steps.refs.outputs.baseline-ref }}" & + .github/scripts/bench-txgen-build.sh baseline "${BASELINE_DIR}" "${INPUT_BASELINE_REF}" & PID_BASELINE=$! - .github/scripts/bench-txgen-build.sh feature "${FEATURE_DIR}" "${{ steps.refs.outputs.feature-ref }}" & + .github/scripts/bench-txgen-build.sh feature "${FEATURE_DIR}" "${INPUT_FEATURE_REF}" & PID_FEATURE=$! FAIL=0 @@ -1519,6 +1609,9 @@ jobs: mkdir -p "$BENCH_WORK_DIR" - name: Initialize benchmark metrics labels + env: + INPUT_BASELINE_NAME: ${{ needs.bench-ack.outputs.baseline-name }} + INPUT_FEATURE_NAME: ${{ needs.bench-ack.outputs.feature-name }} run: | if [ -n "${BENCH_VICTORIAMETRICS_URL:-}" ]; then echo "::add-mask::$BENCH_VICTORIAMETRICS_URL" @@ -1540,8 +1633,8 @@ jobs: derek_cmd+=( blocks="$BENCH_BLOCKS" warmup="$BENCH_WARMUP_BLOCKS" - baseline="${{ needs.bench-ack.outputs.baseline-name }}" - feature="${{ needs.bench-ack.outputs.feature-name }}" + baseline="${INPUT_BASELINE_NAME}" + feature="${INPUT_FEATURE_NAME}" bal="$BENCH_BAL" cores="$BENCH_CORES" run-pairs="$BENCH_RUN_PAIRS" @@ -1751,11 +1844,13 @@ jobs: )" LOGS_URL="$(printf '%s\n' "$OBSERVABILITY_URLS" | sed -n '1p')" TRACES_URL="$(printf '%s\n' "$OBSERVABILITY_URLS" | sed -n '2p')" - echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT" - echo "logs-url=${LOGS_URL}" >> "$GITHUB_OUTPUT" - echo "traces-url=${TRACES_URL}" >> "$GITHUB_OUTPUT" - echo "observability-from-ms=${FROM_MS}" >> "$GITHUB_OUTPUT" - echo "observability-to-ms=${TO_MS}" >> "$GITHUB_OUTPUT" + { + echo "grafana-url=${GRAFANA_URL}" + echo "logs-url=${LOGS_URL}" + echo "traces-url=${TRACES_URL}" + echo "observability-from-ms=${FROM_MS}" + echo "observability-to-ms=${TO_MS}" + } >> "$GITHUB_OUTPUT" echo "Grafana URL: ${GRAFANA_URL}" echo "Logs URL: ${LOGS_URL}" echo "Traces URL: ${TRACES_URL}" @@ -1780,14 +1875,18 @@ jobs: fi printf '
%s: %d panic(s), %d error(s)\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE" if [ "$panics" -gt 0 ]; then - printf '**Panics:**\n```\n' >> "$ERRORS_FILE" - grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE" - printf '```\n' >> "$ERRORS_FILE" + { + printf '**Panics:**\n```\n' + grep -E 'panicked at' "$LOG" | head -10 + printf '```\n' + } >> "$ERRORS_FILE" fi if [ "$errors" -gt 0 ]; then - printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE" - grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE" - printf '```\n' >> "$ERRORS_FILE" + { + printf '**Errors (first 20):**\n```\n' + grep ' ERROR ' "$LOG" | head -20 + printf '```\n' + } >> "$ERRORS_FILE" fi printf '\n
\n\n' >> "$ERRORS_FILE" fi @@ -1855,6 +1954,11 @@ jobs: id: results if: success() env: + INPUT_GRAFANA_URL: ${{ steps.metrics.outputs.grafana-url }} + INPUT_LOGS_URL: ${{ steps.metrics.outputs.logs-url }} + INPUT_OBSERVABILITY_FROM_MS: ${{ steps.metrics.outputs.observability-from-ms }} + INPUT_OBSERVABILITY_TO_MS: ${{ steps.metrics.outputs.observability-to-ms }} + INPUT_TRACES_URL: ${{ steps.metrics.outputs.traces-url }} BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }} BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }} FEATURE_NAME: ${{ steps.refs.outputs.feature-name }} @@ -1897,11 +2001,11 @@ jobs: if [ -n "${BENCH_ID:-}" ]; then CALL_ARGS="$CALL_ARGS --benchmark-id $BENCH_ID" fi - LOGS_URL='${{ steps.metrics.outputs.logs-url }}' + LOGS_URL="${INPUT_LOGS_URL}" if [ -n "$LOGS_URL" ]; then CALL_ARGS="$CALL_ARGS --logs-url $LOGS_URL" fi - TRACES_URL='${{ steps.metrics.outputs.traces-url }}' + TRACES_URL="${INPUT_TRACES_URL}" if [ -n "$TRACES_URL" ]; then CALL_ARGS="$CALL_ARGS --traces-url $TRACES_URL" fi @@ -1948,20 +2052,20 @@ jobs: if [ -n "${BENCH_ID:-}" ]; then SUMMARY_ARGS="$SUMMARY_ARGS --benchmark-id $BENCH_ID" fi - GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}' + GRAFANA_URL="${INPUT_GRAFANA_URL}" if [ "${BENCH_METRICS:-false}" = "true" ] && [ -n "$GRAFANA_URL" ]; then SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL" fi - LOGS_URL='${{ steps.metrics.outputs.logs-url }}' + LOGS_URL="${INPUT_LOGS_URL}" if [ -n "$LOGS_URL" ]; then SUMMARY_ARGS="$SUMMARY_ARGS --logs-url $LOGS_URL" fi - TRACES_URL='${{ steps.metrics.outputs.traces-url }}' + TRACES_URL="${INPUT_TRACES_URL}" if [ -n "$TRACES_URL" ]; then SUMMARY_ARGS="$SUMMARY_ARGS --traces-url $TRACES_URL" fi - OBSERVABILITY_FROM_MS='${{ steps.metrics.outputs.observability-from-ms }}' - OBSERVABILITY_TO_MS='${{ steps.metrics.outputs.observability-to-ms }}' + OBSERVABILITY_FROM_MS="${INPUT_OBSERVABILITY_FROM_MS}" + OBSERVABILITY_TO_MS="${INPUT_OBSERVABILITY_TO_MS}" if [ -n "$OBSERVABILITY_FROM_MS" ] && [ -n "$OBSERVABILITY_TO_MS" ]; then SUMMARY_ARGS="$SUMMARY_ARGS --observability-from-ms $OBSERVABILITY_FROM_MS" SUMMARY_ARGS="$SUMMARY_ARGS --observability-to-ms $OBSERVABILITY_TO_MS" @@ -2105,6 +2209,8 @@ jobs: - name: Compare & comment if: success() && env.BENCH_COMMENT_ID uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_SHA: ${{ steps.push-charts.outputs.sha }} with: github-token: ${{ secrets.DEREK_PAT }} script: | @@ -2117,7 +2223,7 @@ jobs: comment = '⚠️ Benchmark completed but failed to generate comparison.'; } - const sha = '${{ steps.push-charts.outputs.sha }}'; + const sha = process.env.INPUT_SHA; const prNumber = process.env.BENCH_PR || '0'; const runId = '${{ github.run_id }}'; @@ -2187,15 +2293,19 @@ jobs: - name: Write job summary if: success() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INPUT_LOGS_URL: ${{ steps.metrics.outputs.logs-url }} + INPUT_TRACES_URL: ${{ steps.metrics.outputs.traces-url }} + INPUT_SHA: ${{ steps.push-charts.outputs.sha }} with: script: | const jobSummary = require('./.github/scripts/bench-job-summary.js'); await jobSummary({ core, context, - chartSha: '${{ steps.push-charts.outputs.sha }}', - logsUrl: '${{ steps.metrics.outputs.logs-url }}', - tracesUrl: '${{ steps.metrics.outputs.traces-url }}', + chartSha: process.env.INPUT_SHA, + logsUrl: process.env.INPUT_LOGS_URL, + tracesUrl: process.env.INPUT_TRACES_URL, runId: '${{ github.run_id }}', }); @@ -2283,10 +2393,10 @@ jobs: run: | # Restore frequency scaling to full range for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq; do - cat "$(dirname "$f")/cpuinfo_min_freq" | sudo tee "$f" > /dev/null 2>&1 || true + printf '%s\n' "$(< "$(dirname "$f")/cpuinfo_min_freq")" | sudo tee "$f" > /dev/null 2>&1 || true done for f in /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq; do - cat "$(dirname "$f")/cpuinfo_max_freq" | sudo tee "$f" > /dev/null 2>&1 || true + printf '%s\n' "$(< "$(dirname "$f")/cpuinfo_max_freq")" | sudo tee "$f" > /dev/null 2>&1 || true done # Restore amd-pstate to active (EPP) mode with powersave governor echo active | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 07f26676b62..7f838a987fd 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -16,9 +16,15 @@ jobs: build: runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 90 + env: + # Include the system trust store containing Aegis's CA for Bun and Node. + NODE_EXTRA_CA_CERTS: /etc/ssl/certs/ca-certificates.crt steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -39,7 +45,9 @@ jobs: npx playwright install --with-deps chromium - name: Install Rust nightly - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: nightly - name: Build docs run: cd docs/vocs && bash scripts/build-cargo-docs.sh diff --git a/.github/workflows/check-alloy.yml b/.github/workflows/check-alloy.yml index c9dc0d51197..cea21a21b4a 100644 --- a/.github/workflows/check-alloy.yml +++ b/.github/workflows/check-alloy.yml @@ -29,9 +29,12 @@ jobs: name: Check compilation with patched alloy runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 60 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -39,7 +42,9 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -51,23 +56,23 @@ jobs: ALLOY_EVM_BRANCH: ${{ inputs.alloy_evm_branch }} OP_ALLOY_BRANCH: ${{ inputs.op_alloy_branch }} run: | - ARGS="" + ARGS=() if [ -n "$ALLOY_BRANCH" ]; then - ARGS="$ARGS --alloy $ALLOY_BRANCH" + ARGS+=(--alloy "$ALLOY_BRANCH") fi if [ -n "$ALLOY_EVM_BRANCH" ]; then - ARGS="$ARGS --evm $ALLOY_EVM_BRANCH" + ARGS+=(--evm "$ALLOY_EVM_BRANCH") fi if [ -n "$OP_ALLOY_BRANCH" ]; then - ARGS="$ARGS --op $OP_ALLOY_BRANCH" + ARGS+=(--op "$OP_ALLOY_BRANCH") fi - if [ -z "$ARGS" ]; then + if [ "${#ARGS[@]}" -eq 0 ]; then echo "No branches specified, nothing to patch" exit 1 fi - ./scripts/patch-alloy.sh $ARGS + ./scripts/patch-alloy.sh "${ARGS[@]}" echo "=== Final patch section ===" tail -50 Cargo.toml diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 494a0aac872..0e0b498693a 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -22,19 +22,24 @@ jobs: compact-codec: runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read strategy: matrix: bin: - cargo run --bin reth --features "dev" steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - name: Checkout PR uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/.github/workflows/cyclops-audit.yml b/.github/workflows/cyclops-audit.yml index 52e7db317b9..29fcbf6b884 100644 --- a/.github/workflows/cyclops-audit.yml +++ b/.github/workflows/cyclops-audit.yml @@ -60,7 +60,7 @@ jobs: pull-requests: write steps: - name: Handle audit command - uses: tempoxyz/gh-actions/actions/pr-audit-comment@7741a8658354abd422dd1b941d07f809682ca96d + uses: tempoxyz/gh-actions/actions/pr-audit-comment@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: command-regex: '^(?:@decofe\s+)?(?:cyclops\s+(?:private\s+)?audit|derek\s+audit)\b' permission-check-mode: association diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 47b27170690..5d4542d1f2e 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -15,8 +15,6 @@ jobs: update: if: github.repository == 'paradigmxyz/reth' permissions: - contents: write - pull-requests: write - uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main - secrets: - token: ${{ secrets.GITHUB_TOKEN }} + contents: read + id-token: write + uses: tempoxyz/gh-actions/.github/workflows/cargo-update-pr.yml@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 diff --git a/.github/workflows/docker-tag-latest.yml b/.github/workflows/docker-tag-latest.yml index 24d159a4bd6..851619e0ef1 100644 --- a/.github/workflows/docker-tag-latest.yml +++ b/.github/workflows/docker-tag-latest.yml @@ -38,13 +38,13 @@ jobs: env: VERSION: ${{ inputs.version }} run: | - docker pull ghcr.io/${{ github.repository_owner }}/reth:${VERSION} + docker pull "ghcr.io/${{ github.repository_owner }}/reth:${VERSION}" - name: Tag reth as latest env: VERSION: ${{ inputs.version }} run: | - docker tag ghcr.io/${{ github.repository_owner }}/reth:${VERSION} ghcr.io/${{ github.repository_owner }}/reth:latest + docker tag "ghcr.io/${{ github.repository_owner }}/reth:${VERSION}" ghcr.io/${{ github.repository_owner }}/reth:latest - name: Push reth latest tag run: | diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 31edbe19fa0..c009c9e0f0c 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -2,6 +2,9 @@ name: Build test Docker image on: workflow_call: + secrets: + DEPOT_TOKEN: + required: false inputs: hive_target: required: true @@ -23,6 +26,8 @@ jobs: id-token: write contents: read steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -31,8 +36,10 @@ jobs: - name: Get git info id: git run: | - echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT" - echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT" + { + echo "sha=${{ github.sha }}" + echo "describe=$(git describe --always --tags)" + } >> "$GITHUB_OUTPUT" - name: Detect fork id: fork @@ -47,6 +54,15 @@ jobs: echo "is_fork=false" >> "$GITHUB_OUTPUT" fi + - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + if: steps.fork.outputs.is_fork == 'false' + with: + toolchain: "1.95.0" + + - name: Check Reth dependencies through Aegis + if: steps.fork.outputs.is_fork == 'false' + run: cargo fetch --locked --manifest-path Cargo.toml + # Depot build (upstream only) - name: Set up Depot CLI if: steps.fork.outputs.is_fork == 'false' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1b6932b7a4c..812dca24d9c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,10 +41,19 @@ jobs: contents: read id-token: write steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + with: + toolchain: "1.95.0" + + - name: Check Reth dependencies through Aegis + run: cargo fetch --locked --manifest-path Cargo.toml + - name: Set up Depot CLI uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 @@ -58,9 +67,11 @@ jobs: - name: Get git info for vergen id: git run: | - echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT" - echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT" - echo "dirty=false" >> "$GITHUB_OUTPUT" + { + echo "sha=${{ github.sha }}" + echo "describe=$(git describe --always --tags)" + echo "dirty=false" + } >> "$GITHUB_OUTPUT" - name: Determine build parameters id: params @@ -84,13 +95,17 @@ jobs: echo "EOF" } >> "$GITHUB_OUTPUT" else - echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT" - echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT" + { + echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" + echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" + } >> "$GITHUB_OUTPUT" fi elif [[ "${EVENT_NAME}" == "schedule" ]] || [[ "${BUILD_TYPE}" == "nightly" ]]; then - echo "targets=nightly" >> "$GITHUB_OUTPUT" - echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT" + { + echo "targets=nightly" + echo "ethereum_tags=${REGISTRY}/reth:nightly" + } >> "$GITHUB_OUTPUT" { echo "ethereum_set<> "$GITHUB_OUTPUT" - echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT" - echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT" + { + echo "targets=ethereum" + echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" + echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${{ github.sha }}" + } >> "$GITHUB_OUTPUT" fi - name: Build and push images @@ -125,12 +142,14 @@ jobs: - name: Verify image architectures env: + INPUT_ETHEREUM_TAGS: ${{ steps.params.outputs.ethereum_tags }} + INPUT_TARGETS: ${{ steps.params.outputs.targets }} DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} run: | ./.github/scripts/verify_image_arch.sh \ - "${{ steps.params.outputs.targets }}" \ + "${INPUT_TARGETS}" \ "ghcr.io/${{ github.repository_owner }}" \ - "${{ steps.params.outputs.ethereum_tags }}" + "${INPUT_ETHEREUM_TAGS}" notify: name: Notify on failure diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8d539fdc195..6ec638c6b45 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -24,21 +24,29 @@ jobs: name: e2e-testsuite runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_BACKTRACE: 1 timeout-minutes: 90 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - name: Install nextest + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: nextest + tool: nextest@0.9.140 + checksum: true + fallback: none - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: prefix-key: v1-rust-e2e-targets @@ -60,21 +68,29 @@ jobs: name: e2e-rocksdb runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_BACKTRACE: 1 timeout-minutes: 60 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - name: Install nextest + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: nextest + tool: nextest@0.9.140 + checksum: true + fallback: none - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 5e007f154e0..02057d515a8 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -25,13 +25,15 @@ jobs: with: hive_target: hive artifact_name: "reth" - secrets: inherit + secrets: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} prepare-hive: if: github.repository == 'paradigmxyz/reth' timeout-minutes: 45 runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }} permissions: + id-token: write contents: read strategy: fail-fast: false @@ -41,6 +43,8 @@ jobs: - osaka name: Prepare Hive - ${{ matrix.variant == 'amsterdam' && 'Amsterdam' || 'Osaka' }} steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -53,7 +57,7 @@ jobs: - name: Get hive commit hash id: hive-commit - run: echo "hash=$(cd hivetests && git rev-parse HEAD)" >> $GITHUB_OUTPUT + run: echo "hash=$(cd hivetests && git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: @@ -209,9 +213,12 @@ jobs: # Use larger runners for eels tests to avoid OOM runner crashes runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }} permissions: + id-token: write contents: read issues: write steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -270,7 +277,7 @@ jobs: - name: Parse hive output run: | - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" -print0 | xargs -0 -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml - name: Print simulator output if: ${{ failure() }} @@ -395,9 +402,12 @@ jobs: # Use larger runners for eels tests to avoid OOM runner crashes runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }} permissions: + id-token: write contents: read issues: write steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -456,7 +466,7 @@ jobs: - name: Parse hive output run: | - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" -print0 | xargs -0 -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml - name: Print simulator output if: ${{ failure() }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e61f5985d2d..bae574ccfaa 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -28,6 +28,7 @@ jobs: if: github.event_name != 'schedule' runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_BACKTRACE: 1 @@ -36,17 +37,24 @@ jobs: network: ["ethereum"] timeout-minutes: 60 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - name: Install nextest + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: nextest + tool: nextest@0.9.140 + checksum: true + fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -69,7 +77,7 @@ jobs: timeout-minutes: 30 steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # release/v1 + uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # v1.3.0 with: jobs: ${{ toJSON(needs) }} @@ -78,17 +86,25 @@ jobs: if: github.event_name == 'schedule' && github.repository == 'paradigmxyz/reth' runs-on: ubuntu-latest permissions: + id-token: write contents: read steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - name: Install nextest + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: nextest + tool: nextest@0.9.140 + checksum: true + fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index ff8a910e3f0..a39951b9ad0 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -29,7 +29,8 @@ jobs: uses: ./.github/workflows/docker-test.yml with: hive_target: kurtosis - secrets: inherit + secrets: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} test: timeout-minutes: 60 @@ -38,10 +39,13 @@ jobs: name: run kurtosis runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read needs: - build-reth steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml index b2374d13b9e..b74196167d1 100644 --- a/.github/workflows/lint-actions.yml +++ b/.github/workflows/lint-actions.yml @@ -14,15 +14,20 @@ jobs: actionlint: runs-on: ubuntu-latest permissions: + id-token: write contents: read steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Download actionlint - id: get_actionlint - run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) - shell: bash + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.25' + cache: false + - name: Install actionlint + run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 - name: Check workflow files - run: SHELLCHECK_OPTS="-S error" ${{ steps.get_actionlint.outputs.executable }} -color + run: SHELLCHECK_OPTS="-S error" actionlint -color shell: bash diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9972347999f..280bc12d1a0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,6 +22,7 @@ jobs: name: clippy binaries / ${{ matrix.type }} runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 strategy: @@ -31,13 +32,16 @@ jobs: args: --workspace --lib --examples --tests --benches --locked features: "asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@clippy + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: nightly components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -52,16 +56,20 @@ jobs: name: clippy runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: nightly components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -74,19 +82,26 @@ jobs: wasm: runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: stable target: wasm32-wasip1 - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - name: Install cargo-hack + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: cargo-hack + tool: cargo-hack@0.6.45 + checksum: true + fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -100,19 +115,26 @@ jobs: riscv: runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 60 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: stable target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - name: Install cargo-hack + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: cargo-hack + tool: cargo-hack@0.6.45 + checksum: true + fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -125,6 +147,7 @@ jobs: name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }}) runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }} permissions: + id-token: write contents: read strategy: matrix: @@ -132,15 +155,22 @@ jobs: total_partitions: [3] timeout-minutes: 60 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - name: Install cargo-hack + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: cargo-hack + tool: cargo-hack@0.6.45 + checksum: true + fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -151,15 +181,18 @@ jobs: name: MSRV runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@master + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: toolchain: "1.95" # MSRV - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 @@ -174,15 +207,20 @@ jobs: name: docs runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -197,15 +235,19 @@ jobs: name: fmt runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: nightly components: rustfmt - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Run fmt @@ -215,37 +257,50 @@ jobs: name: udeps runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - name: Install cargo-udeps + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: cargo-udeps + tool: cargo-udeps@0.1.61 + checksum: true + fallback: none - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked book: name: book runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: nightly - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -260,9 +315,12 @@ jobs: typos: runs-on: ubuntu-latest permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -271,9 +329,12 @@ jobs: check-toml: runs-on: ubuntu-latest permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -286,28 +347,34 @@ jobs: grafana: runs-on: ubuntu-latest permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Check dashboard JSON with jq - uses: sergeysova/jq-action@a3f0d4ff59cc1dddf023fc0b325dd75b10deec58 # v2.3.0 - with: - cmd: jq empty etc/grafana/dashboards/overview.json + run: jq empty etc/grafana/dashboards/overview.json no-test-deps: runs-on: ubuntu-latest permissions: + id-token: write contents: read timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - name: Ensure no arbitrary or proptest dependency on default build run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0 @@ -315,14 +382,19 @@ jobs: feature-propagation: runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read timeout-minutes: 20 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 @@ -336,7 +408,7 @@ jobs: permissions: contents: read id-token: write - uses: tempoxyz/gh-actions/.github/workflows/rust-deny.yml@d672d5e512852c71bdff2ba3bc48495fae05090b + uses: tempoxyz/gh-actions/.github/workflows/rust-deny.yml@b35994a23aaf6acf4d5e2f91d18235a6e2f90e95 # 2026-09-23T08-07-21Z-b35994a2 with: rust-toolchain: nightly @@ -361,6 +433,6 @@ jobs: timeout-minutes: 30 steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # release/v1 + uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # v1.3.0 with: jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/release-dist.yml b/.github/workflows/release-dist.yml index 5a5a1e12b12..03e25385e70 100644 --- a/.github/workflows/release-dist.yml +++ b/.github/workflows/release-dist.yml @@ -12,8 +12,12 @@ permissions: {} jobs: release-homebrew: runs-on: ubuntu-latest - permissions: {} + permissions: + contents: read + id-token: write steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - name: Update Homebrew formula uses: dawidd6/action-homebrew-bump-formula@b12d2bc99c46444125e606142f28200ec71a1214 # v10 with: diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 98adf750cfb..158dbb2aeb6 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -4,7 +4,7 @@ name: release-reproducible permissions: {} -on: +on: # zizmor: ignore[dangerous-triggers] Only successful same-repository release pushes or manual releases are accepted below. workflow_run: workflows: [release] types: [completed] @@ -15,7 +15,11 @@ env: jobs: extract-version: name: extract version - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name == github.repository && + github.event.workflow_run.path == '.github/workflows/release.yml' && + (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') runs-on: ubuntu-latest permissions: contents: read @@ -37,7 +41,7 @@ jobs: exit 1 fi - echo "VERSION=$TAG" >> $GITHUB_OUTPUT + echo "VERSION=$TAG" >> "$GITHUB_OUTPUT" outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} @@ -46,13 +50,16 @@ jobs: runs-on: ubuntu-latest needs: [extract-version] permissions: + id-token: write packages: write contents: write steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - ref: ${{ needs.extract-version.outputs.VERSION }} + ref: ${{ github.event.workflow_run.head_sha }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 @@ -68,7 +75,7 @@ jobs: id: rust_version run: | RUST_TOOLCHAIN=$(rustc --version | cut -d' ' -f2) - echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_OUTPUT + echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> "$GITHUB_OUTPUT" - name: Build reproducible artifacts uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 @@ -101,29 +108,34 @@ jobs: DOCKER_BUILD_RECORD_UPLOAD: false - name: Prepare artifacts from Docker build + env: + INPUT_VERSION: ${{ needs.extract-version.outputs.VERSION }} run: | mkdir reproducible-artifacts - cp docker-artifacts/reth reproducible-artifacts/reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu + cp docker-artifacts/reth reproducible-artifacts/reth-reproducible-"${INPUT_VERSION}"-x86_64-unknown-linux-gnu cp -R docker-artifacts/LICENSES reproducible-artifacts/LICENSES cp docker-artifacts/README.md reproducible-artifacts/README.md - cp docker-artifacts/*.deb reproducible-artifacts/reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu-reproducible.deb + cp docker-artifacts/*.deb reproducible-artifacts/reth-"${INPUT_VERSION}"-x86_64-unknown-linux-gnu-reproducible.deb - name: Configure GPG and create artifacts env: + INPUT_VERSION: ${{ needs.extract-version.outputs.VERSION }} GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} run: | - export GPG_TTY=$(tty) + GPG_TTY=$(tty) + export GPG_TTY echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import cd reproducible-artifacts - tar -czf reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu LICENSES README.md --remove-files - echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz - echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu-reproducible.deb + tar -czf reth-reproducible-"${INPUT_VERSION}"-x86_64-unknown-linux-gnu.tar.gz reth-reproducible-"${INPUT_VERSION}"-x86_64-unknown-linux-gnu LICENSES README.md --remove-files + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-reproducible-"${INPUT_VERSION}"-x86_64-unknown-linux-gnu.tar.gz + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-"${INPUT_VERSION}"-x86_64-unknown-linux-gnu-reproducible.deb - name: Upload reproducible artifacts to release env: + INPUT_VERSION: ${{ needs.extract-version.outputs.VERSION }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release upload ${{ needs.extract-version.outputs.VERSION }} \ + gh release upload "${INPUT_VERSION}" \ reproducible-artifacts/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 390593f482c..745c5451134 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: - name: Extract version env: REF: ${{ env.RELEASE_REF }} - run: echo "VERSION=${REF#refs/tags/}" >> $GITHUB_OUTPUT + run: echo "VERSION=${REF#refs/tags/}" >> "$GITHUB_OUTPUT" id: extract_version outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} @@ -59,19 +59,26 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + id-token: write needs: extract-version if: ${{ github.event.inputs.dry_run != 'true' }} steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ env.RELEASE_REF }} - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - name: Verify crate version matches tag # Check that the Cargo version starts with the tag, # so that Cargo version 1.4.8 can be matched against both v1.4.8 and v1.4.8-rc.1 + env: + INPUT_VERSION: ${{ needs.extract-version.outputs.VERSION }} run: | - tag="${{ needs.extract-version.outputs.VERSION }}" + tag="${INPUT_VERSION}" tag=${tag#v} cargo_ver=$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version') [[ "$tag" == "$cargo_ver"* ]] || { echo "Tag $tag doesn’t match the Cargo version $cargo_ver"; exit 1; } @@ -80,6 +87,7 @@ jobs: name: build release runs-on: ${{ matrix.configs.os }} permissions: + id-token: write contents: read needs: extract-version continue-on-error: ${{ matrix.configs.allow_fail }} @@ -108,13 +116,16 @@ jobs: - command: build binary: reth steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ env.RELEASE_REF }} - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: stable target: ${{ matrix.configs.target }} - name: Install LLVM if: runner.os == 'Linux' @@ -133,8 +144,8 @@ jobs: - name: Apple M1 setup if: matrix.configs.target == 'aarch64-apple-darwin' run: | - echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV - echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV + echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> "$GITHUB_ENV" + echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> "$GITHUB_ENV" - name: Build Reth env: @@ -154,15 +165,17 @@ jobs: - name: Configure GPG and create artifacts env: + INPUT_VERSION: ${{ needs.extract-version.outputs.VERSION }} GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} run: | - export GPG_TTY=$(tty) + GPG_TTY=$(tty) + export GPG_TTY echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import cd artifacts - tar -czf ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz ${{ matrix.build.binary }}* LICENSES README.md - echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz - mv *tar.gz* .. + tar -czf ${{ matrix.build.binary }}-"${INPUT_VERSION}"-${{ matrix.configs.target }}.tar.gz ${{ matrix.build.binary }}* LICENSES README.md + echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab ${{ matrix.build.binary }}-"${INPUT_VERSION}"-${{ matrix.configs.target }}.tar.gz + mv ./*tar.gz* .. shell: bash - name: Upload artifact @@ -201,11 +214,9 @@ jobs: - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - name: Generate full changelog - id: changelog run: | - echo "CHANGELOG<> $GITHUB_OUTPUT - echo "$(git log --pretty=format:"- %s" $(git describe --tags --abbrev=0 ${{ env.VERSION }}^)..${{ env.VERSION }})" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + previous_tag=$(git describe --tags --abbrev=0 "${VERSION}^") + git log --pretty=format:"- %s" "${previous_tag}..${VERSION}" > "$RUNNER_TEMP/changelog.md" - name: Create release draft env: GITHUB_USER: ${{ github.repository_owner }} @@ -213,11 +224,13 @@ jobs: # The formatting here is borrowed from Lighthouse (which is borrowed from OpenEthereum): # https://github.com/openethereum/openethereum/blob/6c2d392d867b058ff867c4373e40850ca3f96969/.github/workflows/build.yml run: | - prerelease_flag="" - if [[ "${{ env.VERSION }}" == *-rc* ]]; then - prerelease_flag="--prerelease" + prerelease_flags=() + if [[ "$VERSION" == *-rc* ]]; then + prerelease_flags=(--prerelease) fi + CHANGELOG="$(< "$RUNNER_TEMP/changelog.md")" + export CHANGELOG body=$(cat <<- "ENDBODY" ![image](https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-2.png) @@ -256,7 +269,7 @@ jobs: ## All Changes - ${{ steps.changelog.outputs.CHANGELOG }} + ${CHANGELOG} ## Binaries @@ -268,18 +281,19 @@ jobs: | System | Architecture | Binary | PGP Signature | |:---:|:---:|:---:|:---| - | | x86_64 | [reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | - | | aarch64 | [reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | - | | aarch64 | [reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | - | | Docker | [${{ env.IMAGE_NAME }}](${{ env.DOCKER_IMAGE_NAME_URL }}) | - | + | | x86_64 | [reth-${VERSION}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${REPO_NAME}/releases/download/${VERSION}/reth-${VERSION}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${REPO_NAME}/releases/download/${VERSION}/reth-${VERSION}-x86_64-unknown-linux-gnu.tar.gz.asc) | + | | aarch64 | [reth-${VERSION}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${REPO_NAME}/releases/download/${VERSION}/reth-${VERSION}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${REPO_NAME}/releases/download/${VERSION}/reth-${VERSION}-aarch64-unknown-linux-gnu.tar.gz.asc) | + | | aarch64 | [reth-${VERSION}-aarch64-apple-darwin.tar.gz](https://github.com/${REPO_NAME}/releases/download/${VERSION}/reth-${VERSION}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${REPO_NAME}/releases/download/${VERSION}/reth-${VERSION}-aarch64-apple-darwin.tar.gz.asc) | + | | Docker | [${IMAGE_NAME}](${DOCKER_IMAGE_NAME_URL}) | - | ENDBODY ) + body=$(printf '%s' "$body" | python3 -c 'import os, string, sys; print(string.Template(sys.stdin.read()).safe_substitute(os.environ), end="")') assets=() for asset in ./*reth-*.tar.gz*; do assets+=("$asset/$asset") done - tag_name="${{ env.VERSION }}" - echo "$body" | gh release create --draft $prerelease_flag -t "Reth $tag_name" -F "-" "$tag_name" "${assets[@]}" + tag_name="$VERSION" + echo "$body" | gh release create --draft "${prerelease_flags[@]}" -t "Reth $tag_name" -F "-" "$tag_name" "${assets[@]}" dry-run-summary: name: dry run summary diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index 58980f10012..1778ade901d 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -13,6 +13,7 @@ jobs: name: build reproducible binaries runs-on: ${{ matrix.runner }} permissions: + id-token: write contents: read strategy: matrix: @@ -22,12 +23,15 @@ jobs: - runner: ubuntu-22.04 machine: machine-2 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: stable target: x86_64-unknown-linux-gnu - name: Set up Docker Buildx diff --git a/.github/workflows/scan-github-actions.yml b/.github/workflows/scan-github-actions.yml new file mode 100644 index 00000000000..6f251fad0c4 --- /dev/null +++ b/.github/workflows/scan-github-actions.yml @@ -0,0 +1,19 @@ +name: Scan GitHub Actions + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +jobs: + scan: + uses: tempoxyz/gh-actions/.github/workflows/scan-github-actions.yml@b35994a23aaf6acf4d5e2f91d18235a6e2f90e95 # 2026-09-23T08-07-21Z-b35994a2 + permissions: + actions: read + contents: read + id-token: write + with: + secure-runner-fail-on-violation: false diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 54847480ca6..89ac37ff13e 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -27,18 +27,23 @@ jobs: if: github.event_name == 'merge_group' runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_LOG: info,sync=error RUST_BACKTRACE: 1 timeout-minutes: 60 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index 04b90978b77..a573990403f 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -23,6 +23,7 @@ jobs: name: sync (${{ matrix.chain.bin }}) runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_LOG: info,sync=error @@ -38,12 +39,16 @@ jobs: block: 100000 unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a" steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 55cc74c7931..9782ce33043 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -23,6 +23,7 @@ jobs: name: sync (${{ matrix.chain.bin }}) runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_LOG: info,sync=error @@ -38,12 +39,16 @@ jobs: block: 100000 unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a" steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 96aa4d68afd..de697a5395e 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -24,6 +24,7 @@ jobs: name: test / ${{ matrix.type }} runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_BACKTRACE: 1 @@ -36,21 +37,28 @@ jobs: exclude_args: "" timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: prefix-key: v1-rust-unit-targets cache-bin: false cache-on-failure: true - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - name: Install nextest + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: nextest + tool: nextest@0.9.140 + checksum: true + fallback: none - name: Run tests run: | cargo nextest run \ @@ -64,12 +72,15 @@ jobs: name: Ethereum state tests runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_LOG: info,sync=error RUST_BACKTRACE: 1 timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -93,10 +104,15 @@ jobs: curl -L "$URL" | tar -xz --strip-components=1 -C testing/ef-tests/execution-spec-tests - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - name: Install nextest + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 with: - tool: nextest + tool: nextest@0.9.140 + checksum: true + fallback: none - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -107,17 +123,22 @@ jobs: name: doc tests runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} permissions: + id-token: write contents: read env: RUST_BACKTRACE: 1 timeout-minutes: 30 steps: + - name: Secure runner + uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - run: .github/scripts/install_llvm.sh ubuntu - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -133,6 +154,6 @@ jobs: timeout-minutes: 30 steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # release/v1 + uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 # v1.3.0 with: jobs: ${{ toJSON(needs) }} diff --git a/Dockerfile.depot b/Dockerfile.depot index 0ff3f2b736a..a99641d3469 100644 --- a/Dockerfile.depot +++ b/Dockerfile.depot @@ -15,7 +15,7 @@ RUN /tmp/install_llvm.sh && rm /tmp/install_llvm.sh && \ apt-get install -y --no-install-recommends libclang-dev m4 pkg-config # Install sccache for compilation caching -RUN cargo install sccache --locked +RUN cargo install sccache --version "=0.18.0" --locked ENV RUSTC_WRAPPER=sccache ENV SCCACHE_DIR=/sccache ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev From 2bd45d11903fb3ee5775f8c2ad8789dc40b9da05 Mon Sep 17 00:00:00 2001 From: yellowgram Date: Wed, 23 Sep 2026 12:46:48 +0000 Subject: [PATCH 054/133] fix(rpc): apply --rpc.gascap to eth_estimateGas (#27384) Co-authored-by: Cursor Agent Co-authored-by: yellowgram Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/node/core/src/args/rpc_server.rs | 2 +- .../rpc/rpc-eth-api/src/helpers/estimate.rs | 23 ++-- .../rpc/rpc-eth-types/src/builder/config.rs | 5 +- crates/rpc/rpc-eth-types/src/settings.rs | 2 +- crates/rpc/rpc/src/eth/core.rs | 125 +++++++++++++++++- docs/vocs/docs/pages/cli/reth/node.mdx | 2 +- 6 files changed, 136 insertions(+), 23 deletions(-) diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 46aa59c3557..7dbc0aeaa5b 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -636,7 +636,7 @@ pub struct RpcServerArgs { #[arg(long = "rpc.max-logs-per-response", alias = "rpc-max-logs-per-response", value_name = "COUNT", default_value_t = DefaultRpcServerArgs::get_global().rpc_max_logs_per_response)] pub rpc_max_logs_per_response: ZeroAsNoneU64, - /// Maximum gas limit for `eth_call` and call tracing RPC methods. + /// Maximum gas limit for `eth_call`, `eth_estimateGas`, and call tracing RPC methods. #[arg( long = "rpc.gascap", alias = "rpc-gascap", diff --git a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs index 8f66619d4b9..76a05f08de3 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs @@ -96,23 +96,20 @@ pub trait EstimateCall: Call { // the gas limit of the corresponding block let block_gas_limit = evm_env.block_env.gas_limit(); // If EIP-8037 is enabled, the transaction gas limit cap is not applicable - let max_gas_limit = if evm_env.cfg_env.is_amsterdam_eip8037_enabled() { + let mut max_gas_limit = if evm_env.cfg_env.is_amsterdam_eip8037_enabled() { block_gas_limit } else { evm_env.cfg_env.tx_gas_limit_cap().min(block_gas_limit) }; - // Determine the highest possible gas limit, considering both the request's specified limit - // and the block's limit. - let mut highest_gas_limit = tx_request_gas_limit - .map(|mut tx_gas_limit| { - if max_gas_limit < tx_gas_limit { - // requested gas limit is higher than the allowed gas limit, capping - tx_gas_limit = max_gas_limit; - } - tx_gas_limit - }) - .unwrap_or(max_gas_limit); + // Also bound diagnostic retries by the RPC gas cap. Zero means unlimited. + let gas_cap = self.call_gas_limit(); + if gas_cap != 0 { + max_gas_limit = max_gas_limit.min(gas_cap); + } + + let mut highest_gas_limit = + tx_request_gas_limit.unwrap_or(max_gas_limit).min(max_gas_limit); let mut tx_env = self.create_txn_env(&evm_env, request, &mut db)?; @@ -152,7 +149,7 @@ pub trait EstimateCall: Call { // consumed by a successful run is the exact gas required. EIP-2780 can make that less // than 21_000. let mut min_tx_env = tx_env.clone(); - min_tx_env.set_gas_limit(MIN_TRANSACTION_GAS); + min_tx_env.set_gas_limit(MIN_TRANSACTION_GAS.min(max_gas_limit)); // Reuse the same EVM instance if let Ok(res) = evm.transact(min_tx_env).map_err(Self::Error::from_evm_err) && diff --git a/crates/rpc/rpc-eth-types/src/builder/config.rs b/crates/rpc/rpc-eth-types/src/builder/config.rs index edc808bc499..df736ead69a 100644 --- a/crates/rpc/rpc-eth-types/src/builder/config.rs +++ b/crates/rpc/rpc-eth-types/src/builder/config.rs @@ -84,7 +84,7 @@ pub struct EthConfig { pub max_blocks_per_filter: u64, /// Maximum number of logs that can be returned in a single response in `eth_getLogs` calls. pub max_logs_per_response: usize, - /// Gas limit for `eth_call` and call tracing RPC methods. + /// Gas limit for `eth_call`, `eth_estimateGas`, and call tracing RPC methods. /// /// Defaults to [`RPC_DEFAULT_GAS_CAP`] pub rpc_gas_cap: u64, @@ -196,7 +196,8 @@ impl EthConfig { self } - /// Configures the maximum gas limit for `eth_call` and call tracing RPC methods + /// Configures the maximum gas limit for `eth_call`, `eth_estimateGas`, and call tracing RPC + /// methods. pub const fn rpc_gas_cap(mut self, rpc_gas_cap: u64) -> Self { self.rpc_gas_cap = rpc_gas_cap; self diff --git a/crates/rpc/rpc-eth-types/src/settings.rs b/crates/rpc/rpc-eth-types/src/settings.rs index eb150ec0c6f..b0262c6f98f 100644 --- a/crates/rpc/rpc-eth-types/src/settings.rs +++ b/crates/rpc/rpc-eth-types/src/settings.rs @@ -22,7 +22,7 @@ pub struct EthApiSettings { pub max_blocking_io_requests: usize, /// Cache computed block access lists for transaction tracing. pub cache_computed_bals: bool, - /// Maximum gas limit for `eth_call` and call tracing RPC methods. + /// Maximum gas limit for `eth_call`, `eth_estimateGas`, and call tracing RPC methods. pub gas_cap: u64, /// Maximum number of blocks for `eth_simulateV1`. pub max_simulate_blocks: u64, diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index fc9005d877e..fcf25ebe849 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -550,14 +550,18 @@ where mod tests { use crate::{eth::helpers::types::EthRpcConverter, EthApi, EthApiBuilder}; use alloy_consensus::{Block, BlockBody, Header}; - use alloy_eips::BlockNumberOrTag; - use alloy_primitives::{Address, Signature, B256, U256, U64}; + use alloy_eips::{BlockId, BlockNumberOrTag}; + use alloy_network::TransactionBuilder; + use alloy_primitives::{Address, Bytes, Signature, B256, U256, U64}; use alloy_rpc_types::FeeHistory; - use alloy_rpc_types_eth::{Bundle, TransactionRequest}; + use alloy_rpc_types_eth::{ + state::{AccountOverride, EvmOverrides, StateOverride}, + Bundle, TransactionRequest, + }; use jsonrpsee_types::error::INVALID_PARAMS_CODE; use rand::Rng; use reth_chain_state::CanonStateSubscriptions; - use reth_chainspec::{ChainSpec, ChainSpecProvider, EthChainSpec}; + use reth_chainspec::{ChainSpec, ChainSpecBuilder, ChainSpecProvider, EthChainSpec}; use reth_ethereum_primitives::TransactionSigned; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; @@ -565,7 +569,8 @@ mod tests { test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}, PruneCheckpointReader, StageCheckpointReader, }; - use reth_rpc_eth_api::{node::RpcNodeCoreAdapter, EthApiServer}; + use reth_rpc_eth_api::{helpers::EthCall, node::RpcNodeCoreAdapter, EthApiServer}; + use reth_rpc_eth_types::RpcInvalidTransactionError; use reth_storage_api::{ BalProvider, BlockReader, BlockReaderIdExt, NodePrimitivesProvider, StateProviderFactory, }; @@ -606,6 +611,17 @@ mod tests { .build() } + fn build_test_eth_api_with_gas_cap(provider: MockEthProvider, gas_cap: u64) -> FakeEthApi { + EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .gas_cap(gas_cap.into()) + .build() + } + #[tokio::test] async fn test_transaction_by_sender_and_nonce_without_sender_transaction_returns_none() { use reth_rpc_eth_api::helpers::EthTransactions; @@ -1062,4 +1078,103 @@ mod tests { "all: no percentiles were requested, so there should be no rewards result" ); } + + #[tokio::test] + async fn estimate_gas_respects_rpc_gas_cap() { + const LOW_GAS_CAP: u64 = 50_000; + const HIGH_GAS_CAP: u64 = 200_000; + // Revert unless GAS > 60_000. Retrying above the RPC cap would succeed and + // incorrectly turn the revert into an out-of-gas error. + let code = Bytes::from_static(&[ + 0x5a, 0x61, 0xea, 0x60, 0x10, 0x60, 0x0d, 0x57, 0x60, 0x00, 0x60, 0x00, 0xfd, 0x5b, + 0x00, + ]); + let sender = Address::repeat_byte(0x11); + let contract = Address::repeat_byte(0xaa); + let mut state_override = StateOverride::default(); + state_override.insert(contract, AccountOverride { code: Some(code), ..Default::default() }); + let overrides = EvmOverrides::state(Some(state_override)); + let at = BlockId::Number(BlockNumberOrTag::Latest); + + for chain_spec in [ + ChainSpecBuilder::mainnet().cancun_activated().build(), + ChainSpecBuilder::mainnet().osaka_activated().build(), + ChainSpecBuilder::mainnet().amsterdam_activated().build(), + ] { + let provider = MockEthProvider::default().with_chain_spec(chain_spec); + provider.add_account(sender, ExtendedAccount::new(0, U256::MAX)); + provider.add_block( + B256::repeat_byte(0x42), + Block { + header: Header { + number: 1, + gas_limit: 30_000_000, + excess_blob_gas: Some(0), + ..Default::default() + }, + body: BlockBody::default(), + }, + ); + + for gas in [None, Some(1_000_000)] { + let request = TransactionRequest { + gas, + ..TransactionRequest::default().with_from(sender).with_to(contract) + }; + let capped = build_test_eth_api_with_gas_cap(provider.clone(), LOW_GAS_CAP); + let err = EthCall::estimate_gas_at(&capped, request.clone(), at, overrides.clone()) + .await + .expect_err("estimation above the RPC gas cap must fail"); + assert!( + matches!( + err.as_invalid_transaction(), + Some(RpcInvalidTransactionError::Revert(_)) + ), + "{err}" + ); + + for gas_cap in [HIGH_GAS_CAP, 0] { + let relaxed = build_test_eth_api_with_gas_cap(provider.clone(), gas_cap); + let estimated = + EthCall::estimate_gas_at(&relaxed, request.clone(), at, overrides.clone()) + .await + .expect("same call fits under a higher or unlimited RPC gas cap"); + assert!(estimated > U256::from(LOW_GAS_CAP), "{estimated}"); + assert!(estimated <= U256::from(HIGH_GAS_CAP), "{estimated}"); + } + } + } + } + + #[tokio::test] + async fn estimate_gas_transfer_respects_rpc_gas_cap() { + let provider = MockEthProvider::default(); + provider.add_block( + B256::repeat_byte(0x42), + Block { + header: Header { number: 1, gas_limit: 30_000_000, ..Default::default() }, + body: BlockBody::default(), + }, + ); + let request = TransactionRequest::default().with_to(Address::repeat_byte(0xaa)); + let at = BlockId::Number(BlockNumberOrTag::Latest); + + let capped = build_test_eth_api_with_gas_cap(provider.clone(), 20_999); + let err = EthCall::estimate_gas_at(&capped, request.clone(), at, EvmOverrides::default()) + .await + .expect_err("the transfer shortcut must respect the RPC gas cap"); + assert!( + matches!( + err.as_invalid_transaction(), + Some(RpcInvalidTransactionError::GasRequiredExceedsAllowance { gas_limit: 20_999 }) + ), + "{err}" + ); + + let sufficient = build_test_eth_api_with_gas_cap(provider, 21_000); + let estimated = EthCall::estimate_gas_at(&sufficient, request, at, EvmOverrides::default()) + .await + .unwrap(); + assert_eq!(estimated, U256::from(21_000)); + } } diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index e1f1e8f4ee6..4f408d770a8 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -486,7 +486,7 @@ RPC: [default: 20000] --rpc.gascap - Maximum gas limit for `eth_call` and call tracing RPC methods + Maximum gas limit for `eth_call`, `eth_estimateGas`, and call tracing RPC methods [default: 50000000] From fa1677005bee3876fd6565bb808da45d49ba213a Mon Sep 17 00:00:00 2001 From: Karl Yu <43113774+0xKarl98@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:09:13 +0000 Subject: [PATCH 055/133] feat(rpc): add cache byte limits and idle eviction (#27351) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/node/core/src/args/database.rs | 6 +- crates/node/core/src/args/rpc_server.rs | 1 + crates/node/core/src/args/rpc_state_cache.rs | 172 +++++++- crates/rpc/rpc-builder/src/config.rs | 53 ++- crates/rpc/rpc-eth-types/Cargo.toml | 1 + crates/rpc/rpc-eth-types/src/cache/config.rs | 86 +++- crates/rpc/rpc-eth-types/src/cache/mod.rs | 412 ++++++++++++++---- .../rpc-eth-types/src/cache/multi_consumer.rs | 355 +++++++++++---- crates/rpc/rpc-server-types/src/constants.rs | 14 + docs/vocs/docs/pages/cli/reth/node.mdx | 20 + 10 files changed, 954 insertions(+), 166 deletions(-) diff --git a/crates/node/core/src/args/database.rs b/crates/node/core/src/args/database.rs index abe7d17cd9a..6554f2acabc 100644 --- a/crates/node/core/src/args/database.rs +++ b/crates/node/core/src/args/database.rs @@ -189,7 +189,9 @@ impl FromStr for ByteSize { _ => return Err(format!("Invalid unit: {unit}. Use B, KB, MB, GB, or TB.")), }; - Ok(Self(num * multiplier)) + num.checked_mul(multiplier) + .map(Self) + .ok_or_else(|| "Byte size exceeds the maximum supported value".to_string()) } } @@ -217,7 +219,7 @@ impl fmt::Display for ByteSize { } /// Value parser function that supports various formats. -fn parse_byte_size(s: &str) -> Result { +pub(super) fn parse_byte_size(s: &str) -> Result { s.parse::().map(Into::into) } diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 7dbc0aeaa5b..86d01bf434c 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -1262,6 +1262,7 @@ mod tests { prewarm_bals: Some(0), max_concurrent_db_requests: 512, max_cached_tx_hashes: 100_000, + ..Default::default() }, gas_price_oracle: GasPriceOracleArgs { blocks: 20, diff --git a/crates/node/core/src/args/rpc_state_cache.rs b/crates/node/core/src/args/rpc_state_cache.rs index 0c691e3d001..1f61dcdd6a0 100644 --- a/crates/node/core/src/args/rpc_state_cache.rs +++ b/crates/node/core/src/args/rpc_state_cache.rs @@ -1,8 +1,12 @@ +use super::database::parse_byte_size; use clap::Args; use reth_rpc_server_types::constants::cache::{ - DEFAULT_BAL_CACHE_MAX_LEN, DEFAULT_BLOCK_CACHE_MAX_LEN, DEFAULT_CONCURRENT_DB_REQUESTS, - DEFAULT_HEADER_CACHE_MAX_LEN, DEFAULT_MAX_CACHED_TX_HASHES, DEFAULT_RECEIPT_CACHE_MAX_LEN, + DEFAULT_BAL_CACHE_MAX_BYTES, DEFAULT_BAL_CACHE_MAX_LEN, DEFAULT_BLOCK_CACHE_MAX_BYTES, + DEFAULT_BLOCK_CACHE_MAX_LEN, DEFAULT_CACHE_IDLE_TIMEOUT, DEFAULT_CONCURRENT_DB_REQUESTS, + DEFAULT_HEADER_CACHE_MAX_LEN, DEFAULT_MAX_CACHED_TX_HASHES, DEFAULT_RECEIPT_CACHE_MAX_BYTES, + DEFAULT_RECEIPT_CACHE_MAX_LEN, }; +use std::time::Duration; /// Parameters to configure RPC state cache. #[derive(Debug, Clone, Args, PartialEq, Eq)] @@ -38,6 +42,27 @@ pub struct RpcStateCacheArgs { )] pub max_bals: u32, + /// Maximum estimated block cache memory in bytes or with a unit (e.g. 1GB, 500MB). + /// Units use powers of 1024. Zero disables caching. The entry count limit also applies. + #[arg(long = "rpc-cache.max-blocks-bytes", value_name = "BYTES", value_parser = parse_byte_size, default_value_t = DEFAULT_BLOCK_CACHE_MAX_BYTES)] + pub max_blocks_bytes: usize, + + /// Maximum estimated receipts cache memory in bytes or with a unit (e.g. 1GB, 500MB). + /// Units use powers of 1024. Zero disables caching. The entry count limit also applies. + #[arg(long = "rpc-cache.max-receipts-bytes", value_name = "BYTES", value_parser = parse_byte_size, default_value_t = DEFAULT_RECEIPT_CACHE_MAX_BYTES)] + pub max_receipts_bytes: usize, + + /// Maximum estimated block access list cache memory in bytes or with a unit (e.g. 1GB, 500MB). + /// Units use powers of 1024. Zero disables caching. The entry count limit also applies. + #[arg(long = "rpc-cache.max-bals-bytes", value_name = "BYTES", value_parser = parse_byte_size, default_value_t = DEFAULT_BAL_CACHE_MAX_BYTES)] + pub max_bals_bytes: usize, + + /// Evict blocks, receipts, and block access lists after this duration without a cache hit + /// (e.g. 5m, 30s). Zero disables expiration. Cleanup starts every 1 to 60 seconds and removes + /// at most five entries per cache per poll. + #[arg(long = "rpc-cache.idle-timeout", value_name = "DURATION", value_parser = humantime::parse_duration, default_value = "1h")] + pub idle_timeout: Duration, + /// Cache block access lists computed by RPC requests for transaction tracing. #[arg(long = "rpc-cache.cache-computed-bals")] pub cache_computed_bals: bool, @@ -78,6 +103,10 @@ impl RpcStateCacheArgs { self.max_receipts = 0; self.max_headers = 0; self.max_bals = 0; + self.max_blocks_bytes = 0; + self.max_receipts_bytes = 0; + self.max_bals_bytes = 0; + self.idle_timeout = Duration::ZERO; self.cache_computed_bals = false; self.prewarm_bals = None; } @@ -90,6 +119,10 @@ impl Default for RpcStateCacheArgs { max_receipts: DEFAULT_RECEIPT_CACHE_MAX_LEN, max_headers: DEFAULT_HEADER_CACHE_MAX_LEN, max_bals: DEFAULT_BAL_CACHE_MAX_LEN, + max_blocks_bytes: DEFAULT_BLOCK_CACHE_MAX_BYTES, + max_receipts_bytes: DEFAULT_RECEIPT_CACHE_MAX_BYTES, + max_bals_bytes: DEFAULT_BAL_CACHE_MAX_BYTES, + idle_timeout: DEFAULT_CACHE_IDLE_TIMEOUT, cache_computed_bals: false, prewarm_bals: None, max_concurrent_db_requests: DEFAULT_CONCURRENT_DB_REQUESTS, @@ -97,3 +130,138 @@ impl Default for RpcStateCacheArgs { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::node_config::NodeConfig; + use clap::Parser; + + #[derive(Parser)] + struct CommandParser { + #[command(flatten)] + args: RpcStateCacheArgs, + } + + #[test] + fn rpc_cache_defaults() { + let args = CommandParser::parse_from(["reth"]).args; + assert_eq!(args, RpcStateCacheArgs::default()); + + for flag in ["--rpc-cache.max-headers", "--rpc-cache.max-envs"] { + let args = CommandParser::parse_from(["reth", flag, "123"]).args; + assert_eq!(args.max_headers, 123); + } + } + + #[test] + fn rpc_cache_byte_limits_accept_zero_and_maximum() { + for limit in [0, 1024, usize::MAX] { + let value = limit.to_string(); + let args = CommandParser::parse_from([ + "reth", + "--rpc-cache.max-blocks-bytes", + &value, + "--rpc-cache.max-receipts-bytes", + &value, + "--rpc-cache.max-bals-bytes", + &value, + ]) + .args; + assert_eq!(args.max_blocks_bytes, limit); + assert_eq!(args.max_receipts_bytes, limit); + assert_eq!(args.max_bals_bytes, limit); + } + } + + #[test] + fn rpc_cache_byte_limits_reject_invalid_values() { + let overflow = format!("{}0", usize::MAX); + let unit_overflow = format!("{}GB", usize::MAX); + for flag in [ + "--rpc-cache.max-blocks-bytes", + "--rpc-cache.max-receipts-bytes", + "--rpc-cache.max-bals-bytes", + ] { + for value in ["-1", "-1GB", "1XB", "1.5GB", &overflow, &unit_overflow] { + assert!( + CommandParser::try_parse_from(["reth", &format!("{flag}={value}")]).is_err(), + "accepted {flag}={value}" + ); + } + } + } + + #[test] + fn rpc_cache_byte_limits_accept_units() { + for (value, expected) in [ + ("0GB", 0), + ("1024B", 1024), + ("1KB", 1024), + ("500MB", 500 * 1024 * 1024), + ("1GB", 1024 * 1024 * 1024), + ("1gb", 1024 * 1024 * 1024), + (" 500 MB ", 500 * 1024 * 1024), + ("1TB", 1024usize.pow(4)), + ] { + let args = CommandParser::parse_from([ + "reth", + "--rpc-cache.max-blocks-bytes", + value, + "--rpc-cache.max-receipts-bytes", + value, + "--rpc-cache.max-bals-bytes", + value, + ]) + .args; + assert_eq!(args.max_blocks_bytes, expected); + assert_eq!(args.max_receipts_bytes, expected); + assert_eq!(args.max_bals_bytes, expected); + } + } + + #[test] + fn rpc_cache_idle_timeout_parses_durations() { + for (value, expected) in [ + ("0s", Duration::ZERO), + ("500ms", Duration::from_millis(500)), + ("30s", Duration::from_secs(30)), + ("5m", Duration::from_secs(300)), + ] { + let args = CommandParser::parse_from(["reth", "--rpc-cache.idle-timeout", value]).args; + assert_eq!(args.idle_timeout, expected); + } + for value in ["-1s", "invalid", "18446744073709551616s"] { + assert!(CommandParser::try_parse_from([ + "reth", + &format!("--rpc-cache.idle-timeout={value}") + ]) + .is_err()); + } + } + + #[test] + fn disabling_rpc_cache_overrides_positive_byte_limits() { + let mut config = NodeConfig::default(); + config.rpc.rpc_state_cache = RpcStateCacheArgs { + max_blocks_bytes: 1024, + max_receipts_bytes: 2048, + max_bals_bytes: 4096, + idle_timeout: Duration::from_secs(60), + cache_computed_bals: true, + prewarm_bals: Some(10), + ..Default::default() + }; + + let args = config.with_disabled_rpc_cache().rpc.rpc_state_cache; + assert_eq!(args.max_blocks, 0); + assert_eq!(args.max_receipts, 0); + assert_eq!(args.max_bals, 0); + assert_eq!(args.max_blocks_bytes, 0); + assert_eq!(args.max_receipts_bytes, 0); + assert_eq!(args.max_bals_bytes, 0); + assert_eq!(args.idle_timeout, Duration::ZERO); + assert!(!args.cache_computed_bals); + assert_eq!(args.prewarm_bals, None); + } +} diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index de5cfa0bd6c..42e4d35576b 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -126,6 +126,10 @@ impl RethRpcServerConfig for RpcServerArgs { max_blocks: self.rpc_state_cache.max_blocks, max_receipts: self.rpc_state_cache.max_receipts, max_bals: self.rpc_state_cache.max_bals, + max_blocks_bytes: self.rpc_state_cache.max_blocks_bytes, + max_receipts_bytes: self.rpc_state_cache.max_receipts_bytes, + max_bals_bytes: self.rpc_state_cache.max_bals_bytes, + idle_timeout: self.rpc_state_cache.idle_timeout, cache_computed_bals: self.rpc_state_cache.cache_computed_bals, prewarm_bals: self.rpc_state_cache.prewarm_bals, max_concurrent_db_requests: self.rpc_state_cache.max_concurrent_db_requests, @@ -279,10 +283,13 @@ impl RethRpcServerConfig for RpcServerArgs { mod tests { use clap::{Args, Parser}; use reth_node_core::args::RpcServerArgs; - use reth_rpc_eth_types::RPC_DEFAULT_GAS_CAP; + use reth_rpc_eth_types::{EthStateCacheConfig, RPC_DEFAULT_GAS_CAP}; use reth_rpc_layer::JwtSecret; use reth_rpc_server_types::{constants, RethRpcModule, RpcModuleSelection}; - use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; + use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + time::Duration, + }; use crate::config::RethRpcServerConfig; @@ -293,6 +300,48 @@ mod tests { args: T, } + #[test] + fn rpc_cache_limits_reach_eth_config() { + let args = CommandParser::::parse_from(["reth"]).args; + assert_eq!(args.eth_config().cache, EthStateCacheConfig::default()); + + for (timeout, expected_timeout) in + [("0s", Duration::ZERO), ("2500ms", Duration::from_millis(2500))] + { + let args = CommandParser::::parse_from([ + "reth", + "--rpc-cache.max-blocks", + "10", + "--rpc-cache.max-receipts", + "20", + "--rpc-cache.max-bals", + "30", + "--rpc-cache.max-blocks-bytes", + "1024", + "--rpc-cache.max-receipts-bytes", + "0", + "--rpc-cache.max-bals-bytes", + "4096", + "--rpc-cache.idle-timeout", + timeout, + ]) + .args; + assert_eq!( + args.eth_config().cache, + EthStateCacheConfig { + max_blocks: 10, + max_receipts: 20, + max_bals: 30, + max_blocks_bytes: 1024, + max_receipts_bytes: 0, + max_bals_bytes: 4096, + idle_timeout: expected_timeout, + ..Default::default() + } + ); + } + } + #[test] fn test_rpc_gas_cap() { let args = CommandParser::::parse_from(["reth"]).args; diff --git a/crates/rpc/rpc-eth-types/Cargo.toml b/crates/rpc/rpc-eth-types/Cargo.toml index 66179c7c9c5..70996db5870 100644 --- a/crates/rpc/rpc-eth-types/Cargo.toml +++ b/crates/rpc/rpc-eth-types/Cargo.toml @@ -70,6 +70,7 @@ itertools.workspace = true reth-db-models.workspace = true reth-storage-api = { workspace = true, features = ["std"] } serde_json.workspace = true +tokio = { workspace = true, features = ["test-util"] } [features] js-tracer = ["revm-inspectors/js-tracer"] diff --git a/crates/rpc/rpc-eth-types/src/cache/config.rs b/crates/rpc/rpc-eth-types/src/cache/config.rs index 5fa675b276f..19a1868f6a4 100644 --- a/crates/rpc/rpc-eth-types/src/cache/config.rs +++ b/crates/rpc/rpc-eth-types/src/cache/config.rs @@ -1,15 +1,20 @@ //! Configuration for RPC cache. use serde::{Deserialize, Serialize}; +use std::time::Duration; use reth_rpc_server_types::constants::cache::{ - DEFAULT_BAL_CACHE_MAX_LEN, DEFAULT_BLOCK_CACHE_MAX_LEN, DEFAULT_CONCURRENT_DB_REQUESTS, - DEFAULT_MAX_CACHED_TX_HASHES, DEFAULT_RECEIPT_CACHE_MAX_LEN, + DEFAULT_BAL_CACHE_MAX_BYTES, DEFAULT_BAL_CACHE_MAX_LEN, DEFAULT_BLOCK_CACHE_MAX_BYTES, + DEFAULT_BLOCK_CACHE_MAX_LEN, DEFAULT_CACHE_IDLE_TIMEOUT, DEFAULT_CONCURRENT_DB_REQUESTS, + DEFAULT_MAX_CACHED_TX_HASHES, DEFAULT_RECEIPT_CACHE_MAX_BYTES, DEFAULT_RECEIPT_CACHE_MAX_LEN, }; /// Settings for the [`EthStateCache`](super::EthStateCache). +/// +/// Byte budgets cover estimated payload sizes, excluding map and request queue overhead. Shared +/// payloads can remain allocated after eviction while other components still hold references. #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(default, rename_all = "camelCase")] pub struct EthStateCacheConfig { /// Max number of blocks in cache. /// @@ -23,6 +28,24 @@ pub struct EthStateCacheConfig { /// /// Default is 1000. pub max_bals: u32, + /// Maximum estimated memory retained by the block cache, in bytes. + /// + /// Defaults to 2 GiB. Zero disables caching. The entry count limit also applies. + pub max_blocks_bytes: usize, + /// Maximum estimated memory retained by the receipts cache, in bytes. + /// + /// Defaults to 1 GiB. Zero disables caching. The entry count limit also applies. + pub max_receipts_bytes: usize, + /// Maximum estimated memory retained by the BAL cache, in bytes. + /// + /// Defaults to 500 MiB. Zero disables caching. The entry count limit also applies. + pub max_bals_bytes: usize, + /// Duration after which an unused block, receipts collection, or BAL is evicted. + /// + /// Defaults to one hour. Successful lookups refresh the timeout. Zero disables expiration. + /// Periodic cleanup starts at least once per minute and removes expired entries in bounded + /// batches. + pub idle_timeout: Duration, /// Cache BALs computed by RPC requests for transaction tracing. Disabled by default. #[serde(default)] pub cache_computed_bals: bool, @@ -46,6 +69,10 @@ impl Default for EthStateCacheConfig { max_blocks: DEFAULT_BLOCK_CACHE_MAX_LEN, max_receipts: DEFAULT_RECEIPT_CACHE_MAX_LEN, max_bals: DEFAULT_BAL_CACHE_MAX_LEN, + max_blocks_bytes: DEFAULT_BLOCK_CACHE_MAX_BYTES, + max_receipts_bytes: DEFAULT_RECEIPT_CACHE_MAX_BYTES, + max_bals_bytes: DEFAULT_BAL_CACHE_MAX_BYTES, + idle_timeout: DEFAULT_CACHE_IDLE_TIMEOUT, cache_computed_bals: false, prewarm_bals: None, max_concurrent_db_requests: DEFAULT_CONCURRENT_DB_REQUESTS, @@ -53,3 +80,56 @@ impl Default for EthStateCacheConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn legacy_cache_config_uses_default_limits() { + let config: EthStateCacheConfig = serde_json::from_value(serde_json::json!({ + "maxBlocks": 1000, + "maxReceipts": 500, + "maxBals": 1000, + "maxConcurrentDbRequests": 512, + "maxCachedTxHashes": 100000 + })) + .unwrap(); + + assert_eq!(config, EthStateCacheConfig::default()); + assert_eq!(config.max_blocks_bytes, DEFAULT_BLOCK_CACHE_MAX_BYTES); + assert_eq!(config.max_receipts_bytes, DEFAULT_RECEIPT_CACHE_MAX_BYTES); + assert_eq!(config.max_bals_bytes, DEFAULT_BAL_CACHE_MAX_BYTES); + assert_eq!(config.idle_timeout, DEFAULT_CACHE_IDLE_TIMEOUT); + } + + #[test] + fn cache_memory_limits_and_idle_timeout_round_trip() { + for config in [ + EthStateCacheConfig::default(), + EthStateCacheConfig { + max_blocks_bytes: 1024, + max_receipts_bytes: 2048, + max_bals_bytes: 4096, + idle_timeout: Duration::from_millis(1500), + ..Default::default() + }, + EthStateCacheConfig { + max_blocks_bytes: 0, + max_receipts_bytes: 0, + max_bals_bytes: 0, + idle_timeout: Duration::ZERO, + ..Default::default() + }, + ] { + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["maxBlocksBytes"], serde_json::json!(config.max_blocks_bytes)); + assert_eq!( + serialized["maxReceiptsBytes"], + serde_json::json!(config.max_receipts_bytes) + ); + assert_eq!(serialized["maxBalsBytes"], serde_json::json!(config.max_bals_bytes)); + assert_eq!(serde_json::from_value::(serialized).unwrap(), config); + } + } +} diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index 67d5b28bd9d..44d23553f33 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -21,17 +21,22 @@ use reth_revm::{ }; use reth_storage_api::{BalProvider, BlockReader, TransactionVariant}; use reth_tasks::Runtime; -use schnellru::{ByLength, Limiter, LruMap}; +use schnellru::{ByLength, LruMap}; use std::{ + cell::LazyCell, collections::VecDeque, future::Future, pin::Pin, sync::Arc, task::{Context, Poll}, + time::Duration, }; -use tokio::sync::{ - mpsc::{unbounded_channel, UnboundedSender}, - oneshot, OwnedSemaphorePermit, Semaphore, +use tokio::{ + sync::{ + mpsc::{unbounded_channel, UnboundedSender}, + oneshot, OwnedSemaphorePermit, Semaphore, + }, + time::{Instant, Interval, MissedTickBehavior}, }; use tokio_stream::wrappers::UnboundedReceiverStream; @@ -60,13 +65,12 @@ type TransactionHashResponseSender = oneshot::Sender>>; -type BlockLruCache = - MultiConsumerLruCache>, L, BlockWithSendersResponseSender>; +type BlockLruCache = + MultiConsumerLruCache>, BlockWithSendersResponseSender>; -type ReceiptsLruCache = - MultiConsumerLruCache>, L, ReceiptsResponseSender>; +type ReceiptsLruCache = MultiConsumerLruCache>, ReceiptsResponseSender>; -type BalLruCache = MultiConsumerLruCache; +type BalLruCache = MultiConsumerLruCache; /// Provides async access to cached eth data /// @@ -89,7 +93,7 @@ impl EthStateCache { provider: Provider, action_task_spawner: Runtime, config: EthStateCacheConfig, - ) -> (Self, EthStateCacheService) + ) -> (Self, EthStateCacheService) where Provider: BlockReader + BalProvider, { @@ -97,18 +101,33 @@ impl EthStateCache { max_blocks, max_receipts, max_bals, + max_blocks_bytes, + max_receipts_bytes, + max_bals_bytes, + idle_timeout, cache_computed_bals: _, prewarm_bals: _, max_concurrent_db_requests, max_cached_tx_hashes, } = config; + let eviction_interval = { + let _guard = action_task_spawner.handle().enter(); + // Clamp short timeouts to avoid busy polling. Expiration is best effort between ticks. + let period = (idle_timeout / 4).clamp(Duration::from_secs(1), Duration::from_secs(60)); + let mut interval = tokio::time::interval_at(Instant::now() + period, period); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + interval + }; let (to_service, rx) = unbounded_channel(); let service = EthStateCacheService { provider, - full_block_cache: BlockLruCache::new(max_blocks, "blocks"), - receipts_cache: ReceiptsLruCache::new(max_receipts, "receipts"), - bal_cache: BalLruCache::new(max_bals, "bals"), + full_block_cache: BlockLruCache::new(max_blocks, max_blocks_bytes, "blocks"), + receipts_cache: ReceiptsLruCache::new(max_receipts, max_receipts_bytes, "receipts"), + bal_cache: BalLruCache::new(max_bals, max_bals_bytes, "bals"), + idle_timeout, + eviction_interval, + eviction_pending: false, action_tx: to_service.clone(), action_rx: UnboundedReceiverStream::new(rx), action_task_spawner, @@ -123,7 +142,7 @@ impl EthStateCache { /// Creates a new async LRU backed cache service task and spawns it to a new task via the given /// spawner. /// - /// The cache is memory limited by the given max bytes values. + /// Entry counts and estimated payload bytes bound the cache. Idle entries expire periodically. pub fn spawn_with( provider: Provider, config: EthStateCacheConfig, @@ -297,32 +316,30 @@ impl From for ProviderError { /// `reth_rpc::EthApi` which is typically invoked by the RPC server, which already uses /// permits to limit concurrent requests. #[must_use = "Type does nothing unless spawned"] -pub(crate) struct EthStateCacheService< - Provider, - Tasks, - LimitBlocks = ByLength, - LimitReceipts = ByLength, - LimitBals = ByLength, -> where +pub(crate) struct EthStateCacheService +where Provider: BlockReader + BalProvider, - LimitBlocks: Limiter>>, - LimitReceipts: Limiter>>, - LimitBals: Limiter, { /// The type used to lookup data from disk provider: Provider, /// The LRU cache for full blocks grouped by their block hash. - full_block_cache: BlockLruCache, + full_block_cache: BlockLruCache, /// The LRU cache for block receipts grouped by the block hash. - receipts_cache: ReceiptsLruCache, + receipts_cache: ReceiptsLruCache, /// The LRU cache for revm BALs grouped by the block hash. - bal_cache: BalLruCache, + bal_cache: BalLruCache, + /// Maximum time without a cache hit. Zero disables idle eviction. + idle_timeout: Duration, + /// Reclaims idle entries even when there are no requests or new blocks. + eviction_interval: Interval, + /// Whether an idle sweep has more entries to remove in a subsequent poll. + eviction_pending: bool, /// Sender half of the action channel. action_tx: UnboundedSender>, /// Receiver half of the action channel. action_rx: UnboundedReceiverStream>, /// The type that's used to spawn tasks that do the actual work - action_task_spawner: Tasks, + action_task_spawner: Runtime, /// Rate limiter for spawned fetch tasks. /// /// This restricts the max concurrent fetch tasks at the same time. @@ -336,7 +353,7 @@ pub(crate) struct EthStateCacheService< tx_hash_index: LruMap, } -impl EthStateCacheService +impl EthStateCacheService where Provider: BlockReader + BalProvider + Clone + Unpin + 'static, { @@ -424,6 +441,7 @@ where &mut self, block_hash: B256, res: ProviderResult>>>, + now: Instant, ) { if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders @@ -434,7 +452,7 @@ where // cache good block if let Ok(Some(block)) = res { - self.full_block_cache.insert(block_hash, block); + self.full_block_cache.insert_at(block_hash, block, now); } } @@ -442,6 +460,7 @@ where &mut self, block_hash: B256, res: ProviderResult>>>, + now: Instant, ) { if let Some(queued) = self.receipts_cache.remove(&block_hash) { // send the response to queued senders @@ -452,11 +471,16 @@ where // cache good receipts if let Ok(Some(receipts)) = res { - self.receipts_cache.insert(block_hash, receipts); + self.receipts_cache.insert_at(block_hash, receipts, now); } } - fn on_new_bal(&mut self, block_hash: B256, res: ProviderResult>) { + fn on_new_bal( + &mut self, + block_hash: B256, + res: ProviderResult>, + now: Instant, + ) { // A local replay may finish before an older provider lookup returns. if self.bal_cache.contains_key(&block_hash) { return @@ -469,7 +493,7 @@ where } if let Ok(Some(bal)) = res { - self.bal_cache.insert(block_hash, bal); + self.bal_cache.insert_at(block_hash, bal, now); } } @@ -522,7 +546,7 @@ where } } -impl Future for EthStateCacheService +impl Future for EthStateCacheService where Provider: BlockReader + BalProvider + Clone + Unpin + 'static, { @@ -530,6 +554,23 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); + // All inserts and hits in this poll share a clock read, preserving timestamp/LRU order. + let now = LazyCell::new(Instant::now); + + // Poll until pending to register the next timer wakeup before draining requests. + while !this.idle_timeout.is_zero() && this.eviction_interval.poll_tick(cx).is_ready() { + this.eviction_pending = true; + } + if this.eviction_pending { + let now = *now; + this.eviction_pending = this.full_block_cache.evict_expired(now, this.idle_timeout); + this.eviction_pending |= this.receipts_cache.evict_expired(now, this.idle_timeout); + this.eviction_pending |= this.bal_cache.evict_expired(now, this.idle_timeout); + if this.eviction_pending { + // Process requests before continuing the sweep in another poll. + cx.waker().wake_by_ref(); + } + } loop { let Poll::Ready(action) = this.action_rx.poll_next_unpin(cx) else { @@ -546,21 +587,25 @@ where unreachable!("can't close") } Some(action) => { + let now = *now; match action { CacheAction::GetCachedBlock { block_hash, response_tx } => { - let _ = - response_tx.send(this.full_block_cache.get(&block_hash).cloned()); + let _ = response_tx + .send(this.full_block_cache.get_at(&block_hash, now).cloned()); } CacheAction::GetCachedBal { block_hash, response_tx } => { - let _ = response_tx.send(this.bal_cache.get(&block_hash).cloned()); + let _ = + response_tx.send(this.bal_cache.get_at(&block_hash, now).cloned()); } CacheAction::GetCachedBlockAndReceipts { block_hash, response_tx } => { - let block = this.full_block_cache.get(&block_hash).cloned(); - let receipts = this.receipts_cache.get(&block_hash).cloned(); + let block = this.full_block_cache.get_at(&block_hash, now).cloned(); + let receipts = this.receipts_cache.get_at(&block_hash, now).cloned(); let _ = response_tx.send((block, receipts)); } CacheAction::GetBlockWithSenders { block_hash, response_tx } => { - if let Some(block) = this.full_block_cache.get(&block_hash).cloned() { + if let Some(block) = + this.full_block_cache.get_at(&block_hash, now).cloned() + { let _ = response_tx.send(Ok(Some(block))); continue } @@ -572,7 +617,9 @@ where } CacheAction::GetReceipts { block_hash, response_tx } => { // check if block is cached - if let Some(receipts) = this.receipts_cache.get(&block_hash).cloned() { + if let Some(receipts) = + this.receipts_cache.get_at(&block_hash, now).cloned() + { let _ = response_tx.send(Ok(Some(receipts))); continue } @@ -583,7 +630,7 @@ where } } CacheAction::GetBal { block_hash, response_tx } => { - if let Some(bal) = this.bal_cache.get(&block_hash).cloned() { + if let Some(bal) = this.bal_cache.get_at(&block_hash, now).cloned() { let _ = response_tx.send(Ok(Some(bal))); continue } @@ -593,36 +640,29 @@ where } } CacheAction::ReceiptsResult { block_hash, res } => { - this.on_new_receipts(block_hash, res); + this.on_new_receipts(block_hash, res, now); } CacheAction::BalResult { block_hash, res } => { - this.on_new_bal(block_hash, res); + this.on_new_bal(block_hash, res, now); } CacheAction::InsertBal { block_hash, bal } => { - this.on_new_bal(block_hash, Ok(Some(bal))); + this.on_new_bal(block_hash, Ok(Some(bal)), now); + } + CacheAction::BlockWithSendersResult { block_hash, res } => { + this.on_new_block(block_hash, res, now); } - CacheAction::BlockWithSendersResult { block_hash, res } => match res { - Ok(Some(block_with_senders)) => { - this.on_new_block(block_hash, Ok(Some(block_with_senders))); - } - Ok(None) => { - this.on_new_block(block_hash, Ok(None)); - } - Err(e) => { - this.on_new_block(block_hash, Err(e)); - } - }, CacheAction::CacheNewCanonicalChain { chain_change } => { for block in chain_change.blocks { // Index transactions before caching the block this.index_block_transactions(&block); - this.on_new_block(block.hash(), Ok(Some(block))); + this.on_new_block(block.hash(), Ok(Some(block)), now); } for block_receipts in chain_change.receipts { this.on_new_receipts( block_receipts.block_hash, Ok(Some(block_receipts.receipts)), + now, ); } @@ -630,6 +670,7 @@ where this.on_new_bal( block_hash, Ok(Some(CachedRevmBal::from_shared(bal))), + now, ); } } @@ -652,8 +693,10 @@ where CacheAction::GetTransactionByHash { tx_hash, response_tx } => { let result = this.tx_hash_index.get(&tx_hash).and_then(|(block_hash, idx)| { - let block = this.full_block_cache.get(block_hash).cloned()?; - let receipts = this.receipts_cache.get(block_hash).cloned(); + let block = + this.full_block_cache.get_at(block_hash, now).cloned()?; + let receipts = + this.receipts_cache.get_at(block_hash, now).cloned(); Some(CachedTransaction::new(block, *idx, receipts)) }); let _ = response_tx.send(result); @@ -961,11 +1004,21 @@ mod tests { mpsc::{self, Receiver, SyncSender}, Mutex, }, + task::{Wake, Waker}, thread, time::Duration, }; - fn test_service() -> EthStateCacheService { + #[derive(Default)] + struct WakeCounter(AtomicUsize); + + impl Wake for WakeCounter { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + fn test_service() -> EthStateCacheService { let (_cache, service) = EthStateCache::::create( NoopProvider::default(), Runtime::test(), @@ -977,6 +1030,7 @@ mod tests { prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 16, + ..Default::default() }, ); service @@ -1002,6 +1056,190 @@ mod tests { ) } + #[tokio::test(start_paused = true)] + async fn idle_timer_continues_waking_after_its_first_tick() { + let (cache, mut service) = EthStateCache::::create( + NoopProvider::default(), + Runtime::test(), + EthStateCacheConfig { idle_timeout: Duration::from_secs(10), ..Default::default() }, + ); + tokio::time::advance(Duration::from_secs(3)).await; + let block = Arc::new(test_block()); + let hash = block.hash(); + let receipts = Arc::new(vec![]); + let bal = CachedRevmBal::new(test_decoded_revm_bal()); + let retained_block = Arc::downgrade(&block); + let retained_receipts = Arc::downgrade(&receipts); + let retained_bal = Arc::downgrade(&bal.0); + service.on_new_block(hash, Ok(Some(block)), Instant::now()); + service.on_new_receipts(hash, Ok(Some(receipts)), Instant::now()); + service.on_new_bal(hash, Ok(Some(bal)), Instant::now()); + let (response_tx, response_rx) = oneshot::channel(); + service.full_block_cache.queue(hash, response_tx); + let service_task = tokio::spawn(service); + assert!(cache.get_maybe_block(hash).await.unwrap().is_some()); + + tokio::time::advance(Duration::from_secs(7)).await; + tokio::task::yield_now().await; + assert!(retained_block.upgrade().is_some()); + assert!(retained_receipts.upgrade().is_some()); + assert!(retained_bal.upgrade().is_some()); + // No more cache actions: only a later timer tick can release these payloads. + tokio::time::advance(Duration::from_secs(11)).await; + tokio::task::yield_now().await; + let expired = retained_block.upgrade().is_none() && + retained_receipts.upgrade().is_none() && + retained_bal.upgrade().is_none(); + + // Expiration must leave pending consumers available for the next result. + cache + .to_service + .send(CacheAction::BlockWithSendersResult { + block_hash: hash, + res: Ok(Some(Arc::new(test_block()))), + }) + .unwrap(); + let response = response_rx.await.unwrap(); + service_task.abort(); + assert!(expired, "idle service did not register its next timer wakeup"); + assert!(response.unwrap().is_some()); + } + + #[tokio::test(start_paused = true)] + async fn idle_eviction_yields_between_batches_and_serves_requests() { + let (cache, mut service) = EthStateCache::::create( + NoopProvider::default(), + Runtime::test(), + EthStateCacheConfig { idle_timeout: Duration::from_secs(10), ..Default::default() }, + ); + let mut retained = Vec::new(); + for key in 0..12 { + let hash = B256::with_last_byte(key); + let block = Arc::new(test_block()); + let receipts = Arc::new(vec![]); + let bal = CachedRevmBal::new(test_decoded_revm_bal()); + retained.push(( + Arc::downgrade(&block), + Arc::downgrade(&receipts), + Arc::downgrade(&bal.0), + )); + service.on_new_block(hash, Ok(Some(block)), Instant::now()); + service.on_new_receipts(hash, Ok(Some(receipts)), Instant::now()); + service.on_new_bal(hash, Ok(Some(bal)), Instant::now()); + } + let (response_tx, mut response_rx) = oneshot::channel(); + cache + .to_service + .send(CacheAction::GetCachedBlock { block_hash: B256::repeat_byte(0xff), response_tx }) + .unwrap(); + tokio::time::advance(Duration::from_secs(10)).await; + + let wakes = Arc::new(WakeCounter::default()); + let waker = Waker::from(wakes.clone()); + let mut cx = Context::from_waker(&waker); + for (remaining, expected_wakes) in [(7, 1), (2, 2), (0, 2)] { + assert!(Pin::new(&mut service).poll(&mut cx).is_pending()); + assert_eq!(wakes.0.load(Ordering::Relaxed), expected_wakes); + assert_eq!( + retained.iter().filter(|(block, _, _)| block.strong_count() > 0).count(), + remaining + ); + assert_eq!( + retained.iter().filter(|(_, receipts, _)| receipts.strong_count() > 0).count(), + remaining + ); + assert_eq!( + retained.iter().filter(|(_, _, bal)| bal.strong_count() > 0).count(), + remaining + ); + if remaining == 7 { + assert!(response_rx.try_recv().unwrap().is_none()); + } + } + } + + #[tokio::test(start_paused = true)] + async fn disabled_idle_timeout_does_not_schedule_cleanup() { + let (_cache, mut service) = EthStateCache::::create( + NoopProvider::default(), + Runtime::test(), + EthStateCacheConfig { idle_timeout: Duration::ZERO, ..Default::default() }, + ); + let wakes = Arc::new(WakeCounter::default()); + let waker = Waker::from(wakes.clone()); + let mut cx = Context::from_waker(&waker); + assert!(Pin::new(&mut service).poll(&mut cx).is_pending()); + + tokio::time::advance(Duration::from_secs(10)).await; + assert_eq!(wakes.0.load(Ordering::Relaxed), 0); + } + + #[test] + fn oversized_results_reach_queued_consumers() { + let block = Arc::new(test_block()); + let receipts = Arc::new(vec![Receipt::default()]); + let bal = CachedRevmBal::new(test_decoded_revm_bal()); + let hash = block.hash(); + let (_cache, mut service) = EthStateCache::::create( + NoopProvider::default(), + Runtime::test(), + EthStateCacheConfig { + max_blocks_bytes: block.size() - 1, + max_receipts_bytes: receipts.size() - 1, + max_bals_bytes: bal.size() - 1, + ..Default::default() + }, + ); + let (block_tx, mut block_rx) = oneshot::channel(); + let (receipts_tx, mut receipts_rx) = oneshot::channel(); + let (bal_tx, mut bal_rx) = oneshot::channel(); + service.full_block_cache.queue(hash, block_tx); + service.receipts_cache.queue(hash, receipts_tx); + service.bal_cache.queue(hash, bal_tx); + + service.on_new_block(hash, Ok(Some(block.clone())), Instant::now()); + service.on_new_receipts(hash, Ok(Some(receipts.clone())), Instant::now()); + service.on_new_bal(hash, Ok(Some(bal.clone())), Instant::now()); + + assert!(Arc::ptr_eq(&block_rx.try_recv().unwrap().unwrap().unwrap(), &block)); + assert!(Arc::ptr_eq(&receipts_rx.try_recv().unwrap().unwrap().unwrap(), &receipts)); + assert!(Arc::ptr_eq(&bal_rx.try_recv().unwrap().unwrap().unwrap().0, &bal.0)); + assert!(service.full_block_cache.get(&hash).is_none()); + assert!(service.receipts_cache.get(&hash).is_none()); + assert!(service.bal_cache.get(&hash).is_none()); + } + + #[tokio::test(start_paused = true)] + async fn cached_transaction_lookup_renews_idle_timeout() { + let (cache, mut service) = EthStateCache::::create( + NoopProvider::default(), + Runtime::test(), + EthStateCacheConfig { idle_timeout: Duration::from_secs(10), ..Default::default() }, + ); + let block = Arc::new(test_block()); + let hash = block.hash(); + let tx_hash = *block.body().transactions().next().unwrap().tx_hash(); + service.index_block_transactions(&block); + service.on_new_block(hash, Ok(Some(block)), Instant::now()); + service.on_new_receipts(hash, Ok(Some(Arc::new(vec![Receipt::default()]))), Instant::now()); + + for delay in [6, 6, 11] { + tokio::time::advance(Duration::from_secs(delay)).await; + let request = cache.get_transaction_by_hash(tx_hash); + futures::pin_mut!(request); + assert!(futures::poll!(&mut request).is_pending()); + assert!(futures::poll!(&mut service).is_pending()); + let result = request.await; + if delay < 10 { + let result = result.expect("transaction is still cached after a renewed hit"); + assert_eq!(result.block.hash(), hash); + assert!(result.receipts.is_some()); + } else { + assert!(result.is_none()); + } + } + } + #[test] fn reorg_removes_tx_hash_index_entries_unconditionally() { let mut service = test_service(); @@ -1034,9 +1272,13 @@ mod tests { let hash = B256::repeat_byte(0x69); let (tx, mut rx) = oneshot::channel(); assert!(service.bal_cache.queue(hash, tx)); - service.on_new_bal(hash, Ok(Some(CachedRevmBal::new(test_decoded_revm_bal())))); + service.on_new_bal( + hash, + Ok(Some(CachedRevmBal::new(test_decoded_revm_bal()))), + Instant::now(), + ); assert!(rx.try_recv().unwrap().unwrap().is_some()); - service.on_new_bal(hash, Ok(None)); + service.on_new_bal(hash, Ok(None), Instant::now()); assert!(service.bal_cache.contains_key(&hash)); } @@ -1091,6 +1333,7 @@ mod tests { prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, + ..Default::default() }, ); @@ -1158,28 +1401,31 @@ mod tests { } #[tokio::test] - async fn get_bal_uses_cached_revm_bal() { - let fetches = Arc::new(AtomicUsize::default()); - let provider = TestBalProvider::new(fetches.clone()); - let cache = EthStateCache::::spawn_with( - provider, - EthStateCacheConfig { - max_blocks: 0, - max_receipts: 0, - max_bals: 4, - cache_computed_bals: false, - prewarm_bals: None, - max_concurrent_db_requests: 1, - max_cached_tx_hashes: 0, - }, - Runtime::test(), - ); - let block_hash = B256::repeat_byte(0x66); + async fn get_bal_only_caches_results_within_budget() { + for (max_bals_bytes, expected_fetches) in [(usize::MAX, 1), (1, 2), (0, 2)] { + let fetches = Arc::new(AtomicUsize::default()); + let cache = EthStateCache::::spawn_with( + TestBalProvider::new(fetches.clone()), + EthStateCacheConfig { + max_blocks: 0, + max_receipts: 0, + max_bals: 4, + max_bals_bytes, + cache_computed_bals: false, + prewarm_bals: None, + max_concurrent_db_requests: 1, + max_cached_tx_hashes: 0, + ..Default::default() + }, + Runtime::test(), + ); + let block_hash = B256::repeat_byte(0x66); - assert!(cache.get_bal(block_hash).await.unwrap().is_some()); - assert!(cache.get_bal(block_hash).await.unwrap().is_some()); + assert!(cache.get_bal(block_hash).await.unwrap().is_some()); + assert!(cache.get_bal(block_hash).await.unwrap().is_some()); - assert_eq!(fetches.load(Ordering::SeqCst), 1); + assert_eq!(fetches.load(Ordering::SeqCst), expected_fetches); + } } #[tokio::test] @@ -1198,6 +1444,7 @@ mod tests { prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, + ..Default::default() }, Runtime::test(), ); @@ -1281,6 +1528,7 @@ mod tests { prewarm_bals: None, max_concurrent_db_requests: 1, max_cached_tx_hashes: 0, + ..Default::default() }, Runtime::test(), ); diff --git a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs index c727bc11faa..d9013a94dc7 100644 --- a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs +++ b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs @@ -3,36 +3,32 @@ use super::metrics::CacheMetrics; use reth_primitives_traits::InMemorySize; -use schnellru::{ByLength, Limiter, LruMap}; +use schnellru::{ByLength, LruMap}; use std::{ collections::{hash_map::Entry, HashMap}, fmt::{self, Debug, Formatter}, hash::Hash, + time::Duration, }; +use tokio::time::Instant; -/// A multi-consumer LRU cache. -pub struct MultiConsumerLruCache -where - K: Hash + Eq, - L: Limiter, -{ +/// A multi-consumer LRU cache bounded by entry count and estimated payload size. +pub struct MultiConsumerLruCache { /// The LRU cache. - cache: LruMap, + cache: LruMap>, /// All queued consumers. queued: HashMap>, - /// Cache metrics + /// Cache metrics. metrics: CacheMetrics, - // Tracked heap usage + /// Tracked payload usage, excluding map and request queue overhead. memory_usage: usize, + /// Maximum estimated payload bytes retained by the cache. + max_bytes: usize, /// Whether the cached count or memory usage changed since the gauges were last published. metrics_dirty: bool, } -impl Debug for MultiConsumerLruCache -where - K: Hash + Eq, - L: Limiter, -{ +impl Debug for MultiConsumerLruCache { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("MultiConsumerLruCache") .field("cache_length", &self.cache.len()) @@ -43,14 +39,24 @@ where } } -impl MultiConsumerLruCache -where - K: Hash + Eq + Debug, - L: Limiter, -{ +impl MultiConsumerLruCache { + /// Creates an empty cache with entry and payload byte limits and a metric label. + /// + /// Setting either limit to zero disables caching, but consumers can still queue requests. + pub fn new(max_len: u32, max_bytes: usize, cache_id: &str) -> Self { + Self { + cache: LruMap::new(ByLength::new(max_len)), + queued: Default::default(), + metrics: CacheMetrics::new_with_labels(&[("cache", cache_id.to_string())]), + memory_usage: 0, + max_bytes, + metrics_dirty: false, + } + } + /// Adds the sender to the queue for the given key. /// - /// Returns true if this is the first queued sender for the key + /// Returns true if this is the first queued sender for the key. pub fn queue(&mut self, key: K, sender: S) -> bool { self.metrics.queued_consumers_count.increment(1.0); match self.queued.entry(key) { @@ -65,64 +71,91 @@ where } } - /// Remove consumers for a given key, this will also remove the key from the cache. - pub fn remove(&mut self, key: &K) -> Option> - where - V: InMemorySize, - { - self.cache.remove(key).inspect(|value| { - self.memory_usage = self.memory_usage.saturating_sub(value.size()); + /// Removes consumers for a given key, also removing the key from the cache. + pub fn remove(&mut self, key: &K) -> Option> { + if let Some(entry) = self.cache.remove(key) { + self.memory_usage -= entry.size; self.metrics_dirty = true; - }); + } self.queued .remove(key) .inspect(|removed| self.metrics.queued_consumers_count.decrement(removed.len() as f64)) } - /// Returns a reference to the value for a given key and promotes that element to be the most - /// recently used. - pub fn get(&mut self, key: &K) -> Option<&mut V> { - let entry = self.cache.get(key); - if entry.is_some() { + /// Returns a reference to the value and refreshes its recency and idle timestamp. + pub fn get(&mut self, key: &K) -> Option<&V> { + self.get_at(key, Instant::now()) + } + + /// Returns a reference to the value and refreshes its recency using the supplied timestamp. + /// + /// Use nondecreasing timestamps for lookups and [`Self::insert_at`] to keep idle eviction in + /// LRU order. + pub fn get_at(&mut self, key: &K, now: Instant) -> Option<&V> { + if let Some(entry) = self.cache.get(key) { + entry.last_access = now; self.metrics.hits_total.increment(1); + Some(&entry.value) } else { self.metrics.misses_total.increment(1); + None } - entry } - /// Checks for a cached value without changing recency or lookup metrics. + /// Checks for a cached value without changing recency, idle timestamp, or lookup metrics. pub(crate) fn contains_key(&self, key: &K) -> bool { self.cache.peek(key).is_some() } - /// Inserts a new element into the map. + /// Inserts a new element, evicting the oldest entries until both limits are satisfied. /// - /// Can fail if the element is rejected by the limiter or if we fail to grow an empty map. + /// Oversized values are rejected without evicting existing entries or queued consumers. + pub fn insert(&mut self, key: K, value: V) -> bool + where + V: InMemorySize, + { + self.insert_at(key, value, Instant::now()) + } + + /// Inserts an element using the supplied timestamp, enforcing the same limits as + /// [`Self::insert`]. /// - /// See [`LruMap::insert`] for more info. - pub fn insert<'a>(&mut self, key: L::KeyToInsert<'a>, value: V) -> bool + /// Use the same nondecreasing timestamps as [`Self::get_at`] when batching cache operations. + pub fn insert_at(&mut self, key: K, value: V, now: Instant) -> bool where - L::KeyToInsert<'a>: Hash + PartialEq, V: InMemorySize, { + if self.cache.limiter().max_length() == 0 || self.max_bytes == 0 { + return false + } let size = value.size(); + if size > self.max_bytes { + return false + } - if self.cache.limiter().is_over_the_limit(self.cache.len() + 1) && - let Some((_, evicted)) = self.cache.pop_oldest() - { - // update tracked memory with the evicted value - self.memory_usage = self.memory_usage.saturating_sub(evicted.size()); + // Remove replacements before checking the limits so their size is not counted twice. + if let Some(previous) = self.cache.remove(&key) { + self.memory_usage -= previous.size; self.metrics_dirty = true; } + while self.cache.len() >= self.cache.limiter().max_length() as usize || + self.memory_usage > self.max_bytes - size + { + self.pop_oldest(); + } - if self.cache.insert(key, value) { - self.memory_usage = self.memory_usage.saturating_add(size); + let previous_len = self.cache.len(); + let inserted = self.cache.insert(key, CachedValue { value, size, last_access: now }); + if inserted { + self.memory_usage += size; + self.metrics_dirty = true; + } + if self.cache.len() != previous_len + usize::from(inserted) { + // A failed allocation can make the LRU evict internally while trying to insert. + self.memory_usage = self.cache.iter().map(|(_, entry)| entry.size).sum(); self.metrics_dirty = true; - true - } else { - false } + inserted } /// Shrinks the capacity of the queue with a lower limit. @@ -144,45 +177,217 @@ where self.metrics.memory_usage.set(self.memory_usage as f64); true } -} -impl MultiConsumerLruCache -where - K: Hash + Eq, -{ - /// Creates a new empty map with a given `max_len` and metric label. - pub fn new(max_len: u32, cache_id: &str) -> Self { - Self { - cache: LruMap::new(ByLength::new(max_len)), - queued: Default::default(), - metrics: CacheMetrics::new_with_labels(&[("cache", cache_id.to_string())]), - memory_usage: 0, - metrics_dirty: false, + /// Removes at most five idle entries without disturbing consumers waiting for a database + /// result. + /// + /// Returns whether more expired entries remain for the next poll. + pub(crate) fn evict_expired(&mut self, now: Instant, idle_timeout: Duration) -> bool { + if idle_timeout.is_zero() { + return false + } + let mut remaining = 5; + // Hits update timestamps and LRU order together, so the first live entry ends the scan. + while self + .cache + .peek_oldest() + .is_some_and(|(_, entry)| now.duration_since(entry.last_access) >= idle_timeout) + { + if remaining == 0 { + return true + } + self.pop_oldest(); + remaining -= 1; + } + false + } + + fn pop_oldest(&mut self) { + if let Some((_, entry)) = self.cache.pop_oldest() { + self.memory_usage -= entry.size; + self.metrics_dirty = true; } } } +/// Payload metadata measured once at insertion, so eviction never needs to traverse the payload. +struct CachedValue { + value: V, + size: usize, + last_access: Instant, +} + #[cfg(test)] mod tests { use super::*; - use alloy_primitives::B256; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + #[derive(Debug, PartialEq, Eq)] + struct Weighted(usize); + + impl InMemorySize for Weighted { + fn size(&self) -> usize { + self.0 + } + } + + struct Measured(Arc); + + impl InMemorySize for Measured { + fn size(&self) -> usize { + self.0.fetch_add(1, Ordering::Relaxed); + 3 + } + } + + #[test] + fn count_and_byte_limits_evict_oldest_entries() { + let mut cache = MultiConsumerLruCache::::new(3, 10, "test"); + for key in 0..3 { + assert!(cache.insert(key, Weighted(2))); + } + assert!(cache.get(&0).is_some()); + assert!(cache.insert(3, Weighted(2))); + assert!(cache.get(&1).is_none()); + assert_eq!(cache.memory_usage, 6); + + assert!(cache.insert(4, Weighted(8))); + assert!(cache.get(&0).is_none()); + assert!(cache.get(&2).is_none()); + assert!(cache.get(&3).is_some()); + assert!(cache.get(&4).is_some()); + assert_eq!(cache.memory_usage, 10); + } + + #[test] + fn replacement_removal_and_oversized_values() { + let mut cache = MultiConsumerLruCache::::new(2, 10, "test"); + assert!(cache.insert(0, Weighted(6))); + assert!(cache.insert(1, Weighted(4))); + assert!(cache.insert(1, Weighted(2))); + assert!(cache.get(&0).is_some()); + assert_eq!(cache.memory_usage, 8); + + assert!(cache.insert(1, Weighted(7))); + assert!(cache.get(&0).is_none()); + assert_eq!(cache.memory_usage, 7); + assert!(!cache.insert(1, Weighted(11))); + assert!(!cache.insert(2, Weighted(11))); + assert_eq!(cache.get(&1), Some(&Weighted(7))); + assert_eq!(cache.memory_usage, 7); + + assert!(cache.insert(2, Weighted(10))); + assert!(cache.get(&1).is_none()); + assert_eq!(cache.memory_usage, 10); + cache.remove(&2); + assert_eq!(cache.memory_usage, 0); + } + + #[tokio::test(start_paused = true)] + async fn idle_eviction_preserves_hits_and_queued_consumers() { + let mut cache = MultiConsumerLruCache::new(3, 10, "test"); + let timeout = Duration::from_secs(10); + for key in 0..3 { + assert!(cache.insert(key, Weighted(3))); + } + assert!(cache.queue(1, 42)); + assert!(!cache.queue(1, 43)); + tokio::time::advance(timeout / 2).await; + assert!(cache.get(&0).is_some()); + assert!(cache.contains_key(&1)); + tokio::time::advance(timeout / 2).await; + cache.evict_expired(Instant::now(), timeout); + assert_eq!(cache.memory_usage, 3); + assert_eq!(cache.remove(&1), Some(vec![42, 43])); + assert!(cache.contains_key(&0)); + tokio::time::advance(timeout / 2).await; + cache.evict_expired(Instant::now(), Duration::ZERO); + assert_eq!(cache.memory_usage, 3); + cache.evict_expired(Instant::now(), timeout); + assert_eq!(cache.memory_usage, 0); + } + + #[test] + fn payloads_are_only_measured_on_enabled_insertions() { + for (count, bytes) in [(0, 10), (10, 0), (10, 3)] { + let measurements = Arc::new(AtomicUsize::new(0)); + let mut cache = MultiConsumerLruCache::::new(count, bytes, "test"); + let enabled = count > 0 && bytes > 0; + assert!(cache.queue(0, ())); + for key in [0, 1, 1] { + assert_eq!(cache.insert(key, Measured(measurements.clone())), enabled); + assert_eq!(cache.get(&key).is_some(), enabled); + } + cache.evict_expired(Instant::now() + Duration::from_secs(10), Duration::from_secs(10)); + assert_eq!(cache.memory_usage, 0); + assert_eq!(measurements.load(Ordering::Relaxed), if enabled { 3 } else { 0 }); + assert_eq!(cache.remove(&0), Some(vec![()])); + } + } #[test] fn gauges_only_republished_after_cache_changes() { - let mut cache: MultiConsumerLruCache = - MultiConsumerLruCache::new(2, "test"); + let mut cache = MultiConsumerLruCache::::new(2, 10, "test"); assert!(!cache.update_cached_metrics()); - - assert!(cache.insert(B256::ZERO, B256::ZERO)); + assert!(cache.insert(0, Weighted(3))); assert!(cache.update_cached_metrics()); assert!(!cache.update_cached_metrics()); - - // hits do not touch the gauges - assert!(cache.get(&B256::ZERO).is_some()); - assert!(cache.get(&B256::with_last_byte(1)).is_none()); + assert!(cache.get(&0).is_some()); + assert!(cache.get(&1).is_none()); assert!(!cache.update_cached_metrics()); - - assert!(cache.remove(&B256::ZERO).is_none()); + cache.remove(&0); assert!(cache.update_cached_metrics()); } + + #[tokio::test(start_paused = true)] + async fn batched_timestamps_preserve_idle_eviction_order() { + let mut cache = MultiConsumerLruCache::::new(4, 12, "test"); + let start = Instant::now(); + let timeout = Duration::from_secs(10); + for key in 0..3 { + assert!(cache.insert_at(key, Weighted(3), start)); + } + tokio::time::advance(timeout / 2).await; + let now = Instant::now(); + assert!(cache.get_at(&0, now).is_some()); + + // Insertions and later hits in a batch share its timestamp even if the clock advances. + tokio::time::advance(timeout / 2).await; + assert!(cache.insert_at(3, Weighted(3), now)); + assert!(cache.get_at(&2, now).is_some()); + cache.evict_expired(Instant::now(), timeout); + assert!(!cache.contains_key(&1)); + assert_eq!(cache.memory_usage, 9); + + tokio::time::advance(timeout / 2).await; + cache.evict_expired(Instant::now(), timeout); + assert_eq!(cache.memory_usage, 0); + } + + #[test] + fn idle_eviction_is_bounded_and_preserves_renewed_entries() { + let mut cache = MultiConsumerLruCache::new(12, 36, "test"); + let start = Instant::now(); + let timeout = Duration::from_secs(10); + for key in 0..12 { + assert!(cache.insert_at(key, Weighted(3), start)); + } + assert!(cache.queue(0, 42)); + let now = start + timeout; + + assert!(cache.evict_expired(now, timeout)); + assert_eq!(cache.memory_usage, 21); + assert!(!cache.contains_key(&4)); + assert!(cache.get_at(&5, now).is_some()); + + assert!(cache.evict_expired(now, timeout)); + assert_eq!(cache.memory_usage, 6); + assert!(!cache.evict_expired(now, timeout)); + assert_eq!(cache.memory_usage, 3); + assert!(cache.contains_key(&5)); + assert_eq!(cache.remove(&0), Some(vec![42])); + } } diff --git a/crates/rpc/rpc-server-types/src/constants.rs b/crates/rpc/rpc-server-types/src/constants.rs index 1e58d4eeb57..750b37f8cf2 100644 --- a/crates/rpc/rpc-server-types/src/constants.rs +++ b/crates/rpc/rpc-server-types/src/constants.rs @@ -114,6 +114,20 @@ pub mod gas_oracle { /// Cache specific constants pub mod cache { + use std::time::Duration; + + /// Default estimated block payload limit: 2 GiB. + pub const DEFAULT_BLOCK_CACHE_MAX_BYTES: usize = 2 * 1024 * 1024 * 1024; + + /// Default estimated receipts payload limit: 1 GiB. + pub const DEFAULT_RECEIPT_CACHE_MAX_BYTES: usize = 1024 * 1024 * 1024; + + /// Default estimated block access list payload limit: 500 MiB. + pub const DEFAULT_BAL_CACHE_MAX_BYTES: usize = 500 * 1024 * 1024; + + /// Default idle timeout for cached blocks, receipts, and block access lists. + pub const DEFAULT_CACHE_IDLE_TIMEOUT: Duration = Duration::from_secs(60 * 60); + /// Default cache size for the block cache: 1000 blocks. pub const DEFAULT_BLOCK_CACHE_MAX_LEN: u32 = 1000; diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 4f408d770a8..61511b415dc 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -549,6 +549,26 @@ RPC State Cache: [default: 1000] + --rpc-cache.max-blocks-bytes + Maximum estimated block cache memory in bytes or with a unit (e.g. 1GB, 500MB). Units use powers of 1024. Zero disables caching. The entry count limit also applies + + [default: 2147483648] + + --rpc-cache.max-receipts-bytes + Maximum estimated receipts cache memory in bytes or with a unit (e.g. 1GB, 500MB). Units use powers of 1024. Zero disables caching. The entry count limit also applies + + [default: 1073741824] + + --rpc-cache.max-bals-bytes + Maximum estimated block access list cache memory in bytes or with a unit (e.g. 1GB, 500MB). Units use powers of 1024. Zero disables caching. The entry count limit also applies + + [default: 524288000] + + --rpc-cache.idle-timeout + Evict blocks, receipts, and block access lists after this duration without a cache hit (e.g. 5m, 30s). Zero disables expiration. Cleanup starts every 1 to 60 seconds and removes at most five entries per cache per poll + + [default: 1h] + --rpc-cache.cache-computed-bals Cache block access lists computed by RPC requests for transaction tracing From ddfc09a3a66e0856bde7325a67122d6e58b50a02 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 23 Sep 2026 15:27:47 +0000 Subject: [PATCH 056/133] refactor: remove cached state root/proof methods (#27109) --- Cargo.lock | 2 +- crates/cli/commands/src/re_execute.rs | 5 +- crates/engine/execution-cache/Cargo.toml | 4 +- .../execution-cache/src/cached_state.rs | 141 ++---------------- .../engine/invalid-block-hooks/src/witness.rs | 13 +- .../tree/src/tree/instrumented_state.rs | 124 +-------------- .../payload_processor/bal_prewarm_pool.rs | 10 +- .../src/tree/payload_processor/prewarm.rs | 29 ++-- .../engine/tree/src/tree/payload_validator.rs | 57 +++---- .../tree/src/tree/txpool_prewarm/worker.rs | 9 +- crates/engine/util/src/reorg.rs | 4 +- .../ethereum/node/src/engine_ssz_witness.rs | 11 +- crates/ethereum/payload/src/lib.rs | 22 ++- crates/exex/exex/src/backfill/job.rs | 10 +- crates/exex/exex/src/backfill/test_utils.rs | 11 +- crates/revm/src/database.rs | 64 +------- crates/rpc/rpc-eth-api/src/helpers/bal.rs | 4 +- crates/rpc/rpc-eth-api/src/helpers/call.rs | 12 +- .../rpc/rpc-eth-api/src/helpers/estimate.rs | 11 +- .../rpc-eth-api/src/helpers/pending_block.rs | 4 +- crates/rpc/rpc-eth-types/src/cache/db.rs | 4 +- crates/rpc/rpc/src/debug.rs | 20 +-- crates/rpc/rpc/src/reth.rs | 7 +- crates/rpc/rpc/src/testing.rs | 4 +- crates/rpc/rpc/src/validation.rs | 7 +- .../stages/stages/src/stages/execution/mod.rs | 7 +- crates/stages/stages/tests/pipeline.rs | 2 +- crates/stages/stages/tests/preimage.rs | 8 +- crates/storage/storage-api/src/evm.rs | 111 ++++++++++++++ crates/storage/storage-api/src/lib.rs | 3 + crates/storage/storage-api/src/state.rs | 15 +- testing/ef-tests/src/cases/blockchain_test.rs | 4 +- 32 files changed, 305 insertions(+), 434 deletions(-) create mode 100644 crates/storage/storage-api/src/evm.rs diff --git a/Cargo.lock b/Cargo.lock index d316bcdf0e4..ff688151015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9060,7 +9060,7 @@ dependencies = [ "reth-primitives-traits", "reth-provider", "reth-revm", - "reth-trie", + "reth-storage-api", "revm", "tracing", ] diff --git a/crates/cli/commands/src/re_execute.rs b/crates/cli/commands/src/re_execute.rs index a2e134b73ce..5dee49840db 100644 --- a/crates/cli/commands/src/re_execute.rs +++ b/crates/cli/commands/src/re_execute.rs @@ -28,7 +28,7 @@ use reth_revm::{ }, }; use reth_stages::stages::calculate_gas_used_from_headers; -use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader}; +use reth_storage_api::{ChangeSetReader, DBProvider, StateProvider, StorageChangeSetReader}; use std::{ collections::HashMap, sync::{ @@ -163,7 +163,8 @@ impl let hash = provider.block_hash(block_number).unwrap().unwrap(); StateProviderDatabase( state_provider_factory - .state_provider_from_database(provider, hash), + .state_provider_from_database(provider, hash) + .into_evm_state_provider(), ) } }; diff --git a/crates/engine/execution-cache/Cargo.toml b/crates/engine/execution-cache/Cargo.toml index 72ea6e96ea0..f6e482d8a64 100644 --- a/crates/engine/execution-cache/Cargo.toml +++ b/crates/engine/execution-cache/Cargo.toml @@ -19,9 +19,8 @@ parking_lot.workspace = true reth-errors.workspace = true reth-metrics = { workspace = true, features = ["common"] } reth-primitives-traits = { workspace = true, features = ["std"] } -reth-provider.workspace = true reth-revm.workspace = true -reth-trie.workspace = true +reth-storage-api.workspace = true tracing.workspace = true [dev-dependencies] @@ -35,5 +34,4 @@ test-utils = [ "reth-primitives-traits/test-utils", "reth-revm/test-utils", "reth-provider/test-utils", - "reth-trie/test-utils", ] diff --git a/crates/engine/execution-cache/src/cached_state.rs b/crates/engine/execution-cache/src/cached_state.rs index 6c500bbad96..efb05a40217 100644 --- a/crates/engine/execution-cache/src/cached_state.rs +++ b/crates/engine/execution-cache/src/cached_state.rs @@ -10,15 +10,8 @@ use parking_lot::Once; use reth_errors::ProviderResult; use reth_metrics::Metrics; use reth_primitives_traits::{Account, Bytecode}; -use reth_provider::{ - AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider, - StateProvider, StateRootProvider, StorageRootProvider, -}; use reth_revm::db::BundleState; -use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, - MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, -}; +use reth_storage_api::EvmStateProvider; use std::{ cell::Cell, fmt, @@ -844,7 +837,12 @@ impl StatsHandler for CacheStatsHandler { } } -impl AccountReader for CachedStateProvider { +#[inline] +fn nonzero_storage_value(value: StorageValue) -> Option { + (!value.is_zero()).then_some(value) +} + +impl EvmStateProvider for CachedStateProvider { fn basic_account(&self, address: &Address) -> ProviderResult> { if let Some(snapshot) = &self.txpool_snapshot { if let Some(account) = snapshot.account(address) { @@ -875,18 +873,7 @@ impl AccountReader for CachedStateProvider { self.state_provider.basic_account(address) } } -} -#[inline] -fn nonzero_storage_value(value: StorageValue) -> Option { - if value.is_zero() { - None - } else { - Some(value) - } -} - -impl StateProvider for CachedStateProvider { fn storage( &self, account: Address, @@ -921,9 +908,7 @@ impl StateProvider for CachedStateProvider { self.state_provider.storage(account, storage_key) } } -} -impl BytecodeReader for CachedStateProvider { fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { if let Some(snapshot) = &self.txpool_snapshot { if let Some(code) = snapshot.bytecode(code_hash) { @@ -954,117 +939,10 @@ impl BytecodeReader for CachedStateProvider { self.state_provider.bytecode_by_hash(code_hash) } } -} - -impl StateRootProvider for CachedStateProvider { - fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { - self.state_provider.state_root(hashed_state) - } - - fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { - self.state_provider.state_root_from_nodes(input) - } - - fn state_root_with_updates( - &self, - hashed_state: HashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - self.state_provider.state_root_with_updates(hashed_state) - } - fn state_root_from_nodes_with_updates( - &self, - input: TrieInput, - ) -> ProviderResult<(B256, TrieUpdates)> { - self.state_provider.state_root_from_nodes_with_updates(input) - } -} - -impl StateProofProvider for CachedStateProvider { - fn proof( - &self, - input: TrieInput, - address: Address, - slots: &[B256], - ) -> ProviderResult { - self.state_provider.proof(input, address, slots) - } - - fn multiproof( - &self, - input: TrieInput, - targets: MultiProofTargets, - ) -> ProviderResult { - self.state_provider.multiproof(input, targets) - } - - fn multiproof_v2( - &self, - input: TrieInput, - targets: reth_trie::MultiProofTargetsV2, - ) -> ProviderResult { - self.state_provider.multiproof_v2(input, targets) - } - - fn witness( - &self, - input: TrieInput, - target: HashedPostState, - mode: reth_trie::ExecutionWitnessMode, - ) -> ProviderResult> { - self.state_provider.witness(input, target, mode) - } -} - -impl StorageRootProvider for CachedStateProvider { - fn storage_root( - &self, - address: Address, - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.state_provider.storage_root(address, hashed_storage) - } - - fn storage_proof( - &self, - address: Address, - slot: B256, - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.state_provider.storage_proof(address, slot, hashed_storage) - } - - fn storage_multiproof( - &self, - address: Address, - slots: &[B256], - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.state_provider.storage_multiproof(address, slots, hashed_storage) - } -} - -impl BlockHashReader for CachedStateProvider { fn block_hash(&self, number: alloy_primitives::BlockNumber) -> ProviderResult> { self.state_provider.block_hash(number) } - - fn canonical_hashes_range( - &self, - start: alloy_primitives::BlockNumber, - end: alloy_primitives::BlockNumber, - ) -> ProviderResult> { - self.state_provider.canonical_hashes_range(start, end) - } -} - -impl HashedPostStateProvider for CachedStateProvider { - fn hashed_post_state( - &self, - bundle_state: &reth_revm::db::BundleState, - ) -> ProviderResult { - self.state_provider.hashed_post_state(bundle_state) - } } /// Execution cache used during block processing. @@ -1434,6 +1312,7 @@ mod tests { use alloy_primitives::{map::HashMap, U256}; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_revm::db::{AccountStatus, BundleAccount}; + use reth_storage_api::StateProvider; use revm::state::AccountInfo; #[test] @@ -1447,7 +1326,7 @@ mod tests { let caches = ExecutionCache::new(1000); let state_provider = CachedStateProvider::new( - provider, + provider.into_evm_state_provider(), caches, Some(CachedStateMetrics::zeroed(CachedStateMetricsSource::Test)), ); @@ -1470,7 +1349,7 @@ mod tests { let caches = ExecutionCache::new(1000); let state_provider = CachedStateProvider::new( - provider, + provider.into_evm_state_provider(), caches, Some(CachedStateMetrics::zeroed(CachedStateMetricsSource::Test)), ); diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index c7ad116ece7..a0d79857aef 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -5,7 +5,10 @@ use pretty_assertions::Comparison; use reth_engine_primitives::InvalidBlockHook; use reth_evm::{execute::Executor, ConfigureEvm}; use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedHeader}; -use reth_provider::{BlockExecutionOutput, StateProvider, StateProviderBox, StateProviderFactory}; +use reth_provider::{ + BlockExecutionOutput, EvmStateProviderAdapter, StateProvider, StateProviderBox, + StateProviderFactory, +}; use reth_revm::{ database::StateProviderDatabase, db::{BundleState, State}, @@ -116,7 +119,7 @@ fn sort_bundle_state_for_comparison(bundle_state: &BundleState) -> BundleStateSo /// Extracts execution data including codes, preimages, and hashed state from database fn collect_execution_data( - mut db: State>, + mut db: State>>, ) -> eyre::Result { let bundle_state = db.take_bundle(); let mut codes = BTreeMap::new(); @@ -214,7 +217,7 @@ where block: &RecoveredBlock, ) -> eyre::Result<(ExecutionWitness, BundleState)> { let mut executor = self.evm_config.batch_executor(StateProviderDatabase::new( - self.provider.state_by_block_hash(parent_header.hash())?, + self.provider.state_by_block_hash(parent_header.hash())?.into_evm_state_provider(), )); executor.execute_one(block)?; @@ -536,7 +539,9 @@ mod tests { // Create a State with StateProviderTest let state_provider = StateProviderTest::default(); let mut state = State::builder() - .with_database(StateProviderDatabase::new(Box::new(state_provider) as StateProviderBox)) + .with_database(StateProviderDatabase::new( + (Box::new(state_provider) as StateProviderBox).into_evm_state_provider(), + )) .with_bundle_update() .build(); diff --git a/crates/engine/tree/src/tree/instrumented_state.rs b/crates/engine/tree/src/tree/instrumented_state.rs index e60ee6110b2..2e5f0131e09 100644 --- a/crates/engine/tree/src/tree/instrumented_state.rs +++ b/crates/engine/tree/src/tree/instrumented_state.rs @@ -4,14 +4,7 @@ use metrics::{Gauge, Histogram}; use reth_errors::ProviderResult; use reth_metrics::Metrics; use reth_primitives_traits::{Account, Bytecode, FastInstant as Instant}; -use reth_provider::{ - AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider, - StateProvider, StateRootProvider, StorageRootProvider, -}; -use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, - MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, -}; +use reth_provider::EvmStateProvider; use std::{ sync::{ atomic::{AtomicU64, AtomicUsize, Ordering}, @@ -69,7 +62,7 @@ pub struct InstrumentedStateProvider { impl InstrumentedStateProvider where - S: StateProvider, + S: EvmStateProvider, { /// Creates a new [`InstrumentedStateProvider`] from a state provider with the provided label /// for metrics. @@ -155,7 +148,7 @@ impl StateProviderMetrics { } } -impl AccountReader for InstrumentedStateProvider { +impl EvmStateProvider for InstrumentedStateProvider { fn basic_account(&self, address: &Address) -> ProviderResult> { let start = Instant::now(); let res = self.state_provider.basic_account(address); @@ -165,9 +158,7 @@ impl AccountReader for InstrumentedStateProvider { self.stats.total_account_fetch_latency.add_duration(elapsed); res } -} -impl StateProvider for InstrumentedStateProvider { fn storage( &self, account: Address, @@ -181,9 +172,7 @@ impl StateProvider for InstrumentedStateProvider { self.stats.total_storage_fetch_latency.add_duration(elapsed); res } -} -impl BytecodeReader for InstrumentedStateProvider { fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { let start = Instant::now(); let res = self.state_provider.bytecode_by_hash(code_hash); @@ -200,117 +189,10 @@ impl BytecodeReader for InstrumentedStateProvider { ); res } -} - -impl StateRootProvider for InstrumentedStateProvider { - fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { - self.state_provider.state_root(hashed_state) - } - - fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { - self.state_provider.state_root_from_nodes(input) - } - - fn state_root_with_updates( - &self, - hashed_state: HashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - self.state_provider.state_root_with_updates(hashed_state) - } - - fn state_root_from_nodes_with_updates( - &self, - input: TrieInput, - ) -> ProviderResult<(B256, TrieUpdates)> { - self.state_provider.state_root_from_nodes_with_updates(input) - } -} - -impl StateProofProvider for InstrumentedStateProvider { - fn proof( - &self, - input: TrieInput, - address: Address, - slots: &[B256], - ) -> ProviderResult { - self.state_provider.proof(input, address, slots) - } - - fn multiproof( - &self, - input: TrieInput, - targets: MultiProofTargets, - ) -> ProviderResult { - self.state_provider.multiproof(input, targets) - } - - fn multiproof_v2( - &self, - input: TrieInput, - targets: reth_trie::MultiProofTargetsV2, - ) -> ProviderResult { - self.state_provider.multiproof_v2(input, targets) - } - - fn witness( - &self, - input: TrieInput, - target: HashedPostState, - mode: reth_trie::ExecutionWitnessMode, - ) -> ProviderResult> { - self.state_provider.witness(input, target, mode) - } -} - -impl StorageRootProvider for InstrumentedStateProvider { - fn storage_root( - &self, - address: Address, - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.state_provider.storage_root(address, hashed_storage) - } - - fn storage_proof( - &self, - address: Address, - slot: B256, - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.state_provider.storage_proof(address, slot, hashed_storage) - } - fn storage_multiproof( - &self, - address: Address, - slots: &[B256], - hashed_storage: HashedStorage, - ) -> ProviderResult { - self.state_provider.storage_multiproof(address, slots, hashed_storage) - } -} - -impl BlockHashReader for InstrumentedStateProvider { fn block_hash(&self, number: alloy_primitives::BlockNumber) -> ProviderResult> { self.state_provider.block_hash(number) } - - fn canonical_hashes_range( - &self, - start: alloy_primitives::BlockNumber, - end: alloy_primitives::BlockNumber, - ) -> ProviderResult> { - self.state_provider.canonical_hashes_range(start, end) - } -} - -impl HashedPostStateProvider for InstrumentedStateProvider { - fn hashed_post_state( - &self, - bundle_state: &reth_revm::db::BundleState, - ) -> ProviderResult { - self.state_provider.hashed_post_state(bundle_state) - } } /// Accumulated fetch statistics from an [`InstrumentedStateProvider`]. diff --git a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs index fcec9395040..d7e0d0cab93 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs @@ -2,9 +2,7 @@ use alloy_primitives::{Address, StorageKey}; use reth_execution_cache::{CachedStateProvider, ExecutionCache, TxPoolPrewarmCacheSnapshot}; -use reth_provider::{ - AccountReader, BytecodeReader, ProviderResult, StateProvider, StateProviderBox, -}; +use reth_provider::{EvmStateProvider, EvmStateProviderBox, ProviderResult}; use std::{ sync::{ atomic::{AtomicUsize, Ordering}, @@ -15,9 +13,9 @@ use std::{ use tokio::sync::oneshot; use tracing::trace; -/// Builds a fresh `StateProviderBox` over the block's parent state. Type-erased so the pool is not +/// Builds a fresh EVM provider over the block's parent state. Type-erased so the pool is not /// generic over the provider factory; each worker builds its own per block. -pub type BuildProviderFn = dyn Fn() -> ProviderResult + Send + Sync; +pub type BuildProviderFn = dyn Fn() -> ProviderResult + Send + Sync; /// A single warm request: a whole account (basic account + its bytecode) followed by a batch of /// its storage slots, or a batch of storage slots on their own. @@ -161,7 +159,7 @@ const WARM_BATCH_SIZE: usize = 8; fn prewarm_loop(rx: crossbeam_channel::Receiver) { // The provider (and its MDBX read txn) held for the current block, between `BeginBlock` and // `EndBlock`. `None` while idle, so no read txn is pinned across the inter-block gap. - let mut provider: Option> = None; + let mut provider: Option> = None; // Blocks when idle; the channel disconnects (and the loop ends) when the pool is dropped. while let Ok(msg) = rx.recv() { diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index b16ed3f2a1f..23b04f83c99 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -27,9 +27,10 @@ use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, use reth_metrics::Metrics; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ - AccountReader, BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, HistoryReader, PruneCheckpointReader, StageCheckpointReader, - StateProviderBox, StorageChangeSetReader, StorageSettingsCache, + BlockExecutionOutput, BlockNumReader, ChangeSetReader, DatabaseProviderFactory, + DatabaseProviderROFactory, EvmStateProvider, EvmStateProviderBox, HistoryReader, + PruneCheckpointReader, StageCheckpointReader, StateProvider, StorageChangeSetReader, + StorageSettingsCache, }; use reth_revm::database::StateProviderDatabase; use reth_storage_overlay::OverlayStateProviderFactory; @@ -400,8 +401,7 @@ where stream_bal.as_bal().par_iter().for_each(|account_changes| { WorkerPool::with_worker_mut(|worker| { - let provider = - worker.get_or_init::>>(|| None); + let provider = worker.get_or_init::>(|| None); ctx.send_bal_hashed_state( &parent_span, provider, @@ -437,9 +437,9 @@ where let caches = saved_cache.cache().clone(); let state_provider_factory = ctx.provider.clone(); let build = Arc::new(move || { - state_provider_factory - .database_provider_ro() - .map(|provider| Box::new(provider) as _) + state_provider_factory.database_provider_ro().map(|provider| { + Box::new(provider.into_evm_state_provider()) as EvmStateProviderBox + }) }); pool.begin_block(build, caches, ctx.env.txpool_snapshot.clone()); @@ -582,8 +582,7 @@ where /// Per-thread EVM state initialised by [`PrewarmContext::evm_for_ctx`] and stored in /// [`WorkerPool`] workers via [`Worker::get_or_init`](reth_tasks::pool::Worker::get_or_init). -type PrewarmEvmState = - Option>>; +type PrewarmEvmState = Option>>; impl PrewarmContext where @@ -602,8 +601,8 @@ where /// Creates a per-thread EVM for prewarming. #[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)] fn evm_for_ctx(&self) -> PrewarmEvmState { - let mut state_provider: StateProviderBox = match self.provider.database_provider_ro() { - Ok(provider) => Box::new(provider), + let mut state_provider = match self.provider.database_provider_ro() { + Ok(provider) => Box::new(provider.into_evm_state_provider()) as EvmStateProviderBox, Err(err) => { trace!( target: "engine::tree::payload_processor::prewarm", @@ -678,7 +677,7 @@ where fn send_bal_hashed_state( &self, parent_span: &Span, - provider: &mut Option>, + provider: &mut Option, account_changes: &alloy_eip7928::AccountChanges, hashed_update_stream: &StateRootUpdateStream, ) { @@ -722,7 +721,7 @@ where .entered(); let inner = match self.provider.database_provider_ro() { - Ok(p) => p, + Ok(p) => p.into_evm_state_provider(), Err(err) => { warn!( target: "engine::tree::payload_processor::prewarm", @@ -732,7 +731,7 @@ where return; } }; - let boxed: Box = + let boxed: EvmStateProviderBox = match (self.disable_bal_batch_io, &self.saved_cache) { (false, Some(saved)) => { let caches = saved.cache().clone(); diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index e2ba09e7bae..89f5f7fc548 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -154,8 +154,8 @@ use reth_primitives_traits::{ }; use reth_provider::{ BlockExecutionOutput, BlockHashReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, HashedPostStateProvider, HistoryReader, ProviderError, - PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderBox, + DatabaseProviderROFactory, EvmStateProvider, EvmStateProviderBox, HashedPostStateProvider, + HistoryReader, ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderFactory, StateReader, StateRootProvider, StorageChangeSetReader, StorageSettingsCache, }; @@ -679,41 +679,44 @@ where // // The second parameter `instrument_state_provider` controls whether we should // instrument the state provider with metrics. - let make_state_provider = |fill_on_miss: bool| -> ProviderResult { - let provider = state_provider_factory.database_provider_ro()?; - let mut provider = if let Some((caches, cache_metrics)) = &execution_cache { - let fill_mode = if fill_on_miss { - CacheFillMode::FillOnMiss + let make_state_provider = |fill_on_miss: bool| -> ProviderResult { + let provider = state_provider_factory.database_provider_ro()?.into_evm_state_provider(); + let provider: EvmStateProviderBox = + if let Some((caches, cache_metrics)) = &execution_cache { + let fill_mode = if fill_on_miss { + CacheFillMode::FillOnMiss + } else { + CacheFillMode::LookupOnly + }; + Box::new( + CachedStateProvider::new_with_mode( + provider, + caches.clone(), + fill_mode, + cache_metrics.clone(), + cache_stats.clone(), + ) + .with_txpool_snapshot(txpool_snapshot.clone()), + ) } else { - CacheFillMode::LookupOnly + Box::new(provider) }; - Box::new( - CachedStateProvider::new_with_mode( - provider, - caches.clone(), - fill_mode, - cache_metrics.clone(), - cache_stats.clone(), - ) - .with_txpool_snapshot(txpool_snapshot.clone()), - ) as StateProviderBox - } else { - Box::new(provider) as StateProviderBox - }; - if instrument_state_provider { + let provider: EvmStateProviderBox = if instrument_state_provider { let stats = state_provider_stats .as_ref() .expect("instrumented state provider requires shared stats"); let metrics = state_provider_metrics .as_ref() .expect("instrumented state provider requires metrics"); - provider = Box::new(InstrumentedStateProvider::with_stats( + Box::new(InstrumentedStateProvider::with_stats( provider, metrics.clone(), Arc::clone(stats), - )); - } + )) + } else { + provider + }; Ok(provider) }; @@ -1023,7 +1026,7 @@ where InsertBlockErrorKind, > where - S: StateProvider + Send, + S: EvmStateProvider + Send, Err: core::error::Error + Send + Sync + 'static, V: PayloadValidator, T: PayloadTypes>, @@ -1163,7 +1166,7 @@ where where Tx: ExecutableTxFor + Send, Err: core::error::Error + Send + Sync + 'static, - MakeStateProvider: Fn(bool) -> ProviderResult + Sync, + MakeStateProvider: Fn(bool) -> ProviderResult + Sync, Evm: ConfigureEngineEvm, T: PayloadTypes>, V: PayloadValidator, diff --git a/crates/engine/tree/src/tree/txpool_prewarm/worker.rs b/crates/engine/tree/src/tree/txpool_prewarm/worker.rs index 71a6efd9bea..9a44cbf0d18 100644 --- a/crates/engine/tree/src/tree/txpool_prewarm/worker.rs +++ b/crates/engine/tree/src/tree/txpool_prewarm/worker.rs @@ -200,9 +200,12 @@ where return BatchEnd::Rest } }; - let mut state = State::builder() - .with_database(self.cache.as_db_mut(StateProviderDatabase::new(state_provider))) - .build(); + let mut state = + State::builder() + .with_database(self.cache.as_db_mut(StateProviderDatabase::new( + state_provider.into_evm_state_provider(), + ))) + .build(); // The environment is the head block's own, not a predicted next-block one, and execution // is out of context by design: transaction viability is the pool's business, so nonce, // balance and (one-block-stale) basefee checks must not gate which state gets warmed. diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index 4769c729aac..5f31301aa5b 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -20,7 +20,7 @@ use reth_primitives_traits::{ block::Block as _, BlockBody as _, BlockTy, HeaderTy, SealedBlock, SignedTransaction, }; use reth_revm::{database::StateProviderDatabase, db::State}; -use reth_storage_api::{errors::ProviderError, BlockReader, StateProviderFactory}; +use reth_storage_api::{errors::ProviderError, BlockReader, StateProvider, StateProviderFactory}; use std::{ collections::VecDeque, future::Future, @@ -270,7 +270,7 @@ where let has_bal = reorg_target.header().block_access_list_hash().is_some(); let state_provider = provider.state_by_block_hash(reorg_target.header().parent_hash())?; let mut state = State::builder() - .with_database_ref(StateProviderDatabase::new(&state_provider)) + .with_database_ref(StateProviderDatabase::new((&state_provider).into_evm_state_provider())) .with_bundle_update() .with_bal_builder_if(has_bal) .build(); diff --git a/crates/ethereum/node/src/engine_ssz_witness.rs b/crates/ethereum/node/src/engine_ssz_witness.rs index 79c98146d0e..457daa8eac2 100644 --- a/crates/ethereum/node/src/engine_ssz_witness.rs +++ b/crates/ethereum/node/src/engine_ssz_witness.rs @@ -6,7 +6,7 @@ use alloy_rpc_types_engine::ExecutionData; use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; use reth_evm::{execute::Executor, ConfigureEvm}; use reth_primitives_traits::{AlloyBlockHeader, Block}; -use reth_provider::{HeaderProvider, StateProviderFactory}; +use reth_provider::{HeaderProvider, StateProvider, StateProviderBox, StateProviderFactory}; use reth_revm::{database::StateProviderDatabase, witness::ExecutionWitnessRecord}; use reth_tasks::Runtime; use reth_trie_common::ExecutionWitnessMode; @@ -70,8 +70,9 @@ where source: eyre::Report::new(source), } })?; - let block_executor = - evm_config.executor(StateProviderDatabase::new(state_provider)); + let block_executor = evm_config.executor(StateProviderDatabase::new( + state_provider.into_evm_state_provider(), + )); let mut witness = None; let mut first_header = block_number.saturating_sub(1); block_executor @@ -81,8 +82,8 @@ where } witness = Some( ExecutionWitnessRecord::new(statedb) - .into_execution_witness_without_headers( - &statedb.database.0, + .into_execution_witness_without_headers::( + &statedb.database, ExecutionWitnessMode::Canonical, ), ); diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 3cb0ade0fea..ec28b52ffac 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -32,7 +32,7 @@ use reth_payload_builder_primitives::PayloadBuilderError; use reth_payload_primitives::PayloadAttributes; use reth_primitives_traits::transaction::error::InvalidTransactionError; use reth_revm::{database::StateProviderDatabase, db::State}; -use reth_storage_api::StateProviderFactory; +use reth_storage_api::{EvmStateProvider, StateProvider, StateProviderFactory}; use reth_transaction_pool::{ error::{Eip4844PoolTransactionError, InvalidPoolTransactionError}, BestTransactions, BestTransactionsAttributes, PoolTransaction, TransactionPool, @@ -169,17 +169,23 @@ where let PayloadConfig { parent_header, attributes, payload_id, .. } = config; let skip_state_root = builder_config.skip_state_root; - let mut state_provider = client.state_by_block_hash(parent_header.hash())?; - if let Some(execution_cache) = execution_cache { - state_provider = Box::new(CachedStateProvider::new( - state_provider, + let state_provider = client.state_by_block_hash(parent_header.hash())?; + let evm_state_provider = (&state_provider).into_evm_state_provider(); + let cached_state_provider = execution_cache.map(|execution_cache| { + CachedStateProvider::new( + &evm_state_provider, execution_cache.cache().clone(), // It's ok to recreate the cache every time, because it's cheap to do so for a vanilla // Ethereum builder every 12s. Some(CachedStateMetrics::zeroed(CachedStateMetricsSource::Builder)), - )); - } - let state = StateProviderDatabase::new(state_provider.as_ref()); + ) + }); + let state = StateProviderDatabase::new( + cached_state_provider + .as_ref() + .map(|provider| provider as &dyn EvmStateProvider) + .unwrap_or(&evm_state_provider), + ); let chain_spec = client.chain_spec(); let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(attributes.timestamp()); let mut db = State::builder() diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 5473b48dafe..a88c51e143e 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -13,8 +13,8 @@ use reth_evm::execute::{BlockExecutionError, BlockExecutionOutput, Executor}; use reth_node_api::{Block as _, BlockBody as _, NodePrimitives}; use reth_primitives_traits::{format_gas_throughput, RecoveredBlock, SignedTransaction}; use reth_provider::{ - BlockReader, Chain, ExecutionOutcome, HeaderProvider, ProviderError, StateProviderFactory, - TransactionVariant, + BlockReader, Chain, ExecutionOutcome, HeaderProvider, ProviderError, StateProvider, + StateProviderFactory, TransactionVariant, }; use reth_prune_types::PruneModes; use reth_revm::database::StateProviderDatabase; @@ -79,7 +79,8 @@ where let mut executor = self.evm_config.batch_executor(StateProviderDatabase::new( self.provider .history_by_block_number(self.range.start().saturating_sub(1)) - .map_err(BlockExecutionError::other)?, + .map_err(BlockExecutionError::other)? + .into_evm_state_provider(), )); let mut fetch_block_duration = Duration::default(); @@ -217,7 +218,8 @@ where let executor = self.evm_config.batch_executor(StateProviderDatabase::new( self.provider .history_by_block_number(block_number.saturating_sub(1)) - .map_err(BlockExecutionError::other)?, + .map_err(BlockExecutionError::other)? + .into_evm_state_provider(), )); trace!(target: "exex::backfill", number = block_number, txs = block_with_senders.body().transaction_count(), "Executing block"); diff --git a/crates/exex/exex/src/backfill/test_utils.rs b/crates/exex/exex/src/backfill/test_utils.rs index b40637df82a..d09f9e6e715 100644 --- a/crates/exex/exex/src/backfill/test_utils.rs +++ b/crates/exex/exex/src/backfill/test_utils.rs @@ -14,7 +14,7 @@ use reth_node_api::NodePrimitives; use reth_primitives_traits::{Block as _, RecoveredBlock}; use reth_provider::{ providers::ProviderNodeTypes, BlockWriter as _, ExecutionOutcome, LatestStateProvider, - ProviderFactory, + ProviderFactory, StateProvider, }; use reth_revm::database::StateProviderDatabase; use reth_testing_utils::generators::sign_tx_with_key_pair; @@ -70,7 +70,9 @@ where // Execute the block to produce a block execution output let mut block_execution_output = EthEvmConfig::ethereum(chain_spec) - .batch_executor(StateProviderDatabase::new(LatestStateProvider::new(provider))) + .batch_executor(StateProviderDatabase::new( + LatestStateProvider::new(provider).into_evm_state_provider(), + )) .execute(block)?; block_execution_output.state.reverts.sort(); @@ -201,8 +203,9 @@ where let provider = provider_factory.provider()?; let evm_config = EthEvmConfig::new(chain_spec); - let executor = - evm_config.batch_executor(StateProviderDatabase::new(LatestStateProvider::new(provider))); + let executor = evm_config.batch_executor(StateProviderDatabase::new( + LatestStateProvider::new(provider).into_evm_state_provider(), + )); let mut execution_outcome = executor.execute_batch(vec![&block1, &block2])?; execution_outcome.state_mut().reverts.sort(); diff --git a/crates/revm/src/database.rs b/crates/revm/src/database.rs index 110d337cedf..496336933b2 100644 --- a/crates/revm/src/database.rs +++ b/crates/revm/src/database.rs @@ -1,76 +1,22 @@ -use crate::primitives::alloy_primitives::{BlockNumber, StorageKey, StorageValue}; use alloy_primitives::{Address, B256, U256}; use core::ops::{Deref, DerefMut}; use reth_primitives_traits::Account; -use reth_storage_api::{AccountReader, BlockHashReader, BytecodeReader, StateProvider}; +use reth_storage_api::{AccountReader, BytecodeReader, EvmStateProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use revm::{bytecode::Bytecode, state::AccountInfo, Database, DatabaseRef}; -/// A helper trait responsible for providing state necessary for EVM execution. -/// -/// This serves as the data layer for [`Database`]. -pub trait EvmStateProvider { - /// Get basic account information. - /// - /// Returns [`None`] if the account doesn't exist. - fn basic_account(&self, address: &Address) -> ProviderResult>; - - /// Get the hash of the block with the given number. Returns [`None`] if no block with this - /// number exists. - fn block_hash(&self, number: BlockNumber) -> ProviderResult>; - - /// Get account code by hash. - fn bytecode_by_hash( - &self, - code_hash: &B256, - ) -> ProviderResult>; - - /// Get storage of the given account. - fn storage( - &self, - account: Address, - storage_key: StorageKey, - ) -> ProviderResult>; -} - -// Blanket implementation of EvmStateProvider for any type that implements StateProvider. -impl EvmStateProvider for T { - fn basic_account(&self, address: &Address) -> ProviderResult> { - ::basic_account(self, address) - } - - fn block_hash(&self, number: BlockNumber) -> ProviderResult> { - ::block_hash(self, number) - } - - fn bytecode_by_hash( - &self, - code_hash: &B256, - ) -> ProviderResult> { - ::bytecode_by_hash(self, code_hash) - } - - fn storage( - &self, - account: Address, - storage_key: StorageKey, - ) -> ProviderResult> { - ::storage(self, account, storage_key) - } -} - /// A [Database] and [`DatabaseRef`] implementation that uses [`EvmStateProvider`] as the underlying /// data source. #[derive(Clone)] pub struct StateProviderDatabase(pub DB); impl StateProviderDatabase { - /// Create new State with generic `StateProvider`. + /// Creates a database backed by an [`EvmStateProvider`]. pub const fn new(db: DB) -> Self { Self(db) } - /// Consume State and return inner `StateProvider`. + /// Consumes the database and returns its inner provider. pub fn into_inner(self) -> DB { self.0 } @@ -173,8 +119,8 @@ impl DatabaseRef for StateProviderDatabase { /// A [`DatabaseRef`] backed account info reader. /// /// This adapts account and bytecode reads from revm's database interface back into Reth's storage -/// reader traits. It is intentionally not a full [`StateProvider`] because [`DatabaseRef`] does -/// not expose roots or proofs. +/// reader traits. It is intentionally not a full [`reth_storage_api::StateProvider`] because +/// [`DatabaseRef`] does not expose roots or proofs. /// /// Note: [`DatabaseRef::code_by_hash_ref`] returns [`Bytecode`] directly, so this adapter cannot /// distinguish missing bytecode from the database's default bytecode and wraps whatever the diff --git a/crates/rpc/rpc-eth-api/src/helpers/bal.rs b/crates/rpc/rpc-eth-api/src/helpers/bal.rs index ec68eebf995..ef768f77424 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bal.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bal.rs @@ -7,7 +7,7 @@ use reth_errors::RethError; use reth_evm::{block::BlockExecutor, ConfigureEvm, Evm}; use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_eth_types::{error::FromEthApiError, EthApiError}; -use reth_storage_api::StateProviderFactory; +use reth_storage_api::{StateProvider, StateProviderFactory}; use std::sync::Arc; use crate::{ @@ -54,7 +54,7 @@ pub trait GetBlockAccessList: Trace + Call + LoadBlock + RpcNodeCoreExt { .map_err(Self::Error::from_eth_err)?; let mut db = State::builder() - .with_database(StateProviderDatabase::new(state)) + .with_database(StateProviderDatabase::new(state.into_evm_state_provider())) .with_bal_builder() .build(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 48249a286fa..d7e0fe40e14 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -41,7 +41,7 @@ use reth_rpc_eth_types::{ simulate::{self, EthSimulateError}, EthApiError, StateCacheDb, }; -use reth_storage_api::{BlockIdReader, ProviderTx}; +use reth_storage_api::{BlockIdReader, ProviderTx, StateProvider}; use revm::{ context::Block, context_interface::{result::ResultAndState, Cfg, Transaction}, @@ -108,7 +108,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA self.spawn_with_state_at_block(block, move |this, db| { let _permit = permit; - let state_provider = db.database.0; + let state_provider = db.database.into_inner(); let mut db = State::builder() .with_database(StateProviderDatabase::new(&state_provider)) .with_bundle_update() @@ -241,7 +241,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA simulate::execute_transactions( builder, - &state_provider, + &*state_provider, calls, &mut remaining_call_gas_limit, chain_id, @@ -263,7 +263,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA simulate::execute_transactions( builder, - &state_provider, + &*state_provider, calls, &mut remaining_call_gas_limit, chain_id, @@ -650,7 +650,9 @@ pub trait Call: let at = at.into(); self.spawn_blocking_io_fut(async move |this| { let state = this.state_at_block_id(at).await?; - let db = State::builder().with_database(StateProviderDatabase::new(state)).build(); + let db = State::builder() + .with_database(StateProviderDatabase::new(state.into_evm_state_provider())) + .build(); f(this, db) }) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs index 76a05f08de3..faa5b3bd9ca 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs @@ -14,7 +14,7 @@ use reth_evm::{ TxEnvFor, }; use reth_revm::{ - database::{EvmStateProvider, StateProviderDatabase}, + database::StateProviderDatabase, db::{bal::EvmDatabaseError, State}, }; use reth_rpc_convert::{RpcConvert, RpcTxReq}; @@ -26,6 +26,7 @@ use reth_rpc_eth_types::{ EthApiError, RpcInvalidTransactionError, }; use reth_rpc_server_types::constants::gas_oracle::{CALL_STIPEND_GAS, ESTIMATE_GAS_ERROR_RATIO}; +use reth_storage_api::{EvmStateProvider, StateProvider}; use revm::{ context::Block, context_interface::{result::ExecutionResult, Cfg, Transaction}, @@ -314,7 +315,13 @@ pub trait EstimateCall: Call { self.spawn_blocking_io_fut(async move |this| { let state = this.state_at_block_id(at).await?; - EstimateCall::estimate_gas_with(&this, evm_env, request, state, overrides) + EstimateCall::estimate_gas_with( + &this, + evm_env, + request, + state.into_evm_state_provider(), + overrides, + ) }) .await } diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 9fc4f3558fd..1a63562f58f 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -24,7 +24,7 @@ use reth_rpc_eth_types::{ PendingBlockEnv, PendingBlockEnvOrigin, }; use reth_storage_api::{ - noop::NoopProvider, BlockReader, BlockReaderIdExt, ProviderHeader, ProviderTx, + noop::NoopProvider, BlockReader, BlockReaderIdExt, ProviderHeader, ProviderTx, StateProvider, StateProviderBox, StateProviderFactory, }; use reth_transaction_pool::{ @@ -255,7 +255,7 @@ pub trait LoadPendingBlock: .provider() .history_by_block_hash(parent.hash()) .map_err(Self::Error::from_eth_err)?; - let state = StateProviderDatabase::new(state_provider); + let state = StateProviderDatabase::new(state_provider.into_evm_state_provider()); let mut db = State::builder().with_database(state).with_bundle_update().build(); let mut builder = self diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 0a2220e5204..81a4e6b40d1 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -2,12 +2,12 @@ use alloy_eip7928::{bal::DecodedBal, BlockAccessIndex}; use reth_revm::database::StateProviderDatabase; -use reth_storage_api::StateProviderBox; +use reth_storage_api::{EvmStateProviderAdapter, StateProviderBox}; use revm::{database::State, state::bal::Bal as RevmBal, Database}; use std::sync::Arc; /// Helper alias type for the state's [`State`] -pub type StateCacheDb = State>; +pub type StateCacheDb = State>>; /// Attaches `bal` to the database, positioned at the state right before the transaction at /// `tx_index`. diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index f10ee838152..c0bd7cba295 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -35,8 +35,8 @@ use reth_rpc_eth_types::{EthApiError, StateCacheDb}; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use reth_storage_api::{ BlockIdReader, BlockReaderIdExt, HashedPostStateProvider, HeaderProvider, ProviderBlock, - ReceiptProviderIdExt, StateProviderFactory, StateRootProvider, StorageRootProvider, - TransactionVariant, + ReceiptProviderIdExt, StateProviderBox, StateProviderFactory, StateRootProvider, + StorageRootProvider, TransactionVariant, }; use reth_tasks::{pool::BlockingTaskGuard, Runtime}; use reth_transaction_pool::TransactionPool; @@ -632,13 +632,15 @@ where let mut witness = None; let _ = block_executor .execute_with_state_closure(&block, |statedb: &State<_>| { - witness = - Some(ExecutionWitnessRecord::new(statedb).into_execution_witness( - &statedb.database.database.0, - eth_api.provider(), - block_number, - mode, - )); + witness = Some( + ExecutionWitnessRecord::new(statedb) + .into_execution_witness::( + &statedb.database.database, + eth_api.provider(), + block_number, + mode, + ), + ); }) .map_err(|err| EthApiError::Internal(err.into()))?; diff --git a/crates/rpc/rpc/src/reth.rs b/crates/rpc/rpc/src/reth.rs index ab03b422e6a..121a25f9b59 100644 --- a/crates/rpc/rpc/src/reth.rs +++ b/crates/rpc/rpc/src/reth.rs @@ -17,7 +17,8 @@ use reth_primitives_traits::{NodePrimitives, SealedHeader}; use reth_rpc_api::{RethApiServer, RethJitAction}; use reth_rpc_eth_types::{EthApiError, EthResult}; use reth_storage_api::{ - BlockReader, BlockReaderIdExt, ChangeSetReader, StateProviderFactory, TransactionVariant, + BlockReader, BlockReaderIdExt, ChangeSetReader, StateProvider, StateProviderFactory, + TransactionVariant, }; use reth_tasks::{pool::BlockingTaskGuard, Runtime}; use serde::Serialize; @@ -159,7 +160,9 @@ where } let state_provider = self.provider().history_by_block_number(start_block - 1)?; - let db = reth_revm::database::StateProviderDatabase::new(&state_provider); + let db = reth_revm::database::StateProviderDatabase::new( + (&state_provider).into_evm_state_provider(), + ); let mut blocks = Vec::with_capacity(block_count as usize); for block_number in start_block..start_block + block_count { diff --git a/crates/rpc/rpc/src/testing.rs b/crates/rpc/rpc/src/testing.rs index 99206114aa8..24b8381a4be 100644 --- a/crates/rpc/rpc/src/testing.rs +++ b/crates/rpc/rpc/src/testing.rs @@ -125,7 +125,7 @@ where let gas_limit_override = self.gas_limit_override; self.eth_api .spawn_with_state_at_block(request.parent_block_hash, move |eth_api, state| { - let state = state.database.0; + let state = state.database.into_inner(); let parent = eth_api .provider() .sealed_header_by_hash(request.parent_block_hash)? @@ -268,7 +268,7 @@ where block_transactions_rlp_length += tx_rlp_len; total_fees += U256::from(tip) * U256::from(gas_used); } - let outcome = builder.finish(&state, None).map_err(Eth::Error::from_eth_err)?; + let outcome = builder.finish(&*state, None).map_err(Eth::Error::from_eth_err)?; let has_requests = outcome.block.requests_hash().is_some(); let requests = has_requests.then_some(outcome.execution_result.requests); diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index b7b74488b35..884661703b8 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -36,7 +36,9 @@ use reth_primitives_traits::{ use reth_revm::{cached::CachedReads, database::StateProviderDatabase}; use reth_rpc_api::BlockSubmissionValidationApiServer; use reth_rpc_server_types::result::{internal_rpc_err, invalid_params_rpc_err}; -use reth_storage_api::{BlockReaderIdExt, HashedPostStateProvider, StateProviderFactory}; +use reth_storage_api::{ + BlockReaderIdExt, HashedPostStateProvider, StateProvider, StateProviderFactory, +}; use reth_tasks::Runtime; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -196,7 +198,8 @@ where let mut request_cache = self.cached_reads(parent_header_hash).await; let (output, block_access_list_hash) = { - let cached_db = request_cache.as_db_mut(StateProviderDatabase::new(&state_provider)); + let cached_db = request_cache + .as_db_mut(StateProviderDatabase::new((&state_provider).into_evm_state_provider())); let mut executor = self.evm_config.batch_executor(cached_db); let result = executor.execute_one(&block)?; diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index f185df2db23..990f893ecd5 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -15,8 +15,8 @@ use reth_provider::{ providers::{StaticFileProvider, StaticFileWriter}, BlockHashReader, BlockReader, DBProvider, EitherWriter, ExecutionOutcome, HashedPostStateProvider, HeaderProvider, LatestStateProviderRef, OriginalValuesKnown, - ProviderError, StateWriteConfig, StateWriter, StaticFileProviderFactory, StatsReader, - StoragePath, StorageSettingsCache, TransactionVariant, + ProviderError, StateProvider, StateWriteConfig, StateWriter, StaticFileProviderFactory, + StatsReader, StoragePath, StorageSettingsCache, TransactionVariant, }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::{ @@ -305,7 +305,8 @@ where self.ensure_consistency(provider, input.checkpoint().block_number, None)?; - let db = StateProviderDatabase(LatestStateProviderRef::new(provider)); + let db = + StateProviderDatabase(LatestStateProviderRef::new(provider).into_evm_state_provider()); let mut executor = self.evm_config.batch_executor(db); // Progress tracking diff --git a/crates/stages/stages/tests/pipeline.rs b/crates/stages/stages/tests/pipeline.rs index 02ee384bff3..784f2e21b5d 100644 --- a/crates/stages/stages/tests/pipeline.rs +++ b/crates/stages/stages/tests/pipeline.rs @@ -338,7 +338,7 @@ async fn run_pipeline_forward_and_unwind( // Execute in a scope so state_provider is dropped before we use provider for writes let output = { let state_provider = provider.latest(); - let db = StateProviderDatabase::new(&*state_provider); + let db = StateProviderDatabase::new((&*state_provider).into_evm_state_provider()); let executor = evm_config.batch_executor(db); executor.execute(&block_with_senders)? }; diff --git a/crates/stages/stages/tests/preimage.rs b/crates/stages/stages/tests/preimage.rs index 9f7b0436552..d87849ce204 100644 --- a/crates/stages/stages/tests/preimage.rs +++ b/crates/stages/stages/tests/preimage.rs @@ -48,7 +48,9 @@ use reth_stages::{ }; use reth_stages_api::{Pipeline, StageSet}; use reth_static_file::StaticFileProducer; -use reth_storage_api::{StorageChangeSetReader, StorageSettings, StorageSettingsCache}; +use reth_storage_api::{ + StateProvider, StorageChangeSetReader, StorageSettings, StorageSettingsCache, +}; use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair}; use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot}; use reth_trie_db::DatabaseStateRoot; @@ -552,7 +554,7 @@ fn setup_create2_selfdestruct_scenario() -> eyre::Result ProviderResult>; + + /// Returns the block hash, or `None` if the block does not exist. + fn block_hash(&self, number: BlockNumber) -> ProviderResult>; + + /// Returns bytecode by its hash. + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult>; + + /// Returns the storage value of the given account and slot. + fn storage( + &self, + account: Address, + storage_key: StorageKey, + ) -> ProviderResult>; +} + +/// Type-erased provider for EVM execution. +pub type EvmStateProviderBox = Box; + +/// Adapts an owned or borrowed full state provider for EVM execution. +#[derive(Debug, Clone, Copy)] +pub struct EvmStateProviderAdapter

(pub P); + +impl

Deref for EvmStateProviderAdapter

{ + type Target = P; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl EvmStateProvider for EvmStateProviderAdapter

{ + fn basic_account(&self, address: &Address) -> ProviderResult> { + self.0.basic_account(address) + } + + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.0.block_hash(number) + } + + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.0.bytecode_by_hash(code_hash) + } + + fn storage( + &self, + account: Address, + storage_key: StorageKey, + ) -> ProviderResult> { + self.0.storage(account, storage_key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{noop::NoopProvider, StateProviderBox}; + use alloc::sync::Arc; + + #[test] + fn converts_owned_borrowed_and_boxed_state_providers() { + let provider = NoopProvider::default(); + assert_reads((&provider).into_evm_state_provider()); + assert_reads((&provider as &dyn StateProvider).into_evm_state_provider()); + assert_reads(provider.into_evm_state_provider()); + + let boxed = Box::new(NoopProvider::default()) as StateProviderBox; + assert_reads(boxed.into_evm_state_provider()); + assert_reads(Arc::new(NoopProvider::default()).into_evm_state_provider()); + } + + #[test] + fn delegates_execution_only_references_and_boxes() { + let provider = NoopProvider::default(); + let adapter = (&provider).into_evm_state_provider(); + assert_reads(&adapter as &dyn EvmStateProvider); + assert_reads(Box::new(adapter) as Box); + + let boxed = Box::new(provider.into_evm_state_provider()) as EvmStateProviderBox; + assert_reads(boxed); + assert_reads(Arc::new(NoopProvider::default().into_evm_state_provider())); + } + + #[test] + fn exposes_inner_state_provider() { + let provider = NoopProvider::default(); + let adapter = (&provider).into_evm_state_provider(); + assert_eq!(adapter.account_balance(&Address::ZERO).unwrap(), None); + assert!(core::ptr::eq(*adapter, &raw const provider)); + } + + fn assert_reads(provider: impl EvmStateProvider) { + assert_eq!(provider.basic_account(&Address::ZERO).unwrap(), None); + assert_eq!(provider.block_hash(1).unwrap(), None); + assert_eq!(provider.bytecode_by_hash(&B256::ZERO).unwrap(), None); + assert_eq!(provider.storage(Address::ZERO, B256::ZERO).unwrap(), None); + } +} diff --git a/crates/storage/storage-api/src/lib.rs b/crates/storage/storage-api/src/lib.rs index 9651b870dcf..f4f8e53aff9 100644 --- a/crates/storage/storage-api/src/lib.rs +++ b/crates/storage/storage-api/src/lib.rs @@ -112,3 +112,6 @@ mod full; pub use full::*; pub mod macros; + +mod evm; +pub use evm::*; diff --git a/crates/storage/storage-api/src/state.rs b/crates/storage/storage-api/src/state.rs index 200c4c3fd68..f5eb61ebb69 100644 --- a/crates/storage/storage-api/src/state.rs +++ b/crates/storage/storage-api/src/state.rs @@ -1,6 +1,6 @@ use super::{ - AccountReader, BlockHashReader, BlockIdReader, StateProofProvider, StateRootProvider, - StorageRootProvider, + AccountReader, BlockHashReader, BlockIdReader, EvmStateProviderAdapter, StateProofProvider, + StateRootProvider, StorageRootProvider, }; use alloc::boxed::Box; use alloy_consensus::constants::KECCAK_EMPTY; @@ -90,6 +90,17 @@ pub trait StateProvider: // Returns None if acc doesn't exist self.basic_account(addr)?.map_or_else(|| Ok(None), |acc| Ok(Some(acc.nonce))) } + + /// Wraps this provider for EVM execution without allocating or cloning it. + /// + /// Call this on a reference to borrow the provider, or on a box to retain ownership. + #[auto_impl(keep_default_for(&, Arc, Box))] + fn into_evm_state_provider(self) -> EvmStateProviderAdapter + where + Self: Sized, + { + EvmStateProviderAdapter(self) + } } /// Minimal requirements to read a full account, for example, to validate its new transactions diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index 9bc59749f40..15a6ba7cc88 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -16,7 +16,7 @@ use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{ParallelBridgeBuffered, RecoveredBlock, SealedBlock}; use reth_provider::{ test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, DatabaseProviderFactory, - ExecutionOutcome, HashedPostStateProvider, HistoryWriter, OriginalValuesKnown, + ExecutionOutcome, HashedPostStateProvider, HistoryWriter, OriginalValuesKnown, StateProvider, StateWriteConfig, StateWriter, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, StorageSettingsCache, TrieWriter, }; @@ -245,7 +245,7 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { // Execute the block let state_provider = provider.latest(); - let state_db = StateProviderDatabase(&state_provider); + let state_db = StateProviderDatabase((&state_provider).into_evm_state_provider()); let executor = executor_provider.batch_executor(state_db); let output = executor From 58a51b3ee3f6714ded9207b244a273c8afb592fd Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:43:52 +0000 Subject: [PATCH 057/133] fix(ci): enable benchmark frame pointers (#27393) Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- .github/scripts/bench-txgen-build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/bench-txgen-build.sh b/.github/scripts/bench-txgen-build.sh index 4c040558064..e10f9a0e789 100755 --- a/.github/scripts/bench-txgen-build.sh +++ b/.github/scripts/bench-txgen-build.sh @@ -22,10 +22,8 @@ else fi EXTRA_FEATURES="" -EXTRA_RUSTFLAGS="" if [ "${BENCH_TRACY:-off}" != "off" ]; then EXTRA_FEATURES="tracy,tracy-client/ondemand" - EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes" fi build_node_binary() { @@ -39,7 +37,7 @@ build_node_binary() { fi # shellcheck disable=SC2086 - RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \ + RUSTFLAGS="-C target-cpu=native -C force-frame-pointers=yes" \ cargo build --locked --profile profiling $NODE_PKG $workspace_arg $features_arg } From 0d1648572c8df05ccfd6501a1ff7ee8197d49bb9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 10:12:32 +0200 Subject: [PATCH 058/133] fix(rpc): require explicit testing API (#27398) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/ethereum/node/tests/e2e/eth.rs | 15 +++--- crates/rpc/rpc-builder/src/config.rs | 32 ++++++++++++ crates/rpc/rpc-server-types/src/module.rs | 63 ++++++++++++++--------- docs/vocs/docs/pages/jsonrpc/intro.mdx | 8 +-- 4 files changed, 83 insertions(+), 35 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index a246eafb2c5..728ba82b06e 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -31,6 +31,7 @@ use reth_node_ethereum::{ use reth_provider::{BlockNumReader, StateProviderFactory}; use reth_rpc_api::TestingBuildBlockRequestV1; use reth_rpc_layer::secret_to_bearer_header; +use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; use reth_tasks::Runtime; use ssz::{Decode, Encode}; use std::sync::Arc; @@ -224,10 +225,9 @@ async fn test_testing_build_block_v1_osaka() -> eyre::Result<()> { let node_config = NodeConfig::test().with_chain(chain_spec.clone()).with_unused_ports().with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(reth_rpc_server_types::RpcModuleSelection::All), + RpcServerArgs::default().with_unused_ports().with_http().with_http_api( + RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing]), + ), ); let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) @@ -299,10 +299,9 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let genesis_hash = chain_spec.genesis_hash(); let node_config = NodeConfig::test().with_chain(chain_spec.clone()).with_unused_ports().with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(reth_rpc_server_types::RpcModuleSelection::All), + RpcServerArgs::default().with_unused_ports().with_http().with_http_api( + RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing]), + ), ); let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index 42e4d35576b..546cccd8c8b 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -376,6 +376,38 @@ mod tests { ); } + #[test] + fn test_testing_namespace_requires_explicit_selection() { + let args = CommandParser::::parse_from([ + "reth", + "--http", + "--http.api", + "all", + "--ws", + "--ws.api", + "all", + ]) + .args; + let config = args.transport_rpc_module_config(); + assert!(!config.contains_http(&RethRpcModule::Testing)); + assert!(!config.contains_ws(&RethRpcModule::Testing)); + assert!(!config.contains_ipc(&RethRpcModule::Testing)); + + let args = CommandParser::::parse_from([ + "reth", + "--http", + "--http.api", + "eth,testing", + "--ws", + "--ws.api", + "eth,testing", + ]) + .args; + let config = args.transport_rpc_module_config(); + assert!(config.contains_http(&RethRpcModule::Testing)); + assert!(config.contains_ws(&RethRpcModule::Testing)); + } + #[test] fn test_transport_rpc_module_trim_config() { let args = CommandParser::::parse_from([ diff --git a/crates/rpc/rpc-server-types/src/module.rs b/crates/rpc/rpc-server-types/src/module.rs index d41e10bb737..8027c052368 100644 --- a/crates/rpc/rpc-server-types/src/module.rs +++ b/crates/rpc/rpc-server-types/src/module.rs @@ -15,7 +15,7 @@ use strum::{ParseError, VariantNames}; /// ``` #[derive(Debug, Default, Clone, Eq, PartialEq)] pub enum RpcModuleSelection { - /// Use _all_ available modules. + /// Use all modules except `testing`, which requires an explicit selection. All, /// The default modules `eth`, `net`, `web3` #[default] @@ -31,9 +31,9 @@ impl RpcModuleSelection { pub const STANDARD_MODULES: [RethRpcModule; 3] = [RethRpcModule::Eth, RethRpcModule::Net, RethRpcModule::Web3]; - /// Returns a selection of [`RethRpcModule`] with all [`RethRpcModule::all_variants`]. + /// Returns all modules that do not require an explicit selection. pub fn all_modules() -> HashSet { - RethRpcModule::modules().into_iter().collect() + Self::all_modules_iter().collect() } /// Returns the [`RpcModuleSelection::STANDARD_MODULES`] as a selection. @@ -41,9 +41,7 @@ impl RpcModuleSelection { HashSet::from(Self::STANDARD_MODULES) } - /// All modules that are available by default on IPC. - /// - /// By default all modules are available on IPC. + /// All modules that are available by default on IPC, excluding `testing`. pub fn default_ipc_modules() -> HashSet { Self::all_modules() } @@ -84,7 +82,7 @@ impl RpcModuleSelection { /// Returns the number of modules in the selection pub fn len(&self) -> usize { match self { - Self::All => RethRpcModule::variant_count(), + Self::All => Self::all_modules_iter().count(), Self::Standard => Self::STANDARD_MODULES.len(), Self::Selection(s) => s.len(), } @@ -106,7 +104,7 @@ impl RpcModuleSelection { /// Returns an iterator over all configured [`RethRpcModule`] pub fn iter_selection(&self) -> Box + '_> { match self { - Self::All => Box::new(RethRpcModule::modules().into_iter()), + Self::All => Box::new(Self::all_modules_iter()), Self::Standard => Box::new(Self::STANDARD_MODULES.iter().cloned()), Self::Selection(s) => Box::new(s.iter().cloned()), } @@ -133,11 +131,6 @@ impl RpcModuleSelection { /// Returns true if both selections are identical. pub fn are_identical(http: Option<&Self>, ws: Option<&Self>) -> bool { match (http, ws) { - // Shortcut for common case to avoid iterating later - (Some(Self::All), Some(other)) | (Some(other), Some(Self::All)) => { - other.len() == RethRpcModule::variant_count() - } - // If either side is disabled, then the other must be empty (Some(some), None) | (None, Some(some)) => some.is_empty(), @@ -149,7 +142,7 @@ impl RpcModuleSelection { /// Returns true if the selection contains the given module. pub fn contains(&self, module: &RethRpcModule) -> bool { match self { - Self::All => true, + Self::All => !matches!(module, RethRpcModule::Testing), Self::Standard => Self::STANDARD_MODULES.contains(module), Self::Selection(s) => s.contains(module), } @@ -212,6 +205,10 @@ impl RpcModuleSelection { Self::Selection(modules) } } + + fn all_modules_iter() -> impl Iterator { + RethRpcModule::modules().into_iter().filter(|module| *module != RethRpcModule::Testing) + } } impl From<&HashSet> for RpcModuleSelection { @@ -542,7 +539,20 @@ mod test { #[test] fn test_all_modules() { let all_modules = RpcModuleSelection::all_modules(); - assert_eq!(all_modules.len(), RethRpcModule::variant_count()); + assert_eq!(all_modules.len(), RethRpcModule::variant_count() - 1); + assert!(!all_modules.contains(&RethRpcModule::Testing)); + } + + #[test] + fn test_testing_requires_explicit_selection() { + let all = RpcModuleSelection::All; + let explicit = RpcModuleSelection::from([RethRpcModule::Testing]); + + assert!(!all.contains(&RethRpcModule::Testing)); + assert!(!all.to_selection().contains(&RethRpcModule::Testing)); + assert!(!all.into_selection().contains(&RethRpcModule::Testing)); + assert!(explicit.contains(&RethRpcModule::Testing)); + assert!(explicit.iter_selection().any(|module| module == RethRpcModule::Testing)); } #[test] @@ -557,6 +567,7 @@ mod test { fn test_default_ipc_modules() { let default_ipc_modules = RpcModuleSelection::default_ipc_modules(); assert_eq!(default_ipc_modules, RpcModuleSelection::all_modules()); + assert!(!default_ipc_modules.contains(&RethRpcModule::Testing)); } #[test] @@ -572,7 +583,7 @@ mod test { let standard = RpcModuleSelection::Standard; let selection = RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Admin]); - assert_eq!(all_modules.len(), RethRpcModule::variant_count()); + assert_eq!(all_modules.len(), RethRpcModule::variant_count() - 1); assert_eq!(standard.len(), 3); assert_eq!(selection.len(), 2); } @@ -592,7 +603,8 @@ mod test { let standard = RpcModuleSelection::Standard; let selection = RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Admin]); - assert_eq!(all_modules.iter_selection().count(), RethRpcModule::variant_count()); + assert_eq!(all_modules.iter_selection().count(), RethRpcModule::variant_count() - 1); + assert!(!all_modules.iter_selection().any(|module| module == RethRpcModule::Testing)); assert_eq!(standard.iter_selection().count(), 3); assert_eq!(selection.iter_selection().count(), 2); } @@ -615,8 +627,7 @@ mod test { fn test_rpc_module_selection_are_identical() { // Test scenario: both selections are `All` // - // Since both selections include all possible RPC modules, they should be considered - // identical. + // Both selections include the same modules, excluding `testing`. let all_modules = RpcModuleSelection::All; assert!(RpcModuleSelection::are_identical(Some(&all_modules), Some(&all_modules))); @@ -668,12 +679,16 @@ mod test { // Test scenario: full selection vs `All` // - // If the other selection explicitly selects all available modules, it should be identical - // to `All`. - let full_selection = - RpcModuleSelection::from(RethRpcModule::modules().into_iter().collect::>()); + // An explicit selection of the same modules is identical to `All`. + let full_selection = RpcModuleSelection::from(RpcModuleSelection::all_modules()); assert!(RpcModuleSelection::are_identical(Some(&all_modules), Some(&full_selection))); + // Explicitly including `testing` makes the selections different. + let mut testing_selection = RpcModuleSelection::all_modules(); + testing_selection.insert(RethRpcModule::Testing); + let testing_selection = RpcModuleSelection::from(testing_selection); + assert!(!RpcModuleSelection::are_identical(Some(&all_modules), Some(&testing_selection),)); + // Test scenario: different non-empty selections // // If the two selections contain different sets of modules, they should not be considered @@ -850,7 +865,7 @@ mod test { // But All doesn't explicitly contain custom modules // (though contains() returns true for all modules when selection is All) - assert_eq!(all_selection.len(), RethRpcModule::variant_count()); + assert_eq!(all_selection.len(), RethRpcModule::variant_count() - 1); } #[test] diff --git a/docs/vocs/docs/pages/jsonrpc/intro.mdx b/docs/vocs/docs/pages/jsonrpc/intro.mdx index be67fb854a7..78a0aae04c0 100644 --- a/docs/vocs/docs/pages/jsonrpc/intro.mdx +++ b/docs/vocs/docs/pages/jsonrpc/intro.mdx @@ -67,7 +67,7 @@ To enable JSON-RPC namespaces on the HTTP server, pass each namespace separated reth node --http --http.api eth,net,trace ``` -You can pass the `all` option, which is a convenient wrapper for all the JSON-RPC namespaces `admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev,testing` on the HTTP server: +You can pass the `all` option, which enables the JSON-RPC namespaces `admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev` on the HTTP server. The `testing` namespace is excluded and must be selected explicitly: ```bash reth node --http --http.api all @@ -77,6 +77,8 @@ reth node --http --http.api all reth node --http --http.api All ``` +To enable the testing namespace on a trusted endpoint, include `testing` in an explicit list, for example `--http.api eth,testing`. + Similarly, you can pass the `none` option (also case-insensitive) to start the HTTP server without exposing any JSON-RPC namespaces on that transport: ```bash @@ -116,7 +118,7 @@ The configuration of the WebSocket server follows the same pattern as the HTTP s IPC is a simpler transport protocol for use in local environments where the node and the client exist on the same machine. -The IPC transport is enabled by default and has access to all namespaces, unless explicitly disabled with `--ipcdisable`. +The IPC transport is enabled by default and has access to all namespaces except `testing`, unless explicitly disabled with `--ipcdisable`. Reth creates a UNIX socket on Linux and macOS at `/tmp/reth.ipc`. On Windows, IPC is provided using named pipes at `\\.\pipe\reth.ipc`. @@ -131,7 +133,7 @@ You can use `curl`, a programming language with a low-level library, or a tool l As a reminder, you need to run the command below to enable all of these APIs using an HTTP transport: ```bash -reth node --http --http.api "admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev,testing" +reth node --http --http.api "admin,debug,eth,net,trace,txpool,web3,rpc,reth,ots,flashbots,miner,mev" ``` This allows you to then call: From 9b8e2480feb8514a0b31279bf25eaa30dfd38d8a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 10:19:56 +0200 Subject: [PATCH 059/133] feat(engine): make the backfill threshold configurable (#27329) Co-authored-by: Rodi-Can Bozman Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/engine/primitives/src/config.rs | 68 ++++++++++++- crates/engine/tree/src/tree/mod.rs | 15 +-- crates/engine/tree/src/tree/tests.rs | 127 ++++++++++++++++++++----- crates/node/core/src/args/engine.rs | 50 +++++++++- docs/vocs/docs/pages/cli/reth/node.mdx | 8 ++ 5 files changed, 227 insertions(+), 41 deletions(-) diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index 7493e0f0594..a1f1b5324d6 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -16,6 +16,22 @@ pub const MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 16; /// How close to the canonical head we persist blocks. pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 5; +/// The largest gap for which the tree will be used to sync individual blocks by downloading them. +/// +/// This is the default threshold, and represents the distance (gap) from the local head to a +/// new (canonical) block, e.g. the forkchoice head block. If the block distance from the local head +/// exceeds this threshold, the pipeline will be used to backfill the gap more efficiently. +/// +/// E.g.: Local head `block.number` is 100 and the forkchoice head `block.number` is 133 (more than +/// an epoch has slots), then this exceeds the threshold at which the pipeline should be used to +/// backfill this gap. +pub const DEFAULT_BACKFILL_RUN_THRESHOLD: u64 = EPOCH_SLOTS; + +/// Largest supported live-sync gap, matching the peer header response limit of 1024. +/// +/// The range downloader requires a complete response and cannot yet split larger requests. +pub const MAX_BACKFILL_RUN_THRESHOLD: u64 = 1024; + /// The size of proof targets chunk to spawn in one multiproof calculation. pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 5; @@ -116,6 +132,9 @@ pub struct TreeConfig { /// Number of pending blocks that cannot be executed due to missing parent and /// are kept in cache. block_buffer_limit: u32, + /// Largest gap between the local head and a new canonical block that is closed by downloading + /// the missing blocks. Larger gaps trigger a pipeline (backfill) run instead. + backfill_run_threshold: u64, /// Number of invalid headers to keep in cache. max_invalid_header_cache_length: u32, /// Number of cache hits before an invalid header entry is evicted and reprocessed. @@ -238,6 +257,7 @@ impl Default for TreeConfig { memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, persistence_backpressure_threshold: DEFAULT_PERSISTENCE_THRESHOLD * 2, block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT, + backfill_run_threshold: DEFAULT_BACKFILL_RUN_THRESHOLD, max_invalid_header_cache_length: DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH, invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, max_execute_block_batch_size: DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE, @@ -274,6 +294,8 @@ impl Default for TreeConfig { impl TreeConfig { /// Create engine tree configuration. + /// + /// The backfill run threshold is clamped to [`MAX_BACKFILL_RUN_THRESHOLD`]. #[expect(clippy::too_many_arguments)] pub const fn new( persistence_threshold: u64, @@ -281,6 +303,7 @@ impl TreeConfig { memory_block_buffer_target: u64, persistence_backpressure_threshold: u64, block_buffer_limit: u32, + backfill_run_threshold: u64, max_invalid_header_cache_length: u32, invalid_header_hit_eviction_threshold: u8, max_execute_block_batch_size: usize, @@ -320,6 +343,7 @@ impl TreeConfig { memory_block_buffer_target, persistence_backpressure_threshold, block_buffer_limit, + backfill_run_threshold: clamp_backfill_run_threshold(backfill_run_threshold), max_invalid_header_cache_length, invalid_header_hit_eviction_threshold, max_execute_block_batch_size, @@ -378,6 +402,11 @@ impl TreeConfig { self.block_buffer_limit } + /// Return the backfill run threshold. + pub const fn backfill_run_threshold(&self) -> u64 { + self.backfill_run_threshold + } + /// Return the maximum invalid cache header length. pub const fn max_invalid_header_cache_length(&self) -> u32 { self.max_invalid_header_cache_length @@ -537,6 +566,20 @@ impl TreeConfig { self } + /// Setter for backfill run threshold. + /// + /// Clamps the threshold to [`MAX_BACKFILL_RUN_THRESHOLD`] and grows the block buffer to twice + /// the clamped threshold so it can retain disconnected blocks while downloading the gap, + /// with room for incoming payloads. + pub const fn with_backfill_run_threshold(mut self, backfill_run_threshold: u64) -> Self { + self.backfill_run_threshold = clamp_backfill_run_threshold(backfill_run_threshold); + let buffer_limit = self.backfill_run_threshold as u32 * 2; + if buffer_limit > self.block_buffer_limit { + self.block_buffer_limit = buffer_limit; + } + self + } + /// Setter for maximum invalid header cache length. pub const fn with_max_invalid_header_cache_length( mut self, @@ -819,9 +862,32 @@ impl TreeConfig { } } +/// Keep live-sync requests within a single peer header response until chunking is supported. +const fn clamp_backfill_run_threshold(threshold: u64) -> u64 { + if threshold > MAX_BACKFILL_RUN_THRESHOLD { + MAX_BACKFILL_RUN_THRESHOLD + } else { + threshold + } +} + #[cfg(test)] mod tests { - use super::TreeConfig; + use super::*; + + #[test] + fn backfill_threshold_grows_block_buffer() { + let default = TreeConfig::default(); + for (threshold, expected, buffer_limit) in + [(0, 0, 64), (500, 500, 1000), (1024, 1024, 2048), (u64::MAX, 1024, 2048)] + { + let config = default.clone().with_backfill_run_threshold(threshold); + assert_eq!(config.backfill_run_threshold(), expected); + assert_eq!(config.block_buffer_limit(), buffer_limit); + } + let config = default.with_block_buffer_limit(2000).with_backfill_run_threshold(500); + assert_eq!(config.block_buffer_limit(), 2000); + } #[test] fn txpool_prewarming_is_disabled_by_default_and_can_be_enabled() { diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 4a39a4b76fd..03676493f8b 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -6,7 +6,7 @@ use crate::{ tree::{error::InsertPayloadError, payload_validator::TreeCtx}, }; use alloy_consensus::BlockHeader; -use alloy_eips::{eip1898::BlockWithParent, merge::EPOCH_SLOTS, BlockNumHash, NumHash}; +use alloy_eips::{eip1898::BlockWithParent, BlockNumHash, NumHash}; use alloy_primitives::{map::B256Map, B256}; use alloy_rpc_types_engine::{ ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError, @@ -96,17 +96,6 @@ pub use types::{ExecutionEnv, ValidationOutcome, ValidationOutput}; pub mod state; -/// The largest gap for which the tree will be used to sync individual blocks by downloading them. -/// -/// This is the default threshold, and represents the distance (gap) from the local head to a -/// new (canonical) block, e.g. the forkchoice head block. If the block distance from the local head -/// exceeds this threshold, the pipeline will be used to backfill the gap more efficiently. -/// -/// E.g.: Local head `block.number` is 100 and the forkchoice head `block.number` is 133 (more than -/// an epoch has slots), then this exceeds the threshold at which the pipeline should be used to -/// backfill this gap. -pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS; - /// The minimum number of blocks to retain in the changeset cache after eviction. /// /// This ensures that recent changesets are kept in memory for potential reorgs, @@ -2730,7 +2719,7 @@ where /// If the `local_tip` is greater than the `block`, then this will return false. #[inline] const fn exceeds_backfill_run_threshold(&self, local_tip: u64, block: u64) -> bool { - block > local_tip && block - local_tip > MIN_BLOCKS_FOR_PIPELINE_RUN + block > local_tip && block - local_tip > self.config.backfill_run_threshold() } /// Returns how far the local tip is from the given block. If the local tip is at the same diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 2eeddcac802..f7db108819e 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -24,7 +24,9 @@ use alloy_rpc_types_engine::{ use assert_matches::assert_matches; use reth_chain_state::test_utils::TestBlockBuilder; use reth_chainspec::{ChainSpec, HOLESKY, MAINNET}; -use reth_engine_primitives::{EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook}; +use reth_engine_primitives::{ + EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook, DEFAULT_BACKFILL_RUN_THRESHOLD, +}; use reth_ethereum_consensus::EthBeaconConsensus; use reth_ethereum_engine_primitives::{EthEngineTypes, EthPayloadAttributes}; use reth_ethereum_primitives::{Block, EthPrimitives}; @@ -742,8 +744,9 @@ fn backfill_action_waits_while_payload_build_is_active() { } fn deferred_backfill_harness() -> (TestHarness, Vec, BackfillAction) { - let all_blocks: Vec<_> = - TestBlockBuilder::eth().get_executed_blocks(1..MIN_BLOCKS_FOR_PIPELINE_RUN + 10).collect(); + let all_blocks: Vec<_> = TestBlockBuilder::eth() + .get_executed_blocks(1..DEFAULT_BACKFILL_RUN_THRESHOLD + 10) + .collect(); let canonical_blocks = all_blocks[..6].to_vec(); let target = all_blocks.last().unwrap().recovered_block().clone_sealed_block(); let target_hash = target.hash(); @@ -855,7 +858,7 @@ fn deferred_backfill_uses_latest_sync_target() { let newer_chain = test_harness .block_builder - .create_fork(blocks[0].recovered_block(), MIN_BLOCKS_FOR_PIPELINE_RUN + 20); + .create_fork(blocks[0].recovered_block(), DEFAULT_BACKFILL_RUN_THRESHOLD + 20); let newer_target = newer_chain.last().unwrap().clone_sealed_block(); let newer_target_hash = newer_target.hash(); test_harness.tree.state.buffer.insert_block(newer_target.into()); @@ -1635,7 +1638,7 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() { // extend main chain with enough blocks to trigger pipeline run but don't insert them let main_chain = test_harness .block_builder - .create_fork(base_chain[0].recovered_block(), MIN_BLOCKS_FOR_PIPELINE_RUN + 10); + .create_fork(base_chain[0].recovered_block(), DEFAULT_BACKFILL_RUN_THRESHOLD + 10); let main_chain_last_hash = main_chain.last().unwrap().hash(); test_harness.send_fcu(main_chain_last_hash, ForkchoiceStatus::Syncing).await; @@ -1643,7 +1646,7 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() { test_harness.check_fcu(main_chain_last_hash, ForkchoiceStatus::Syncing).await; // create event for backfill finished - let backfill_finished_block_number = MIN_BLOCKS_FOR_PIPELINE_RUN + 1; + let backfill_finished_block_number = DEFAULT_BACKFILL_RUN_THRESHOLD + 1; let backfill_finished = FromOrchestrator::BackfillSyncFinished(ControlFlow::Continue { block_number: backfill_finished_block_number, }); @@ -3153,11 +3156,24 @@ fn test_backfill_target_hash_opstack_returns_head() { assert_eq!(test_harness.tree.backfill_target_hash(state), head); } +#[test] +fn test_exceeds_backfill_run_threshold_uses_configured_value() { + let mut test_harness = TestHarness::new(MAINNET.clone()); + let default_threshold = DEFAULT_BACKFILL_RUN_THRESHOLD; + assert!(!test_harness.tree.exceeds_backfill_run_threshold(0, default_threshold)); + assert!(test_harness.tree.exceeds_backfill_run_threshold(0, default_threshold + 1)); + + test_harness.tree.config = TreeConfig::default().with_backfill_run_threshold(100); + assert!(!test_harness.tree.exceeds_backfill_run_threshold(10, 110)); + assert!(test_harness.tree.exceeds_backfill_run_threshold(10, 111)); + assert!(!test_harness.tree.exceeds_backfill_run_threshold(10, 9)); +} + #[test] fn test_backfill_sync_target_without_sync_state_returns_none() { let test_harness = TestHarness::new(MAINNET.clone()); assert_eq!( - test_harness.tree.backfill_sync_target(0, MIN_BLOCKS_FOR_PIPELINE_RUN + 100, None), + test_harness.tree.backfill_sync_target(0, DEFAULT_BACKFILL_RUN_THRESHOLD + 100, None), None ); } @@ -3182,9 +3198,9 @@ fn test_on_disconnected_downloaded_block_opstack_targets_head() { let canonical_head = BlockNumHash::new(0, B256::ZERO); let downloaded_block = - BlockNumHash::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 100, B256::from([0xCC; 32])); + BlockNumHash::new(DEFAULT_BACKFILL_RUN_THRESHOLD + 100, B256::from([0xCC; 32])); let missing_parent = - BlockNumHash::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 99, B256::from([0xDD; 32])); + BlockNumHash::new(DEFAULT_BACKFILL_RUN_THRESHOLD + 99, B256::from([0xDD; 32])); let event = test_harness.tree.on_disconnected_downloaded_block( downloaded_block, @@ -3223,9 +3239,9 @@ fn test_on_disconnected_downloaded_block_eth_targets_finalized() { let canonical_head = BlockNumHash::new(0, B256::ZERO); let downloaded_block = - BlockNumHash::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 100, B256::from([0xCC; 32])); + BlockNumHash::new(DEFAULT_BACKFILL_RUN_THRESHOLD + 100, B256::from([0xCC; 32])); let missing_parent = - BlockNumHash::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 99, B256::from([0xDD; 32])); + BlockNumHash::new(DEFAULT_BACKFILL_RUN_THRESHOLD + 99, B256::from([0xDD; 32])); let event = test_harness.tree.on_disconnected_downloaded_block( downloaded_block, @@ -3263,9 +3279,9 @@ fn test_on_disconnected_downloaded_block_eth_zero_finalized_targets_head() { let canonical_head = BlockNumHash::new(0, B256::ZERO); let downloaded_block = - BlockNumHash::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 100, B256::from([0xCC; 32])); + BlockNumHash::new(DEFAULT_BACKFILL_RUN_THRESHOLD + 100, B256::from([0xCC; 32])); let missing_parent = - BlockNumHash::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 99, B256::from([0xDD; 32])); + BlockNumHash::new(DEFAULT_BACKFILL_RUN_THRESHOLD + 99, B256::from([0xDD; 32])); let event = test_harness.tree.on_disconnected_downloaded_block( downloaded_block, @@ -3285,16 +3301,19 @@ fn test_on_disconnected_downloaded_block_eth_zero_finalized_targets_head() { } } -/// Verifies that the post-backfill recheck path in `on_backfill_sync_finished` retriggers a -/// new backfill targeting whichever block `backfill_target_hash` resolves to — head on OP -/// Stack, finalized on Ethereum — when that block is buffered far ahead of where the -/// just-finished pipeline landed. -async fn assert_post_backfill_recheck_retriggers_to_buffered_target(engine_kind: EngineApiKind) { +/// Verifies that the post-backfill recheck chooses between backfill and live sync using the +/// configured threshold and the buffered target (head on OP Stack, finalized on Ethereum). +async fn assert_post_backfill_recheck_uses_threshold( + engine_kind: EngineApiKind, + threshold: u64, + expect_backfill: bool, +) { reth_tracing::init_test_tracing(); let chain_spec = MAINNET.clone(); let mut test_harness = TestHarness::new(chain_spec.clone()); test_harness.tree.engine_kind = engine_kind; + test_harness.tree.config = TreeConfig::default().with_backfill_run_threshold(threshold); let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); test_harness = test_harness.with_blocks(base_chain.clone()); @@ -3305,7 +3324,7 @@ async fn assert_post_backfill_recheck_retriggers_to_buffered_target(engine_kind: // Long unsynced chain. The last block is what the helper should resolve to. let main_chain = test_harness .block_builder - .create_fork(base_chain[0].recovered_block(), MIN_BLOCKS_FOR_PIPELINE_RUN + 50); + .create_fork(base_chain[0].recovered_block(), DEFAULT_BACKFILL_RUN_THRESHOLD + 50); let target_block = main_chain.last().unwrap().clone(); let target_hash = target_block.hash(); @@ -3326,8 +3345,8 @@ async fn assert_post_backfill_recheck_retriggers_to_buffered_target(engine_kind: ForkchoiceStatus::Syncing, ); - // Simulate backfill finishing far below the buffered target (gap > threshold). - let backfill_finished_block_number = MIN_BLOCKS_FOR_PIPELINE_RUN + 1; + // Simulate backfill finishing with a gap larger than the default threshold. + let backfill_finished_block_number = DEFAULT_BACKFILL_RUN_THRESHOLD + 1; let backfill_tip_block = main_chain[(backfill_finished_block_number - 1) as usize].clone(); test_harness.provider.add_block(backfill_tip_block.hash(), backfill_tip_block.into_block()); let backfill_finished = FromOrchestrator::BackfillSyncFinished(ControlFlow::Continue { @@ -3337,23 +3356,81 @@ async fn assert_post_backfill_recheck_retriggers_to_buffered_target(engine_kind: let event = test_harness.from_tree_rx.recv().await.unwrap(); match event { - EngineApiEvent::BackfillAction(BackfillAction::Start(emitted_target)) => { + EngineApiEvent::BackfillAction(BackfillAction::Start(emitted_target)) + if expect_backfill => + { assert_eq!( emitted_target.sync_target(), Some(target_hash), "post-backfill recheck should retrigger backfill to the buffered target" ); } - _ => panic!("Expected BackfillAction(Start), got: {event:#?}"), + EngineApiEvent::Download(_) if !expect_backfill => {} + _ => panic!("Unexpected post-backfill event: {event:#?}"), } } #[tokio::test] async fn test_on_backfill_sync_finished_opstack_retriggers_backfill_to_buffered_head() { - assert_post_backfill_recheck_retriggers_to_buffered_target(EngineApiKind::OpStack).await; + assert_post_backfill_recheck_uses_threshold( + EngineApiKind::OpStack, + DEFAULT_BACKFILL_RUN_THRESHOLD, + true, + ) + .await; } #[tokio::test] async fn test_on_backfill_sync_finished_eth_retriggers_backfill_to_buffered_finalized() { - assert_post_backfill_recheck_retriggers_to_buffered_target(EngineApiKind::Ethereum).await; + assert_post_backfill_recheck_uses_threshold( + EngineApiKind::Ethereum, + DEFAULT_BACKFILL_RUN_THRESHOLD, + true, + ) + .await; +} + +#[tokio::test] +async fn test_on_backfill_sync_finished_resumes_live_sync_with_higher_threshold() { + for engine_kind in [EngineApiKind::Ethereum, EngineApiKind::OpStack] { + assert_post_backfill_recheck_uses_threshold(engine_kind, 100, false).await; + } +} + +#[test] +fn test_backfill_threshold_above_header_limit_triggers_backfill() { + let mut test_harness = TestHarness::new(MAINNET.clone()); + let target_hash = B256::from([0xAA; 32]); + test_harness.tree.state.forkchoice_state_tracker.set_latest( + ForkchoiceState { + head_block_hash: target_hash, + safe_block_hash: B256::ZERO, + finalized_block_hash: target_hash, + }, + ForkchoiceStatus::Syncing, + ); + let local_tip = BlockNumHash::new(10, B256::ZERO); + let parent_hash = B256::from([0xBB; 32]); + test_harness.tree.config = TreeConfig::default().with_backfill_run_threshold(2048); + for gap in [1024, 1025] { + let missing_parent = BlockNumHash::new(local_tip.number + gap, parent_hash); + let downloaded = BlockNumHash::new(missing_parent.number + 1, B256::from([0xCC; 32])); + let event = test_harness.tree.on_disconnected_downloaded_block( + downloaded, + missing_parent, + local_tip, + ); + match gap { + 1024 => assert!(matches!( + event, + Some(TreeEvent::Download(DownloadRequest::BlockRange { hash, count: 1024, .. })) + if hash == parent_hash + )), + _ => assert!(matches!( + event, + Some(TreeEvent::BackfillAction(BackfillAction::Start(target))) + if target.sync_target() == Some(target_hash) + )), + } + } } diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index 6a688686615..6b6358b0e52 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -7,8 +7,9 @@ use clap::{ use eyre::ensure; use reth_cli_util::{parse_duration_from_secs_or_ms, parsers::format_duration_as_secs_or_ms}; use reth_engine_primitives::{ - TreeConfig, DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, - DEFAULT_NUM_STATE_MASKING_BLOCKS, MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD, + TreeConfig, DEFAULT_BACKFILL_RUN_THRESHOLD, DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, + DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, DEFAULT_NUM_STATE_MASKING_BLOCKS, + MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD, }; use std::{sync::OnceLock, time::Duration}; @@ -29,6 +30,7 @@ pub struct DefaultEngineValues { persistence_backpressure_threshold: u64, num_state_masking_blocks: u64, memory_block_buffer_target: u64, + backfill_run_threshold: u64, invalid_header_hit_eviction_threshold: u8, state_cache_disabled: bool, prewarming_disabled: bool, @@ -93,6 +95,12 @@ impl DefaultEngineValues { self } + /// Set the default backfill run threshold + pub const fn with_backfill_run_threshold(mut self, v: u64) -> Self { + self.backfill_run_threshold = v; + self + } + /// Set the invalid header cache hit eviction threshold pub const fn with_invalid_header_hit_eviction_threshold(mut self, v: u8) -> Self { self.invalid_header_hit_eviction_threshold = v; @@ -266,6 +274,7 @@ impl Default for DefaultEngineValues { persistence_backpressure_threshold: MIN_PERSISTENCE_BACKPRESSURE_THRESHOLD, num_state_masking_blocks: DEFAULT_NUM_STATE_MASKING_BLOCKS, memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, + backfill_run_threshold: DEFAULT_BACKFILL_RUN_THRESHOLD, invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, state_cache_disabled: false, prewarming_disabled: false, @@ -346,6 +355,20 @@ pub struct EngineArgs { #[arg(long = "engine.memory-block-buffer-target")] pub memory_block_buffer_target: Option, + /// Configure the largest gap, in blocks, between the local head and the forkchoice head that + /// is closed by downloading the missing blocks directly. Larger gaps trigger a full pipeline + /// (backfill) run instead. + /// + /// On chains with short block times a pipeline run may take longer than it takes the chain to + /// produce another gap of this size, in which case raising this value lets the node catch up + /// via live sync. Values above 1024 are clamped to the peer header response limit. + #[arg( + long = "engine.backfill-threshold", + env = "RETH_ENGINE_BACKFILL_THRESHOLD", + default_value_t = DefaultEngineValues::get_global().backfill_run_threshold + )] + pub backfill_run_threshold: u64, + /// Configure how many cache hits an invalid header can accumulate before it is evicted and /// reprocessed. /// @@ -586,6 +609,7 @@ impl Default for EngineArgs { persistence_backpressure_threshold: _, num_state_masking_blocks, memory_block_buffer_target: _, + backfill_run_threshold, invalid_header_hit_eviction_threshold, state_cache_disabled, prewarming_disabled, @@ -619,6 +643,7 @@ impl Default for EngineArgs { persistence_backpressure_threshold: None, num_state_masking_blocks, memory_block_buffer_target: None, + backfill_run_threshold, invalid_header_hit_eviction_threshold, state_root_task_compare_updates, legacy_state_root_task_enabled: false, @@ -737,6 +762,7 @@ impl EngineArgs { .with_persistence_threshold(self.persistence_threshold) .with_memory_block_buffer_target(self.memory_block_buffer_target()) .with_num_state_masking_blocks(self.num_state_masking_blocks()) + .with_backfill_run_threshold(self.backfill_run_threshold) .with_invalid_header_hit_eviction_threshold(self.invalid_header_hit_eviction_threshold) .without_state_cache(self.state_cache_disabled) .without_prewarming(self.prewarming_disabled) @@ -896,6 +922,7 @@ mod tests { persistence_backpressure_threshold: Some(101), num_state_masking_blocks: DEFAULT_NUM_STATE_MASKING_BLOCKS, memory_block_buffer_target: Some(50), + backfill_run_threshold: 100, invalid_header_hit_eviction_threshold: 7, legacy_state_root_task_enabled: true, caching_and_prewarming_enabled: true, @@ -942,6 +969,8 @@ mod tests { "101", "--engine.memory-block-buffer-target", "50", + "--engine.backfill-threshold", + "100", "--engine.invalid-header-cache-hit-eviction-threshold", "7", "--engine.legacy-state-root", @@ -1023,6 +1052,23 @@ mod tests { assert_eq!(args.tree_config().num_state_masking_blocks(), 7); } + #[test] + fn test_parse_backfill_threshold() { + let args = CommandParser::::parse_from(["reth"]).args; + let config = args.tree_config(); + assert_eq!(config.backfill_run_threshold(), DEFAULT_BACKFILL_RUN_THRESHOLD); + + for (threshold, expected) in [("500", 500), ("2048", 1024)] { + let args = CommandParser::::parse_from([ + "reth", + "--engine.backfill-threshold", + threshold, + ]) + .args; + assert_eq!(args.tree_config().backfill_run_threshold(), expected); + } + } + #[test] fn validate_rejects_state_masking_window_at_or_above_threshold() { let args = EngineArgs { diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 61511b415dc..b1c063a0bdb 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1051,6 +1051,14 @@ Engine: If omitted, this defaults to the lesser of `--engine.persistence-threshold` and the configured default memory block buffer target. + --engine.backfill-threshold + Configure the largest gap, in blocks, between the local head and the forkchoice head that is closed by downloading the missing blocks directly. Larger gaps trigger a full pipeline (backfill) run instead. + + On chains with short block times a pipeline run may take longer than it takes the chain to produce another gap of this size, in which case raising this value lets the node catch up via live sync. Values above 1024 are clamped to the peer header response limit. + + [env: RETH_ENGINE_BACKFILL_THRESHOLD=] + [default: 32] + --engine.invalid-header-cache-hit-eviction-threshold Configure how many cache hits an invalid header can accumulate before it is evicted and reprocessed. From 33265aa49f7105774923b5109bc2bc7d6948d5aa Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:46:34 +0000 Subject: [PATCH 060/133] chore(ci): remove separate benchmark hosting repository (#27069) Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- .github/scripts/bench-job-summary-test.js | 39 +++++ .github/scripts/bench-job-summary.js | 29 +--- .github/scripts/bench-scheduled-refs.sh | 44 ++---- .github/scripts/bench-slack-notify.js | 22 +-- .github/scripts/bench-state-test.sh | 151 ++++++++++++++++++++ .github/scripts/bench-state.sh | 45 ++++++ .github/scripts/bench-upload-charts-test.sh | 57 ++++++++ .github/scripts/bench-upload-charts.sh | 66 +++++++++ .github/scripts/bench-utils.js | 5 - .github/workflows/bench-scheduled.yml | 119 ++++----------- .github/workflows/bench.yml | 145 +++---------------- .github/workflows/lint-actions.yml | 5 + 12 files changed, 425 insertions(+), 302 deletions(-) create mode 100644 .github/scripts/bench-job-summary-test.js create mode 100644 .github/scripts/bench-state-test.sh create mode 100644 .github/scripts/bench-state.sh create mode 100644 .github/scripts/bench-upload-charts-test.sh create mode 100644 .github/scripts/bench-upload-charts.sh diff --git a/.github/scripts/bench-job-summary-test.js b/.github/scripts/bench-job-summary-test.js new file mode 100644 index 00000000000..f24c0468522 --- /dev/null +++ b/.github/scripts/bench-job-summary-test.js @@ -0,0 +1,39 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const jobSummary = require('./bench-job-summary'); + +for (const mode of ['engine', 'call']) { + test(`${mode} summary preserves attachments and trace downloads`, async (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-summary-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const previous = { ...process.env }; + t.after(() => { process.env = previous; }); + process.env.BENCH_WORK_DIR = directory; + process.env.BENCH_TRACING_CHROME = 'true'; + process.env.BENCH_RESULTS_URL = 'https://github.com/owner/repo/actions/runs/1/artifacts/2'; + const stats = { mean_ms: 1, stddev_ms: 0, p50_ms: 1, p90_ms: 1, p99_ms: 1, mean_mgas_s: 1 }; + fs.writeFileSync(path.join(directory, 'summary.json'), JSON.stringify({ + mode, changes: {}, blocks: 20, + baseline: { name: 'main', ref: 'a'.repeat(40), stats }, + feature: { name: 'feature', ref: 'b'.repeat(40), stats }, + })); + const charts = '\n### Charts\n\n![Latency](https://github.com/user-attachments/assets/chart)\n'; + if (mode === 'engine') fs.writeFileSync(path.join(directory, 'charts.md'), charts); + let markdown; + await jobSummary({ + core: { summary: { addRaw(value) { markdown = value; return { async write() {} }; } } }, + context: { repo: { owner: 'owner', repo: 'repo' } }, + logsUrl: 'https://example.com/logs', + tracesUrl: 'https://example.com/traces', + runId: '1', + }); + assert.equal(markdown.includes(charts), mode === 'engine'); + assert.ok(markdown.includes(`[Download bench-results](${process.env.BENCH_RESULTS_URL})`)); + assert.ok(markdown.includes('[Logs](https://example.com/logs)')); + assert.ok(markdown.includes('[Traces](https://example.com/traces)')); + assert.ok(!markdown.includes('reth-bench-charts')); + }); +} diff --git a/.github/scripts/bench-job-summary.js b/.github/scripts/bench-job-summary.js index 129507b7afa..7102a9b2055 100644 --- a/.github/scripts/bench-job-summary.js +++ b/.github/scripts/bench-job-summary.js @@ -12,13 +12,12 @@ // // Usage from actions/github-script: // const jobSummary = require('./.github/scripts/bench-job-summary.js'); -// await jobSummary({ core, context, chartSha, grafanaUrl, logsUrl, tracesUrl, runId }); +// await jobSummary({ core, context, grafanaUrl, logsUrl, tracesUrl, runId }); const fs = require('fs'); const { verdict, loadSamplyUrls, - loadTracingChromeUrls, blocksLabel, metricRows, waitTimeRows, @@ -39,7 +38,7 @@ function fmtTargetMetricValue(metric, v) { return fmtMetricValue(v); } -module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, tracesUrl, runId }) { +module.exports = async function ({ core, context, grafanaUrl, logsUrl, tracesUrl, runId }) { let summary; try { summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8')); @@ -116,21 +115,8 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, md += '\n'; } - // Charts (call mode does not generate any) - if (chartSha && summary.mode !== 'call') { - const prNum = prNumber || '0'; - const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`; - const charts = [ - { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' }, - { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' }, - { file: 'gas_vs_latency.png', label: 'Gas vs Latency' }, - ]; - md += `### Charts\n\n`; - for (const chart of charts) { - md += `

${chart.label}\n\n`; - md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`; - md += `
\n\n`; - } + if (summary.mode !== 'call') { + md += fs.readFileSync(process.env.BENCH_WORK_DIR + '/charts.md', 'utf8'); } // Samply profiles @@ -140,11 +126,8 @@ module.exports = async function ({ core, context, chartSha, grafanaUrl, logsUrl, md += `### Samply Profiles\n\n${samplyLinks.join('\n')}\n\n`; } - const tracingChromeUrls = loadTracingChromeUrls(process.env.BENCH_WORK_DIR); - const tracingChromeLinks = Object.entries(tracingChromeUrls) - .map(([run, url]) => `- **${run}**: [Perfetto](${url})`); - if (tracingChromeLinks.length > 0) { - md += `### Chrome Traces\n\n${tracingChromeLinks.join('\n')}\n\n`; + if (process.env.BENCH_TRACING_CHROME === 'true') { + md += `### Chrome Traces\n\n[Download bench-results](${process.env.BENCH_RESULTS_URL}), extract the archive, and open each run's \`tracing-chrome-profile.json\` in [Perfetto](https://ui.perfetto.dev/). GitHub sign-in is required; downloads are available until the artifact expires.\n\n`; } // Observability diff --git a/.github/scripts/bench-scheduled-refs.sh b/.github/scripts/bench-scheduled-refs.sh index 6040ed30c8a..44122a0be47 100755 --- a/.github/scripts/bench-scheduled-refs.sh +++ b/.github/scripts/bench-scheduled-refs.sh @@ -26,12 +26,8 @@ # nightly-created — ISO timestamp of the nightly build (nightly only) # release-tag — release tag name (release mode only, e.g. "v2.0.0") # -# Reads: -# state/nightly-last-feature-ref (nightly, from decofe/reth-bench-charts repo) -# state/hourly-last-feature-ref (hourly, from decofe/reth-bench-charts repo) -# state/release-last-feature-ref (release, from decofe/reth-bench-charts repo) -# -# Requires: gh (GitHub CLI), jq, date, git (hourly mode), curl, DEREK_TOKEN env +# Reads bench-state- Actions artifacts from successful runs on this branch. +# Requires: gh, jq, unzip, date, git (hourly mode), GH_TOKEN with actions:read. set -euxo pipefail FORCE="${1:-false}" @@ -74,16 +70,10 @@ if [ "$MODE" = "hourly" ]; then fi echo "::endgroup::" - # --- Step 3: Read last successful feature ref from charts repo --- + # --- Step 3: Read last successful feature ref from Actions artifacts --- echo "::group::Reading persisted state" - LAST_FEATURE_REF="" - STATE_URL="https://raw.githubusercontent.com/decofe/reth-bench-charts/state/state/hourly-last-feature-ref" - if RAW=$(curl -sfL -H "Authorization: token ${DEREK_TOKEN}" "$STATE_URL"); then - LAST_FEATURE_REF=$(echo "$RAW" | tr -d '[:space:]') - echo "Previous feature ref: $LAST_FEATURE_REF" - else - echo "No persisted state found (first run)" - fi + LAST_FEATURE_REF=$(bash "$(dirname "$0")/bench-state.sh" "$REPO" hourly) + echo "Previous feature ref: ${LAST_FEATURE_REF:-none}" echo "::endgroup::" # --- Step 4: Determine baseline and skip logic --- @@ -181,16 +171,10 @@ if [ "$MODE" = "release" ]; then echo "Release commit (baseline): $BASELINE_REF" echo "::endgroup::" - # --- Step 3: Read last successful feature ref from charts repo --- + # --- Step 3: Read last successful feature ref from Actions artifacts --- echo "::group::Reading persisted state" - LAST_FEATURE_REF="" - STATE_URL="https://raw.githubusercontent.com/decofe/reth-bench-charts/state/state/release-last-feature-ref" - if RAW=$(curl -sfL -H "Authorization: token ${DEREK_TOKEN}" "$STATE_URL"); then - LAST_FEATURE_REF=$(echo "$RAW" | tr -d '[:space:]') - echo "Previous feature ref: $LAST_FEATURE_REF" - else - echo "No persisted state found (first run)" - fi + LAST_FEATURE_REF=$(bash "$(dirname "$0")/bench-state.sh" "$REPO" release) + echo "Previous feature ref: ${LAST_FEATURE_REF:-none}" echo "::endgroup::" # --- Step 4: Skip logic --- @@ -278,16 +262,10 @@ else fi echo "::endgroup::" -# --- Step 3: Read last successful feature ref from charts repo --- +# --- Step 3: Read last successful feature ref from Actions artifacts --- echo "::group::Reading persisted state" -LAST_FEATURE_REF="" -STATE_URL="https://raw.githubusercontent.com/decofe/reth-bench-charts/state/state/nightly-last-feature-ref" -if RAW=$(curl -sfL -H "Authorization: token ${DEREK_TOKEN}" "$STATE_URL"); then - LAST_FEATURE_REF=$(echo "$RAW" | tr -d '[:space:]') - echo "Previous feature ref: $LAST_FEATURE_REF" -else - echo "No persisted state found (first run)" -fi +LAST_FEATURE_REF=$(bash "$(dirname "$0")/bench-state.sh" "$REPO" nightly) + echo "Previous feature ref: ${LAST_FEATURE_REF:-none}" echo "::endgroup::" # --- Step 4: Determine baseline and skip logic --- diff --git a/.github/scripts/bench-slack-notify.js b/.github/scripts/bench-slack-notify.js index e71a9b8aa68..ffd631f9d7f 100644 --- a/.github/scripts/bench-slack-notify.js +++ b/.github/scripts/bench-slack-notify.js @@ -25,7 +25,6 @@ const { verdict, isWin, loadSamplyUrls, - loadTracingChromeUrls, blocksLabel, metricRows, } = require('./bench-utils'); @@ -82,16 +81,6 @@ function profileLinks(samplyUrls, prefix) { }); } -function chromeTraceLinks(tracingChromeUrls, prefix) { - return Object.entries(tracingChromeUrls) - .filter(([run]) => run.startsWith(`${prefix}-`)) - .sort(([a], [b]) => a.localeCompare(b, undefined, { numeric: true })) - .map(([run, url]) => { - const index = run.slice(prefix.length + 1); - return `<${url}|Chrome ${index}>`; - }); -} - // Slack shortcodes for verdict (Block Kit header doesn't support unicode emoji) const SLACK_VERDICT = { '⚠️': ':warning:', @@ -135,7 +124,6 @@ function buildSuccessBlocks({ jobUrl, repo, samplyUrls, - tracingChromeUrls, }) { const { emoji, label } = verdict(summary.changes); const headerEmoji = SLACK_VERDICT[emoji] || emoji; @@ -151,18 +139,18 @@ function buildSuccessBlocks({ if (prNumber) metaParts.push(`*<${prUrl}|PR #${prNumber}>*`); metaParts.push(`triggered by ${actorSlackId ? `<@${actorSlackId}>` : `@${actor}`}`); + if (process.env.BENCH_TRACING_CHROME === 'true' && process.env.BENCH_RESULTS_URL) { + metaParts.push(`<${process.env.BENCH_RESULTS_URL}|Download Chrome traces> (extract and open in Perfetto)`); + } + // Baseline/feature lines with samply profile links let baselineLine = `*Baseline:* ${baselineLink}`; const baselineProfiles = profileLinks(samplyUrls, 'baseline'); if (baselineProfiles.length) baselineLine += ` | ${baselineProfiles.join(' | ')}`; - const baselineTraces = chromeTraceLinks(tracingChromeUrls, 'baseline'); - if (baselineTraces.length) baselineLine += ` | ${baselineTraces.join(' | ')}`; let featureLine = `*Feature:* ${featureLink}`; const featureProfiles = profileLinks(samplyUrls, 'feature'); if (featureProfiles.length) featureLine += ` | ${featureProfiles.join(' | ')}`; - const featureTraces = chromeTraceLinks(tracingChromeUrls, 'feature'); - if (featureTraces.length) featureLine += ` | ${featureTraces.join(' | ')}`; const countsLine = blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '); const configLine = benchConfigLine(); @@ -288,7 +276,6 @@ async function success({ core, context }) { `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR); - const tracingChromeUrls = loadTracingChromeUrls(process.env.BENCH_WORK_DIR); const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.'); const actorSlackId = slackUsers[actor]; @@ -301,7 +288,6 @@ async function success({ core, context }) { jobUrl, repo, samplyUrls, - tracingChromeUrls, }); const text = `Bench: ${summary.baseline.name} vs ${summary.feature.name}`; diff --git a/.github/scripts/bench-state-test.sh b/.github/scripts/bench-state-test.sh new file mode 100644 index 00000000000..f72e998c8cc --- /dev/null +++ b/.github/scripts/bench-state-test.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail +scripts="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +export STATE_TEST_DIR="$scratch" +export GITHUB_OUTPUT="$scratch/output" +export GITHUB_REPOSITORY=owner/repo +export GITHUB_REF_NAME=main +export STATE_TEST_VALUE="" +export STATE_TEST_FAILURE="" +export GH_TOKEN=workflow-test-token +feature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +previous=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +export STATE_TEST_DATE +STATE_TEST_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +gh() { + case "$1" in + api) + if [[ "$2" == *'/git/'* ]]; then + if [[ "$*" == *'.object.type'* ]]; then printf 'commit\n'; else printf '%s\n' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb; fi + return + fi + [[ "$GH_TOKEN" == workflow-test-token ]] || return 1 + if [[ "$*" == *'--paginate --slurp'* ]]; then + [[ "$STATE_TEST_FAILURE" != list ]] || return 1 + if [[ -n "${STATE_TEST_ARTIFACTS:-}" ]]; then + printf '%s\n' "$STATE_TEST_ARTIFACTS" + elif [[ -z "$STATE_TEST_VALUE" ]]; then + printf '[{"artifacts":[]}]\n' + else + local name="${!#}" + name="${name#*name=}" + name="${name%%&*}" + jq -nc --arg name "$name" '[{artifacts:[{id:10,name:$name,expired:false, + workflow_run:{id:100,head_branch:"main",repository_id:1,head_repository_id:1}}]}]' + fi + elif [[ "$2" == */runs/* ]]; then + [[ "$STATE_TEST_FAILURE" != run ]] || return 1 + jq -nc --arg conclusion "${STATE_TEST_CONCLUSION:-success}" \ + --arg path "${STATE_TEST_PATH:-.github/workflows/bench-scheduled.yml}" \ + --arg event "${STATE_TEST_EVENT:-schedule}" --arg status "${STATE_TEST_STATUS:-completed}" \ + '{path:$path,head_branch:"main",status:$status,conclusion:$conclusion,event:$event}' + elif [[ "$2" == */zip ]]; then + [[ "$STATE_TEST_FAILURE" != download ]] || return 1 + printf '%s\n' "$2" >> "$STATE_TEST_DIR/downloads" + if [[ "$STATE_TEST_FAILURE" == archive ]]; then printf 'invalid zip'; return; fi + printf '%s\n' "$STATE_TEST_VALUE" > "$STATE_TEST_DIR/feature-ref" + (cd "$STATE_TEST_DIR" && zip -q - feature-ref) + else + return 1 + fi + ;; + release) printf '{"tagName":"v1.0.0"}\n' ;; + run) + if [[ "$*" == *'--status=in_progress'* ]]; then + printf '0\n' + else + printf '[{"headSha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","createdAt":"%s","conclusion":"success"}]\n' "$STATE_TEST_DATE" + fi + ;; + *) return 1 ;; + esac +} +git() { + case "$*" in + fetch*) ;; + 'rev-parse origin/main') printf '%s\n' aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;; + 'rev-parse origin/main~1') printf '%s\n' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb ;; + 'log -1 --format=%cI '* ) printf '%s\n' "$STATE_TEST_DATE" ;; + *) return 1 ;; + esac +} +export -f gh git + +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +export STATE_TEST_VALUE="$previous" +[[ "$(bash "$scripts/bench-state.sh" owner/repo hourly)" == "$previous" ]] +export STATE_TEST_VALUE=invalid +if bash "$scripts/bench-state.sh" owner/repo hourly 2>/dev/null; then exit 1; fi +if bash "$scripts/bench-state.sh" owner/repo invalid 2>/dev/null; then exit 1; fi +export STATE_TEST_VALUE="$previous" +for failure in list run download archive; do + export STATE_TEST_FAILURE="$failure" + if bash "$scripts/bench-state.sh" owner/repo hourly 2>/dev/null; then exit 1; fi +done +export STATE_TEST_FAILURE="" +echo 'PASS: missing state, real ZIP restore, SHA validation, and API/archive failures' + +# Newer unusable artifacts must not hide the newest usable artifact, even when +# it is on another page. The workflow SHA is deliberately not the feature SHA. +export STATE_TEST_ARTIFACTS +STATE_TEST_ARTIFACTS=$(jq -nc ' + def artifact($id): {id:$id,name:"bench-state-hourly",expired:false, + workflow_run:{id:100,head_branch:"main",head_sha:"cccccccccccccccccccccccccccccccccccccccc", + repository_id:1,head_repository_id:1}}; + [{artifacts:[ + (artifact(90) | .expired=true), + (artifact(80) | .workflow_run.head_branch="test"), + (artifact(70) | .workflow_run.head_repository_id=2), + (artifact(60) | .name="bench-state-nightly")]}, + {artifacts:[artifact(10),artifact(20)]}]') +: > "$scratch/downloads" +[[ "$(bash "$scripts/bench-state.sh" owner/repo hourly)" == "$previous" ]] +[[ "$(cat "$scratch/downloads")" == repos/owner/repo/actions/artifacts/20/zip ]] +export GITHUB_REF_NAME=other-branch +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +export GITHUB_REF_NAME=main +export STATE_TEST_ARTIFACTS='[{"artifacts":[]}]' +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_ARTIFACTS +for conclusion in failure cancelled skipped; do + export STATE_TEST_CONCLUSION="$conclusion" + [[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +done +unset STATE_TEST_CONCLUSION +export STATE_TEST_STATUS=in_progress +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_STATUS +export STATE_TEST_PATH=.github/workflows/other.yml +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_PATH +export STATE_TEST_EVENT=pull_request +[[ -z "$(bash "$scripts/bench-state.sh" owner/repo hourly)" ]] +unset STATE_TEST_EVENT +echo 'PASS: pagination, newest state, expiry, branch/mode/fork/workflow/event isolation, and run outcomes' + +for mode in hourly nightly release; do + for value in "" "$feature" "$previous"; do + export STATE_TEST_VALUE="$value" + : > "$GITHUB_OUTPUT" + bash "$scripts/bench-scheduled-refs.sh" false "$mode" > "$scratch/log" 2>&1 + if [[ "$value" == "$feature" ]]; then + grep -q '^should-skip=true$' "$GITHUB_OUTPUT" + else + grep -q '^should-skip=false$' "$GITHUB_OUTPUT" + fi + if [[ "$value" == "$previous" ]]; then + grep -q "^baseline-ref=$previous$" "$GITHUB_OUTPUT" + fi + done + export STATE_TEST_VALUE="$feature" + : > "$GITHUB_OUTPUT" + bash "$scripts/bench-scheduled-refs.sh" true "$mode" > "$scratch/log" 2>&1 + grep -q '^should-skip=false$' "$GITHUB_OUTPUT" + export STATE_TEST_FAILURE=list + if bash "$scripts/bench-scheduled-refs.sh" false "$mode" > "$scratch/log" 2>&1; then exit 1; fi + export STATE_TEST_FAILURE="" + echo "PASS: $mode first run, unchanged/changed commit, force, and API failure" +done diff --git a/.github/scripts/bench-state.sh b/.github/scripts/bench-state.sh new file mode 100644 index 00000000000..a8f39977095 --- /dev/null +++ b/.github/scripts/bench-state.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Restore the last successful benchmark commit from an Actions artifact. +# Usage: bench-state.sh owner/repo hourly|nightly|release +# Requires GH_TOKEN with actions:read. State is scoped to GITHUB_REF_NAME (main +# outside Actions). Missing/expired state bootstraps; API/corruption errors fail. +set -euo pipefail + +repo=${1:?expected owner/repo} +mode=${2:?expected benchmark mode} +case "$mode" in hourly|nightly|release) ;; *) echo "Invalid benchmark mode: $mode" >&2; exit 1 ;; esac +name="bench-state-$mode" +branch=${GITHUB_REF_NAME:-main} + +# Filter before downloading: dispatches on test branches must not change main's +# baseline. Pagination matters when hourly state accumulates over its retention. +artifacts=$(gh api --paginate --slurp "repos/$repo/actions/artifacts?name=$name&per_page=100") +candidates=$(jq -r --arg name "$name" --arg branch "$branch" ' + [.[].artifacts[] | select(.name == $name and .expired == false + and .workflow_run.head_branch == $branch + and .workflow_run.head_repository_id == .workflow_run.repository_id)] + | sort_by(.id) | reverse | .[] | [.id, .workflow_run.id] | @tsv +' <<< "$artifacts") + +while IFS=$'\t' read -r artifact_id run_id; do + [[ -n "$artifact_id" ]] || continue + run=$(gh api "repos/$repo/actions/runs/$run_id") + if ! jq -e --arg branch "$branch" ' + .path == ".github/workflows/bench-scheduled.yml" and .head_branch == $branch + and .status == "completed" and .conclusion == "success" + and (.event == "schedule" or .event == "workflow_dispatch") + ' <<< "$run" >/dev/null; then + continue + fi + scratch=$(mktemp -d) + trap 'rm -rf "$scratch"' EXIT + gh api "repos/$repo/actions/artifacts/$artifact_id/zip" > "$scratch/state.zip" + value=$(unzip -p "$scratch/state.zip" feature-ref) + if [[ ! "$value" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid commit SHA in $name artifact $artifact_id" >&2 + exit 1 + fi + echo "Restored $name from run $run_id" >&2 + printf '%s\n' "$value" + exit 0 +done <<< "$candidates" diff --git a/.github/scripts/bench-upload-charts-test.sh b/.github/scripts/bench-upload-charts-test.sh new file mode 100644 index 00000000000..9485a8304cd --- /dev/null +++ b/.github/scripts/bench-upload-charts-test.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail +script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/bench-upload-charts.sh" +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/charts with spaces" "$scratch/empty" +# A 1x1 PNG; no credentials or network are needed by these tests. +printf '%s' 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=' | + base64 -d > "$scratch/charts with spaces/latency_throughput.png" +export UPLOAD_TEST_LOG="$scratch/calls" +export UPLOAD_TEST_MODE=dry +gh() { + printf '%s\n' "$*" >> "$UPLOAD_TEST_LOG" + [[ "$UPLOAD_TEST_MODE" != dry ]] || return 1 + if [[ "$2" == repos/owner/repo ]]; then + printf '123\n' + return + fi + [[ "$2" == https://uploads.github.com/user-attachments/assets ]] || return 1 + [[ "$*" == *'content_type=image/png'* && "$*" == *'repository_id=123'* ]] || return 1 + [[ "$*" == *'Content-Type: application/octet-stream'* ]] || return 1 + if [[ "$UPLOAD_TEST_MODE" == failure ]]; then + echo 'HTTP 401' >&2 + return 1 + fi + printf '{"url":"https://github.com/user-attachments/assets/test-image"}\n' +} +export -f gh +bash "$script" --dry-run owner/repo "$scratch/charts with spaces" > "$scratch/dry-run" +[[ ! -e "$UPLOAD_TEST_LOG" ]] +[[ $(wc -l < "$scratch/dry-run") == 2 ]] +grep -q 'repository_id=REPOSITORY_ID' "$scratch/dry-run" +grep -q 'uploads.github.com/user-attachments/assets' "$scratch/dry-run" +echo 'PASS: dry run validates files and prints requests without calling gh' + +export UPLOAD_TEST_MODE=success +bash "$script" owner/repo "$scratch/charts with spaces" > "$scratch/charts.md" +grep -Fq '![Latency, Throughput & Diff](https://github.com/user-attachments/assets/test-image)' "$scratch/charts.md" +[[ $(wc -l < "$UPLOAD_TEST_LOG") == 2 ]] +echo 'PASS: upload request and rendered Markdown' + +export UPLOAD_TEST_MODE=failure +if bash "$script" owner/repo "$scratch/charts with spaces" > /dev/null 2>&1; then + echo 'Expected upload failure' >&2; exit 1 +fi +[[ $(wc -l < "$UPLOAD_TEST_LOG") == 4 ]] +echo 'PASS: upload errors propagate without retries' + +if bash "$script" --dry-run owner/repo "$scratch/empty" > /dev/null 2>&1; then + echo 'Expected missing charts failure' >&2; exit 1 +fi +printf invalid > "$scratch/charts with spaces/bad.png" +if bash "$script" --dry-run owner/repo "$scratch/charts with spaces" > /dev/null 2>&1; then + echo 'Expected invalid PNG failure' >&2; exit 1 +fi +[[ $(wc -l < "$UPLOAD_TEST_LOG") == 4 ]] +echo 'PASS: missing or invalid charts fail before network calls' diff --git a/.github/scripts/bench-upload-charts.sh b/.github/scripts/bench-upload-charts.sh new file mode 100644 index 00000000000..e9ea1e7916d --- /dev/null +++ b/.github/scripts/bench-upload-charts.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Upload benchmark PNGs using the endpoint behind gh --attach. +# Usage: bash bench-upload-charts.sh [--dry-run] owner/repo charts-directory +# Requires gh authentication with a user token that can write to the repository. +set -euo pipefail + +dry_run=false +if [[ "${1:-}" == --dry-run ]]; then + dry_run=true + shift +fi +if [[ $# != 2 ]]; then + echo "Usage: $0 [--dry-run] owner/repo charts-directory" >&2 + exit 1 +fi +repo=$1 +directory=$2 +shopt -s nullglob +charts=("$directory"/*.png) +if (( ${#charts[@]} == 0 )); then + echo "No PNG charts found in $directory" >&2 + exit 1 +fi +# Validate the whole batch before uploading anything. +for chart in "${charts[@]}"; do + if [[ ! -f "$chart" || ! -s "$chart" ]] || (( $(wc -c < "$chart") > 10485760 )); then + echo "Expected a nonempty PNG no larger than 10 MiB: $chart" >&2 + exit 1 + fi + if [[ "$(od -An -tx1 -N8 "$chart" | tr -d ' \n')" != 89504e470d0a1a0a ]]; then + echo "Invalid PNG: $chart" >&2 + exit 1 + fi +done + +if $dry_run; then + printf '%q ' gh api "repos/$repo" --jq .id + printf '\n' + repository_id=REPOSITORY_ID +else + repository_id=$(gh api "repos/$repo" --jq .id) + [[ "$repository_id" =~ ^[1-9][0-9]*$ ]] + printf '\n\n### Charts\n\n' +fi +for chart in "${charts[@]}"; do + name=${chart##*/} + command=(gh api https://uploads.github.com/user-attachments/assets + --method POST -H 'Content-Type: application/octet-stream' + -H 'Accept: application/vnd.github+json' --input "$chart" + -f "name=$name" -f content_type=image/png -F "repository_id=$repository_id") + if $dry_run; then + printf '%q ' "${command[@]}" + printf '\n' + continue + fi + # Print successful URLs immediately; never automatically retry an upload. + response=$("${command[@]}") + url=$(jq -er '.url | select(type == "string" and startswith("https://github.com/user-attachments/assets/"))' <<< "$response") + case "$name" in + latency_throughput.png) label='Latency, Throughput & Diff' ;; + wait_breakdown.png) label='Wait Time Breakdown' ;; + gas_vs_latency.png) label='Gas vs Latency' ;; + *) label='Benchmark chart' ;; + esac + printf '
%s\n\n![%s](%s)\n\n
\n\n' "$label" "$label" "$url" +done diff --git a/.github/scripts/bench-utils.js b/.github/scripts/bench-utils.js index 877be57807f..1e7baabc471 100644 --- a/.github/scripts/bench-utils.js +++ b/.github/scripts/bench-utils.js @@ -45,10 +45,6 @@ function loadSamplyUrls(workDir) { return loadProfileUrls(workDir, 'samply-profile-url.txt'); } -function loadTracingChromeUrls(workDir) { - return loadProfileUrls(workDir, 'tracing-chrome-profile-url.txt'); -} - function loadProfileUrls(workDir, fileName) { const urls = {}; let runs = []; @@ -169,7 +165,6 @@ module.exports = { verdict, isWin, loadSamplyUrls, - loadTracingChromeUrls, blocksLabel, metricRows, waitTimeRows, diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 6c2936f0207..2ade0528b95 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -9,9 +9,9 @@ # release — Compares the latest GitHub release tag against the current nightly # Docker build. Runs daily to track nightly vs release performance. # -# State is persisted between runs via the decofe/reth-bench-charts repo: each -# successful run saves the feature commit SHA so the next run knows what to -# compare against. +# State is an Actions artifact containing the feature SHA from a successful run, +# scoped by mode and branch. After 90 days without retained state, the existing +# first-run baseline rules apply. No PAT or separate repository is needed. on: schedule: @@ -145,7 +145,6 @@ jobs: env: INPUT_MODE: ${{ steps.mode.outputs.mode }} GH_TOKEN: ${{ github.token }} - DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ID: ${{ github.run_id }} INPUT_FORCE: ${{ inputs.force || 'false' }} @@ -370,6 +369,7 @@ jobs: # apt packages sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends gh sudo apt-get install -y --no-install-recommends \ python3 make jq zstd curl dmsetup m4 \ linux-tools-"$(uname -r)" || \ @@ -411,7 +411,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" missing=() - for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv jq m4; do + for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv jq m4 gh; do command -v "$cmd" &>/dev/null || missing+=("$cmd") done if [ ${#missing[@]} -gt 0 ]; then @@ -747,58 +747,18 @@ jobs: name: bench-scheduled-results path: ${{ env.BENCH_WORK_DIR }} - - name: Push charts - id: push-charts + - name: Upload chart attachments + id: upload-charts if: success() && env.BENCH_MODE != 'hourly' env: - DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }} - RUN_ID: ${{ github.run_id }} - run: | - CHART_DIR="${BENCH_MODE}/${RUN_ID}" - CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git" - - TMP_DIR="" - prepare_charts() { - if [ -n "${TMP_DIR}" ]; then - rm -rf "${TMP_DIR}" - fi - - TMP_DIR=$(mktemp -d) - if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then - true - else - git init "${TMP_DIR}" - git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}" - fi - .github/scripts/configure-git-token-user.sh "${TMP_DIR}" "${DEREK_TOKEN}" - - mkdir -p "${TMP_DIR}/${CHART_DIR}" - cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/" - git -C "${TMP_DIR}" add "${CHART_DIR}" - } - - for attempt in 1 2 3 4 5; do - prepare_charts - git -C "${TMP_DIR}" commit -m "nightly bench charts for run ${RUN_ID}" - if git -C "${TMP_DIR}" push origin HEAD:main; then - break - fi - if [ "$attempt" -eq 5 ]; then - echo "::error::Failed to push charts after ${attempt} attempts" - rm -rf "${TMP_DIR}" - exit 1 - fi - sleep "$attempt" - done - echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT" - rm -rf "${TMP_DIR}" + GH_TOKEN: ${{ secrets.DEREK_PAT }} + run: bash .github/scripts/bench-upload-charts.sh "$GITHUB_REPOSITORY" "$BENCH_WORK_DIR/charts" > "$BENCH_WORK_DIR/charts.md" - name: Write job summary if: success() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: INPUT_GRAFANA_URL: ${{ steps.metrics.outputs.grafana-url }} - INPUT_SHA: ${{ steps.push-charts.outputs.sha }} with: script: | const fs = require('fs'); @@ -845,22 +805,8 @@ jobs: md += '\n'; } - // Charts - const chartSha = process.env.INPUT_SHA; - if (chartSha) { - const runId = '${{ github.run_id }}'; - const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`; - const charts = [ - { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' }, - { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' }, - { file: 'gas_vs_latency.png', label: 'Gas vs Latency' }, - ]; - md += `### Charts\n\n`; - for (const chart of charts) { - md += `
${chart.label}\n\n`; - md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`; - md += `
\n\n`; - } + if (process.env.BENCH_MODE !== 'hourly') { + md += fs.readFileSync(process.env.BENCH_WORK_DIR + '/charts.md', 'utf8'); } const grafanaUrl = process.env.INPUT_GRAFANA_URL; @@ -1116,36 +1062,21 @@ jobs: if: success() name: save-state runs-on: ubuntu-latest + permissions: {} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - sparse-checkout: .github/scripts - sparse-checkout-cone-mode: true - - - name: Push state to charts repo + - name: Prepare benchmark state env: - INPUT_FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }} - INPUT_MODE: ${{ needs.resolve-refs.outputs.mode }} - DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }} + FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }} run: | - MODE="${INPUT_MODE}" - FEATURE_REF="${INPUT_FEATURE_REF}" - CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git" + [[ "$FEATURE_REF" =~ ^[0-9a-f]{40}$ ]] || exit 1 + mkdir -p "$RUNNER_TEMP/bench-state" + printf '%s\n' "$FEATURE_REF" > "$RUNNER_TEMP/bench-state/feature-ref" - TMP_DIR=$(mktemp -d) - if git clone --depth 1 --branch state "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then - true - else - git init "${TMP_DIR}" - git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}" - fi - .github/scripts/configure-git-token-user.sh "${TMP_DIR}" "${DEREK_TOKEN}" - - mkdir -p "${TMP_DIR}/state" - echo "${FEATURE_REF}" > "${TMP_DIR}/state/${MODE}-last-feature-ref" - git -C "${TMP_DIR}" add state/ - git -C "${TMP_DIR}" diff --cached --quiet && echo "No state change" && exit 0 - git -C "${TMP_DIR}" commit -m "bench: update ${MODE} state to ${FEATURE_REF}" - git -C "${TMP_DIR}" push origin HEAD:state - rm -rf "${TMP_DIR}" + - name: Save benchmark state + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-state-${{ needs.resolve-refs.outputs.mode }} + path: ${{ runner.temp }}/bench-state/feature-ref + retention-days: 90 + if-no-files-found: error + overwrite: true diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9530bef873d..dcc760fbc21 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1339,6 +1339,7 @@ jobs: # apt packages sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends gh sudo apt-get install -y --no-install-recommends \ python3 make jq zstd curl dmsetup m4 \ linux-tools-"$(uname -r)" || \ @@ -1387,7 +1388,7 @@ jobs: echo "$HOME/.local/bin" >> "$GITHUB_PATH" echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" missing=() - for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq llvm-config m4; do + for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq llvm-config m4 gh; do command -v "$cmd" &>/dev/null || missing+=("$cmd") done if [ ${#missing[@]} -gt 0 ]; then @@ -2097,7 +2098,15 @@ jobs: # shellcheck disable=SC2086 uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS + - name: Upload chart attachments + id: upload-charts + if: success() && env.BENCH_EXECUTION_MODE != 'call' + env: + GH_TOKEN: ${{ secrets.DEREK_PAT }} + run: bash .github/scripts/bench-upload-charts.sh "$GITHUB_REPOSITORY" "$BENCH_WORK_DIR/charts" > "$BENCH_WORK_DIR/charts.md" + - name: Upload results + id: upload-results if: "!cancelled()" uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -2109,108 +2118,11 @@ jobs: !${{ env.BENCH_WORK_DIR }}/txgen-payloads/corpus.jsonl !${{ env.BENCH_WORK_DIR }}/txgen-payloads/corpus.jsonl.gz - - name: Push benchmark artifacts - id: push-charts - if: success() - env: - DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }} - RUN_ID: ${{ github.run_id }} - run: | - PR_NUMBER="${BENCH_PR:-0}" - CHART_DIR="pr/${PR_NUMBER}/${RUN_ID}" - CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git" - - TMP_DIR="" - prepare_artifacts() { - if [ -n "${TMP_DIR}" ]; then - rm -rf "${TMP_DIR}" - fi - - TMP_DIR=$(mktemp -d) - if git clone --depth 1 --filter=blob:none --sparse "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then - git -C "${TMP_DIR}" sparse-checkout set --no-cone "${CHART_DIR}" - true - else - git init "${TMP_DIR}" - git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}" - fi - .github/scripts/configure-git-token-user.sh "${TMP_DIR}" "${DEREK_TOKEN}" - - mkdir -p "${TMP_DIR}/${CHART_DIR}" - if compgen -G "$BENCH_WORK_DIR/charts/*.png" > /dev/null; then - cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/" - fi - if [ "${BENCH_TRACING_CHROME:-false}" = "true" ]; then - TRACE_DIR="${CHART_DIR}/tracing-chrome" - mkdir -p "${TMP_DIR}/${TRACE_DIR}" - mapfile -t RUN_DIRS < <(find "$BENCH_WORK_DIR" -mindepth 1 -maxdepth 1 -type d \( -name 'baseline-*' -o -name 'feature-*' \) -printf '%f\n' | sort -V) - for run_dir in "${RUN_DIRS[@]}"; do - TRACE="$BENCH_WORK_DIR/$run_dir/tracing-chrome-profile.json" - if [ ! -f "$TRACE" ]; then continue; fi - - TRACE_SIZE=$(du -h "$TRACE" | cut -f1) - echo "Adding $run_dir Chrome trace (${TRACE_SIZE}) to benchmark artifacts..." - gzip -c "$TRACE" > "${TMP_DIR}/${TRACE_DIR}/${run_dir}.json.gz" - done - fi - git -C "${TMP_DIR}" add "${CHART_DIR}" - } - - prepare_artifacts - if git -C "${TMP_DIR}" diff --cached --quiet; then - echo "Artifacts for ${CHART_DIR} are already present, skipping push" - PUSH_SHA="$(git -C "${TMP_DIR}" rev-parse HEAD)" - else - for attempt in 1 2 3 4 5; do - git -C "${TMP_DIR}" commit -m "bench artifacts for PR #${PR_NUMBER} run ${RUN_ID}" - if git -C "${TMP_DIR}" push origin HEAD:main; then - break - fi - if [ "$attempt" -eq 5 ]; then - echo "::error::Failed to push benchmark artifacts after ${attempt} attempts" - rm -rf "${TMP_DIR}" - exit 1 - fi - sleep "$attempt" - prepare_artifacts - done - - PUSH_SHA="$(git -C "${TMP_DIR}" rev-parse HEAD)" - fi - - if [ "${BENCH_TRACING_CHROME:-false}" = "true" ]; then - RAW_BASE="https://raw.githubusercontent.com/decofe/reth-bench-charts/${PUSH_SHA}/${CHART_DIR}/tracing-chrome" - mapfile -t RUN_DIRS < <(find "$BENCH_WORK_DIR" -mindepth 1 -maxdepth 1 -type d \( -name 'baseline-*' -o -name 'feature-*' \) -printf '%f\n' | sort -V) - for run_dir in "${RUN_DIRS[@]}"; do - if [ ! -f "${TMP_DIR}/${CHART_DIR}/tracing-chrome/${run_dir}.json.gz" ]; then continue; fi - - RAW_URL="${RAW_BASE}/${run_dir}.json.gz" - PERFETTO_URL="$(python3 - "$RAW_URL" "${run_dir}.json.gz" <<'PY' - import sys - from urllib.parse import quote - - raw_url, file_name = sys.argv[1:3] - print( - "https://ui.perfetto.dev/#!/?url=" - + quote(raw_url, safe="") - + "&fileName=" - + quote(file_name, safe="") - ) - PY - )" - echo "$PERFETTO_URL" > "$BENCH_WORK_DIR/$run_dir/tracing-chrome-profile-url.txt" - echo "Perfetto URL for $run_dir: $PERFETTO_URL" - done - fi - - echo "sha=${PUSH_SHA}" >> "$GITHUB_OUTPUT" - rm -rf "${TMP_DIR}" - - name: Compare & comment if: success() && env.BENCH_COMMENT_ID uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - INPUT_SHA: ${{ steps.push-charts.outputs.sha }} + BENCH_RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} with: github-token: ${{ secrets.DEREK_PAT }} script: | @@ -2223,25 +2135,8 @@ jobs: comment = '⚠️ Benchmark completed but failed to generate comparison.'; } - const sha = process.env.INPUT_SHA; - const prNumber = process.env.BENCH_PR || '0'; - const runId = '${{ github.run_id }}'; - - // Call mode does not generate charts, so never link them. - if (sha && process.env.BENCH_EXECUTION_MODE !== 'call') { - const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${sha}/pr/${prNumber}/${runId}`; - const charts = [ - { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' }, - { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' }, - { file: 'gas_vs_latency.png', label: 'Gas vs Latency' }, - ]; - let chartMarkdown = '\n\n### Charts\n\n'; - for (const chart of charts) { - chartMarkdown += `
${chart.label}\n\n`; - chartMarkdown += `![${chart.label}](${baseUrl}/${chart.file})\n\n`; - chartMarkdown += `
\n\n`; - } - comment += chartMarkdown; + if (process.env.BENCH_EXECUTION_MODE !== 'call') { + comment += fs.readFileSync(process.env.BENCH_WORK_DIR + '/charts.md', 'utf8'); } // Samply profile links (URLs point directly to Firefox Profiler) @@ -2258,15 +2153,7 @@ jobs: } if (process.env.BENCH_TRACING_CHROME === 'true') { - const { loadTracingChromeUrls } = require('./.github/scripts/bench-utils'); - const tracingChromeUrls = loadTracingChromeUrls(process.env.BENCH_WORK_DIR); - const links = []; - for (const [run, url] of Object.entries(tracingChromeUrls)) { - links.push(`- **${run}**: [Perfetto](${url})`); - } - if (links.length > 0) { - comment += `\n\n### Chrome Traces\n\n${links.join('\n')}\n`; - } + comment += `\n\n### Chrome Traces\n\n[Download bench-results](${process.env.BENCH_RESULTS_URL}), extract the archive, and open each run's \`tracing-chrome-profile.json\` in [Perfetto](https://ui.perfetto.dev/). GitHub sign-in is required; downloads are available until the artifact expires.\n`; } // Node errors (panics / ERROR logs) @@ -2296,14 +2183,13 @@ jobs: env: INPUT_LOGS_URL: ${{ steps.metrics.outputs.logs-url }} INPUT_TRACES_URL: ${{ steps.metrics.outputs.traces-url }} - INPUT_SHA: ${{ steps.push-charts.outputs.sha }} + BENCH_RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} with: script: | const jobSummary = require('./.github/scripts/bench-job-summary.js'); await jobSummary({ core, context, - chartSha: process.env.INPUT_SHA, logsUrl: process.env.INPUT_LOGS_URL, tracesUrl: process.env.INPUT_TRACES_URL, runId: '${{ github.run_id }}', @@ -2313,6 +2199,7 @@ jobs: if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win') uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: + BENCH_RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }} SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }} with: diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml index b74196167d1..f40d236c33d 100644 --- a/.github/workflows/lint-actions.yml +++ b/.github/workflows/lint-actions.yml @@ -31,3 +31,8 @@ jobs: - name: Check workflow files run: SHELLCHECK_OPTS="-S error" actionlint -color shell: bash + - name: Test benchmark CI helpers + run: | + bash .github/scripts/bench-state-test.sh + bash .github/scripts/bench-upload-charts-test.sh + node --test .github/scripts/bench-job-summary-test.js From 5166eb7dab0ef0a589e8911e09307953941b66f2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 11:00:07 +0000 Subject: [PATCH 061/133] chore: bump revm and reth-core deps (#27401) Co-authored-by: Chase Wright Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 90 ++++++++++--------- Cargo.toml | 12 +-- crates/rpc/rpc-convert/src/transaction.rs | 14 +-- crates/rpc/rpc-eth-api/src/helpers/block.rs | 8 +- .../rpc-eth-api/src/helpers/subscriptions.rs | 3 +- crates/rpc/rpc-eth-types/src/simulate.rs | 2 +- crates/rpc/rpc/src/debug.rs | 6 +- crates/rpc/rpc/src/eth/core.rs | 34 ++++++- 8 files changed, 107 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff688151015..9abbe664105 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,9 +252,9 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cb19c9d53b507960f649e5689f7098e556696023fe1b57b67f2181ed54bc5b7" +checksum = "6a12e1dc70236cfb8e79b96dd8544aed47c39e2f45dc6b51c3703e969e052faa" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8117,9 +8117,9 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6bac09d9d84a412ae151149ec870d6eb7770335c5eb63f209453d81deff8d39" +checksum = "ef769807575055ccafe9db3a9d7eca3b97ca8213122942e71c47915776ec52b2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8138,9 +8138,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0101b1f474563c4c48c04fc2477199279a53d9c19b65301c6fd2131cf6e4a330" +checksum = "0652b5da6dcce54dc8bcfb69c5241b0ef09211963f76929d4d63e23767b43bf6" dependencies = [ "proc-macro2", "quote", @@ -9867,9 +9867,9 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18341fba74d004c7fad32f784488baaea9adebfde7a4f231a4d8d445aea70347" +checksum = "c6835d57ed17047c8ba42992550b3cd1232cb92deb119715d32f63079fb5844d" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10398,9 +10398,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f410d6bff1fd059ebaec2edeff1326f103f61162158411325d78158f94b2b4a9" +checksum = "8d6c3c55bf695994b2ce87e05eaf49c1135c84d3d1f5e774df6077a8173ebdb0" dependencies = [ "alloy-consensus", "alloy-network", @@ -10987,18 +10987,18 @@ dependencies = [ [[package]] name = "reth-zstd-compressors" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbdee2f0ad09dc32b43ab52641f8239766a66de4345834bad27d5cfa2f62864d" +checksum = "253f7d5772f3c8d0dcc3fdac398db6bca4f6f6e602d240c3bc47338dde0910dd" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "43.0.2" +version = "43.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f2322f5ea197fac72c2cdc88cb501b2a7db0b0009911133fd3dda45bfceedd" +checksum = "62b2fbb3bdce72fef4f156e2164bdfde44dc3aeebc5b4f31a413fba0c09c524c" dependencies = [ "revm-bytecode", "revm-context", @@ -11028,9 +11028,9 @@ dependencies = [ [[package]] name = "revm-context" -version = "43.0.2" +version = "43.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c16835bc2ead203219a18fcc84c2c71879ad6b890e8a71150cf7f10e4e355dc4" +checksum = "9b06efea511a15ff8309acaa4c283239b444b461bbd2e9f0326c8586f45fb1da" dependencies = [ "bitvec", "cfg-if", @@ -11045,9 +11045,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "43.0.1" +version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b4109bdceaca5ee6e27bf41f6b156f2b15892dc5bb47d3530bc4d3b3060077" +checksum = "3068385c68f6a3bd2e781dd81cefd25c98dd0a574de6c8ec2a90acc33318b2da" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -11061,9 +11061,9 @@ dependencies = [ [[package]] name = "revm-database" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d764afa9a1c9278c05d40461b07319d21ccb39c1bdf0e9dde7e5ea7d9be53d" +checksum = "8db6fb10912d7fba0161e91d4577ce71fa9726ec38a4d420e4242e504a47aa8f" dependencies = [ "alloy-eips", "derive_more", @@ -11077,9 +11077,9 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2a27e4440c2ef4932afdaa28a1d1ac8601504c7030c1a387b5e307d07d864d1" +checksum = "5581ecadb3d0d1c18a91f1797d6b7c4c17720bea596e9e9ec90b95803db2bc77" dependencies = [ "auto_impl", "either", @@ -11091,9 +11091,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "43.0.2" +version = "43.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73a7409031d87ec281b74efff65d7a4dbea9d80727a55dbc7e4b3fd77b5e03fb" +checksum = "ba5102e9c02b9d27e7c98f449b495a2c2b2ec011209f4eeca16e054520ec5ac4" dependencies = [ "auto_impl", "derive-where", @@ -11110,9 +11110,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "43.0.2" +version = "43.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a56f37224afd628f7244aa6965609ada04b440ed8ab7a19f55e4291df5a0112" +checksum = "825cf946c1bf3178f6a93aae4e8064a0ebd505bc764a7fe3d49411e7242e4091" dependencies = [ "auto_impl", "either", @@ -11148,9 +11148,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "43.0.1" +version = "43.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "932f7fe05a2cdeb08195dd7e88a09a4da7c3660cdc6e62720839c77ba77f65aa" +checksum = "aa32f5099a266329d1869d6f9bf4bb7894345d2b58c57d05396f67385d65f299" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -11161,9 +11161,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "43.0.2" +version = "43.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4dc0279f2a44770faa8c93d008df684440cf573c91b8e0e39d0d581abf7b3d" +checksum = "aa80b8cef09bead8c8b1afa7733b09fc1428f49158d8657c239268409d2a2bf7" dependencies = [ "ark-bls12-381", "ark-bn254", @@ -11199,9 +11199,9 @@ dependencies = [ [[package]] name = "revm-state" -version = "43.0.0" +version = "43.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd02082cf1baca01e8bc17456c45136bdc66632395a98db1c1a2afe4c6ea87f" +checksum = "3b2d2ec0be925c4c87ff4afae33af4b9f693d9b011f74d6b4d52267a62df0c49" dependencies = [ "alloy-eip7928", "bitflags 2.13.1", @@ -11209,12 +11209,13 @@ dependencies = [ "revm-bytecode", "revm-primitives", "serde", + "triomphe", ] [[package]] name = "revmc" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -11234,7 +11235,7 @@ dependencies = [ [[package]] name = "revmc-backend" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "eyre", "ruint", @@ -11243,12 +11244,12 @@ dependencies = [ [[package]] name = "revmc-build" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" [[package]] name = "revmc-builtins" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "paste", "revm-bytecode", @@ -11263,7 +11264,7 @@ dependencies = [ [[package]] name = "revmc-codegen" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "alloy-primitives", "bitflags 2.13.1", @@ -11294,7 +11295,7 @@ dependencies = [ [[package]] name = "revmc-context" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "revm-context", "revm-context-interface", @@ -11309,7 +11310,7 @@ dependencies = [ [[package]] name = "revmc-llvm" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "alloy-primitives", "cc", @@ -11322,7 +11323,7 @@ dependencies = [ [[package]] name = "revmc-runtime" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#68835cf19661307d293c26dd06bd1e59ccae5144" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "alloy-evm", "alloy-primitives", @@ -13295,6 +13296,15 @@ dependencies = [ "rlp", ] +[[package]] +name = "triomphe" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" +dependencies = [ + "serde", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 79892fa6e46..a5a5544d01f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -323,8 +323,8 @@ reth-cli = { path = "crates/cli/cli" } reth-cli-commands = { path = "crates/cli/commands" } reth-cli-runner = { path = "crates/cli/runner" } reth-cli-util = { path = "crates/cli/util" } -reth-codecs = { version = "0.7.0", default-features = false } -reth-codecs-derive = "0.7.0" +reth-codecs = { version = "0.8.1", default-features = false } +reth-codecs-derive = "0.8.1" reth-config = { path = "crates/config", default-features = false } reth-consensus = { path = "crates/consensus/consensus", default-features = false } reth-consensus-common = { path = "crates/consensus/common", default-features = false } @@ -390,7 +390,7 @@ reth-payload-builder = { path = "crates/payload/builder" } reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } reth-payload-primitives = { path = "crates/payload/primitives" } reth-payload-validator = { path = "crates/payload/validator" } -reth-primitives-traits = { version = "0.7.1", default-features = false } +reth-primitives-traits = { version = "0.8.1", default-features = false } reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } reth-prune-types = { path = "crates/prune/types", default-features = false } @@ -404,7 +404,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal reth-rpc-layer = { path = "crates/rpc/rpc-layer" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } reth-rpc-convert = { path = "crates/rpc/rpc-convert" } -reth-rpc-traits = { version = "0.7.0", default-features = false } +reth-rpc-traits = { version = "0.8.1", default-features = false } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } @@ -424,10 +424,10 @@ reth-trie-common = { path = "crates/trie/common", default-features = false } reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } -reth-zstd-compressors = { version = "0.7.0", default-features = false } +reth-zstd-compressors = { version = "0.8.1", default-features = false } # revm -revm = { version = "43.0.2", default-features = false } +revm = { version = "43.0.3", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } revm-inspectors = "0.43.0" diff --git a/crates/rpc/rpc-convert/src/transaction.rs b/crates/rpc/rpc-convert/src/transaction.rs index 22d70029111..ec3efeb3c58 100644 --- a/crates/rpc/rpc-convert/src/transaction.rs +++ b/crates/rpc/rpc-convert/src/transaction.rs @@ -72,11 +72,11 @@ pub trait HeaderConverter: Send + Sync + Unpin + Clone + 'static /// An associated RPC conversion error. type Err: error::Error; - /// Converts a consensus header into an RPC header. + /// Converts a consensus header into an RPC header. `block_size` is set for block responses. fn convert_header( &self, header: SealedHeader, - block_size: usize, + block_size: Option, ) -> Result; } @@ -91,7 +91,7 @@ where fn convert_header( &self, header: SealedHeader, - block_size: usize, + block_size: Option, ) -> Result { Ok(Rpc::from_consensus_header(header, block_size)) } @@ -99,14 +99,14 @@ where impl HeaderConverter for F where - F: Fn(SealedHeader, usize) -> Rpc + Send + Sync + Unpin + Clone + 'static, + F: Fn(SealedHeader, Option) -> Rpc + Send + Sync + Unpin + Clone + 'static, { type Err = Infallible; fn convert_header( &self, header: SealedHeader, - block_size: usize, + block_size: Option, ) -> Result { Ok(self(header, block_size)) } @@ -197,7 +197,7 @@ pub trait RpcConvert: Send + Sync + Unpin + Debug + DynClone + 'static { fn convert_header( &self, header: SealedHeaderFor, - block_size: usize, + block_size: Option, ) -> Result, Self::Error>; } @@ -777,7 +777,7 @@ where fn convert_header( &self, header: SealedHeaderFor, - block_size: usize, + block_size: Option, ) -> Result, Self::Error> { Ok(self.header_converter.convert_header(header, block_size)?) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 755be4eda00..f2965d0051b 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -43,8 +43,8 @@ pub trait EthBlocks: LoadBlock Some(header), Err(err) => { diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index 903eda2d872..6f44f39d089 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -578,7 +578,7 @@ where let block = block.into_rpc_block( txs_kind, |tx, tx_info| converter.fill(tx, tx_info), - |header, size| converter.convert_header(header, size), + |header, block_size| converter.convert_header(header, Some(block_size)), )?; Ok(SimulatedBlock { inner: block, calls }) } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index c0bd7cba295..3c137519b83 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -940,7 +940,7 @@ where } for entry in entries { - let rlp = alloy_rlp::encode(entry.block.sealed_block()).into(); + let rlp = Bytes::from(alloy_rlp::encode(entry.block.sealed_block())); let hash = entry.block.hash(); let block = entry @@ -948,7 +948,9 @@ where .clone_into_rpc_block( BlockTransactionsKind::Full, |tx, tx_info| self.eth_api().converter().fill(tx, tx_info), - |header, size| self.eth_api().converter().convert_header(header, size), + |header, block_size| { + self.eth_api().converter().convert_header(header, Some(block_size)) + }, ) .map_err(|err| Eth::Error::from(err).into())?; diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index fcf25ebe849..b71d80baba4 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -569,7 +569,11 @@ mod tests { test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}, PruneCheckpointReader, StageCheckpointReader, }; - use reth_rpc_eth_api::{helpers::EthCall, node::RpcNodeCoreAdapter, EthApiServer}; + use reth_rpc_eth_api::{ + helpers::{EthBlocks, EthCall}, + node::RpcNodeCoreAdapter, + EthApiServer, + }; use reth_rpc_eth_types::RpcInvalidTransactionError; use reth_storage_api::{ BalProvider, BlockReader, BlockReaderIdExt, NodePrimitivesProvider, StateProviderFactory, @@ -1177,4 +1181,32 @@ mod tests { .unwrap(); assert_eq!(estimated, U256::from(21_000)); } + + #[tokio::test] + async fn header_responses_omit_size_while_blocks_keep_it() { + let provider = MockEthProvider::default(); + let block = Block { + header: Header { number: 1, ..Default::default() }, + body: BlockBody::default(), + }; + let hash = block.header.hash_slow(); + let block_size = alloy_rlp::encode(&block).len(); + provider.add_block(hash, block); + + let api = build_test_eth_api(provider); + for block_id in [BlockId::Number(BlockNumberOrTag::Number(1)), BlockId::Hash(hash.into())] { + let header = EthBlocks::rpc_block_header(&api, block_id).await.unwrap().unwrap(); + let response = serde_json::to_value(&header).unwrap(); + assert!(response.get("size").is_none()); + } + + for full in [false, true] { + let block = + EthBlocks::rpc_block(&api, BlockId::Number(BlockNumberOrTag::Number(1)), full) + .await + .unwrap() + .unwrap(); + assert_eq!(block.header.size, Some(U256::from(block_size))); + } + } } From 2b0b905bc8e28bb08c07ef9def8d5b676c60d06e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 14:56:57 +0200 Subject: [PATCH 062/133] fix(net): reject invalid block header ranges (#27399) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/net/p2p/src/full_block.rs | 38 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/crates/net/p2p/src/full_block.rs b/crates/net/p2p/src/full_block.rs index df8d3f8fba2..c6b260f96b3 100644 --- a/crates/net/p2p/src/full_block.rs +++ b/crates/net/p2p/src/full_block.rs @@ -79,8 +79,8 @@ where /// Note: this future is cancel safe. /// /// Caution: This does no validation of body (transactions) responses but guarantees that - /// the starting [`SealedHeader`] matches the requested hash, and that the number of headers and - /// bodies received matches the requested limit. + /// the headers form a valid range starting at the requested hash, and that the number of + /// headers and bodies received matches the requested limit. /// /// The returned future yields bodies in falling order, i.e. with descending block numbers. pub fn get_full_block_range( @@ -831,6 +831,7 @@ where if let Err(err) = self.consensus.validate_header_range(&headers_rising) { debug!(target: "downloaders", %err, ?self.start_hash, "Received bad header response"); self.client.report_bad_message(peer); + return } // get the bodies request so it can be polled later @@ -2025,22 +2026,23 @@ mod tests { } #[tokio::test] - async fn download_full_block_range_with_invalid_header() { + async fn download_full_block_range_rejects_invalid_headers() { let client = TestFullBlockClient::default(); - let range_length: usize = 3; - let (header, _) = insert_headers_into_client(&client, 0..range_length); - - let test_consensus = reth_consensus::test_utils::TestConsensus::default(); - test_consensus.set_fail_validation(true); - test_consensus.set_fail_body_against_header(false); - let client = FullBlockClient::new(client, Arc::new(test_consensus)); - - let received = client.get_full_block_range(header.hash(), range_length as u64).await; - - assert_eq!(received.len(), range_length); - for (i, block) in received.iter().enumerate() { - let expected_number = header.number - i as u64; - assert_eq!(block.number, expected_number); - } + let (header, _) = insert_headers_into_client(&client, 0..3); + let consensus = reth_consensus::test_utils::TestConsensus::default(); + consensus.set_fail_validation(true); + let client = FullBlockClient::new(client, Arc::new(consensus)); + let mut request = client.get_full_block_range(header.hash(), 3); + let headers = client + .client + .get_headers(HeadersRequest::falling(header.hash().into(), 3)) + .await + .unwrap(); + + request.on_headers_response(headers); + + assert!(request.headers.is_none()); + assert!(request.pending_headers.is_empty()); + assert!(request.request.bodies.is_none()); } } From 973b1e7ea0a9757c3a7b3ddc2226d7e3904e4de0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 12:51:45 +0000 Subject: [PATCH 063/133] perf: specialize RLP encoded lengths (#27408) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/net/discv4/src/proto.rs | 41 +++++++++++++++++-- crates/net/eth-wire-types/src/capability.rs | 27 +++++++++++++ crates/trie/common/src/trie_node_v2.rs | 45 +++++++++++++++++++++ examples/bsc-p2p/src/upgrade_status.rs | 25 +++++++++++- 4 files changed, 132 insertions(+), 6 deletions(-) diff --git a/crates/net/discv4/src/proto.rs b/crates/net/discv4/src/proto.rs index dff1601d97a..ca7e15e08b0 100644 --- a/crates/net/discv4/src/proto.rs +++ b/crates/net/discv4/src/proto.rs @@ -436,6 +436,15 @@ pub struct Ping { } impl Encodable for Ping { + fn length(&self) -> usize { + let mut payload_length = + 4u32.length() + self.from.length() + self.to.length() + self.expire.length(); + if let Some(enr_seq) = self.enr_sq { + payload_length += enr_seq.length(); + } + payload_length + alloy_rlp::length_of_length(payload_length) + } + fn encode(&self, out: &mut dyn BufMut) { #[derive(RlpEncodable)] struct V4PingMessage<'a> { @@ -528,6 +537,14 @@ pub struct Pong { } impl Encodable for Pong { + fn length(&self) -> usize { + let mut payload_length = self.to.length() + self.echo.length() + self.expire.length(); + if let Some(enr_seq) = self.enr_sq { + payload_length += enr_seq.length(); + } + payload_length + alloy_rlp::length_of_length(payload_length) + } + fn encode(&self, out: &mut dyn BufMut) { #[derive(RlpEncodable)] struct PongMessageEIP868<'a> { @@ -648,8 +665,12 @@ mod tests { enr_sq: None, }; - let decoded = Ping::decode(&mut alloy_rlp::encode(&msg).as_slice()).unwrap(); + let encoded = alloy_rlp::encode(&msg); + assert_eq!(msg.length(), encoded.len()); + let mut buf = encoded.as_slice(); + let decoded = Ping::decode(&mut buf).unwrap(); assert_eq!(msg, decoded); + assert!(buf.is_empty()); } } @@ -666,8 +687,12 @@ mod tests { enr_sq: Some(rng.r#gen()), }; - let decoded = Ping::decode(&mut alloy_rlp::encode(&msg).as_slice()).unwrap(); + let encoded = alloy_rlp::encode(&msg); + assert_eq!(msg.length(), encoded.len()); + let mut buf = encoded.as_slice(); + let decoded = Ping::decode(&mut buf).unwrap(); assert_eq!(msg, decoded); + assert!(buf.is_empty()); } } @@ -684,8 +709,12 @@ mod tests { enr_sq: None, }; - let decoded = Pong::decode(&mut alloy_rlp::encode(&msg).as_slice()).unwrap(); + let encoded = alloy_rlp::encode(&msg); + assert_eq!(msg.length(), encoded.len()); + let mut buf = encoded.as_slice(); + let decoded = Pong::decode(&mut buf).unwrap(); assert_eq!(msg, decoded); + assert!(buf.is_empty()); } } @@ -702,8 +731,12 @@ mod tests { enr_sq: Some(rng.r#gen()), }; - let decoded = Pong::decode(&mut alloy_rlp::encode(&msg).as_slice()).unwrap(); + let encoded = alloy_rlp::encode(&msg); + assert_eq!(msg.length(), encoded.len()); + let mut buf = encoded.as_slice(); + let decoded = Pong::decode(&mut buf).unwrap(); assert_eq!(msg, decoded); + assert!(buf.is_empty()); } } diff --git a/crates/net/eth-wire-types/src/capability.rs b/crates/net/eth-wire-types/src/capability.rs index 6d7fb9890eb..d1247bbdce7 100644 --- a/crates/net/eth-wire-types/src/capability.rs +++ b/crates/net/eth-wire-types/src/capability.rs @@ -348,6 +348,10 @@ impl From> for Capabilities { } impl Encodable for Capabilities { + fn length(&self) -> usize { + self.inner.length() + } + fn encode(&self, out: &mut dyn BufMut) { self.inner.encode(out) } @@ -369,3 +373,26 @@ impl Decodable for Capabilities { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + proptest! { + #[test] + fn capabilities_rlp_roundtrip_and_length( + versions in proptest::collection::vec(66usize..=72, 0..32), + ) { + let capabilities = Capabilities::new( + versions.into_iter().map(|version| Capability::new_static("eth", version)).collect(), + ); + let encoded = alloy_rlp::encode(&capabilities); + prop_assert_eq!(capabilities.length(), encoded.len()); + let mut buf = encoded.as_slice(); + let decoded = Capabilities::decode(&mut buf)?; + prop_assert_eq!(decoded, capabilities); + prop_assert!(buf.is_empty()); + } + } +} diff --git a/crates/trie/common/src/trie_node_v2.rs b/crates/trie/common/src/trie_node_v2.rs index bbf1d0daefc..88ada6a66e2 100644 --- a/crates/trie/common/src/trie_node_v2.rs +++ b/crates/trie/common/src/trie_node_v2.rs @@ -117,6 +117,15 @@ pub enum TrieNodeV2 { } impl Encodable for TrieNodeV2 { + fn length(&self) -> usize { + match self { + Self::EmptyRoot => 1, + Self::Leaf(leaf) => leaf.as_ref().length(), + Self::Branch(branch) => branch.length(), + Self::Extension(ext) => ext.length(), + } + } + fn encode(&self, out: &mut dyn bytes::BufMut) { match self { Self::EmptyRoot => { @@ -236,3 +245,39 @@ impl Encodable for BranchNodeV2 { ExtensionNodeRef::new(&self.key, branch_rlp_node.as_slice()).length() } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use proptest::prelude::*; + + fn assert_roundtrip_and_length(node: TrieNodeV2) { + let encoded = alloy_rlp::encode(&node); + assert_eq!(node.length(), encoded.len()); + let mut buf = encoded.as_slice(); + assert_eq!(TrieNodeV2::decode(&mut buf).unwrap(), node); + assert!(buf.is_empty()); + } + + #[test] + fn trie_node_variants_rlp_roundtrip_and_length() { + assert_roundtrip_and_length(TrieNodeV2::EmptyRoot); + assert_roundtrip_and_length(TrieNodeV2::Branch(BranchNodeV2::default())); + assert_roundtrip_and_length(TrieNodeV2::Extension(ExtensionNode::new( + Nibbles::from_nibbles([1]), + RlpNode::word_rlp(&B256::repeat_byte(0xaa)), + ))); + } + + proptest! { + #[test] + fn leaf_rlp_roundtrip_and_length( + key in proptest::collection::vec(0u8..16, 0..64), + value in proptest::collection::vec(any::(), 0..128), + ) { + let node = TrieNodeV2::Leaf(LeafNode::new(Nibbles::from_nibbles(key), value)); + assert_roundtrip_and_length(node); + } + } +} diff --git a/examples/bsc-p2p/src/upgrade_status.rs b/examples/bsc-p2p/src/upgrade_status.rs index c31cf1751ac..bdfb8cf7836 100644 --- a/examples/bsc-p2p/src/upgrade_status.rs +++ b/examples/bsc-p2p/src/upgrade_status.rs @@ -1,7 +1,7 @@ //! Implement BSC upgrade message which is required during handshake with other BSC clients, e.g., //! geth. use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable}; -use bytes::{Buf, BufMut, Bytes, BytesMut}; +use bytes::{BufMut, Bytes, BytesMut}; /// The message id for the upgrade status message, used in the BSC handshake. const UPGRADE_STATUS_MESSAGE_ID: u8 = 0x0b; @@ -16,6 +16,10 @@ pub struct UpgradeStatus { } impl Encodable for UpgradeStatus { + fn length(&self) -> usize { + UPGRADE_STATUS_MESSAGE_ID.length() + self.extension.length() + } + fn encode(&self, out: &mut dyn BufMut) { UPGRADE_STATUS_MESSAGE_ID.encode(out); self.extension.encode(out); @@ -28,7 +32,6 @@ impl Decodable for UpgradeStatus { if message_id != UPGRADE_STATUS_MESSAGE_ID { return Err(alloy_rlp::Error::Custom("Invalid message ID")); } - buf.advance(1); let extension = UpgradeStatusExtension::decode(buf)?; Ok(Self { extension }) } @@ -52,3 +55,21 @@ pub struct UpgradeStatusExtension { /// To notify a peer to disable the broadcast of transactions or not. pub disable_peer_tx_broadcast: bool, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upgrade_status_rlp_roundtrip_and_length() { + for disable_peer_tx_broadcast in [false, true] { + let status = + UpgradeStatus { extension: UpgradeStatusExtension { disable_peer_tx_broadcast } }; + let encoded = alloy_rlp::encode(&status); + assert_eq!(status.length(), encoded.len()); + let mut buf = encoded.as_slice(); + assert_eq!(UpgradeStatus::decode(&mut buf).unwrap(), status); + assert!(buf.is_empty()); + } + } +} From d8d704cf7d83e3b159910bd8bd0d35b41b2813cc Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 15:11:40 +0200 Subject: [PATCH 064/133] fix(rpc): default trace_callMany to latest (#27410) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-api/src/trace.rs | 6 ++-- crates/rpc/rpc/src/trace.rs | 53 +++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 56f824700ef..70a458f2437 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -23,9 +23,9 @@ pub trait TraceApi { block_overrides: Option>, ) -> RpcResult; - /// Performs multiple call traces on top of the same block. i.e. transaction n will be executed - /// on top of a pending block with all n-1 transactions applied (traced) first. Allows to trace - /// dependent transactions. + /// Performs multiple call traces on top of the same block, defaulting to latest when no block + /// is specified. Each call is executed with the preceding calls applied first, allowing + /// dependent transactions to be traced. #[method(name = "callMany")] async fn trace_call_many( &self, diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 3185a3c9aab..5628a7baaf9 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -143,8 +143,8 @@ where .await } - /// Performs multiple call traces on top of the same block. i.e. transaction n will be executed - /// on top of a pending block with all n-1 transactions applied (traced) first. + /// Performs multiple call traces on top of the same block. Defaults to the latest block when + /// no block is specified. Each call is executed with the preceding calls applied first. /// /// Note: Allows tracing dependent transactions, hence all transactions are traced in sequence pub async fn trace_call_many( @@ -152,7 +152,7 @@ where calls: Vec<(RpcTxReq, HashSet)>, block_id: Option, ) -> Result, Eth::Error> { - let at = block_id.unwrap_or(BlockId::pending()); + let at = block_id.unwrap_or_default(); let (evm_env, at) = self.eth_api().evm_env_at(at).await?; // execute all transactions on top of each other and record the traces @@ -894,6 +894,53 @@ fn reward_trace( #[cfg(test)] mod tests { use super::*; + use crate::EthApiBuilder; + use alloy_consensus::Header; + use alloy_rpc_types_eth::TransactionRequest; + use reth_ethereum_primitives::{Block, BlockBody}; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; + use reth_transaction_pool::test_utils::testing_pool; + + #[tokio::test] + async fn trace_call_many_defaults_to_latest() { + let provider = MockEthProvider::default(); + let target = Address::with_last_byte(0x42); + // Return NUMBER as a 32-byte word. + provider.add_account( + target, + ExtendedAccount::new(0, U256::ZERO) + .with_bytecode("4360005260206000f3".parse().unwrap()), + ); + let header = Header { number: 1, gas_limit: 30_000_000, ..Default::default() }; + provider.add_block(header.hash_slow(), Block { header, body: BlockBody::default() }); + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + let api = TraceApi::new(eth_api, BlockingTaskGuard::new(1), EthConfig::default()); + let calls = vec![(TransactionRequest::default().to(target), HashSet::default())]; + + let omitted = api.trace_call_many(calls.clone(), None).await.unwrap(); + let latest = api.trace_call_many(calls.clone(), Some(BlockId::latest())).await.unwrap(); + let request = serde_json::json!({ + "jsonrpc": "2.0", "id": 1, "method": "trace_callMany", "params": [calls.clone()], + }); + let pending = api.trace_call_many(calls, Some(BlockId::pending())).await.unwrap(); + assert_eq!(omitted, latest); + assert_ne!(omitted, pending); + assert_eq!(U256::from_be_slice(&omitted[0].output), U256::from(1)); + assert_eq!(U256::from_be_slice(&pending[0].output), U256::from(2)); + + let module = api.into_rpc(); + let (response, _) = module.raw_json_request(&request.to_string(), 1).await.unwrap(); + let response: serde_json::Value = serde_json::from_str(response.get()).unwrap(); + assert_eq!(response["result"], serde_json::to_value(latest).unwrap()); + } #[tokio::test] async fn trace_get_selects_tree_paths() { From 6351fbb92e80d8370092ff4d6f002ef6a7576996 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 13:32:44 +0000 Subject: [PATCH 065/133] perf(rpc): reuse shared sender recovery cache (#27368) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/ethereum/node/tests/e2e/rpc.rs | 32 ++++ crates/evm/evm/src/sender_recovery.rs | 72 +++++++- crates/net/network/src/transactions/mod.rs | 9 +- crates/node/api/src/node.rs | 4 +- crates/node/builder/src/launch/common.rs | 10 +- crates/node/builder/src/launch/engine.rs | 1 + crates/node/builder/src/rpc.rs | 13 +- .../rpc-eth-api/src/helpers/transaction.rs | 42 ++++- crates/rpc/rpc-eth-types/src/settings.rs | 6 +- crates/rpc/rpc-eth-types/src/utils.rs | 14 +- crates/rpc/rpc/src/eth/builder.rs | 17 ++ crates/rpc/rpc/src/eth/bundle.rs | 9 +- crates/rpc/rpc/src/eth/core.rs | 31 +++- crates/rpc/rpc/src/eth/helpers/transaction.rs | 162 +++++++++++++++++- crates/rpc/rpc/src/eth/sim_bundle.rs | 6 +- crates/rpc/rpc/src/trace.rs | 6 +- .../transaction-pool/src/test_utils/mock.rs | 19 ++ crates/transaction-pool/src/traits.rs | 45 ++++- 18 files changed, 452 insertions(+), 46 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 4df2ccb6d1b..a15960c3630 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -65,6 +65,38 @@ alloy_sol_types::sol! { } } +#[tokio::test] +async fn test_rpc_shares_sender_recovery_cache_with_execution() -> eyre::Result<()> { + use reth_primitives_traits::SignedTransaction; + use reth_transaction_pool::test_utils::TransactionGenerator; + + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().cancun_activated().build()); + let (mut nodes, _) = + E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) + .with_node_config_modifier(|mut config| { + config.engine.sender_recovery_cache_enabled = true; + config + }) + .build() + .await?; + let node = nodes.pop().unwrap(); + let cache = node.inner.evm_config.sender_recovery_cache.as_ref().unwrap(); + let client = node.rpc_client().unwrap(); + let transaction = + TransactionGenerator::new(rand::rng()).transaction().chain_id(2).into_legacy(); + let hash = *transaction.tx_hash(); + let sender = transaction.try_recover()?; + let raw = Bytes::from(transaction.encoded_2718()); + assert_eq!(cache.get(&hash), None); + + for _ in 0..2 { + let result = client.request::("eth_sendRawTransaction", (raw.clone(),)).await; + assert!(result.unwrap_err().to_string().contains("chain ID")); + assert_eq!(cache.get(&hash), Some(sender)); + } + Ok(()) +} + async fn inject_blob_transaction( node: &NodeHelperType, wallet: &Wallet, diff --git a/crates/evm/evm/src/sender_recovery.rs b/crates/evm/evm/src/sender_recovery.rs index de9d1f51280..2ddb0cec726 100644 --- a/crates/evm/evm/src/sender_recovery.rs +++ b/crates/evm/evm/src/sender_recovery.rs @@ -2,14 +2,17 @@ use alloc::sync::Arc; use alloy_primitives::{map::FbBuildHasher, Address, B256}; use reth_primitives_traits::{transaction::signed::RecoveryError, SignedTransaction}; -/// Number of entries retained in the default sender recovery cache. +/// Number of entries retained in the default sender recovery cache: 131,072. const SENDER_RECOVERY_CACHE_CAPACITY: usize = 1 << 17; /// Shared cache of recovered transaction senders. /// -/// Sender recovery is performed when a transaction enters the pool and again when the same -/// transaction is received in an execution payload. Sharing this bounded, lock-free cache lets -/// payload prewarming reuse the result produced by transaction-pool ingress. +/// This bounded, lock-free cache shares recovered senders across transaction ingress and payload +/// execution. Repeated transactions reuse the recovered sender while still undergoing the +/// validation required by each caller. +/// +/// The default cache retains up to 131,072 transaction hash-to-sender entries. Cloning the cache +/// shares these entries rather than allocating another cache. #[derive(Clone, Debug)] pub struct SenderRecoveryCache { cache: Arc, SenderRecoveryCacheConfig>>, @@ -34,9 +37,22 @@ impl SenderRecoveryCache { /// Failed recoveries are not cached. #[inline] pub fn recover(&self, transaction: &T) -> Result { + self.recover_with(transaction, SignedTransaction::try_recover) + } + + /// Returns the cached sender or uses the given recovery function on a cache miss. + /// + /// The recovery function must validate the transaction's signature. It can also prepare + /// transaction-specific metadata alongside sender recovery. Failed recoveries are not cached. + #[inline] + pub fn recover_with( + &self, + transaction: &T, + recover: impl FnOnce(&T) -> Result, + ) -> Result { self.cache.get_or_try_insert_with_ref( transaction.tx_hash(), - |_| transaction.try_recover(), + |_| recover(transaction), |hash| *hash, ) } @@ -87,4 +103,50 @@ mod tests { assert!(cache.recover(&transaction).is_err()); assert_eq!(cache.get(transaction.tx_hash()), None); } + + #[test] + fn custom_recovery_runs_only_on_cache_miss() { + let transaction = TransactionSigned::new_unhashed( + Transaction::Legacy(TxLegacy::default()), + Signature::test_signature(), + ); + let cache = SenderRecoveryCache::new(4); + let mut recovered = None; + + let sender = cache + .recover_with(&transaction, |tx| { + let signer = tx.try_recover()?; + recovered = Some(signer); + Ok(signer) + }) + .unwrap(); + + assert_eq!(recovered, Some(sender)); + assert_eq!(cache.get(transaction.tx_hash()), Some(sender)); + assert_eq!( + cache.recover_with(&transaction, |_| panic!("cache hit must skip recovery")).unwrap(), + sender + ); + } + + #[test] + fn failed_custom_recovery_is_retried() { + let transaction = TransactionSigned::new_unhashed( + Transaction::Legacy(TxLegacy::default()), + Signature::new(U256::ZERO, U256::ZERO, false), + ); + let cache = SenderRecoveryCache::new(4); + let mut attempts = 0; + + for _ in 0..2 { + assert!(cache + .recover_with(&transaction, |tx| { + attempts += 1; + tx.try_recover() + }) + .is_err()); + assert_eq!(cache.get(transaction.tx_hash()), None); + } + assert_eq!(attempts, 2); + } } diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 3bf96181d8b..42752eb9e1e 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1430,11 +1430,10 @@ where let txs_len = transactions.len(); let recover = |tx| { - let recovered = if let Some(cache) = &self.sender_recovery_cache { - Pool::Transaction::try_recover_with_cache(tx, cache) - } else { - Pool::Transaction::try_recover(tx) - }; + let recovered = Pool::Transaction::try_recover_with_cache_opt( + tx, + self.sender_recovery_cache.as_ref(), + ); match recovered { Ok(tx) => Some(tx), Err(badtx) => { diff --git a/crates/node/api/src/node.rs b/crates/node/api/src/node.rs index c98c84063d3..1cff727ebda 100644 --- a/crates/node/api/src/node.rs +++ b/crates/node/api/src/node.rs @@ -6,7 +6,7 @@ use reth_basic_payload_builder::PayloadBuilder; use reth_consensus::FullConsensus; use reth_db_api::{database_metrics::DatabaseMetrics, Database}; use reth_engine_primitives::{ConsensusEngineEvent, ConsensusEngineHandle}; -use reth_evm::ConfigureEvm; +use reth_evm::{ConfigureEvm, SenderRecoveryCache}; use reth_network_api::FullNetwork; use reth_node_core::node_config::NodeConfig; use reth_node_types::{NodeTypes, NodeTypesWithDBAdapter, TxTy}; @@ -115,6 +115,8 @@ pub struct AddOnsContext<'a, N: FullNodeComponents> { pub engine_events: EventSender::Primitives>>, /// JWT secret for the node. pub jwt_secret: JwtSecret, + /// Cache of recovered transaction senders shared by node components, if enabled. + pub sender_recovery_cache: Option, } /// Customizable node add-on types. diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 70287d252f2..6f54bfca5a3 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -49,7 +49,7 @@ use reth_db_common::init::{ }; use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; use reth_engine_local::MiningMode; -use reth_evm::{noop::NoopEvmConfig, ConfigureEvm}; +use reth_evm::{noop::NoopEvmConfig, ConfigureEvm, SenderRecoveryCache}; use reth_exex::ExExManagerHandle; use reth_fs_util as fs; use reth_network_p2p::headers::client::HeadersClient; @@ -945,6 +945,7 @@ where }, node_adapter, head, + sender_recovery_cache: builder_ctx.sender_recovery_cache().cloned(), }; let ctx = LaunchContextWith { @@ -1007,6 +1008,11 @@ where &mut self.right_mut().node_adapter } + /// Returns the cache of recovered transaction senders shared by node components, if enabled. + pub const fn sender_recovery_cache(&self) -> Option<&SenderRecoveryCache> { + self.right().sender_recovery_cache.as_ref() + } + /// Returns a reference to the blockchain provider. pub const fn blockchain_db(&self) -> &T::Provider { &self.node_adapter().provider @@ -1370,6 +1376,8 @@ where db_provider_container: WithMeteredProvider>, node_adapter: NodeAdapter, head: Head, + /// Cache of recovered transaction senders shared by node components, if enabled. + sender_recovery_cache: Option, } /// Returns the metrics hooks for the node. diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 4bc37c3de4f..4380c35622f 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -203,6 +203,7 @@ impl EngineNodeLauncher { beacon_engine_handle: beacon_engine_handle.clone(), jwt_secret, engine_events: event_sender.clone(), + sender_recovery_cache: ctx.sender_recovery_cache().cloned(), }; let validator_builder = add_ons.engine_validator_builder(); diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 6cc2b6973b2..e8bbf40ec9e 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -1141,7 +1141,14 @@ where let Self { eth_api_builder, engine_api_builder, hooks, .. } = self; let engine_api = engine_api_builder.build_engine_api(&ctx).await?; - let AddOnsContext { node, config, beacon_engine_handle, jwt_secret, engine_events } = ctx; + let AddOnsContext { + node, + config, + beacon_engine_handle, + jwt_secret, + engine_events, + sender_recovery_cache, + } = ctx; info!(target: "reth::cli", "Engine API handler initialized"); @@ -1170,6 +1177,7 @@ where config: eth_config, cache, engine_handle: beacon_engine_handle.clone(), + sender_recovery_cache, }; let eth_api = eth_api_builder.build_eth_api(ctx).await?; @@ -1348,6 +1356,8 @@ pub struct EthApiCtx<'a, N: FullNodeTypes> { pub config: EthConfig, /// Cache for eth state pub cache: EthStateCache>, + /// Cache of recovered transaction senders shared by node components, if enabled. + pub sender_recovery_cache: Option, /// Handle to the beacon consensus engine pub engine_handle: ConsensusEngineHandle<::Payload>, } @@ -1359,6 +1369,7 @@ impl<'a, N: FullNodeComponents reth_rpc::EthApiBuilder> { reth_rpc::EthApiBuilder::new_with_components(self.components.clone()) .eth_cache(self.cache) + .sender_recovery_cache(self.sender_recovery_cache) .eth_state_cache_config(self.config.cache) .task_spawner(self.components.task_executor().clone()) .gas_cap(self.config.rpc_gas_cap.into()) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 1222b5ffbb8..2a2ed08ee89 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -25,9 +25,9 @@ use reth_primitives_traits::{ use reth_rpc_convert::{transaction::RpcConvert, RpcTxReq, TransactionConversionError}; use reth_rpc_eth_types::{ block::convert_transaction_receipt, - utils::binary_search, + utils::{binary_search, decode_raw_transaction}, EthApiError::{self, TransactionConfirmationTimeout}, - FillTransaction, SignError, TransactionSource, + EthResult, FillTransaction, SignError, TransactionSource, }; use reth_storage_api::{ BlockNumReader, BlockReaderIdExt, ProviderBlock, ProviderReceipt, ProviderTx, ReceiptProvider, @@ -86,9 +86,7 @@ pub trait EthTransactions: LoadTransaction { tx: Bytes, ) -> impl Future> + Send { async move { - let pool_transaction = - as PoolTransaction>::recover_raw_transaction(&tx) - .map_err(Self::Error::from_eth_err)?; + let pool_transaction = self.recover_raw_pool_transaction(&tx)?; self.send_pool_transaction( TransactionOrigin::Local, WithEncoded::new(tx, pool_transaction), @@ -97,6 +95,19 @@ pub trait EthTransactions: LoadTransaction { } } + /// Decodes and recovers a raw pool transaction, sharing sender recovery when configured. + fn recover_raw_pool_transaction(&self, tx: &[u8]) -> Result, Self::Error> { + match self.eth_api_settings().sender_recovery_cache.as_ref() { + Some(cache) => { + as PoolTransaction>::recover_raw_transaction_with_cache( + tx, cache, + ) + } + None => as PoolTransaction>::recover_raw_transaction(tx), + } + .map_err(Self::Error::from_eth_err) + } + /// Submits the transaction to the pool with the given [`TransactionOrigin`]. fn send_transaction( &self, @@ -736,6 +747,27 @@ pub trait EthTransactions: LoadTransaction { /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` transactions RPC /// methods. pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes + RpcNodeCoreExt { + /// Decodes and recovers a raw transaction, sharing sender recovery when configured. + fn recover_raw_transaction(&self, tx: &[u8]) -> EthResult> { + let transaction = decode_raw_transaction::(tx)?; + let signer = match self.eth_api_settings().sender_recovery_cache.as_ref() { + Some(cache) => cache.recover(&transaction), + None => transaction.try_recover(), + } + .map_err(|_| EthApiError::InvalidTransactionSignature)?; + Ok(Recovered::new_unchecked(transaction, signer)) + } + + /// Decodes and recovers raw transactions in order, stopping at the first error. + /// + /// Uses [`Self::recover_raw_transaction`] to share sender recovery when configured. + fn recover_raw_transactions( + &self, + txs: impl IntoIterator>, + ) -> EthResult>> { + txs.into_iter().map(|tx| self.recover_raw_transaction(tx.as_ref())).collect() + } + /// Returns the transaction by hash. /// /// Checks the pool and state. diff --git a/crates/rpc/rpc-eth-types/src/settings.rs b/crates/rpc/rpc-eth-types/src/settings.rs index b0262c6f98f..48cfc674081 100644 --- a/crates/rpc/rpc-eth-types/src/settings.rs +++ b/crates/rpc/rpc-eth-types/src/settings.rs @@ -1,6 +1,7 @@ //! Settings for initializing and serving the `eth` RPC API. use crate::{builder::config::PendingBlockKind, RPC_DEFAULT_GAS_CAP}; +use reth_evm::SenderRecoveryCache; use reth_rpc_server_types::constants::{ DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_BLOCKING_IO_REQUEST, DEFAULT_MAX_SIMULATE_BLOCKS, DEFAULT_PROOF_PERMITS, RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, @@ -12,8 +13,10 @@ use std::time::Duration; /// These settings are shared by the API's helper traits so additional settings can be exposed /// without adding individual trait methods. Concurrency and batching limits take effect when the /// API is constructed. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone)] pub struct EthApiSettings { + /// Cache of recovered transaction senders shared with transaction ingress and execution. + pub sender_recovery_cache: Option, /// Maximum number of concurrent proof requests. pub proof_permits: usize, /// Maximum batch size for transaction insertions. @@ -43,6 +46,7 @@ pub struct EthApiSettings { impl Default for EthApiSettings { fn default() -> Self { Self { + sender_recovery_cache: None, proof_permits: DEFAULT_PROOF_PERMITS, max_batch_size: 1, max_blocking_io_requests: DEFAULT_MAX_BLOCKING_IO_REQUEST, diff --git a/crates/rpc/rpc-eth-types/src/utils.rs b/crates/rpc/rpc-eth-types/src/utils.rs index 5df161d33b5..fee5d4f24f8 100644 --- a/crates/rpc/rpc-eth-types/src/utils.rs +++ b/crates/rpc/rpc-eth-types/src/utils.rs @@ -33,15 +33,19 @@ pub fn calculate_gas_used_and_next_log_index( /// /// See [`alloy_eips::eip2718::Decodable2718::decode_2718_exact`] pub fn recover_raw_transaction(data: &[u8]) -> EthResult> { + SignedTransaction::try_into_recovered(decode_raw_transaction::(data)?) + .or(Err(EthApiError::InvalidTransactionSignature)) +} + +/// Decodes an EIP-2718 transaction without recovering its sender. +/// +/// Rejects empty input, malformed transactions, and trailing bytes with RPC-specific errors. +pub fn decode_raw_transaction(data: &[u8]) -> EthResult { if data.is_empty() { return Err(EthApiError::EmptyRawTransactionData) } - let transaction = - T::decode_2718_exact(data).map_err(|_| EthApiError::FailedToDecodeSignedTransaction)?; - - SignedTransaction::try_into_recovered(transaction) - .or(Err(EthApiError::InvalidTransactionSignature)) + T::decode_2718_exact(data).map_err(|_| EthApiError::FailedToDecodeSignedTransaction) } /// Performs a binary search within a given block range to find the desired block number. diff --git a/crates/rpc/rpc/src/eth/builder.rs b/crates/rpc/rpc/src/eth/builder.rs index d54f92037f9..17c78657272 100644 --- a/crates/rpc/rpc/src/eth/builder.rs +++ b/crates/rpc/rpc/src/eth/builder.rs @@ -4,6 +4,7 @@ use crate::{eth::core::EthApiInner, EthApi}; use alloy_network::Ethereum; use reth_chain_state::CanonStateSubscriptions; use reth_chainspec::ChainSpecProvider; +use reth_evm::SenderRecoveryCache; use reth_primitives_traits::HeaderTy; use reth_rpc_convert::{RpcConvert, RpcConverter}; use reth_rpc_eth_api::{ @@ -47,6 +48,7 @@ pub struct EthApiBuilder { max_blocking_io_requests: usize, pending_block_kind: PendingBlockKind, raw_tx_forwarder: ForwardConfig, + sender_recovery_cache: Option, send_raw_transaction_sync_timeout: Duration, evm_memory_limit: u64, force_blob_sidecar_upcasting: bool, @@ -101,6 +103,7 @@ impl EthApiBuilder { max_blocking_io_requests, pending_block_kind, raw_tx_forwarder, + sender_recovery_cache, send_raw_transaction_sync_timeout, evm_memory_limit, force_blob_sidecar_upcasting, @@ -125,6 +128,7 @@ impl EthApiBuilder { max_blocking_io_requests, pending_block_kind, raw_tx_forwarder, + sender_recovery_cache, send_raw_transaction_sync_timeout, evm_memory_limit, force_blob_sidecar_upcasting, @@ -160,6 +164,7 @@ where max_blocking_io_requests: DEFAULT_MAX_BLOCKING_IO_REQUEST, pending_block_kind: PendingBlockKind::Full, raw_tx_forwarder: ForwardConfig::default(), + sender_recovery_cache: None, send_raw_transaction_sync_timeout: Duration::from_secs(30), evm_memory_limit: (1 << 32) - 1, force_blob_sidecar_upcasting: false, @@ -202,6 +207,7 @@ where max_blocking_io_requests, pending_block_kind, raw_tx_forwarder, + sender_recovery_cache, send_raw_transaction_sync_timeout, evm_memory_limit, force_blob_sidecar_upcasting, @@ -226,6 +232,7 @@ where max_blocking_io_requests, pending_block_kind, raw_tx_forwarder, + sender_recovery_cache, send_raw_transaction_sync_timeout, evm_memory_limit, force_blob_sidecar_upcasting, @@ -257,6 +264,7 @@ where max_blocking_io_requests, pending_block_kind, raw_tx_forwarder, + sender_recovery_cache, send_raw_transaction_sync_timeout, evm_memory_limit, force_blob_sidecar_upcasting, @@ -281,6 +289,7 @@ where max_blocking_io_requests, pending_block_kind, raw_tx_forwarder, + sender_recovery_cache, send_raw_transaction_sync_timeout, evm_memory_limit, force_blob_sidecar_upcasting, @@ -379,6 +388,12 @@ where self } + /// Sets the sender recovery cache shared with transaction ingress and execution. + pub fn sender_recovery_cache(mut self, cache: Option) -> Self { + self.sender_recovery_cache = cache; + self + } + /// Sets the raw transaction forwarder. pub fn raw_tx_forwarder(mut self, tx_forwarder: ForwardConfig) -> Self { self.raw_tx_forwarder = tx_forwarder; @@ -529,6 +544,7 @@ where max_blocking_io_requests, pending_block_kind, raw_tx_forwarder, + sender_recovery_cache, send_raw_transaction_sync_timeout, evm_memory_limit, force_blob_sidecar_upcasting, @@ -559,6 +575,7 @@ where ); let settings = EthApiSettings { + sender_recovery_cache, proof_permits, max_batch_size, max_blocking_io_requests, diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index 37f8d1f8afc..43b776afece 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -13,8 +13,7 @@ use reth_rpc_eth_api::{ EthCallBundleApiServer, FromEthApiError, FromEvmError, }; use reth_rpc_eth_types::{ - utils::recover_raw_transaction, EthApiError, PendingBlockEnv, PendingBlockEnvOrigin, - RpcInvalidTransactionError, + EthApiError, PendingBlockEnv, PendingBlockEnvOrigin, RpcInvalidTransactionError, }; use reth_storage_api::BlockReaderIdExt; use reth_tasks::pool::BlockingTaskGuard; @@ -91,10 +90,8 @@ where ) } - let transactions = txs - .into_iter() - .map(|tx| recover_raw_transaction::>(&tx)) - .collect::, _>>()?; + let transactions = + self.eth_api().recover_raw_transactions::>(txs)?; let block_id: alloy_rpc_types_eth::BlockId = state_block_number.into(); // Note: the block number is considered the `parent` block: diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index b71d80baba4..4fb2731cf14 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -715,10 +715,36 @@ mod tests { api.eth_api_settings() } + fn assert_settings(actual: &EthApiSettings, expected: &EthApiSettings) { + assert_eq!(actual.proof_permits, expected.proof_permits); + assert_eq!(actual.max_batch_size, expected.max_batch_size); + assert_eq!(actual.max_blocking_io_requests, expected.max_blocking_io_requests); + assert_eq!(actual.cache_computed_bals, expected.cache_computed_bals); + assert_eq!(actual.gas_cap, expected.gas_cap); + assert_eq!(actual.max_simulate_blocks, expected.max_simulate_blocks); + assert_eq!( + actual.compute_state_root_for_eth_simulate, + expected.compute_state_root_for_eth_simulate + ); + assert_eq!(actual.eth_proof_window, expected.eth_proof_window); + assert_eq!(actual.pending_block_kind, expected.pending_block_kind); + assert_eq!( + actual.send_raw_transaction_sync_timeout, + expected.send_raw_transaction_sync_timeout + ); + assert_eq!(actual.evm_memory_limit, expected.evm_memory_limit); + assert_eq!(actual.force_blob_sidecar_upcasting, expected.force_blob_sidecar_upcasting); + assert_eq!( + actual.sender_recovery_cache.is_some(), + expected.sender_recovery_cache.is_some() + ); + } + let default_api = build_test_eth_api(MockEthProvider::default()); - assert_eq!(settings(&default_api), &EthApiSettings::default()); + assert_settings(settings(&default_api), &EthApiSettings::default()); let expected = EthApiSettings { + sender_recovery_cache: Some(reth_evm::SenderRecoveryCache::new(16)), proof_permits: 3, max_batch_size: 5, max_blocking_io_requests: 7, @@ -742,6 +768,7 @@ mod tests { cache_computed_bals: true, ..Default::default() }) + .sender_recovery_cache(expected.sender_recovery_cache.clone()) .proof_permits(expected.proof_permits) .max_batch_size(expected.max_batch_size) .max_blocking_io_requests(expected.max_blocking_io_requests) @@ -755,7 +782,7 @@ mod tests { .force_blob_sidecar_upcasting(expected.force_blob_sidecar_upcasting) .build(); - assert_eq!(settings(&api), &expected); + assert_settings(settings(&api), &expected); assert_eq!( api.inner.blocking_io_request_semaphore().available_permits(), expected.max_blocking_io_requests diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 247f814f5dd..15546b68442 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -131,19 +131,23 @@ mod tests { use alloy_consensus::{ BlobTransactionSidecar, Block, Header, SidecarBuilder, SimpleCoder, Transaction, }; - use alloy_primitives::{map::AddressMap, Address, Bytes, U256}; + use alloy_eips::eip2718::Encodable2718; + use alloy_primitives::{map::AddressMap, Address, Bytes, Signature, U256}; use alloy_rpc_types_eth::request::TransactionRequest; use reth_chainspec::{ChainSpec, ChainSpecBuilder}; + use reth_ethereum_primitives::TransactionSigned; + use reth_evm::SenderRecoveryCache; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; + use reth_primitives_traits::SignedTransaction; use reth_provider::{ test_utils::{ExtendedAccount, MockEthProvider}, ChainSpecProvider, }; use reth_rpc_eth_api::node::RpcNodeCoreAdapter; use reth_transaction_pool::{ - test_utils::{testing_pool, TestPool}, - TransactionOrigin, TransactionPool, + test_utils::{testing_pool, TestPool, TransactionGenerator}, + EthPooledTransaction, PoolPooledTx, TransactionOrigin, TransactionPool, }; fn mock_eth_api( @@ -161,6 +165,17 @@ mod tests { ) -> EthApi< RpcNodeCoreAdapter, EthRpcConverter, + > { + mock_eth_api_builder(accounts) + .send_raw_transaction_sync_timeout(send_raw_transaction_sync_timeout) + .build() + } + + fn mock_eth_api_builder( + accounts: AddressMap, + ) -> crate::EthApiBuilder< + RpcNodeCoreAdapter, + EthRpcConverter, > { let mock_provider = MockEthProvider::default() .with_chain_spec(ChainSpecBuilder::mainnet().cancun_activated().build()); @@ -183,8 +198,6 @@ mod tests { mock_provider.add_block(genesis_hash, Block::new(genesis_header, Default::default())); EthApi::builder(mock_provider, pool, NoopNetwork::default(), evm_config) - .send_raw_transaction_sync_timeout(send_raw_transaction_sync_timeout) - .build() } fn raw_transfer_tx() -> Bytes { @@ -194,6 +207,145 @@ mod tests { )) } + #[tokio::test] + async fn raw_transaction_recovery_shares_sender_cache() { + let cache = SenderRecoveryCache::new(16); + let eth_api = mock_eth_api_builder(Default::default()) + .sender_recovery_cache(Some(cache.clone())) + .build(); + let mut generator = TransactionGenerator::new(rand::rng()); + + for (transaction, warm_cache) in [ + (generator.transaction().into_legacy(), false), + (generator.transaction().nonce(1).into_eip1559(), true), + ] { + let raw = Bytes::from(transaction.encoded_2718()); + let hash = *transaction.tx_hash(); + let sender = transaction.try_recover().unwrap(); + assert_eq!(cache.get(&hash), None); + if warm_cache { + let recovered = + eth_api.recover_raw_transaction::>(&raw).unwrap(); + assert_eq!(recovered.signer(), sender); + assert_eq!(cache.get(&hash), Some(sender)); + } + + assert_eq!(eth_api.send_raw_transaction(raw.clone()).await.unwrap(), hash); + assert_eq!(cache.get(&hash), Some(sender)); + let recovered = + eth_api.recover_raw_transaction::>(&raw).unwrap(); + assert_eq!(recovered.signer(), sender); + assert_eq!(eth_api.recover_raw_pool_transaction(&raw).unwrap().sender(), sender); + } + } + + #[tokio::test] + async fn raw_transaction_recovery_uses_pool_overrides() { + let raw = raw_transfer_tx(); + for enabled in [false, true] { + let cache = SenderRecoveryCache::new(16); + let eth_api = mock_eth_api_builder(Default::default()) + .sender_recovery_cache(enabled.then(|| cache.clone())) + .build(); + + // MockTransaction's raw hooks retain the input length instead of its in-memory size. + for _ in 0..2 { + let transaction = eth_api.recover_raw_pool_transaction(&raw).unwrap(); + assert_eq!(transaction.encoded_length(), raw.len()); + assert_eq!(cache.get(transaction.hash()), enabled.then_some(transaction.sender())); + } + } + } + + #[tokio::test] + async fn raw_transaction_recovery_preserves_errors() { + let invalid = TransactionSigned::new_unhashed( + alloy_consensus::TxLegacy::default().into(), + Signature::new(U256::ZERO, U256::ZERO, false), + ); + let mut trailing = raw_transfer_tx().to_vec(); + trailing.push(0); + + for enabled in [false, true] { + let cache = SenderRecoveryCache::new(16); + let eth_api = mock_eth_api_builder(Default::default()) + .sender_recovery_cache(enabled.then(|| cache.clone())) + .build(); + let valid = eth_api.recover_raw_pool_transaction(&raw_transfer_tx()).unwrap(); + assert_eq!(cache.get(valid.hash()), enabled.then_some(valid.sender())); + for (raw, expected) in [ + (vec![], EthApiError::EmptyRawTransactionData), + (vec![2], EthApiError::FailedToDecodeSignedTransaction), + (trailing.clone(), EthApiError::FailedToDecodeSignedTransaction), + (invalid.encoded_2718(), EthApiError::InvalidTransactionSignature), + ] { + let err = eth_api.send_raw_transaction(raw.clone().into()).await.unwrap_err(); + assert_eq!(err.to_string(), expected.to_string()); + let err = + eth_api.recover_raw_transaction::>(&raw).unwrap_err(); + assert_eq!(err.to_string(), expected.to_string()); + } + assert!(eth_api.pool().is_empty()); + assert_eq!(cache.get(invalid.tx_hash()), None); + } + } + + #[tokio::test] + async fn cached_raw_transaction_preserves_blob_sidecar() { + let cache = SenderRecoveryCache::new(16); + let provider = MockEthProvider::default(); + let evm_config = EthEvmConfig::new(provider.chain_spec()); + let eth_api = EthApi::builder( + provider, + reth_transaction_pool::noop::NoopTransactionPool::default(), + NoopNetwork::default(), + evm_config, + ) + .sender_recovery_cache(Some(cache.clone())) + .build(); + let mut builder = SidecarBuilder::::new(); + builder.ingest(b"cached blob transaction"); + let sidecar = + BlobTransactionSidecarVariant::from(builder.build::().unwrap()); + let transaction = TransactionSigned::new_unhashed( + alloy_consensus::TxEip4844 { + blob_versioned_hashes: sidecar.versioned_hashes().collect(), + ..Default::default() + } + .into(), + Signature::test_signature(), + ); + let hash = *transaction.tx_hash(); + let sender = transaction.try_recover().unwrap(); + let raw = transaction.try_into_pooled_eip4844(sidecar.clone()).unwrap().encoded_2718(); + + for _ in 0..2 { + let mut recovered = eth_api.recover_raw_pool_transaction(&raw).unwrap(); + assert_eq!(recovered.sender(), sender); + assert_eq!(cache.get(&hash), Some(sender)); + let EthBlobTransactionSidecar::Present(actual) = recovered.take_blob() else { + panic!("missing blob sidecar") + }; + assert_eq!(actual.into_sidecar(), sidecar); + } + } + + #[tokio::test] + async fn send_raw_transaction_sync_populates_sender_cache() { + let cache = SenderRecoveryCache::new(16); + let eth_api = mock_eth_api_builder(Default::default()) + .sender_recovery_cache(Some(cache.clone())) + .build(); + let raw = raw_transfer_tx(); + let transaction = EthPooledTransaction::decode_raw_transaction(&raw).unwrap(); + let sender = transaction.try_recover().unwrap(); + + let err = eth_api.send_raw_transaction_sync(raw, Some(1)).await.unwrap_err(); + assert!(matches!(err, EthApiError::TransactionConfirmationTimeout { .. })); + assert_eq!(cache.get(transaction.tx_hash()), Some(sender)); + assert_eq!(eth_api.pool().len(), 1); + } + #[tokio::test] async fn send_raw_transaction() { let eth_api = mock_eth_api(Default::default()); diff --git a/crates/rpc/rpc/src/eth/sim_bundle.rs b/crates/rpc/rpc/src/eth/sim_bundle.rs index e1959705f7f..288230d4851 100644 --- a/crates/rpc/rpc/src/eth/sim_bundle.rs +++ b/crates/rpc/rpc/src/eth/sim_bundle.rs @@ -18,7 +18,7 @@ use reth_rpc_eth_api::{ helpers::{block::LoadBlock, Call, EthTransactions}, FromEthApiError, FromEvmError, }; -use reth_rpc_eth_types::{utils::recover_raw_transaction, EthApiError}; +use reth_rpc_eth_types::EthApiError; use reth_storage_api::ProviderTx; use reth_tasks::pool::BlockingTaskGuard; use reth_transaction_pool::{PoolPooledTx, PoolTransaction, TransactionPool}; @@ -229,7 +229,9 @@ where while idx < body.len() { match &body[idx] { BundleItem::Tx { tx, can_revert } => { - let recovered_tx = recover_raw_transaction::>(tx)?; + let recovered_tx = self + .eth_api() + .recover_raw_transaction::>(tx)?; let tx = recovered_tx.map( ::Transaction::pooled_into_consensus, ); diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 5628a7baaf9..5a0abdc6be6 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -26,7 +26,7 @@ use reth_rpc_eth_api::{ helpers::{Call, LoadPendingBlock, LoadTransaction, Trace, TraceExt}, FromEthApiError, RpcNodeCore, }; -use reth_rpc_eth_types::{error::EthApiError, utils::recover_raw_transaction, EthConfig}; +use reth_rpc_eth_types::{error::EthApiError, EthConfig}; use reth_storage_api::{BlockNumReader, BlockReader}; use reth_tasks::pool::BlockingTaskGuard; use reth_transaction_pool::{PoolPooledTx, PoolTransaction, TransactionPool}; @@ -124,7 +124,9 @@ where trace_types: HashSet, block_id: Option, ) -> Result { - let tx = recover_raw_transaction::>(&tx)? + let tx = self + .eth_api() + .recover_raw_transaction::>(&tx)? .map(::Transaction::pooled_into_consensus); let (evm_env, at) = self.eth_api().evm_env_at(block_id.unwrap_or_default()).await?; diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 83ca3c81741..d7072348793 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -1,6 +1,7 @@ //! Mock types. use crate::{ + error::RawPoolTransactionError, identifier::{SenderIdentifiers, TransactionId}, pool::txpool::TxPool, traits::TransactionOrigin, @@ -729,6 +730,24 @@ impl PoolTransaction for MockTransaction { pooled.into() } + fn recover_raw_transaction(data: &[u8]) -> Result { + let mut transaction = Self::try_recover(Self::decode_raw_transaction(data)?) + .map_err(|_| RawPoolTransactionError::InvalidTransactionSignature)?; + transaction.set_size(data.len()); + Ok(transaction) + } + + fn recover_raw_transaction_with_cache( + data: &[u8], + cache: &reth_evm::SenderRecoveryCache, + ) -> Result { + let mut transaction = + Self::try_recover_with_cache(Self::decode_raw_transaction(data)?, cache) + .map_err(|_| RawPoolTransactionError::InvalidTransactionSignature)?; + transaction.set_size(data.len()); + Ok(transaction) + } + fn hash(&self) -> &TxHash { self.get_hash() } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index a41152ad8cb..bbb5797d428 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1415,20 +1415,55 @@ pub trait PoolTransaction: } } + /// Recovers and converts a pooled transaction, using the sender recovery cache if provided. + /// + /// Delegates to [`Self::try_recover`] when no cache is configured. + fn try_recover_with_cache_opt( + pooled: Self::Pooled, + cache: Option<&reth_evm::SenderRecoveryCache>, + ) -> Result { + match cache { + Some(cache) => Self::try_recover_with_cache(pooled, cache), + None => Self::try_recover(pooled), + } + } + /// Decodes and recovers a raw transaction into this pool transaction type. /// /// Implementations can override this to avoid constructing the pooled transaction as an /// intermediate value when the raw representation can be converted directly into `Self`. + /// RPC uses this hook when no sender recovery cache is configured. Override + /// [`Self::recover_raw_transaction_with_cache`] to specialize the cached path as well. fn recover_raw_transaction(data: &[u8]) -> Result { + Self::try_recover(Self::decode_raw_transaction(data)?) + .map_err(|_| RawPoolTransactionError::InvalidTransactionSignature) + } + + /// Decodes and recovers a raw pool transaction using the provided sender recovery cache. + /// + /// RPC uses this hook when a sender recovery cache is configured. Implementations can override + /// it to reuse the raw bytes or encoded length during recovery and construction. The default + /// implementation decodes and recovers separately; it does not call + /// [`Self::recover_raw_transaction`]. + fn recover_raw_transaction_with_cache( + data: &[u8], + cache: &reth_evm::SenderRecoveryCache, + ) -> Result { + Self::try_recover_with_cache(Self::decode_raw_transaction(data)?, cache) + .map_err(|_| RawPoolTransactionError::InvalidTransactionSignature) + } + + /// Decodes a raw pooled transaction without recovering its sender. + /// + /// The entire input must be consumed; trailing bytes are rejected. Implementations can override + /// this to customize raw decoding independently of sender recovery. + fn decode_raw_transaction(data: &[u8]) -> Result { if data.is_empty() { return Err(RawPoolTransactionError::EmptyRawTransactionData) } - let transaction = Self::Pooled::decode_2718_exact(data) - .map_err(|_| RawPoolTransactionError::FailedToDecodeSignedTransaction)?; - - Self::try_recover(transaction) - .map_err(|_| RawPoolTransactionError::InvalidTransactionSignature) + Self::Pooled::decode_2718_exact(data) + .map_err(|_| RawPoolTransactionError::FailedToDecodeSignedTransaction) } /// Tries to convert the `Consensus` type into the `Pooled` type. From a120f00d47e43427816e2fe7e38229fabbb2db87 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 13:53:27 +0000 Subject: [PATCH 066/133] fix(engine): validate forkchoice state before applying updates (#27247) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/chain-state/src/in_memory.rs | 16 +- crates/engine/tree/src/tree/mod.rs | 73 +++++++++ crates/engine/tree/src/tree/tests.rs | 36 +++++ crates/ethereum/node/tests/e2e/forkchoice.rs | 161 +++++++++++++++++++ crates/ethereum/node/tests/e2e/main.rs | 1 + 5 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 crates/ethereum/node/tests/e2e/forkchoice.rs diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index 6a095c7eccb..7f2ad5ac92c 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -1012,12 +1012,22 @@ impl> NewCanonicalChain { /// Returns the new tip for [`Self::Reorg`] and [`Self::Commit`] variants which commit at least /// 1 new block. pub fn tip(&self) -> &RecoveredBlock { + self.new_blocks().last().expect("non empty blocks").recovered_block() + } + + /// Returns the blocks added to the canonical chain by this update. + pub fn new_blocks(&self) -> &[ExecutedBlock] { match self { - Self::Commit { new } | Self::Reorg { new, .. } => { - new.last().expect("non empty blocks").recovered_block() - } + Self::Commit { new } | Self::Reorg { new, .. } => new, } } + + /// Returns whether the newly canonicalized blocks contain the given hash. + /// + /// This does not include the unchanged canonical prefix before the first new block. + pub fn contains(&self, hash: B256) -> bool { + self.new_blocks().iter().any(|block| block.recovered_block().hash() == hash) + } } #[cfg(test)] diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 03676493f8b..545ea65e88b 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -1241,6 +1241,10 @@ where trace!(target: "engine::tree", "fcu head hash is already canonical"); + if !self.is_consistent_forkchoice_state(state, None)? { + return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::invalid_state()))); + } + // Update the safe and finalized blocks and ensure their values are valid if let Err(outcome) = self.ensure_consistent_forkchoice_state(state) { // safe or finalized hashes are invalid @@ -1309,6 +1313,10 @@ where return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::too_deep_reorg()))); } + if !self.is_consistent_forkchoice_state(state, None)? { + return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::invalid_state()))); + } + // We need to effectively unwind the _canonical_ chain to the FCU's head, which is // part of the canonical chain. We need to update the latest block state to reflect // the canonical ancestor. This ensures that state providers and the transaction @@ -1337,6 +1345,10 @@ where // Ensure we can apply a new chain update for the head block if let Some(chain_update) = self.on_new_head(state.head_block_hash)? { + if !self.is_consistent_forkchoice_state(state, Some(&chain_update))? { + return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::invalid_state()))); + } + let tip = chain_update.tip().clone_sealed_header(); self.on_canonical_chain_update(chain_update); @@ -3363,6 +3375,67 @@ where Ok(canonical) } + /// Checks that nonzero safe and finalized hashes belong to the chain defined by the FCU head. + /// + /// The [Engine API forkchoiceUpdated specification] requires `-38002` when a `VALID` head's + /// safe or finalized hash is outside that chain, and requires all forkchoice state updates to + /// be atomic. In direct FCU processing, this check must therefore run before canonicalizing + /// the head or updating either marker; checking only after canonicalization would leave an + /// invalid reorg applied. + /// + /// `chain_update` describes a proposed commit or reorg that has not yet been applied. Its new + /// blocks and the canonical prefix below its first block form the proposed chain. Without a + /// chain update, the proposed head is already canonical, so only canonical blocks through its + /// height are eligible. For a canonical-prefix hash, the current in-memory or persisted + /// canonical hash must match too: a stale persisted header may still be found by hash while + /// disk reorg cleanup is pending. A zero safe or finalized hash leaves that marker unchanged. + /// Returns `Ok(false)` for an unknown or off-chain hash and propagates provider errors. + /// + /// [Engine API forkchoiceUpdated specification]: https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#specification-1 + fn is_consistent_forkchoice_state( + &self, + state: ForkchoiceState, + chain_update: Option<&NewCanonicalChain>, + ) -> ProviderResult { + let canonical_head_number = match chain_update { + Some(chain_update) => { + let new = chain_update.new_blocks(); + // Only the canonical prefix below the new branch remains on the proposed chain. + new.first().expect("non empty chain").block_number() - 1 + } + None => { + let Some(head) = self.find_canonical_header(state.head_block_hash)? else { + return Ok(false) + }; + head.number() + } + }; + + for hash in [state.finalized_block_hash, state.safe_block_hash] { + if hash.is_zero() || chain_update.is_some_and(|update| update.contains(hash)) { + continue + } + let Some(header) = self.find_canonical_header(hash)? else { return Ok(false) }; + if header.number() > canonical_head_number { + return Ok(false) + } + + // A persisted header can still be found by hash while its disk reorg is pending. + // Prefer the in-memory canonical hash at this height over the persisted one. + let canonical_hash = if let Some(hash) = + self.canonical_in_memory_state.hash_by_number(header.number()) + { + Some(hash) + } else { + self.provider.block_hash(header.number())? + }; + if canonical_hash != Some(hash) { + return Ok(false) + } + } + Ok(true) + } + /// Updates the tracked finalized block if we have it. fn update_finalized_block( &self, diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index f7db108819e..0932dccf7f0 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -3434,3 +3434,39 @@ fn test_backfill_threshold_above_header_limit_triggers_backfill() { } } } + +/// A stale persisted hash must not count as an ancestor after an in-memory reorg. +#[test] +fn test_forkchoice_rejects_stale_persisted_prefix_hash() { + let mut test_harness = TestHarness::new(MAINNET.clone()); + let old: Vec<_> = test_harness.block_builder.get_executed_blocks(0..3).collect(); + test_harness = test_harness.with_blocks(old.clone()); + + let new: Vec<_> = test_harness + .block_builder + .create_fork(old[0].recovered_block(), 2) + .into_iter() + .map(|block| { + ExecutedBlock::new( + Arc::new(block), + Arc::new(BlockExecutionOutput::default()), + ComputedTrieData::default(), + ) + }) + .collect(); + test_harness.tree.canonical_in_memory_state.update_chain(NewCanonicalChain::Reorg { + new: vec![new[0].clone()], + old: old[1..].to_vec(), + }); + + let state = ForkchoiceState { + head_block_hash: new[1].recovered_block().hash(), + safe_block_hash: old[1].recovered_block().hash(), + finalized_block_hash: old[0].recovered_block().hash(), + }; + let update = NewCanonicalChain::Commit { new: vec![new[1].clone()] }; + assert!(!test_harness.tree.is_consistent_forkchoice_state(state, Some(&update)).unwrap()); + + let state = ForkchoiceState { safe_block_hash: new[0].recovered_block().hash(), ..state }; + assert!(test_harness.tree.is_consistent_forkchoice_state(state, Some(&update)).unwrap()); +} diff --git a/crates/ethereum/node/tests/e2e/forkchoice.rs b/crates/ethereum/node/tests/e2e/forkchoice.rs new file mode 100644 index 00000000000..e9ff2ef457f --- /dev/null +++ b/crates/ethereum/node/tests/e2e/forkchoice.rs @@ -0,0 +1,161 @@ +//! Tests for atomic forkchoice state updates via the Engine API. + +use crate::utils::eth_payload_attributes; +use alloy_eips::BlockNumberOrTag; +use alloy_primitives::B256; +use alloy_provider::{Provider, ProviderBuilder}; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; +use jsonrpsee_core::client::Error; +use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_e2e_test_utils::E2ETestSetupBuilder; +use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_rpc_api::{EngineApiClient, TestingBuildBlockRequestV1}; +use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; +use std::sync::Arc; + +#[tokio::test] +async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + let (mut nodes, _) = + E2ETestSetupBuilder::::new(2, chain_spec, eth_payload_attributes) + .with_connect_nodes(false) + .with_node_config_modifier(|mut config| { + config.rpc.http_api = + Some(RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing])); + config + }) + .build() + .await?; + let node = nodes.pop().unwrap(); + let producer = nodes.pop().unwrap(); + let genesis = node.block_hash(0); + let engine = node.auth_server_handle().http_client(); + let producer_engine = producer.auth_server_handle().http_client(); + let rpc = ProviderBuilder::new().connect_http(node.rpc_url()); + + // Build a1 <- a2 <- a3 and b1 <- b2, both rooted at genesis. Only chain A is canonical. + let mut chains = [vec![genesis], vec![genesis]]; + for (branch, length) in [3, 2].into_iter().enumerate() { + for number in 1..=length { + let envelope = producer + .testing_build_block_v1(TestingBuildBlockRequestV1 { + parent_block_hash: *chains[branch].last().unwrap(), + payload_attributes: eth_payload_attributes(number + branch as u64 * 10), + transactions: vec![], + extra_data: None, + }) + .await?; + let payload = envelope.execution_payload; + let hash = payload.payload_inner.payload_inner.block_hash; + for client in [&producer_engine, &engine] { + let status = EngineApiClient::::new_payload_v3( + client, + payload.clone(), + vec![], + B256::ZERO, + ) + .await?; + assert_eq!(status.status, PayloadStatusEnum::Valid); + } + producer.update_forkchoice(genesis, hash).await?; + chains[branch].push(hash); + } + if branch == 0 { + let status = EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState { + head_block_hash: chains[0][3], + safe_block_hash: genesis, + finalized_block_hash: genesis, + }, + None, + ) + .await?; + assert_eq!(status.payload_status.status, PayloadStatusEnum::Valid); + } + } + let [a, b] = chains; + + for (head, safe, finalized) in [ + (b[2], a[2], a[1]), + (b[2], a[2], b[1]), + (b[2], b[1], a[1]), + (b[2], B256::repeat_byte(0xff), b[1]), + (a[3], b[1], a[1]), + (a[2], a[3], genesis), + ] { + let state = ForkchoiceState { + head_block_hash: head, + safe_block_hash: safe, + finalized_block_hash: finalized, + }; + let err = EngineApiClient::::fork_choice_updated_v3(&engine, state, None) + .await + .unwrap_err(); + let Error::Call(err) = err else { panic!("Expected an RPC error, got {err:?}") }; + assert_eq!(err.code(), -38002, "{state:?}"); + + for (tag, hash) in [ + (BlockNumberOrTag::Latest, a[3]), + (BlockNumberOrTag::Safe, genesis), + (BlockNumberOrTag::Finalized, genesis), + (BlockNumberOrTag::Number(1), a[1]), + (BlockNumberOrTag::Number(2), a[2]), + (BlockNumberOrTag::Number(3), a[3]), + ] { + assert_eq!( + rpc.get_block_by_number(tag).await?.unwrap().header.hash, + hash, + "{tag:?} changed after rejecting {state:?}" + ); + } + } + + // Safe/finalized hashes on the proposed branch must be accepted before it is canonical. + let status = EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState { + head_block_hash: b[2], + safe_block_hash: b[2], + finalized_block_hash: b[1], + }, + None, + ) + .await?; + assert_eq!(status.payload_status.status, PayloadStatusEnum::Valid); + for (tag, hash) in [ + (BlockNumberOrTag::Latest, b[2]), + (BlockNumberOrTag::Safe, b[2]), + (BlockNumberOrTag::Finalized, b[1]), + (BlockNumberOrTag::Number(1), b[1]), + (BlockNumberOrTag::Number(2), b[2]), + ] { + assert_eq!(rpc.get_block_by_number(tag).await?.unwrap().header.hash, hash); + } + assert!(rpc.get_block_by_number(BlockNumberOrTag::Number(3)).await?.is_none()); + + // Invalid payload attributes must not roll back an otherwise valid forkchoice update. + let err = EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState::same_hash(b[2]), + Some(eth_payload_attributes(1)), + ) + .await + .unwrap_err(); + let Error::Call(err) = err else { panic!("Expected an RPC error, got {err:?}") }; + assert_eq!(err.code(), -38003); + assert_eq!( + rpc.get_block_by_number(BlockNumberOrTag::Finalized).await?.unwrap().header.hash, + b[2] + ); + + Ok(()) +} diff --git a/crates/ethereum/node/tests/e2e/main.rs b/crates/ethereum/node/tests/e2e/main.rs index 045998ffedc..7272b4fade8 100644 --- a/crates/ethereum/node/tests/e2e/main.rs +++ b/crates/ethereum/node/tests/e2e/main.rs @@ -5,6 +5,7 @@ mod custom_genesis; mod dev; mod eip6780; mod eth; +mod forkchoice; mod invalid_payload; mod p2p; mod pool; From 074bf32ef193987b41ee79a93f07e2251ff6fe81 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 14:04:03 +0000 Subject: [PATCH 067/133] fix(rpc): default trace_filter to latest (#27411) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-api/src/trace.rs | 2 +- crates/rpc/rpc/src/trace.rs | 45 +++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 70a458f2437..3b004e9796b 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -69,7 +69,7 @@ pub trait TraceApi { /// Returns traces matching given filter. /// - /// This is similar to `eth_getLogs` but for traces. + /// This is similar to `eth_getLogs` but for traces. Omitted range bounds default to latest. #[method(name = "filter")] async fn trace_filter(&self, filter: TraceFilter) -> RpcResult>; diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 5a0abdc6be6..7913bb248ae 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -368,7 +368,7 @@ where /// Returns all transaction traces that match the given filter. /// /// This is similar to [`Self::trace_block`] but only returns traces for transactions that match - /// the filter. + /// the filter. Omitted range bounds default to the latest block. pub async fn trace_filter( &self, filter: TraceFilter, @@ -376,9 +376,9 @@ where // We'll reuse the matcher across multiple blocks that are traced in parallel let matcher = Arc::new(filter.matcher()); let TraceFilter { from_block, to_block, mut after, count, .. } = filter; - let start = from_block.unwrap_or(0); let latest_block = self.provider().best_block_number().map_err(Eth::Error::from_eth_err)?; + let start = from_block.unwrap_or(latest_block); if start > latest_block { // can't trace that range return Err(EthApiError::HeaderNotFound(start.into()).into()); @@ -944,6 +944,47 @@ mod tests { assert_eq!(response["result"], serde_json::to_value(latest).unwrap()); } + #[tokio::test] + async fn trace_filter_defaults_to_latest() { + let provider = MockEthProvider::default(); + for number in [1, 2] { + let header = Header { number, gas_limit: 30_000_000, ..Default::default() }; + provider.add_block(header.hash_slow(), Block { header, body: BlockBody::default() }); + } + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + let api = TraceApi::new( + eth_api, + BlockingTaskGuard::new(1), + EthConfig::default().max_trace_filter_blocks(1), + ); + + let omitted = api.trace_filter(TraceFilter::default()).await.unwrap(); + let latest = + api.trace_filter(TraceFilter::default().from_block(2).to_block(2)).await.unwrap(); + assert_eq!(omitted, latest); + assert!(api.trace_filter(TraceFilter::default().from_block(1).to_block(2)).await.is_ok()); + + let module = api.into_rpc(); + let request = serde_json::json!({ + "jsonrpc": "2.0", "id": 1, "method": "trace_filter", "params": [{"toBlock": "0x1"}], + }); + let (response, _) = module.raw_json_request(&request.to_string(), 1).await.unwrap(); + let response: serde_json::Value = serde_json::from_str(response.get()).unwrap(); + assert_eq!( + response["error"], + serde_json::json!({ + "code": -32602, + "message": "invalid parameters: fromBlock cannot be greater than toBlock", + }) + ); + } + #[tokio::test] async fn trace_get_selects_tree_paths() { use crate::EthApiBuilder; From c0c2b5663991d0b4095f86485f08ee55e54c4af6 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:16:29 +0000 Subject: [PATCH 068/133] refactor(txpool): use alloy transaction helpers (#27406) Co-authored-by: Claude Opus 5.5 --- crates/transaction-pool/src/maintain.rs | 14 +++++--- crates/transaction-pool/src/pool/best.rs | 4 +-- crates/transaction-pool/src/pool/mod.rs | 4 +-- .../transaction-pool/src/test_utils/mock.rs | 33 +++++-------------- crates/transaction-pool/src/traits.rs | 21 ++++-------- crates/transaction-pool/src/validate/eth.rs | 23 +++++++------ 6 files changed, 40 insertions(+), 59 deletions(-) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index f0e7d780c9e..a7a26f52b83 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -9,9 +9,9 @@ use crate::{ TransactionOrigin, }; use alloy_consensus::{transaction::TxHashRef, BlockHeader, Typed2718}; -use alloy_eips::{BlockNumberOrTag, Decodable2718}; +use alloy_eips::{merge::SLOT_DURATION_SECS, BlockNumberOrTag, Decodable2718}; use alloy_primitives::{ - map::{AddressSet, HashSet}, + map::{AddressSet, B256Set, HashSet}, Address, BlockHash, BlockNumber, Bytes, }; use alloy_rlp::Encodable; @@ -382,7 +382,7 @@ pub async fn maintain_transaction_pool( // all transactions mined in the new chain let mined_transactions = new_blocks.transaction_hashes_vec(); let new_mined_transactions = - mined_transactions.iter().copied().collect::>(); + mined_transactions.iter().copied().collect::(); // update the pool then re-inject the pruned transactions // find all transactions that were mined in the old chain but not in the new chain @@ -512,8 +512,12 @@ pub async fn maintain_transaction_pool( // If Osaka activates in 2 slots we need to convert blobs to new format. if !chain_spec.is_osaka_active_at_timestamp(tip.timestamp()) && - !chain_spec.is_osaka_active_at_timestamp(tip.timestamp().saturating_add(12)) && - chain_spec.is_osaka_active_at_timestamp(tip.timestamp().saturating_add(24)) + !chain_spec.is_osaka_active_at_timestamp( + tip.timestamp().saturating_add(SLOT_DURATION_SECS), + ) && + chain_spec.is_osaka_active_at_timestamp( + tip.timestamp().saturating_add(2 * SLOT_DURATION_SECS), + ) { let pool = pool.clone(); let spawner = task_spawner.clone(); diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index e3e214dd9d8..5468a71a0c9 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -1225,9 +1225,7 @@ mod tests { let mut included_txs = 0u64; while let Some(tx) = best.next() { - if let Some(blob_hashes) = tx.transaction.blob_versioned_hashes() { - let tx_blob_count = blob_hashes.len() as u64; - + if let Some(tx_blob_count) = tx.transaction.blob_count() { if block_blob_count + tx_blob_count > max_blob_count { crate::traits::BestTransactions::mark_invalid( &mut best, diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 3b1ec21404d..10ec0143a2a 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -89,7 +89,7 @@ use crate::{ }; use alloy_primitives::{ - map::{AddressSet, HashSet}, + map::{AddressSet, B256Set}, Address, TxHash, B256, }; use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; @@ -721,7 +721,7 @@ where // Linear search avoids allocating a hash set for small eviction batches. const MAX_LINEAR_SEARCH_DISCARDS: usize = 4; let discarded_hashes = (discarded.len() > MAX_LINEAR_SEARCH_DISCARDS) - .then(|| discarded.iter().map(|tx| *tx.hash()).collect::>()); + .then(|| discarded.iter().map(|tx| *tx.hash()).collect::()); let is_discarded = |hash: &TxHash| match &discarded_hashes { Some(hashes) => hashes.contains(hash), None => discarded.iter().any(|tx| tx.hash() == hash), diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index d7072348793..68d06548583 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -17,7 +17,7 @@ use alloy_consensus::{ TxLegacy, TxType, Typed2718, }; use alloy_eips::{ - eip1559::MIN_PROTOCOL_BASE_FEE, + eip1559::{calc_effective_gas_price, MIN_PROTOCOL_BASE_FEE}, eip2718::Encodable2718, eip2930::AccessList, eip4844::{BlobTransactionSidecar, BlobTransactionValidationError, DATA_GAS_PER_BLOB}, @@ -859,24 +859,10 @@ impl alloy_consensus::Transaction for MockTransaction { } fn effective_gas_price(&self, base_fee: Option) -> u128 { - base_fee.map_or_else( - || self.max_fee_per_gas(), - |base_fee| { - // if the tip is greater than the max priority fee per gas, set it to the max - // priority fee per gas + base fee - let tip = self.max_fee_per_gas().saturating_sub(base_fee as u128); - if let Some(max_tip) = self.max_priority_fee_per_gas() { - if tip > max_tip { - max_tip + base_fee as u128 - } else { - // otherwise return the max fee per gas - self.max_fee_per_gas() - } - } else { - self.max_fee_per_gas() - } - }, - ) + match self.max_priority_fee_per_gas() { + Some(max_tip) => calc_effective_gas_price(self.max_fee_per_gas(), max_tip, base_fee), + None => self.max_fee_per_gas(), + } } fn is_dynamic_fee(&self) -> bool { @@ -985,8 +971,7 @@ impl TryFrom> for MockTransaction { type Error = TryFromRecoveredTransactionError; fn try_from(tx: Recovered) -> Result { - let sender = tx.signer(); - let transaction = tx.into_inner(); + let (transaction, sender) = tx.into_parts(); let hash = *transaction.tx_hash(); let size = transaction.size(); @@ -1129,8 +1114,7 @@ impl TryFrom>>, ) -> Result { - let sender = tx.signer(); - let transaction = tx.into_inner(); + let (transaction, sender) = tx.into_parts(); let hash = *transaction.tx_hash(); let size = transaction.size(); @@ -1234,8 +1218,7 @@ impl TryFrom> for MockTransaction { fn from(tx: Recovered) -> Self { - let (tx, signer) = tx.into_parts(); - Recovered::::new_unchecked(tx.into(), signer).try_into().expect( + tx.convert::().try_into().expect( "Failed to convert from PooledTransactionsElementEcRecovered to MockTransaction", ) } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index bbb5797d428..ef4d74953fb 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1361,8 +1361,7 @@ pub trait PoolTransaction: fn try_from_consensus( tx: Recovered, ) -> Result { - let (tx, signer) = tx.into_parts(); - Ok(Self::from_pooled(Recovered::new_unchecked(tx.try_into()?, signer))) + Ok(Self::from_pooled(tx.try_convert()?)) } /// Clone the transaction into a consensus variant. @@ -1468,17 +1467,13 @@ pub trait PoolTransaction: /// Tries to convert the `Consensus` type into the `Pooled` type. fn try_into_pooled(self) -> Result, Self::TryFromConsensusError> { - let consensus = self.into_consensus(); - let (tx, signer) = consensus.into_parts(); - Ok(Recovered::new_unchecked(tx.try_into()?, signer)) + self.into_consensus().try_convert() } /// Clones the consensus transactions and tries to convert the `Consensus` type into the /// `Pooled` type. fn clone_into_pooled(&self) -> Result, Self::TryFromConsensusError> { - let consensus = self.clone_into_consensus(); - let (tx, signer) = consensus.into_parts(); - Ok(Recovered::new_unchecked(tx.try_into()?, signer)) + self.clone_into_consensus().try_convert() } /// Converts the `Pooled` type into the `Consensus` type. @@ -1697,7 +1692,7 @@ impl PoolTransaction for EthPooledTransaction { } fn consensus_ref(&self) -> Recovered<&Self::Consensus> { - Recovered::new_unchecked(&*self.transaction, self.transaction.signer()) + self.transaction.as_recovered_ref() } fn into_consensus(self) -> Recovered { @@ -1862,11 +1857,9 @@ impl EthPoolTransaction for EthPooledTransaction { self, sidecar: Arc, ) -> Option> { - let (signed_transaction, signer) = self.into_consensus().into_parts(); - let pooled_transaction = - signed_transaction.try_into_pooled_eip4844(Arc::unwrap_or_clone(sidecar)).ok()?; - - Some(Recovered::new_unchecked(pooled_transaction, signer)) + self.into_consensus() + .try_map(|tx| tx.try_into_pooled_eip4844(Arc::unwrap_or_clone(sidecar))) + .ok() } fn try_from_eip4844( diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index ecea4e38a59..21c9065d270 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -22,7 +22,7 @@ use alloy_consensus::{ }; use alloy_eips::{ eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M, eip4844::env_settings::EnvKzgSettings, - eip7840::BlobParams, BlockId, + eip7840::BlobParams, merge::SLOT_DURATION_SECS, BlockId, }; use alloy_primitives::U256; use alloy_rlp::Encodable; @@ -744,7 +744,7 @@ where { let is_eip7702 = if self.fork_tracker.is_prague_activated() { match state.bytecode_by_hash(code_hash) { - Ok(bytecode) => bytecode.unwrap_or_default().is_eip7702(), + Ok(bytecode) => bytecode.is_some_and(|b| b.is_eip7702()), Err(err) => { return Err(TransactionValidationOutcome::Error( *transaction.hash(), @@ -978,9 +978,15 @@ where let tip_timestamp = self.fork_tracker.tip_timestamp(); // If next block is Osaka, allow 7594 sidecars - if self.chain_spec().is_osaka_active_at_timestamp(tip_timestamp.saturating_add(12)) { + if self + .chain_spec() + .is_osaka_active_at_timestamp(tip_timestamp.saturating_add(SLOT_DURATION_SECS)) + { true - } else if self.chain_spec().is_osaka_active_at_timestamp(tip_timestamp.saturating_add(24)) { + } else if self + .chain_spec() + .is_osaka_active_at_timestamp(tip_timestamp.saturating_add(2 * SLOT_DURATION_SECS)) + { let current_timestamp = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs(); @@ -1539,16 +1545,13 @@ pub fn ensure_intrinsic_gas( transaction.input(), transaction.is_create(), transaction.access_list().map(|l| l.len()).unwrap_or_default() as u64, - transaction - .access_list() - .map(|l| l.iter().map(|i| i.storage_keys.len()).sum::()) - .unwrap_or_default() as u64, - transaction.authorization_list().map(|l| l.len()).unwrap_or_default() as u64, + transaction.access_list().map(|l| l.storage_keys_count()).unwrap_or_default() as u64, + transaction.authorization_count().unwrap_or_default(), eip2780, ); let gas_limit = transaction.gas_limit(); - if gas_limit < gas.initial_total_gas() || gas_limit < gas.floor_gas { + if gas_limit < gas.initial_total_gas() || gas_limit < gas.floor_gas() { Err(InvalidPoolTransactionError::IntrinsicGasTooLow) } else { Ok(()) From 06a7b16bcb55fe0313fbbb2e119b5b92372f1f0a Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:29:23 +0000 Subject: [PATCH 069/133] refactor(trie): remove storage root cache (#27409) Co-authored-by: Claude Opus 5.5 Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> --- .github/scripts/bench-txgen-build.sh | 3 ++ crates/trie/parallel/Cargo.toml | 2 +- crates/trie/parallel/src/proof_task.rs | 26 ++---------- .../trie/parallel/src/proof_task_metrics.rs | 3 -- crates/trie/parallel/src/value_encoder.rs | 40 +++---------------- 5 files changed, 12 insertions(+), 62 deletions(-) diff --git a/.github/scripts/bench-txgen-build.sh b/.github/scripts/bench-txgen-build.sh index e10f9a0e789..8803641548c 100755 --- a/.github/scripts/bench-txgen-build.sh +++ b/.github/scripts/bench-txgen-build.sh @@ -8,6 +8,9 @@ # payload format. set -euxo pipefail +# Use the runner's Git authentication configuration when fetching dependencies. +export CARGO_NET_GIT_FETCH_WITH_CLI=true + MODE="$1" SOURCE_DIR="$2" COMMIT="$3" diff --git a/crates/trie/parallel/Cargo.toml b/crates/trie/parallel/Cargo.toml index 7a67d2fffa8..806bcc6c75b 100644 --- a/crates/trie/parallel/Cargo.toml +++ b/crates/trie/parallel/Cargo.toml @@ -13,7 +13,7 @@ workspace = true [dependencies] # reth -reth-primitives-traits = { workspace = true, features = ["dashmap", "std"] } +reth-primitives-traits = { workspace = true, features = ["std"] } reth-execution-errors.workspace = true reth-provider.workspace = true reth-storage-errors.workspace = true diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 1e77d654fac..4851eb6264f 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -39,7 +39,7 @@ use alloy_primitives::{ }; use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use reth_execution_errors::StateProofError; -use reth_primitives_traits::{dashmap::DashMap, FastInstant as Instant}; +use reth_primitives_traits::FastInstant as Instant; use reth_provider::{DatabaseProviderROFactory, ProviderError, ProviderResult}; use reth_storage_errors::db::DatabaseError; use reth_tasks::Runtime; @@ -181,7 +181,6 @@ impl ProofWorkerHandle { { let (storage_work_tx, storage_work_rx) = unbounded::(); let (account_work_tx, account_work_rx) = unbounded::(); - let cached_storage_roots = Arc::>::default(); let divisor = if halve_workers { 2 } else { 1 }; let storage_worker_count = @@ -205,7 +204,6 @@ impl ProofWorkerHandle { let storage_rt = runtime.clone(); let storage_task_ctx = task_ctx.clone(); let storage_avail = storage_availability.clone(); - let storage_roots = cached_storage_roots.clone(); let storage_result_tx = proof_result_tx.clone(); let storage_parent_span = tracing::Span::current(); runtime.spawn_blocking_named("storage-workers", move || { @@ -225,7 +223,6 @@ impl ProofWorkerHandle { storage_work_rx.clone(), worker_id, storage_avail.clone(), - storage_roots.clone(), #[cfg(feature = "metrics")] metrics, #[cfg(feature = "metrics")] @@ -272,7 +269,6 @@ impl ProofWorkerHandle { worker_id, account_tx.clone(), account_avail.clone(), - cached_storage_roots.clone(), #[cfg(feature = "metrics")] metrics, #[cfg(feature = "metrics")] @@ -589,8 +585,6 @@ struct StorageProofWorker { worker_id: usize, /// Per-worker availability flags availability: Arc, - /// Cached storage roots - cached_storage_roots: Arc>, /// Metrics collector for this worker #[cfg(feature = "metrics")] metrics: ProofTaskTrieMetrics, @@ -609,7 +603,6 @@ where work_rx: CrossbeamReceiver, worker_id: usize, availability: Arc, - cached_storage_roots: Arc>, #[cfg(feature = "metrics")] metrics: ProofTaskTrieMetrics, #[cfg(feature = "metrics")] cursor_metrics: ProofTaskCursorMetrics, ) -> Self { @@ -618,7 +611,6 @@ where work_rx, worker_id, availability, - cached_storage_roots, #[cfg(feature = "metrics")] metrics, #[cfg(feature = "metrics")] @@ -773,10 +765,6 @@ where ); } - if let Some(root) = root { - self.cached_storage_roots.insert(hashed_address, root); - } - trace!( target: "trie::proof_task", worker_id = self.worker_id, @@ -804,8 +792,6 @@ struct AccountProofWorker { storage_work_tx: CrossbeamSender, /// Per-worker availability flags availability: Arc, - /// Cached storage roots - cached_storage_roots: Arc>, /// Metrics collector for this worker #[cfg(feature = "metrics")] metrics: ProofTaskTrieMetrics, @@ -819,14 +805,12 @@ where Factory: DatabaseProviderROFactory, { /// Creates a new account proof worker. - #[expect(clippy::too_many_arguments)] const fn new( task_ctx: ProofTaskCtx, work_rx: CrossbeamReceiver, worker_id: usize, storage_work_tx: CrossbeamSender, availability: Arc, - cached_storage_roots: Arc>, #[cfg(feature = "metrics")] metrics: ProofTaskTrieMetrics, #[cfg(feature = "metrics")] cursor_metrics: ProofTaskCursorMetrics, ) -> Self { @@ -836,7 +820,6 @@ where worker_id, storage_work_tx, availability, - cached_storage_roots, #[cfg(feature = "metrics")] metrics, #[cfg(feature = "metrics")] @@ -1010,11 +993,8 @@ where let storage_proof_receivers = dispatch_v2_storage_proofs(&self.storage_work_tx, &account_targets, storage_targets)?; - let mut value_encoder = AsyncAccountValueEncoder::new( - storage_proof_receivers, - self.cached_storage_roots.clone(), - v2_storage_calculator, - ); + let mut value_encoder = + AsyncAccountValueEncoder::new(storage_proof_receivers, v2_storage_calculator); let account_proofs = v2_account_calculator.proof(&mut value_encoder, &mut account_targets)?; diff --git a/crates/trie/parallel/src/proof_task_metrics.rs b/crates/trie/parallel/src/proof_task_metrics.rs index a915b3bd4ac..92e35c7938a 100644 --- a/crates/trie/parallel/src/proof_task_metrics.rs +++ b/crates/trie/parallel/src/proof_task_metrics.rs @@ -18,8 +18,6 @@ pub struct ProofTaskTrieMetrics { account_worker_idle_time_seconds: Histogram, /// Histogram for `Dispatched` deferred encoder variant count. deferred_encoder_dispatched: Histogram, - /// Histogram for `FromCache` deferred encoder variant count. - deferred_encoder_from_cache: Histogram, /// Histogram for `Sync` deferred encoder variant count. deferred_encoder_sync: Histogram, /// Histogram for dispatched storage proofs that fell back to sync due to missing root. @@ -44,7 +42,6 @@ impl ProofTaskTrieMetrics { /// Record value encoder stats (deferred encoder variant counts and storage wait time). pub(crate) fn record_value_encoder_stats(&self, stats: &ValueEncoderStats) { self.deferred_encoder_dispatched.record(stats.dispatched_count as f64); - self.deferred_encoder_from_cache.record(stats.from_cache_count as f64); self.deferred_encoder_sync.record(stats.sync_count as f64); self.deferred_encoder_dispatched_missing_root .record(stats.dispatched_missing_root_count as f64); diff --git a/crates/trie/parallel/src/value_encoder.rs b/crates/trie/parallel/src/value_encoder.rs index 14912ed99af..139ba0e03b9 100644 --- a/crates/trie/parallel/src/value_encoder.rs +++ b/crates/trie/parallel/src/value_encoder.rs @@ -4,7 +4,7 @@ use alloy_rlp::Encodable; use core::cell::RefCell; use crossbeam_channel::Receiver as CrossbeamReceiver; use reth_execution_errors::trie::StateProofError; -use reth_primitives_traits::{dashmap::DashMap, Account}; +use reth_primitives_traits::Account; use reth_storage_errors::db::DatabaseError; use reth_trie::{ hashed_cursor::HashedStorageCursor, @@ -14,7 +14,6 @@ use reth_trie::{ }; use std::{ rc::Rc, - sync::Arc, time::{Duration, Instant}, }; @@ -27,8 +26,6 @@ pub(crate) struct ValueEncoderStats { pub(crate) storage_wait_time: Duration, /// Number of times the `Dispatched` variant was used (proof pre-dispatched to workers). pub(crate) dispatched_count: u64, - /// Number of times the `FromCache` variant was used (storage root already cached). - pub(crate) from_cache_count: u64, /// Number of times the `Sync` variant was used (synchronous computation). pub(crate) sync_count: u64, /// Number of times a dispatched storage proof had no root node and fell back to sync @@ -41,7 +38,6 @@ impl ValueEncoderStats { pub(crate) fn extend(&mut self, other: &Self) { self.storage_wait_time += other.storage_wait_time; self.dispatched_count += other.dispatched_count; - self.from_cache_count += other.from_cache_count; self.sync_count += other.sync_count; self.dispatched_missing_root_count += other.dispatched_missing_root_count; } @@ -65,19 +61,13 @@ pub(crate) enum AsyncAccountDeferredValueEncoder { /// Shared storage proof calculator for synchronous fallback when dispatched proof has no /// root. storage_calculator: Rc>>, - /// Cache to store computed storage roots for future reuse. - cached_storage_roots: Arc>, }, - /// The storage root was found in cache. - FromCache { account: Account, root: B256 }, /// Synchronous storage root computation. Sync { /// Shared storage proof calculator for computing storage roots. storage_calculator: Rc>>, hashed_address: B256, account: Account, - /// Cache to store computed storage roots for future reuse. - cached_storage_roots: Arc>, }, } @@ -136,7 +126,6 @@ where storage_proof_results, stats, storage_calculator, - cached_storage_roots, } => { let hashed_address = *hashed_address; let account = *account; @@ -170,19 +159,15 @@ where let mut calculator = storage_calculator.borrow_mut(); let root_node = calculator.storage_root_node(hashed_address)?; - let storage_root = calculator + calculator .compute_root_hash(&[root_node])? - .expect("storage_root_node returns a node at empty path"); - - cached_storage_roots.insert(hashed_address, storage_root); - storage_root + .expect("storage_root_node returns a node at empty path") } }; (account, root) } - Self::FromCache { account, root } => (*account, *root), - Self::Sync { storage_calculator, hashed_address, account, cached_storage_roots } => { + Self::Sync { storage_calculator, hashed_address, account } => { let hashed_address = *hashed_address; let account = *account; let mut calculator = storage_calculator.borrow_mut(); @@ -191,7 +176,6 @@ where .compute_root_hash(&[root_node])? .expect("storage_root_node returns a node at empty path"); - cached_storage_roots.insert(hashed_address, storage_root); (account, storage_root) } }; @@ -207,15 +191,12 @@ where /// Accepts a set of pre-dispatched storage proof receivers for accounts whose storage roots are /// being computed asynchronously by worker threads. /// -/// For accounts without pre-dispatched proofs or cached roots, uses a shared +/// For accounts without pre-dispatched proofs, uses a shared /// [`StorageProofCalculator`] to compute storage roots synchronously, reusing cursors across /// multiple accounts. pub(crate) struct AsyncAccountValueEncoder { /// Storage proof jobs which were dispatched ahead of time. dispatched: B256Map>, - /// Storage roots which have already been computed. This can be used only if a storage proof - /// wasn't dispatched for an account, otherwise we must consume the proof result. - cached_storage_roots: Arc>, /// Tracks storage proof results received from the storage workers. [`Rc`] + [`RefCell`] is /// required because [`DeferredValueEncoder`] cannot have a lifetime. storage_proof_results: Rc>>>, @@ -232,16 +213,13 @@ impl AsyncAccountValueEncoder { /// /// # Parameters /// - `dispatched`: Pre-dispatched storage proof receivers for target accounts - /// - `cached_storage_roots`: Shared cache of already-computed storage roots /// - `storage_calculator`: Shared storage proof calculator for synchronous computation pub(crate) fn new( dispatched: B256Map>, - cached_storage_roots: Arc>, storage_calculator: Rc>>, ) -> Self { Self { dispatched, - cached_storage_roots, storage_proof_results: Default::default(), storage_calculator, stats: Default::default(), @@ -313,26 +291,18 @@ where storage_proof_results: self.storage_proof_results.clone(), stats: self.stats.clone(), storage_calculator: self.storage_calculator.clone(), - cached_storage_roots: self.cached_storage_roots.clone(), } } // If the address didn't have a job dispatched for it then we can assume it has no targets, // and we only need its root. - // If the root is already calculated then just use it directly - if let Some(root) = self.cached_storage_roots.get(&hashed_address) { - self.stats.borrow_mut().from_cache_count += 1; - return AsyncAccountDeferredValueEncoder::FromCache { account, root: *root } - } - // Compute storage root synchronously using the shared calculator self.stats.borrow_mut().sync_count += 1; AsyncAccountDeferredValueEncoder::Sync { storage_calculator: self.storage_calculator.clone(), hashed_address, account, - cached_storage_roots: self.cached_storage_roots.clone(), } } } From 861a56167cd4eee8ed66e2bfb5d54a6b0620b40d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 16:46:46 +0200 Subject: [PATCH 070/133] perf(rpc): share sender cache with validation (#27308) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/engine/primitives/src/lib.rs | 3 +- crates/ethereum/node/src/node.rs | 1 + crates/ethereum/node/tests/e2e/rpc.rs | 66 ++++++++++++++++++++ crates/evm/evm/src/sender_recovery.rs | 89 ++++++++++++++++++++++++++- crates/rpc/rpc/src/validation.rs | 89 +++++++++++++++++++++++++-- 5 files changed, 239 insertions(+), 9 deletions(-) diff --git a/crates/engine/primitives/src/lib.rs b/crates/engine/primitives/src/lib.rs index 7855b803871..8a6a633b0b4 100644 --- a/crates/engine/primitives/src/lib.rs +++ b/crates/engine/primitives/src/lib.rs @@ -203,7 +203,8 @@ pub trait PayloadValidator: Send + Sync + Unpin + 'static { /// fields. /// /// Implementers should ensure that the checks are done in the order that conforms with the - /// engine-API specification. + /// engine-API specification. Callers that recover senders separately use + /// [`Self::convert_payload_to_block`] directly, so required payload checks belong there. fn ensure_well_formed_payload( &self, payload: Types::ExecutionData, diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 4907343e441..9c268e05d3a 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -342,6 +342,7 @@ where ctx.config.rpc.flashbots_config(), ctx.node.task_executor().clone(), Arc::new(EthereumEngineValidator::new(ctx.config.chain.clone())), + ctx.sender_recovery_cache.clone(), ); let eth_config = diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index a15960c3630..b8e520c1ef4 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -587,6 +587,72 @@ async fn test_flashbots_validate_v3() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn test_flashbots_validate_uses_shared_sender_recovery_cache() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + + let (mut nodes, wallet) = + E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) + .with_node_config_modifier(|mut config| { + config.engine.sender_recovery_cache_enabled = true; + config + }) + .build() + .await?; + let mut node = nodes.pop().unwrap(); + let cache = node.inner.evm_config.sender_recovery_cache.clone().expect("cache is enabled"); + + let signer = wallet.wallet_gen().swap_remove(0); + let sender = signer.address(); + let provider = + ProviderBuilder::new().wallet(EthereumWallet::new(signer)).connect_http(node.rpc_url()); + + let tx_hash = *provider + .send_transaction(TransactionRequest::default().to(Address::ZERO)) + .await? + .tx_hash(); + // Transaction ingress has already recovered the sender through the shared cache. + let payload = node.new_payload().await?; + assert!(payload.block().body().transactions.iter().any(|tx| *tx.tx_hash() == tx_hash)); + assert_eq!(cache.get(&tx_hash), Some(sender)); + + let request = BuilderBlockValidationRequestV3 { + request: SignedBidSubmissionV3 { + message: BidTrace { + parent_hash: payload.block().parent_hash, + block_hash: payload.block().hash(), + gas_used: payload.block().gas_used, + gas_limit: payload.block().gas_limit, + ..Default::default() + }, + execution_payload: ExecutionPayloadV3::from_block_unchecked( + payload.block().hash(), + &payload.block().clone().into_block(), + ), + blobs_bundle: BlobsBundleV1::new([]), + signature: Default::default(), + }, + parent_beacon_block_root: payload.block().parent_beacon_block_root.unwrap(), + registered_gas_limit: payload.block().gas_limit, + }; + + provider + .raw_request::<_, ()>("flashbots_validateBuilderSubmissionV3".into(), (&request,)) + .await?; + // Validation accepts the submission while reusing the sender cached by transaction ingress. + assert_eq!(cache.get(&tx_hash), Some(sender)); + + Ok(()) +} + #[tokio::test] async fn test_flashbots_validate_v4() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/evm/evm/src/sender_recovery.rs b/crates/evm/evm/src/sender_recovery.rs index 2ddb0cec726..b3665bc4ed2 100644 --- a/crates/evm/evm/src/sender_recovery.rs +++ b/crates/evm/evm/src/sender_recovery.rs @@ -1,6 +1,9 @@ -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use alloy_primitives::{map::FbBuildHasher, Address, B256}; -use reth_primitives_traits::{transaction::signed::RecoveryError, SignedTransaction}; +use reth_primitives_traits::{ + transaction::{recover::recover_signers, signed::RecoveryError}, + SignedTransaction, +}; /// Number of entries retained in the default sender recovery cache: 131,072. const SENDER_RECOVERY_CACHE_CAPACITY: usize = 1 << 17; @@ -13,6 +16,9 @@ const SENDER_RECOVERY_CACHE_CAPACITY: usize = 1 << 17; /// /// The default cache retains up to 131,072 transaction hash-to-sender entries. Cloning the cache /// shares these entries rather than allocating another cache. +/// +/// Cached senders are trusted without re-checking the signature, so only senders recovered from +/// the transaction with the keyed hash are ever inserted. #[derive(Clone, Debug)] pub struct SenderRecoveryCache { cache: Arc, SenderRecoveryCacheConfig>>, @@ -56,6 +62,42 @@ impl SenderRecoveryCache { |hash| *hash, ) } + + /// Recovers the senders of the given transactions, in transaction order. + /// + /// Cached senders are reused. The remaining transactions are recovered together, in parallel + /// when the `rayon` feature of `reth-primitives-traits` is enabled, and their senders are + /// cached. Nothing is cached if any recovery fails. + pub fn recover_signers( + &self, + transactions: &[T], + ) -> Result, RecoveryError> { + let mut senders = Vec::with_capacity(transactions.len()); + let mut misses = Vec::new(); + for (index, transaction) in transactions.iter().enumerate() { + match self.get(transaction.tx_hash()) { + Some(sender) => senders.push(sender), + None => { + // Placeholder that is overwritten once the miss has been recovered below. + senders.push(Address::ZERO); + misses.push(index); + } + } + } + + if misses.is_empty() { + return Ok(senders) + } + + let recovered = + recover_signers(misses.iter().map(|&index| &transactions[index]).collect::>())?; + for (index, sender) in misses.into_iter().zip(recovered) { + self.cache.insert(*transactions[index].tx_hash(), sender); + senders[index] = sender; + } + + Ok(senders) + } } impl Default for SenderRecoveryCache { @@ -149,4 +191,47 @@ mod tests { } assert_eq!(attempts, 2); } + + /// Returns a validly signed transaction whose sender depends on the nonce. + fn signed_transaction(nonce: u64) -> TransactionSigned { + TransactionSigned::new_unhashed( + Transaction::Legacy(TxLegacy { nonce, ..Default::default() }), + Signature::test_signature(), + ) + } + + #[test] + fn recover_signers_reuses_cached_senders_and_caches_misses() { + let transactions: Vec<_> = (0..4).map(signed_transaction).collect(); + let recovered: Vec<_> = + transactions.iter().map(|transaction| transaction.try_recover().unwrap()).collect(); + let cache = SenderRecoveryCache::default(); + + // Stands in for a sender recovered by another component: it can only be returned if + // the hit skipped signature recovery. + let cached_sender = Address::repeat_byte(0xaa); + cache.cache.insert(*transactions[1].tx_hash(), cached_sender); + + let senders = cache.recover_signers(&transactions).unwrap(); + + assert_eq!(senders, [recovered[0], cached_sender, recovered[2], recovered[3]]); + for (transaction, sender) in transactions.iter().zip(&senders) { + assert_eq!(cache.get(transaction.tx_hash()), Some(*sender)); + } + assert!(cache.recover_signers::(&[]).unwrap().is_empty()); + } + + #[test] + fn recover_signers_rejects_invalid_signature_without_caching() { + let valid = signed_transaction(0); + let invalid = TransactionSigned::new_unhashed( + Transaction::Legacy(TxLegacy::default()), + Signature::new(U256::ZERO, U256::ZERO, false), + ); + let cache = SenderRecoveryCache::default(); + + assert!(cache.recover_signers(&[valid.clone(), invalid.clone()]).is_err()); + assert_eq!(cache.get(valid.tx_hash()), None); + assert_eq!(cache.get(invalid.tx_hash()), None); + } } diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index 884661703b8..b9e087ce041 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -22,7 +22,7 @@ use reth_consensus::{Consensus, FullConsensus}; use reth_consensus_common::validation::MAX_RLP_BLOCK_SIZE; use reth_engine_primitives::PayloadValidator; use reth_errors::{BlockExecutionError, ConsensusError, ProviderError}; -use reth_evm::{execute::Executor, ConfigureEvm}; +use reth_evm::{execute::Executor, ConfigureEvm, SenderRecoveryCache}; use reth_execution_types::BlockExecutionOutput; use reth_metrics::{ metrics, @@ -31,7 +31,8 @@ use reth_metrics::{ }; use reth_node_api::{NewPayloadError, PayloadTypes}; use reth_primitives_traits::{ - BlockBody, GotExpected, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeaderFor, + block::error::SealedBlockRecoveryError, BlockBody, GotExpected, NodePrimitives, RecoveredBlock, + SealedBlock, SealedHeaderFor, }; use reth_revm::{cached::CachedReads, database::StateProviderDatabase}; use reth_rpc_api::BlockSubmissionValidationApiServer; @@ -59,6 +60,9 @@ where T: PayloadTypes, { /// Create a new instance of the [`ValidationApi`] + /// + /// If a `sender_recovery_cache` is given, the senders of submitted blocks are recovered + /// through it, reusing senders that other node components have already recovered. pub fn new( provider: Provider, consensus: Arc>, @@ -68,6 +72,7 @@ where payload_validator: Arc< dyn PayloadValidator::Block>, >, + sender_recovery_cache: Option, ) -> Self { let ValidationApiConfig { disallow, validation_window } = config; @@ -80,6 +85,7 @@ where validation_window, cached_state: Default::default(), task_spawner, + sender_recovery_cache, metrics: Default::default(), }); @@ -379,12 +385,39 @@ where Ok(versioned_hashes) } + /// Converts the payload into a block and recovers the transaction senders. + /// + /// Like the engine's `newPayload` handling, this leaves payload validation and conversion to + /// the payload validator and recovers senders through the shared [`SenderRecoveryCache`] if + /// one is configured, so senders already recovered on transaction ingress or payload + /// execution are reused. Without a cache, this calls + /// [`PayloadValidator::ensure_well_formed_payload`] so validator overrides are preserved. + /// + /// Senders recovered here are cached before the block is validated. Competing submissions for + /// the same slot share most of their transactions, so the senders are likely to be needed again + /// even if this submission is rejected, and only successfully recovered senders can be cached. + fn recover_payload( + &self, + payload: ExecutionData, + ) -> Result::Block>, ValidationApiError> { + let Some(cache) = &self.sender_recovery_cache else { + return self.payload_validator.ensure_well_formed_payload(payload).map_err(Into::into) + }; + + let block = self.payload_validator.convert_payload_to_block(payload)?; + let recovered = match cache.recover_signers(block.body().transactions()) { + Ok(senders) => Ok(RecoveredBlock::new_sealed(block, senders)), + Err(_) => Err(SealedBlockRecoveryError::new(block)), + }; + recovered.map_err(|err| NewPayloadError::Other(err.into()).into()) + } + /// Core logic for validating the builder submission v3 async fn validate_builder_submission_v3( &self, request: BuilderBlockValidationRequestV3, ) -> Result<(), ValidationApiError> { - let block = self.payload_validator.ensure_well_formed_payload(ExecutionData { + let block = self.recover_payload(ExecutionData { payload: ExecutionPayload::V3(request.request.execution_payload), sidecar: ExecutionPayloadSidecar::v3(CancunPayloadFields { parent_beacon_block_root: request.parent_beacon_block_root, @@ -406,7 +439,7 @@ where &self, request: BuilderBlockValidationRequestV4, ) -> Result<(), ValidationApiError> { - let block = self.payload_validator.ensure_well_formed_payload(ExecutionData { + let block = self.recover_payload(ExecutionData { payload: ExecutionPayload::V3(request.request.execution_payload), sidecar: ExecutionPayloadSidecar::v4( CancunPayloadFields { @@ -438,7 +471,7 @@ where let payload = ExecutionPayload::V3(request.request.execution_payload); validate_message_against_payload(&request.request.message, &payload)?; - let block = self.payload_validator.ensure_well_formed_payload(ExecutionData { + let block = self.recover_payload(ExecutionData { payload, sidecar: ExecutionPayloadSidecar::v4( CancunPayloadFields { @@ -487,7 +520,7 @@ where DecodedBal::from_rlp_bytes(payload.as_v4().unwrap().block_access_list.clone()) .map_err(ValidationApiError::InvalidBlockAccessList)?; - let block = self.payload_validator.ensure_well_formed_payload(ExecutionData { + let block = self.recover_payload(ExecutionData { payload, sidecar: ExecutionPayloadSidecar::v4( CancunPayloadFields { @@ -645,6 +678,9 @@ pub struct ValidationApiInner { cached_state: RwLock<(B256, CachedReads)>, /// Task spawner for blocking operations task_spawner: Runtime, + /// Cache of recovered transaction senders shared with transaction ingress and payload + /// execution, if enabled. + sender_recovery_cache: Option, /// Validation metrics metrics: ValidationMetrics, } @@ -983,6 +1019,46 @@ mod tests { } } + #[derive(Debug)] + struct SpecializedPayloadValidator; + + impl PayloadValidator for SpecializedPayloadValidator { + type Block = Block; + + fn convert_payload_to_block( + &self, + _payload: ExecutionData, + ) -> Result, NewPayloadError> { + panic!("the specialized recovery method must be used without a cache") + } + + fn ensure_well_formed_payload( + &self, + _payload: ExecutionData, + ) -> Result, NewPayloadError> { + Ok(SealedBlock::seal_slow(Block::default()).try_recover().unwrap()) + } + } + + #[test] + fn recover_payload_without_cache_uses_validator_override() { + let api = ValidationApi::new( + MockEthProvider::default(), + NoopConsensus::arc(), + EthEvmConfig::mainnet(), + ValidationApiConfig::default(), + Runtime::test(), + Arc::new(SpecializedPayloadValidator), + None, + ); + + api.recover_payload(ExecutionData { + payload: test_execution_payload(), + sidecar: Default::default(), + }) + .unwrap(); + } + fn test_validation_api( provider: MockEthProvider, ) -> ValidationApi { @@ -993,6 +1069,7 @@ mod tests { ValidationApiConfig::default(), Runtime::test(), Arc::new(UnusedPayloadValidator), + None, ) } From 54cd0da45d468762688d38640a791e92ad0dcbcb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 24 Sep 2026 16:36:19 +0000 Subject: [PATCH 071/133] fix(net): offload blocking NAT resolver work (#27360) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/net/nat/Cargo.toml | 4 +- crates/net/nat/src/lib.rs | 135 +++++++++++++++++++++++++++++++++----- 2 files changed, 121 insertions(+), 18 deletions(-) diff --git a/crates/net/nat/Cargo.toml b/crates/net/nat/Cargo.toml index 6a072e491a6..aa5516811eb 100644 --- a/crates/net/nat/Cargo.toml +++ b/crates/net/nat/Cargo.toml @@ -16,13 +16,13 @@ futures-util.workspace = true reqwest.workspace = true serde_with = { workspace = true, optional = true } thiserror.workspace = true -tokio = { workspace = true, features = ["time"] } +tokio = { workspace = true, features = ["rt", "time"] } if-addrs.workspace = true tracing.workspace = true [dev-dependencies] reth-tracing.workspace = true -tokio = { workspace = true, features = ["macros"] } +tokio = { workspace = true, features = ["macros", "sync", "test-util"] } [features] default = ["serde"] diff --git a/crates/net/nat/src/lib.rs b/crates/net/nat/src/lib.rs index 8f2303b30e5..a02f10c53d4 100644 --- a/crates/net/nat/src/lib.rs +++ b/crates/net/nat/src/lib.rs @@ -66,6 +66,11 @@ pub enum NatResolver { impl NatResolver { /// Attempts to produce an IP address (best effort). + /// + /// # Panics + /// + /// Network and interface resolution require an active Tokio runtime. Only + /// [`Self::ExternalIp`] and [`Self::None`] are guaranteed to work without one. pub async fn external_addr(self) -> Option { external_addr_with(self).await } @@ -73,7 +78,8 @@ impl NatResolver { /// Returns the fixed ip, if it is [`NatResolver::ExternalIp`] or [`NatResolver::ExternalAddr`]. /// /// In the case of [`NatResolver::ExternalAddr`], it will return the first IP address found for - /// the domain. + /// the domain. This performs blocking DNS resolution; async callers should use + /// [`Self::external_addr`] instead. pub fn as_external_ip(self, port: u16) -> Option { match self { Self::ExternalIp(ip) => Some(ip), @@ -138,6 +144,8 @@ impl FromStr for NatResolver { } /// With this type you can resolve the external public IP address on an interval basis. +/// +/// Keeps at most one resolution in flight and skips missed interval ticks. #[must_use = "Does nothing unless polled"] pub struct ResolveNatInterval { resolver: NatResolver, @@ -156,7 +164,9 @@ impl fmt::Debug for ResolveNatInterval { } impl ResolveNatInterval { - fn with_interval(resolver: NatResolver, interval: tokio::time::Interval) -> Self { + fn with_interval(resolver: NatResolver, mut interval: tokio::time::Interval) -> Self { + // Resolving once is sufficient after a delay; do not replay missed attempts in a burst. + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); Self { resolver, future: None, interval } } @@ -198,7 +208,8 @@ impl ResolveNatInterval { /// * `Poll::Ready(Option)` if the next [`IpAddr`] has been resolved. This returns /// `None` if the attempt was unsuccessful. pub fn poll_tick(&mut self, cx: &mut Context<'_>) -> Poll> { - if self.interval.poll_tick(cx).is_ready() { + // Dropping a resolution future cannot cancel blocking work it has already started. + if self.interval.poll_tick(cx).is_ready() && self.future.is_none() { self.future = Some(Box::pin(self.resolver.clone().external_addr())); } @@ -214,23 +225,39 @@ impl ResolveNatInterval { } /// Attempts to produce an IP address with all builtin resolvers (best effort). +/// +/// # Panics +/// +/// Panics if polled outside a Tokio runtime. pub async fn external_ip() -> Option { external_addr_with(NatResolver::Any).await } /// Given a [`NatResolver`] attempts to produce an IP address (best effort). +/// +/// # Panics +/// +/// Network and interface resolution require an active Tokio runtime. Only +/// [`NatResolver::ExternalIp`] and [`NatResolver::None`] are guaranteed to work without one. pub async fn external_addr_with(resolver: NatResolver) -> Option { match resolver { NatResolver::Any | NatResolver::Upnp | NatResolver::PublicIp => resolve_external_ip().await, NatResolver::ExternalIp(ip) => Some(ip), - NatResolver::NetIf => resolve_net_if_ip(DEFAULT_NET_IF_NAME) - .inspect_err(|err| { - debug!(target: "net::nat", - %err, - "Failed to resolve network interface IP" - ); - }) - .ok(), + NatResolver::NetIf => tokio::task::spawn_blocking(|| { + resolve_net_if_ip(DEFAULT_NET_IF_NAME) + }) + .await + .inspect_err(|err| { + debug!(target: "net::nat", %err, "Failed to join network interface resolution task"); + }) + .ok()? + .inspect_err(|err| { + debug!(target: "net::nat", + %err, + "Failed to resolve network interface IP" + ); + }) + .ok(), NatResolver::ExternalAddr(domain) => tokio::net::lookup_host(format!("{domain}:0")) .await .inspect_err(|err| { @@ -243,7 +270,22 @@ pub async fn external_addr_with(resolver: NatResolver) -> Option { } async fn resolve_external_ip() -> Option { - let futures = EXTERNAL_IP_APIS.iter().copied().map(resolve_external_ip_url_res).map(Box::pin); + // Client setup can read system proxy and TLS configuration. Keep it off the task polling + // discovery; the requests themselves use async I/O and share the same client. + let client = tokio::task::spawn_blocking(|| { + reqwest::Client::builder().timeout(Duration::from_secs(10)).build() + }) + .await + .inspect_err(|err| { + debug!(target: "net::nat", %err, "Failed to join external IP client setup task"); + }) + .ok()? + .inspect_err(|err| { + debug!(target: "net::nat", %err, "Failed to build external IP client"); + }) + .ok()?; + let futures = + EXTERNAL_IP_APIS.iter().map(|url| resolve_external_ip_url_res(&client, url)).map(Box::pin); futures_util::future::select_ok(futures) .await .inspect_err(|err| { @@ -256,12 +298,11 @@ async fn resolve_external_ip() -> Option { .map(|(ip, _)| ip) } -async fn resolve_external_ip_url_res(url: &str) -> Result { - resolve_external_ip_url(url).await.ok_or(()) +async fn resolve_external_ip_url_res(client: &reqwest::Client, url: &str) -> Result { + resolve_external_ip_url(client, url).await.ok_or(()) } -async fn resolve_external_ip_url(url: &str) -> Option { - let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().ok()?; +async fn resolve_external_ip_url(client: &reqwest::Client, url: &str) -> Option { let response = client.get(url).send().await.ok()?; let response = response.error_for_status().ok()?; let text = response.text().await.ok()?; @@ -271,6 +312,7 @@ async fn resolve_external_ip_url(url: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use futures_util::FutureExt; use std::net::{Ipv4Addr, Ipv6Addr}; #[tokio::test] @@ -293,6 +335,67 @@ mod tests { dbg!(ip); } + #[tokio::test(start_paused = true)] + async fn interval_preserves_pending_resolution() { + let period = Duration::from_secs(5); + let next_ip: IpAddr = "203.0.113.7".parse().unwrap(); + for result in [Some("203.0.113.8".parse().unwrap()), None] { + let mut interval = + ResolveNatInterval::interval(NatResolver::ExternalIp(next_ip), period); + assert_eq!(interval.tick().await, Some(next_ip)); + + let (tx, rx) = tokio::sync::oneshot::channel(); + interval.future = Some(Box::pin(async move { rx.await.unwrap() })); + assert!(interval.tick().now_or_never().is_none()); + + tokio::time::advance(period * 3).await; + assert!(interval.tick().now_or_never().is_none()); + tx.send(result).expect("the pending resolution must not be dropped"); + assert_eq!(interval.tick().await, result); + + // The next attempt waits for its interval after either success or failure. + assert!(interval.tick().now_or_never().is_none()); + tokio::time::advance(period).await; + assert_eq!(interval.tick().await, Some(next_ip)); + } + } + + #[tokio::test(start_paused = true)] + async fn interval_skips_missed_attempts() { + let period = Duration::from_secs(5); + let ip: IpAddr = "203.0.113.7".parse().unwrap(); + let mut interval = ResolveNatInterval::interval(NatResolver::ExternalIp(ip), period); + assert_eq!(interval.tick().await, Some(ip)); + + tokio::time::advance(period * 3).await; + assert_eq!(interval.tick().await, Some(ip)); + assert!(interval.tick().now_or_never().is_none()); + tokio::time::advance(period).await; + assert_eq!(interval.tick().await, Some(ip)); + } + + #[test] + fn netif_resolution_does_not_block_the_runtime() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + // Occupy the blocking pool so a lookup must yield until the worker is released. + let blocker = runtime.spawn_blocking(move || { + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }); + runtime.block_on(async { + let mut lookup = std::pin::pin!(NatResolver::NetIf.external_addr()); + let pending = futures_util::poll!(&mut lookup).is_pending(); + release_tx.send(()).unwrap(); + assert!(pending, "network interface lookup must run on a blocking worker"); + assert_eq!(lookup.await, resolve_net_if_ip(DEFAULT_NET_IF_NAME).ok()); + blocker.await.unwrap(); + }); + } + #[test] fn as_external_ip_test() { let resolver = NatResolver::ExternalAddr("localhost".to_string()); From a63d346128473d64167926f7c167b5d9c6c1ac41 Mon Sep 17 00:00:00 2001 From: banteg <4562643+banteg@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:05:30 +0000 Subject: [PATCH 072/133] fix(rpc): omit genesis block reward traces (#27423) --- crates/rpc/rpc/src/trace.rs | 63 +++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 7913bb248ae..eee781fef8e 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -302,6 +302,7 @@ where /// Calculates the base block reward for the given block: /// + /// - the genesis block is not mined, so no block rewards are given /// - if Paris hardfork is activated, no block rewards are given /// - if Paris hardfork is not activated, calculate block rewards with block number only fn calculate_base_block_reward( @@ -310,7 +311,7 @@ where ) -> Result, Eth::Error> { let chain_spec = self.provider().chain_spec(); - if chain_spec.is_paris_active_at_block(header.number()) { + if header.number() == 0 || chain_spec.is_paris_active_at_block(header.number()) { return Ok(None) } @@ -488,7 +489,8 @@ where } else { // Blocks are processed in ascending order, so once a historical range // reaches post-Paris blocks, later blocks in the range have no rewards. - include_reward_traces = false; + // The genesis block has no reward, but later pre-Paris blocks do. + include_reward_traces = block.number() == 0; Vec::new() } } else { @@ -898,11 +900,21 @@ mod tests { use super::*; use crate::EthApiBuilder; use alloy_consensus::Header; + use alloy_genesis::Genesis; use alloy_rpc_types_eth::TransactionRequest; + use reth_chainspec::ChainSpecBuilder; + use reth_db_common::init::init_genesis; use reth_ethereum_primitives::{Block, BlockBody}; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; - use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; + use reth_primitives_traits::Block as _; + use reth_provider::{ + providers::BlockchainProvider, + test_utils::{ + create_test_provider_factory_with_chain_spec, ExtendedAccount, MockEthProvider, + }, + BlockWriter, StageCheckpointWriter, + }; use reth_transaction_pool::test_utils::testing_pool; #[tokio::test] @@ -1196,6 +1208,51 @@ mod tests { } } + #[tokio::test] + async fn genesis_block_has_no_reward_trace() { + // Paris is not active at genesis, so blocks after genesis carry block rewards. + let coinbase = Address::repeat_byte(0x11); + let genesis = Genesis::default().with_gas_limit(30_000_000).with_coinbase(coinbase); + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().genesis(genesis).build()); + let factory = create_test_provider_factory_with_chain_spec(chain_spec); + let genesis_hash = init_genesis(&factory).unwrap(); + let block = Block { + header: Header { + parent_hash: genesis_hash, + number: 1, + beneficiary: coinbase, + gas_limit: 30_000_000, + ..Default::default() + }, + body: BlockBody::default(), + }; + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.insert_block(&block.seal_slow().try_recover().unwrap()).unwrap(); + provider_rw.update_pipeline_stages(1, false).unwrap(); + provider_rw.commit().unwrap(); + + let provider = BlockchainProvider::new(factory).unwrap(); + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + let api = TraceApi::new(eth_api, BlockingTaskGuard::new(1), EthConfig::default()); + + assert_eq!(api.trace_block(0.into()).await.unwrap(), Some(vec![])); + assert_eq!( + trace_order(&api.trace_block(1.into()).await.unwrap().unwrap()), + [(1, None, true)] + ); + for (to_block, expected) in [(0, vec![]), (1, vec![(1, None, true)])] { + let filter = + TraceFilter { from_block: Some(0), to_block: Some(to_block), ..Default::default() }; + assert_eq!(trace_order(&api.trace_filter(filter).await.unwrap()), expected); + } + } + fn localized_transaction_trace( block_number: u64, transaction_position: u64, From 67b65de2b78fb7a918fffd6d32949131f5cec6cf Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:43:20 +0000 Subject: [PATCH 073/133] fix(net): resolve the external ip when discv4 is disabled (#27335) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 1 + crates/net/network/Cargo.toml | 1 + crates/net/network/src/discovery.rs | 158 ++++++++++++++++++++++++- crates/net/network/src/manager.rs | 1 + crates/net/network/tests/it/startup.rs | 4 +- 5 files changed, 162 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9abbe664105..b4c6c9b7f65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9354,6 +9354,7 @@ dependencies = [ "reth-fs-util", "reth-metrics", "reth-net-banlist", + "reth-net-nat", "reth-network", "reth-network-api", "reth-network-p2p", diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index a8fca7bcd8f..bf8ba554ebf 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -22,6 +22,7 @@ reth-network-api.workspace = true reth-network-p2p.workspace = true reth-discv4.workspace = true reth-discv5.workspace = true +reth-net-nat.workspace = true reth-dns-discovery.workspace = true reth-ethereum-forks.workspace = true reth-evm.workspace = true diff --git a/crates/net/network/src/discovery.rs b/crates/net/network/src/discovery.rs index 0006ad45ac3..bdaa96cb323 100644 --- a/crates/net/network/src/discovery.rs +++ b/crates/net/network/src/discovery.rs @@ -12,6 +12,7 @@ use reth_dns_discovery::{ DnsDiscoveryConfig, DnsDiscoveryHandle, DnsDiscoveryService, DnsNodeRecordUpdate, DnsResolver, }; use reth_ethereum_forks::{EnrForkIdEntry, ForkId}; +use reth_net_nat::{NatResolver, ResolveNatInterval}; use reth_network_api::{DiscoveredEvent, DiscoveryEvent}; use reth_network_peers::{NodeRecord, PeerId}; use reth_network_types::PeerAddr; @@ -22,6 +23,7 @@ use std::{ pin::Pin, sync::Arc, task::{ready, Context, Poll}, + time::Duration, }; use tokio::{net::UdpSocket, sync::mpsc, task::JoinHandle}; use tokio_stream::{wrappers::ReceiverStream, Stream}; @@ -32,6 +34,11 @@ use tracing::{debug, trace}; /// Default is 10 000 peers. pub const DEFAULT_MAX_CAPACITY_DISCOVERED_PEERS_CACHE: u32 = 10_000; +/// How often discv5-only nodes refresh their external IP. +/// +/// Matches the default interval of discv4. +const RESOLVE_EXTERNAL_IP_INTERVAL: Duration = Duration::from_secs(60 * 5); + /// An abstraction over the configured discovery protocol. /// /// Listens for new discovered nodes and emits events for discovered nodes and their @@ -67,13 +74,18 @@ pub struct Discovery { queued_events: VecDeque, /// List of listeners subscribed to discovery events. discovery_listeners: Vec>, + /// Resolves the external IP for discv5 when discv4 is disabled. + nat_resolver: Option, } impl Discovery { /// Spawns the discovery service. /// /// This will spawn the [`reth_discv4::Discv4Service`] onto a new task and establish a listener - /// channel to receive all discovered nodes. + /// channel to receive all discovered nodes. When only discv5 is enabled, `nat` periodically + /// resolves its advertised external IP without waiting for peer votes. When ENR updates are + /// disabled or discv5 checks inbound connectivity, only explicit `extip` or `extaddr` settings + /// are applied. pub async fn new( tcp_addr: SocketAddr, discovery_v4_addr: SocketAddr, @@ -81,6 +93,7 @@ impl Discovery { discv4_config: Option, mut discv5_config: Option, // contains discv5 listen address dns_discovery_config: Option, + nat: Option, ) -> Result { // setup discv4 with the discovery address and tcp port let local_enr = @@ -159,6 +172,24 @@ impl Discovery { (None, None, None, None, None) }; + // Discv4 resolves its own external IP when enabled. + let nat_resolver = if discv4.is_none() && + let Some(config) = &mut discv5_config + { + let config = config.discv5_config_mut(); + nat.filter(|nat| match nat { + NatResolver::None => false, + // Use the address or hostname explicitly configured by the operator. + NatResolver::ExternalIp(_) | NatResolver::ExternalAddr(_) => true, + // Preserve manual addresses and let discv5 handle reachability checks. Its check + // for incoming connections starts when peer votes change the advertised address. + _ => config.enr_update && config.auto_nat_listen_duration.is_none(), + }) + .map(|nat| ResolveNatInterval::interval(nat, RESOLVE_EXTERNAL_IP_INTERVAL)) + } else { + None + }; + // Start discv5, wiring in the shared socket if in shared-port mode. let (discv5, discv5_updates) = if let Some(mut config) = discv5_config { // Set OS-assigned advertised RLPx ports to the bound listener port. @@ -243,6 +274,7 @@ impl Discovery { _dns_disc_service, _dns_discovery, dns_discovery_updates, + nat_resolver, }) } @@ -400,11 +432,45 @@ impl Discovery { self.on_node_record_update(update.node_record, update.fork_id); } + while let Some(Poll::Ready(ip)) = + self.nat_resolver.as_mut().map(|resolver| resolver.poll_tick(cx)) + { + if let Some(ip) = ip { + self.on_external_ip(ip); + } + } + if self.queued_events.is_empty() { return Poll::Pending } } } + + /// Updates discv5's advertised IP without changing ports or address families. + /// + /// Discv4 owns its NAT resolution and updates its own node record and ENR, so only + /// discv5 needs an update here. + fn on_external_ip(&self, external_ip: IpAddr) { + let Some(discv5) = &self.discv5 else { return }; + let enr = discv5.local_enr(); + // TCP and UDP share the ENR IP fields. Preserve their ports, including ports learned + // from peer votes, and only update address families the service is listening on. + let result = match external_ip { + IpAddr::V4(ip) if enr.udp4().is_some() && enr.ip4() != Some(ip) => { + discv5.with_discv5(|discv5| discv5.enr_insert("ip", &ip)) + } + IpAddr::V6(ip) if enr.udp6().is_some() && enr.ip6() != Some(ip) => { + discv5.with_discv5(|discv5| discv5.enr_insert("ip6", &ip)) + } + _ => return, + }; + match result { + Ok(_) => debug!(target: "net::discovery", ?external_ip, "Updated external IP"), + Err(err) => { + debug!(target: "net::discovery", ?external_ip, %err, "Failed to update external IP"); + } + } + } } const fn set_bound_rlpx_port_if_unset(config: &mut reth_discv5::Config, port: u16) { @@ -466,6 +532,7 @@ impl Discovery { dns_discovery_updates: None, _dns_disc_service: None, discovery_listeners: Default::default(), + nat_resolver: None, } } } @@ -487,6 +554,7 @@ mod tests { Default::default(), None, Default::default(), + None, ) .await .unwrap(); @@ -517,11 +585,97 @@ mod tests { Some(discv4_config), Some(discv5_config), None, + None, ) .await .expect("should build discv5 with discv4 downgrade") } + async fn start_discv5_only(config: reth_discv5::Config, nat: Option) -> Discovery { + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); + Discovery::new( + "127.0.0.1:30303".parse().unwrap(), + "127.0.0.1:0".parse().unwrap(), + secret_key, + None, + Some(config), + None, + nat, + ) + .await + .expect("should start discv5") + } + + #[tokio::test] + async fn discv5_only_advertises_resolved_external_ip() { + let socket = Arc::new(UdpSocket::bind("0.0.0.0:0").await.unwrap()); + let udp_port = socket.local_addr().unwrap().port(); + let external_ip = "203.0.113.7".parse().unwrap(); + // The advertised TCP port may differ from the listener port behind NAT. + let mut config = reth_discv5::Config::builder("127.0.0.1:30309".parse().unwrap()).build(); + // Install pre-bound sockets after the builder normalizes its listen addresses. + let discv5_config = config.discv5_config_mut(); + discv5_config.listen_config = + discv5::ListenConfig::FromSockets { ipv4: Some(socket), ipv6: None }; + // Explicit NAT addresses still apply when peer-driven ENR updates are disabled. + discv5_config.enr_update = false; + let mut discovery = + start_discv5_only(config, Some(NatResolver::ExternalIp(external_ip))).await; + let discv5 = discovery.discv5().unwrap(); + assert!(discv5.local_enr().ip4().is_none()); + + tokio::time::timeout( + Duration::from_secs(5), + futures::future::poll_fn(|cx| { + let _ = discovery.poll(cx); + if discv5.node_record().is_some_and(|record| record.address == external_ip) { + Poll::Ready(()) + } else { + Poll::Pending + } + }), + ) + .await + .expect("external IP should be advertised"); + + let record = discv5.node_record().unwrap(); + assert_eq!(record.address, external_ip); + assert_eq!(record.udp_port, udp_port); + assert_eq!(record.tcp_port, 30309); + + let seq = discv5.local_enr().seq(); + discovery.on_external_ip(external_ip); + assert_eq!(discv5.local_enr().seq(), seq); + discovery.on_external_ip("2001:db8::7".parse().unwrap()); + assert_eq!(discv5.local_enr().seq(), seq); + + // Peer votes can change the address and mapped UDP port between NAT resolutions. + discv5.with_discv5(|discv5| { + discv5.update_local_enr_socket((Ipv4Addr::LOCALHOST, 30310).into(), false); + }); + discovery.on_external_ip(external_ip); + assert_eq!(discv5.node_record().unwrap(), NodeRecord { udp_port: 30310, ..record }); + } + + #[tokio::test] + async fn discv5_only_respects_enr_update_policy() { + for (enr_update, auto_nat_listen_duration) in + [(false, None), (true, Some(Duration::from_secs(300)))] + { + let mut config = reth_discv5::Config::builder((Ipv4Addr::LOCALHOST, 0).into()).build(); + let discv5_config = config.discv5_config_mut(); + discv5_config.listen_config = + discv5::ListenConfig::Ipv4 { ip: Ipv4Addr::LOCALHOST, port: 0 }; + discv5_config.enr_update = enr_update; + // Set this after building: the upstream builder clears the reachability timer. + discv5_config.auto_nat_listen_duration = auto_nat_listen_duration; + let discovery = start_discv5_only(config, Some(NatResolver::Any)).await; + + // No automatic lookup may overwrite a pinned address or bypass reachability checks. + assert!(discovery.nat_resolver.is_none()); + } + } + #[test] fn discv5_enr_advertises_bound_rlpx_port() { let secret_key = SecretKey::new(&mut rand_08::thread_rng()); @@ -659,6 +813,7 @@ mod tests { Some(discv4_config), Some(discv5_config), None, + None, ) .await .expect("should start with shared port") @@ -806,6 +961,7 @@ mod tests { Some(discv4_config), Some(discv5_config), None, + None, ) .await .expect("discovery should start with shared port + dual-stack"); diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index 63255c41daa..733f9ef6ba4 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -298,6 +298,7 @@ impl NetworkManager { discovery_v4_config, discovery_v5_config, dns_discovery_config, + nat.clone(), ) .await?; // need to retrieve the addr here since provided port could be `0` diff --git a/crates/net/network/tests/it/startup.rs b/crates/net/network/tests/it/startup.rs index 46fb714d2aa..1574a2e03f0 100644 --- a/crates/net/network/tests/it/startup.rs +++ b/crates/net/network/tests/it/startup.rs @@ -66,9 +66,9 @@ async fn test_discovery_addr_in_use() { let port = any_port_listener.local_addr().unwrap().port(); let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, port)); let _discovery = - Discovery::new(addr, addr, secret_key, Some(disc_config), None, None).await.unwrap(); + Discovery::new(addr, addr, secret_key, Some(disc_config), None, None, None).await.unwrap(); let disc_config = Discv4Config::default(); - let result = Discovery::new(addr, addr, secret_key, Some(disc_config), None, None).await; + let result = Discovery::new(addr, addr, secret_key, Some(disc_config), None, None, None).await; assert!(is_addr_in_use_kind(&result.err().unwrap(), ServiceKind::Discovery(addr))); } From 4630cc588430fcf3b2846e27100a5d10cff9f949 Mon Sep 17 00:00:00 2001 From: valvecitydev <271573222+valvecitydev@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:18:40 +0000 Subject: [PATCH 074/133] fix(rpc): resolve pending state before blocking (#27415) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 9 +- .../rpc/rpc-eth-api/src/helpers/estimate.rs | 3 +- .../rpc-eth-api/src/helpers/pending_block.rs | 86 +++++++-- crates/rpc/rpc-eth-api/src/helpers/state.rs | 102 +++++++---- crates/rpc/rpc/src/eth/helpers/state.rs | 173 +++++++++++++++++- 5 files changed, 310 insertions(+), 63 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index d7e0fe40e14..e323aaec0bb 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -464,10 +464,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let block_id = block_number.unwrap_or_default(); let (evm_env, at) = self.evm_env_at(block_id).await?; - self.spawn_blocking_io_fut(async move |this| { - this.create_access_list_with(evm_env, at, request, state_override).await - }) - .await + self.create_access_list_with(evm_env, at, request, state_override).await } } @@ -647,9 +644,7 @@ pub trait Call: F: FnOnce(Self, StateCacheDb) -> Result + Send + 'static, R: Send + 'static, { - let at = at.into(); - self.spawn_blocking_io_fut(async move |this| { - let state = this.state_at_block_id(at).await?; + self.spawn_blocking_io_with_state(at.into(), move |this, state| { let db = State::builder() .with_database(StateProviderDatabase::new(state.into_evm_state_provider())) .build(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs index faa5b3bd9ca..7f4fe4f5b71 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs @@ -313,8 +313,7 @@ pub trait EstimateCall: Call { async move { let (evm_env, at) = self.evm_env_at(at).await?; - self.spawn_blocking_io_fut(async move |this| { - let state = this.state_at_block_id(at).await?; + self.spawn_blocking_io_with_state(at, move |this, state| { EstimateCall::estimate_gas_with( &this, evm_env, diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 1a63562f58f..ddecf65fcd7 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -1,5 +1,5 @@ -//! Loads a pending block from database. Helper trait for `eth_` block, transaction, call and trace -//! RPC methods. +//! Loads provider pending blocks or builds them from the transaction pool for `eth_` block, +//! transaction, call and trace RPC methods. use super::SpawnBlocking; use crate::{EthApiTypes, FromEthApiError, FromEvmError, RpcNodeCore}; @@ -16,7 +16,9 @@ use reth_evm::{ execute::{BlockBuilder, BlockBuilderOutcome, BlockExecutionOutput}, ConfigureEvm, Evm, EvmEnvFor, NextBlockEnvAttributes, }; -use reth_primitives_traits::{transaction::error::InvalidTransactionError, HeaderTy, SealedHeader}; +use reth_primitives_traits::{ + transaction::error::InvalidTransactionError, HeaderTy, NodePrimitives, SealedHeader, +}; use reth_revm::{database::StateProviderDatabase, db::State}; use reth_rpc_convert::RpcConvert; use reth_rpc_eth_types::{ @@ -40,7 +42,7 @@ use std::{ use tokio::sync::Mutex; use tracing::debug; -/// Loads a pending block from database. +/// Loads a provider pending block or builds one from the transaction pool. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` blocks RPC methods. pub trait LoadPendingBlock: @@ -57,14 +59,15 @@ pub trait LoadPendingBlock: /// Returns a [`PendingEnvBuilder`] for the pending block. fn pending_env_builder(&self) -> &dyn PendingEnvBuilder; - /// Returns the pending block kind + /// Returns the pending block kind. fn pending_block_kind(&self) -> PendingBlockKind { self.eth_api_settings().pending_block_kind } - /// Configures the [`PendingBlockEnv`] for the pending block + /// Returns the pending block environment and its origin. /// - /// If no pending block is available, this will derive it from the `latest` block + /// Uses the provider's pending block when available; otherwise derives the next block + /// environment from the latest header. This method performs synchronous provider reads. fn pending_block_env_and_cfg(&self) -> Result, Self::Error> { if let Some((block, receipts)) = self.provider().pending_block_and_receipts().map_err(Self::Error::from_eth_err)? @@ -110,6 +113,10 @@ pub trait LoadPendingBlock: } /// Returns a [`StateProviderBox`] on a mem-pool built pending block overlaying latest. + /// + /// This is used by `state_at_block_id` and, by default, by + /// [`Self::local_pending_block_or_state`]. This method's default opens the provider state on a + /// blocking task after building the pending block. fn local_pending_state( &self, ) -> impl Future, Self::Error>> + Send @@ -121,18 +128,40 @@ pub trait LoadPendingBlock: return Ok(None); }; - Ok(Some( - self.provider() - .state_with_block_appended( - pending_block.block().parent_hash(), - pending_block.executed_block, - ) - .map_err(Self::Error::from_eth_err)?, - )) + self.spawn_blocking_io(move |this| { + Ok(Some( + this.provider() + .state_with_block_appended( + pending_block.block().parent_hash(), + pending_block.executed_block, + ) + .map_err(Self::Error::from_eth_err)?, + )) + }) + .await } } - /// Returns a mem-pool built pending block. + /// Selects the local pending source for state RPCs. + /// + /// By default, delegates to [`Self::local_pending_state`] so existing chain-specific overrides + /// continue to apply. Implementations can return [`PendingStateSource::Block`] to defer opening + /// the block's state provider until the state RPC's blocking task. Returning `None` falls back + /// to the provider's pending state. + fn local_pending_block_or_state( + &self, + ) -> impl Future>, Self::Error>> + Send + where + Self: SpawnBlocking, + { + async move { Ok(self.local_pending_state().await?.map(PendingStateSource::State)) } + } + + /// Returns a cached or newly built pending block from the transaction pool. + /// + /// Returns `None` if local pending blocks are disabled or the provider has an actual pending + /// block. Resolves the environment before awaiting the cache lock, then may spawn a blocking + /// task to build the block. Callers must await this outside a blocking task. fn pool_pending_block( &self, ) -> impl Future>, Self::Error>> + Send @@ -153,10 +182,10 @@ pub trait LoadPendingBlock: } } - /// Builds or returns a cached pending block from the transaction pool. + /// Returns a cached or newly built pending block for the given parent and environment. /// - /// This is the shared implementation used by both [`Self::pool_pending_block`] and - /// [`Self::local_pending_block`] to avoid resolving the pending block environment twice. + /// Holds the async cache lock while rebuilding, which may spawn blocking work. The resolved + /// parent and environment let callers reuse the result of their provider lookup. fn build_pool_pending_block( &self, parent: SealedHeader>, @@ -207,7 +236,7 @@ pub trait LoadPendingBlock: } } - /// Returns the locally built pending block + /// When enabled, returns the provider's pending block or builds one from the pool. fn local_pending_block( &self, ) -> impl Future>, Self::Error>> + Send @@ -446,6 +475,23 @@ pub trait LoadPendingBlock: } } +/// Local pending state selected for a state RPC. +pub enum PendingStateSource { + /// A block whose state provider must be opened on the state RPC's blocking task. + Block(PendingBlock), + /// A chain-specific state provider that is already open. + State(StateProviderBox), +} + +impl std::fmt::Debug for PendingStateSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Block(_) => f.write_str("PendingStateSource::Block(..)"), + Self::State(_) => f.write_str("PendingStateSource::State(..)"), + } + } +} + /// A type that knows how to build a [`ConfigureEvm::NextBlockEnvCtx`] for a pending block. pub trait PendingEnvBuilder: Send + Sync + Unpin + 'static { /// Builds a [`ConfigureEvm::NextBlockEnvCtx`] for a pending block. diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 5679706c53b..ecf933eacac 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -1,9 +1,11 @@ //! Loads a pending block from database. Helper trait for `eth_` block, transaction, call and trace //! RPC methods. -use super::{EthApiSpec, LoadBlock, LoadPendingBlock, SpawnBlocking}; +use super::{ + pending_block::PendingStateSource, EthApiSpec, LoadBlock, LoadPendingBlock, SpawnBlocking, +}; use crate::{EthApiTypes, FromEthApiError, RpcNodeCore, RpcNodeCoreExt}; -use alloy_consensus::constants::KECCAK_EMPTY; +use alloy_consensus::{constants::KECCAK_EMPTY, BlockHeader}; use alloy_eips::BlockId; use alloy_primitives::{keccak256, Address, Bytes, B256, U256}; use alloy_rpc_types_eth::{Account, AccountInfo, EIP1186AccountProofResponse}; @@ -79,10 +81,8 @@ pub trait EthState: LoadState + SpawnBlocking { address: Address, block_id: Option, ) -> impl Future> + Send { - self.spawn_blocking_io_fut(async move |this| { - Ok(this - .state_at_block_id_or_latest(block_id) - .await? + self.spawn_blocking_io_with_state(block_id.unwrap_or_default(), move |_, state| { + Ok(state .account_balance(&address) .map_err(Self::Error::from_eth_err)? .unwrap_or_default()) @@ -96,10 +96,9 @@ pub trait EthState: LoadState + SpawnBlocking { index: JsonStorageKey, block_id: Option, ) -> impl Future> + Send { - self.spawn_blocking_io_fut(async move |this| { + self.spawn_blocking_io_with_state(block_id.unwrap_or_default(), move |_, state| { Ok(B256::new( - this.state_at_block_id_or_latest(block_id) - .await? + state .storage(address, index.as_b256()) .map_err(Self::Error::from_eth_err)? .unwrap_or_default() @@ -132,9 +131,7 @@ pub trait EthState: LoadState + SpawnBlocking { ))); } - self.spawn_blocking_io_fut(async move |this| { - let state = this.state_at_block_id_or_latest(block_id).await?; - + self.spawn_blocking_io_with_state(block_id.unwrap_or_default(), move |_, state| { let mut result = HashMap::with_capacity(requests.len()); for (address, slots) in requests { let mut values = Vec::with_capacity(slots.len()); @@ -177,9 +174,8 @@ pub trait EthState: LoadState + SpawnBlocking { let block_id = block_id.unwrap_or_default(); self.ensure_within_proof_window(block_id)?; - self.spawn_blocking_io_fut(move |this| async move { + self.spawn_blocking_io_with_state(block_id, move |_, state| { let _permit = permit; - let state = this.state_at_block_id(block_id).await?; let storage_keys = keys.iter().map(|key| key.as_b256()).collect::>(); let proof = state .proof(Default::default(), address, &storage_keys) @@ -212,9 +208,8 @@ pub trait EthState: LoadState + SpawnBlocking { let block_id = block_id.unwrap_or_default(); self.ensure_within_proof_window(block_id)?; - self.spawn_blocking_io_fut(move |this| async move { + self.spawn_blocking_io_with_state(block_id, move |_, state| { let _permit = permit; - let state = this.state_at_block_id(block_id).await?; let mut proof_targets = MultiProofTargetsV2::default(); proof_targets.account_targets.reserve(targets.len()); proof_targets.storage_targets.reserve(targets.len()); @@ -261,8 +256,7 @@ pub trait EthState: LoadState + SpawnBlocking { async move { self.ensure_within_proof_window(block_id)?; - self.spawn_blocking_io_fut(async move |this| { - let state = this.state_at_block_id(block_id).await?; + self.spawn_blocking_io_with_state(block_id, move |_, state| { let account = state.basic_account(&address).map_err(Self::Error::from_eth_err)?; let Some(account) = account else { return Ok(None) }; @@ -288,8 +282,7 @@ pub trait EthState: LoadState + SpawnBlocking { address: Address, block_id: BlockId, ) -> impl Future> + Send { - self.spawn_blocking_io_fut(async move |this| { - let state = this.state_at_block_id(block_id).await?; + self.spawn_blocking_io_with_state(block_id, move |_, state| { let account = state .basic_account(&address) .map_err(Self::Error::from_eth_err)? @@ -327,10 +320,15 @@ pub trait LoadState: self.provider().history_by_block_hash(block_hash).map_err(Self::Error::from_eth_err) } - /// Returns the state at the given [`BlockId`] enum. + /// Returns the state at the given [`BlockId`], preferring locally built state for `pending`. + /// + /// If local pending state is unavailable or fails to build, falls back to the provider's + /// pending state. Other block IDs resolve only canonical state. See + /// . /// - /// Note: if not [`BlockNumberOrTag::Pending`](alloy_eips::BlockNumberOrTag) then this - /// will only return canonical state. See also + /// Pending block construction may spawn blocking work, so await this outside a blocking task. + /// The fallback provider lookup is synchronous on the calling task; RPC handlers should use + /// [`Self::spawn_blocking_io_with_state`] to keep that read on the blocking pool. fn state_at_block_id( &self, at: BlockId, @@ -373,6 +371,49 @@ pub trait LoadState: } } + /// Executes `f` with the state at the given [`BlockId`] on a blocking IO task. + /// + /// Resolves the chain's pending-state source before occupying a blocking thread, since + /// building a pending block may need a blocking thread of its own. + fn spawn_blocking_io_with_state( + &self, + at: BlockId, + f: F, + ) -> impl Future> + Send + where + Self: SpawnBlocking, + F: FnOnce(Self, StateProviderBox) -> Result + Send + 'static, + R: Send + 'static, + { + async move { + let pending = if at.is_pending() { + self.local_pending_block_or_state().await.ok().flatten() + } else { + None + }; + + self.spawn_blocking_io(move |this| { + let state = match pending { + Some(PendingStateSource::Block(pending)) => this + .provider() + .state_with_block_appended( + pending.block().parent_hash(), + pending.executed_block, + ) + .or_else(|_| this.provider().state_by_block_id(BlockId::pending())) + .map_err(Self::Error::from_eth_err)?, + Some(PendingStateSource::State(state)) => state, + None if at.is_latest() => this.latest_state()?, + None => { + this.provider().state_by_block_id(at).map_err(Self::Error::from_eth_err)? + } + }; + f(this, state) + }) + .await + } + } + /// Returns the EVM environment for the given sealed header. fn evm_env_for_header( &self, @@ -511,16 +552,15 @@ pub trait LoadState: where Self: SpawnBlocking, { - self.spawn_blocking_io_fut(async move |this| { + let at = block_id.unwrap_or_default(); + self.spawn_blocking_io_with_state(at, move |this, state| { // first fetch the on chain nonce of the account - let on_chain_account_nonce = this - .state_at_block_id_or_latest(block_id) - .await? + let on_chain_account_nonce = state .account_nonce(&address) .map_err(Self::Error::from_eth_err)? .unwrap_or_default(); - if block_id == Some(BlockId::pending()) { + if at.is_pending() { // for pending tag we need to find the highest nonce of txn in the pending state. if let Some(highest_pool_tx) = this .pool() @@ -557,10 +597,8 @@ pub trait LoadState: where Self: SpawnBlocking, { - self.spawn_blocking_io_fut(async move |this| { - Ok(this - .state_at_block_id_or_latest(block_id) - .await? + self.spawn_blocking_io_with_state(block_id.unwrap_or_default(), move |_, state| { + Ok(state .account_code(&address) .map_err(Self::Error::from_eth_err)? .unwrap_or_default() diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 390424db3ed..ee30428095b 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -29,19 +29,34 @@ mod tests { use crate::eth::helpers::types::EthRpcConverter; use super::*; + use alloy_eips::BlockId; use alloy_primitives::{ map::{AddressMap, B256Map}, - Address, StorageKey, StorageValue, U256, + Address, StorageKey, StorageValue, B256, U256, }; + use alloy_rpc_types_eth::TransactionRequest; use reth_chainspec::ChainSpec; + use reth_ethereum_primitives::Block; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; use reth_provider::{ test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}, ChainSpecProvider, }; - use reth_rpc_eth_api::{helpers::EthState, node::RpcNodeCoreAdapter}; + use reth_rpc_eth_api::{ + helpers::{pending_block::PendingEnvBuilder, EthCall, EthState, SpawnBlocking}, + node::{RpcNodeCoreAdapter, RpcNodeCoreExt}, + EthApiTypes, + }; + use reth_rpc_eth_types::{EthApiSettings, EthStateCache, PendingBlock}; + use reth_storage_api::{StateProviderBox, StateProviderFactory}; + use reth_tasks::{ + pool::{BlockingTaskGuard, BlockingTaskPool}, + Runtime, + }; use reth_transaction_pool::test_utils::{testing_pool, TestPool}; + use std::{future::Future, sync::Arc, time::Duration}; + use tokio::sync::{Mutex, Semaphore}; fn noop_eth_api() -> EthApi< RpcNodeCoreAdapter, @@ -100,4 +115,158 @@ mod tests { let account = eth_api.get_account(address, Default::default()).await.unwrap(); assert!(account.is_none()); } + + #[test] + fn pending_state_and_access_list_do_not_deadlock() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + + let result = runtime.block_on(async { + let address = Address::random(); + let accounts = + AddressMap::from_iter([(address, ExtendedAccount::new(0, U256::from(1337)))]); + let eth_api = mock_eth_api(accounts); + // Use a block after access-list transactions became valid. + let mut block = Block::default(); + block.header.number = 13_000_000; + block.header.timestamp = 1_629_000_000; + block.header.gas_limit = 30_000_000; + block.header.base_fee_per_gas = Some(1_000_000_000); + eth_api.provider().add_block(B256::ZERO, block); + tokio::time::timeout(Duration::from_secs(10), async { + let balance = eth_api.balance(address, Some(BlockId::pending())).await?; + eth_api + .create_access_list_at( + TransactionRequest { gas_price: Some(0), ..Default::default() }, + Some(BlockId::latest()), + None, + ) + .await?; + Ok::<_, EthApiError>(balance) + }) + .await + }); + // A deadlocked blocking thread would also block a regular runtime drop. + runtime.shutdown_background(); + assert_eq!( + result.expect("RPC timed out on one blocking thread").expect("RPC returned an error"), + U256::from(1337) + ); + } + + type MockEthApi = EthApi< + RpcNodeCoreAdapter, + EthRpcConverter, + >; + + #[derive(Clone)] + struct CustomPendingState { + inner: MockEthApi, + pending: MockEthProvider, + } + + impl EthApiTypes for CustomPendingState { + type Error = EthApiError; + type NetworkTypes = ::NetworkTypes; + type RpcConvert = ::RpcConvert; + + fn eth_api_settings(&self) -> &EthApiSettings { + self.inner.eth_api_settings() + } + + fn converter(&self) -> &Self::RpcConvert { + self.inner.converter() + } + } + + impl RpcNodeCore for CustomPendingState { + type Primitives = ::Primitives; + type Provider = ::Provider; + type Pool = ::Pool; + type Evm = ::Evm; + type Network = ::Network; + + fn pool(&self) -> &Self::Pool { + self.inner.pool() + } + + fn evm_config(&self) -> &Self::Evm { + self.inner.evm_config() + } + + fn network(&self) -> &Self::Network { + self.inner.network() + } + + fn provider(&self) -> &Self::Provider { + self.inner.provider() + } + } + + impl RpcNodeCoreExt for CustomPendingState { + fn cache(&self) -> &EthStateCache { + self.inner.cache() + } + } + + impl SpawnBlocking for CustomPendingState { + fn io_task_spawner(&self) -> &Runtime { + self.inner.io_task_spawner() + } + + fn tracing_task_pool(&self) -> &BlockingTaskPool { + self.inner.tracing_task_pool() + } + + fn tracing_task_guard(&self) -> &BlockingTaskGuard { + self.inner.tracing_task_guard() + } + + fn blocking_io_task_guard(&self) -> &Arc { + self.inner.blocking_io_task_guard() + } + } + + impl LoadPendingBlock for CustomPendingState { + fn pending_block(&self) -> &Mutex>> { + self.inner.pending_block() + } + + fn pending_env_builder(&self) -> &dyn PendingEnvBuilder { + self.inner.pending_env_builder() + } + + fn local_pending_state( + &self, + ) -> impl Future, Self::Error>> + Send { + let state = self.pending.latest().map_err(EthApiError::from); + async move { state.map(Some) } + } + } + + impl LoadState for CustomPendingState {} + + impl EthState for CustomPendingState {} + + #[tokio::test] + async fn pending_state_reads_use_existing_override() { + let address = Address::random(); + let chain = AddressMap::from_iter([(address, ExtendedAccount::new(0, U256::from(1337)))]); + let eth_api = mock_eth_api(chain); + eth_api.provider().add_block(B256::ZERO, Block::default()); + + let pending = MockEthProvider::default(); + pending.extend_accounts([(address, ExtendedAccount::new(0, U256::from(42)))]); + let eth_api = CustomPendingState { inner: eth_api, pending }; + + assert_eq!( + eth_api.balance(address, Some(BlockId::pending())).await.unwrap(), + U256::from(42) + ); + assert_eq!(eth_api.balance(address, None).await.unwrap(), U256::from(1337)); + } } From eb1214b550d6b58bef70e64a5f72a6e23bde5586 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 07:28:03 +0000 Subject: [PATCH 075/133] perf: share pending block and receipts (#27427) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/chain-state/src/in_memory.rs | 34 +++++------ crates/evm/execution-types/src/execute.rs | 37 +++++++++++- crates/rpc/rpc-eth-api/src/helpers/block.rs | 14 +++-- .../rpc-eth-api/src/helpers/pending_block.rs | 6 +- crates/rpc/rpc-eth-types/src/block.rs | 60 +++++++++++++++---- crates/rpc/rpc-eth-types/src/cache/mod.rs | 7 ++- crates/rpc/rpc-eth-types/src/pending_block.rs | 11 ++-- .../src/providers/blockchain_provider.rs | 18 +++--- .../provider/src/providers/consistent.rs | 5 +- .../provider/src/providers/database/mod.rs | 5 +- .../src/providers/database/provider.rs | 9 ++- .../src/providers/static_file/manager.rs | 5 +- .../storage/provider/src/test_utils/mock.rs | 6 +- crates/storage/rpc-provider/src/lib.rs | 10 ++-- crates/storage/storage-api/src/block.rs | 22 +++---- crates/storage/storage-api/src/noop.rs | 6 +- 16 files changed, 169 insertions(+), 86 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index 7f2ad5ac92c..1e6b2af6338 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -11,6 +11,7 @@ use reth_chainspec::ChainInfo; use reth_ethereum_primitives::EthPrimitives; use reth_execution_types::{ BlockExecutionOutput, BlockExecutionResult, Chain, DecodedRevmBal, ExecutionOutcome, + RecoveredBlockAndExecutionOutput, }; use reth_metrics::{metrics::Gauge, Metrics}; use reth_primitives_traits::{ @@ -168,9 +169,6 @@ impl CanonicalInMemoryStateInner { } } -type PendingBlockAndReceipts = - (RecoveredBlock<::Block>, Vec>); - /// This type is responsible for providing the blocks, receipts, and state for /// all canonical blocks not on disk yet and keeps track of the block range that /// is in memory. @@ -498,20 +496,18 @@ impl CanonicalInMemoryState { } /// Returns the `RecoveredBlock` corresponding to the pending state. - pub fn pending_recovered_block(&self) -> Option> - where - N::SignedTx: SignedTransaction, - { - self.pending_state().map(|block_state| block_state.block_ref().recovered_block().clone()) + pub fn pending_recovered_block(&self) -> Option>> { + self.pending_state().map(|block_state| Arc::clone(&block_state.block_ref().recovered_block)) } - /// Returns a tuple with the `SealedBlock` corresponding to the pending - /// state and a vector of its `Receipt`s. - pub fn pending_block_and_receipts(&self) -> Option> { + /// Returns the pending recovered block and its execution output, which contains the receipts. + pub fn pending_block_and_receipts( + &self, + ) -> Option> { self.pending_state().map(|block_state| { - ( - block_state.block_ref().recovered_block().clone(), - block_state.executed_block_receipts(), + RecoveredBlockAndExecutionOutput::new( + Arc::clone(&block_state.block_ref().recovered_block), + Arc::clone(&block_state.block_ref().execution_output), ) }) } @@ -1254,13 +1250,13 @@ mod tests { ); // Check the pending block with senders - assert_eq!(state.pending_recovered_block().unwrap(), block2.recovered_block().clone()); + let pending_block = state.pending_recovered_block().unwrap(); + assert!(Arc::ptr_eq(&pending_block, &block2.recovered_block)); // Check the pending block and receipts - assert_eq!( - state.pending_block_and_receipts().unwrap(), - (block2.recovered_block().clone(), vec![]) - ); + let pending = state.pending_block_and_receipts().unwrap(); + assert!(Arc::ptr_eq(pending.block(), &block2.recovered_block)); + assert!(Arc::ptr_eq(pending.execution_output(), &block2.execution_output)); } #[test] diff --git a/crates/evm/execution-types/src/execute.rs b/crates/evm/execution-types/src/execute.rs index a5149b8f466..572349cb50a 100644 --- a/crates/evm/execution-types/src/execute.rs +++ b/crates/evm/execution-types/src/execute.rs @@ -1,5 +1,6 @@ +use alloc::sync::Arc; use alloy_primitives::{Address, B256, U256}; -use reth_primitives_traits::{Account, Bytecode}; +use reth_primitives_traits::{Account, Block, Bytecode, RecoveredBlock}; use revm::database::{states::BundleState, BundleAccount}; pub use alloy_evm::block::BlockExecutionResult; @@ -63,3 +64,37 @@ impl Default for BlockExecutionOutput { } } } + +/// A recovered block and its shared execution output. +/// +/// The execution output contains the block's receipts. +#[derive(Debug)] +pub struct RecoveredBlockAndExecutionOutput { + block: Arc>, + execution_output: Arc>, +} + +impl RecoveredBlockAndExecutionOutput { + /// Creates a block with its execution output. + pub const fn new( + block: Arc>, + execution_output: Arc>, + ) -> Self { + Self { block, execution_output } + } + + /// Returns the recovered block. + pub const fn block(&self) -> &Arc> { + &self.block + } + + /// Returns the execution output. + pub const fn execution_output(&self) -> &Arc> { + &self.execution_output + } + + /// Consumes the container and returns its shared parts. + pub fn into_parts(self) -> (Arc>, Arc>) { + (self.block, self.execution_output) + } +} diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index f2965d0051b..1a0269985bd 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -14,6 +14,7 @@ use futures::Future; use reth_node_api::BlockBody; use reth_primitives_traits::{AlloyBlockHeader, RecoveredBlock, SealedHeader, TransactionMeta}; use reth_rpc_convert::{transaction::ConvertReceiptInput, RpcConvert, RpcHeader}; +use reth_rpc_eth_types::block::SharedReceipts; use reth_storage_api::{BlockIdReader, BlockReader, ProviderHeader, ProviderReceipt, ProviderTx}; use reth_transaction_pool::{PoolTransaction, TransactionPool}; use revm::state::bal::Bal as RevmBal; @@ -25,7 +26,7 @@ pub type BlockReceiptsResult = Result>>, E>; pub type BlockAndReceiptsResult = Result< Option<( Arc::Provider as BlockReader>::Block>>, - Arc::Provider>>>, + SharedReceipts::Provider>>, )>, ::Error, >; @@ -105,7 +106,7 @@ pub trait EthBlocks: LoadBlock Result, Self::Error> { - if let Some((block, receipts)) = + if let Some(pending) = self.provider().pending_block_and_receipts().map_err(Self::Error::from_eth_err)? { + let (block, output) = pending.into_parts(); + // Note: for the PENDING block we assume it is past the known merge block and // thus this will not fail when looking up the total // difficulty value for the blockenv. @@ -83,7 +85,7 @@ pub trait LoadPendingBlock: return Ok(PendingBlockEnv::new( evm_env, - PendingBlockEnvOrigin::ActualPending(Arc::new(block), Arc::new(receipts)), + PendingBlockEnvOrigin::ActualPending(block, output.into()), )); } diff --git a/crates/rpc/rpc-eth-types/src/block.rs b/crates/rpc/rpc-eth-types/src/block.rs index f47ecc0b534..51ade8aacfc 100644 --- a/crates/rpc/rpc-eth-types/src/block.rs +++ b/crates/rpc/rpc-eth-types/src/block.rs @@ -1,12 +1,13 @@ //! Block related types for RPC API. -use std::sync::Arc; +use std::{ops::Deref, sync::Arc}; use alloy_consensus::{ transaction::{TransactionMeta, TxHashRef}, BlockHeader, TxReceipt, }; use alloy_primitives::TxHash; +use reth_execution_types::BlockExecutionOutput; use reth_primitives_traits::{ Block, BlockBody, BlockTy, IndexedTx, NodePrimitives, ReceiptTy, Recovered, RecoveredBlock, SealedBlock, @@ -114,7 +115,7 @@ pub struct BlockAndReceipts { /// The recovered block. pub block: Arc>>, /// The receipts for the block. - pub receipts: Arc>>, + pub receipts: SharedReceipts>, } impl BlockAndReceipts { @@ -123,7 +124,7 @@ impl BlockAndReceipts { block: Arc>>, receipts: Arc>>, ) -> Self { - Self { block, receipts } + Self { block, receipts: SharedReceipts::Receipts(receipts) } } /// Finds a transaction by hash and returns it along with its corresponding receipt. @@ -156,13 +157,52 @@ impl BlockAndReceipts { C: RpcConvert, { let (tx, receipt) = self.find_transaction_and_receipt_by_hash(tx_hash)?; - convert_transaction_receipt( - self.block.as_ref(), - self.receipts.as_ref(), - tx, - receipt, - converter, - ) + convert_transaction_receipt(self.block.as_ref(), &self.receipts, tx, receipt, converter) + } +} + +/// Shared receipts, either loaded directly or held by a block's execution output. +#[derive(Debug, Clone)] +pub enum SharedReceipts { + /// Receipts loaded from storage or a locally built block. + Receipts(Arc>), + /// Receipts belonging to an in-memory executed block. + ExecutionOutput(Arc>), +} + +impl Deref for SharedReceipts { + type Target = [R]; + + fn deref(&self) -> &Self::Target { + match self { + Self::Receipts(receipts) => receipts, + Self::ExecutionOutput(output) => &output.receipts, + } + } +} + +impl SharedReceipts { + /// Returns owned receipts for consumers that need to move them into a converter. + pub fn into_vec(self) -> Vec { + match self { + Self::Receipts(receipts) => Arc::unwrap_or_clone(receipts), + Self::ExecutionOutput(output) => match Arc::try_unwrap(output) { + Ok(output) => output.result.receipts, + Err(output) => output.receipts.clone(), + }, + } + } +} + +impl From>> for SharedReceipts { + fn from(receipts: Arc>) -> Self { + Self::Receipts(receipts) + } +} + +impl From>> for SharedReceipts { + fn from(output: Arc>) -> Self { + Self::ExecutionOutput(output) } } diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index 44d23553f33..9c4ce896245 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -991,7 +991,7 @@ mod tests { use reth_ethereum_primitives::{ Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned, }; - use reth_execution_types::ExecutionOutcome; + use reth_execution_types::{ExecutionOutcome, RecoveredBlockAndExecutionOutput}; use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_storage_api::{ noop::NoopProvider, BalProvider, BalStore, BalStoreHandle, BlockBodyIndicesProvider, @@ -1797,13 +1797,14 @@ mod tests { Ok(None) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { Ok(None) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> + { Ok(None) } diff --git a/crates/rpc/rpc-eth-types/src/pending_block.rs b/crates/rpc/rpc-eth-types/src/pending_block.rs index 0cc01eee15a..7e8f52bdc6f 100644 --- a/crates/rpc/rpc-eth-types/src/pending_block.rs +++ b/crates/rpc/rpc-eth-types/src/pending_block.rs @@ -4,7 +4,7 @@ use std::{sync::Arc, time::Instant}; -use crate::block::BlockAndReceipts; +use crate::block::{BlockAndReceipts, SharedReceipts}; use alloy_consensus::BlockHeader; use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_primitives::{BlockHash, TxHash, B256}; @@ -30,7 +30,7 @@ pub struct PendingBlockEnv { #[derive(Clone, Debug)] pub enum PendingBlockEnvOrigin { /// The pending block as received from the CL. - ActualPending(Arc>, Arc>), + ActualPending(Arc>, SharedReceipts), /// The _modified_ header of the latest block. /// /// This derives the pending state based on the latest header by modifying: @@ -78,8 +78,7 @@ impl PendingBlockEnvOrigin { } } -/// A type alias for a pair of an [`Arc`] wrapped [`RecoveredBlock`] and a vector of -/// [`NodePrimitives::Receipt`]. +/// A type alias for a shared [`RecoveredBlock`] and its receipts. pub type PendingBlockAndReceipts = BlockAndReceipts; /// Locally built pending block for `pending` tag. @@ -88,7 +87,7 @@ pub struct PendingBlock { /// Timestamp when the pending block is considered outdated. pub expires_at: Instant, /// The receipts for the pending block - pub receipts: Arc>>, + pub receipts: SharedReceipts>, /// The locally built pending block with execution output. pub executed_block: ExecutedBlock, } @@ -99,7 +98,7 @@ impl PendingBlock { pub fn with_executed_block(expires_at: Instant, executed_block: ExecutedBlock) -> Self { Self { expires_at, - receipts: Arc::new(executed_block.execution_output.receipts.clone()), + receipts: Arc::clone(&executed_block.execution_output).into(), executed_block, } } diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index eb8ca1ca2e4..4bbee9ba45e 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -21,7 +21,7 @@ use reth_chain_state::{ }; use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; -use reth_execution_types::ExecutionOutcome; +use reth_execution_types::{ExecutionOutcome, RecoveredBlockAndExecutionOutput}; use reth_node_types::{BlockTy, HeaderTy, NodeTypes, NodeTypesWithDB, ReceiptTy, TxTy}; use reth_primitives_traits::{ Account, RecoveredBlock, SealedHeader, SealedOrRecoveredBlock, StorageEntry, @@ -521,13 +521,13 @@ impl BlockReader for BlockchainProvider { self.consistent_provider()?.block(id) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { Ok(self.canonical_in_memory_state.pending_recovered_block()) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { Ok(self.canonical_in_memory_state.pending_block_and_receipts()) } @@ -1656,15 +1656,15 @@ mod tests { // Assertions related to the pending block + let pending_block = provider.pending_block()?.unwrap(); assert_eq!( - provider.pending_block()?, - Some(RecoveredBlock::new_sealed(block.clone(), block.senders().unwrap())) + *pending_block, + RecoveredBlock::new_sealed(block.clone(), block.senders().unwrap()) ); - assert_eq!( - provider.pending_block_and_receipts()?, - Some((RecoveredBlock::new_sealed(block.clone(), block.senders().unwrap()), vec![])) - ); + let pending = provider.pending_block_and_receipts()?.unwrap(); + assert!(Arc::ptr_eq(&pending_block, pending.block())); + assert!(pending.execution_output().receipts.is_empty()); Ok(()) } diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index 9fe14d05317..f951c5f902a 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -15,6 +15,7 @@ use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use reth_chain_state::{BlockState, CanonicalInMemoryState}; use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; +use reth_execution_types::RecoveredBlockAndExecutionOutput; use reth_node_types::{BlockTy, HeaderTy, ReceiptTy, TxTy}; use reth_primitives_traits::{ BlockBody, RecoveredBlock, SealedHeader, SealedOrRecoveredBlock, StorageEntry, @@ -627,13 +628,13 @@ impl BlockReader for ConsistentProvider { ) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { Ok(self.canonical_in_memory_state.pending_recovered_block()) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { Ok(self.canonical_in_memory_state.pending_block_and_receipts()) } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index f4cae91d56b..fa55e1a946c 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -21,6 +21,7 @@ use reth_chainspec::ChainInfo; use reth_db::{init_db, mdbx::DatabaseArguments, DatabaseEnv}; use reth_db_api::{database::Database, models::StoredBlockBodyIndices}; use reth_errors::{RethError, RethResult}; +use reth_execution_types::RecoveredBlockAndExecutionOutput; use reth_node_types::{ BlockTy, HeaderTy, NodeTypesWithDB, NodeTypesWithDBAdapter, ReceiptTy, TxTy, }; @@ -720,13 +721,13 @@ impl BlockReader for ProviderFactory { self.provider()?.block(id) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { self.provider()?.pending_block() } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { self.provider()?.pending_block_and_receipts() } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 53169e8932a..53ea7f2e54d 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -52,7 +52,10 @@ use reth_db_api::{ transaction::{DbTx, DbTxMut}, BlockNumberList, }; -use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome}; +use reth_execution_types::{ + BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome, + RecoveredBlockAndExecutionOutput, +}; use reth_node_types::{BlockTy, BodyTy, HeaderTy, NodeTypes, ReceiptTy, TxTy}; use reth_primitives_traits::{ Account, Block as _, BlockBody as _, Bytecode, FastInstant as Instant, RecoveredBlock, @@ -1896,13 +1899,13 @@ impl BlockReader for DatabaseProvid Ok(None) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { Ok(None) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { Ok(None) } diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index bef0e1421f4..4a382aef359 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -33,6 +33,7 @@ use reth_db_api::{ transaction::DbTx, }; use reth_ethereum_primitives::{Receipt, TransactionSigned}; +use reth_execution_types::RecoveredBlockAndExecutionOutput; use reth_nippy_jar::{NippyJar, NippyJarChecker}; use reth_node_types::NodePrimitives; use reth_primitives_traits::{ @@ -3025,14 +3026,14 @@ impl> Blo Err(ProviderError::UnsupportedProvider) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 75860cf7334..2640f4a775c 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -27,7 +27,7 @@ use reth_db_api::{ models::{AccountBeforeTx, StorageSettings, StoredBlockBodyIndices}, }; use reth_ethereum_primitives::EthPrimitives; -use reth_execution_types::ExecutionOutcome; +use reth_execution_types::{ExecutionOutcome, RecoveredBlockAndExecutionOutput}; use reth_primitives_traits::{ Account, Block, BlockBody, Bytecode, GotExpected, NodePrimitives, RecoveredBlock, SealedHeader, SignerRecoverable, StorageEntry, @@ -935,13 +935,13 @@ impl BlockRe } } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { Ok(None) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { Ok(None) } diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs index c87a87bd4aa..91449d0f9a6 100644 --- a/crates/storage/rpc-provider/src/lib.rs +++ b/crates/storage/rpc-provider/src/lib.rs @@ -39,6 +39,7 @@ use reth_db_api::{ models::StoredBlockBodyIndices, }; use reth_errors::{ProviderError, ProviderResult}; +use reth_execution_types::RecoveredBlockAndExecutionOutput; use reth_node_types::{ Block, BlockBody, BlockTy, HeaderTy, NodeTypes, PrimitivesTy, ReceiptTy, TxTy, }; @@ -494,13 +495,13 @@ where Ok(Some(block)) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { Err(ProviderError::UnsupportedProvider) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { Err(ProviderError::UnsupportedProvider) } @@ -1527,13 +1528,14 @@ where Err(ProviderError::UnsupportedProvider) } - fn pending_block(&self) -> Result>, ProviderError> { + fn pending_block(&self) -> Result>>, ProviderError> { Err(ProviderError::UnsupportedProvider) } fn pending_block_and_receipts( &self, - ) -> Result, Vec)>, ProviderError> { + ) -> Result>, ProviderError> + { Err(ProviderError::UnsupportedProvider) } diff --git a/crates/storage/storage-api/src/block.rs b/crates/storage/storage-api/src/block.rs index b93d022b817..020d6752305 100644 --- a/crates/storage/storage-api/src/block.rs +++ b/crates/storage/storage-api/src/block.rs @@ -6,6 +6,7 @@ use alloc::{sync::Arc, vec::Vec}; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; use alloy_primitives::{BlockNumber, TxNumber, B256}; use core::ops::RangeInclusive; +use reth_execution_types::RecoveredBlockAndExecutionOutput; use reth_primitives_traits::{Block as _, RecoveredBlock, SealedHeader, SealedOrRecoveredBlock}; use reth_storage_errors::provider::ProviderResult; @@ -95,17 +96,16 @@ pub trait BlockReader: /// Returns `None` if block is not found. fn block(&self, id: BlockHashOrNumber) -> ProviderResult>; - /// Returns the pending block if available + /// Returns the pending block if available. /// - /// Note: This returns a [`RecoveredBlock`] because it's expected that this is sealed by - /// the provider and the caller does not know the hash. - fn pending_block(&self) -> ProviderResult>>; + /// Returns a shared [`RecoveredBlock`] because the provider has already sealed it and the + /// caller does not know the hash. + fn pending_block(&self) -> ProviderResult>>>; - /// Returns the pending block and receipts if available. - #[expect(clippy::type_complexity)] + /// Returns the pending block and its execution output, including receipts, if available. fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>>; + ) -> ProviderResult>>; /// Returns the block with matching hash from the database. /// @@ -186,12 +186,12 @@ impl BlockReader for Arc { fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { T::block(self, id) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { T::pending_block(self) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { T::pending_block_and_receipts(self) } fn block_by_hash(&self, hash: B256) -> ProviderResult> { @@ -254,12 +254,12 @@ impl BlockReader for &T { fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { T::block(self, id) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { T::pending_block(self) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { T::pending_block_and_receipts(self) } fn block_by_hash(&self, hash: B256) -> ProviderResult> { diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index a1779868cd9..2a6c0d74cf0 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -38,7 +38,7 @@ use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec, MAINNET}; use reth_db_api::mock::{DatabaseMock, TxMock}; use reth_db_models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_ethereum_primitives::EthPrimitives; -use reth_execution_types::ExecutionOutcome; +use reth_execution_types::{ExecutionOutcome, RecoveredBlockAndExecutionOutput}; use reth_primitives_traits::{Account, Bytecode, NodePrimitives, RecoveredBlock, SealedHeader}; #[cfg(feature = "db-api")] use reth_prune_types::PruneModes; @@ -222,13 +222,13 @@ impl BlockReader for NoopProvider { Ok(None) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>>> { Ok(None) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult>> { Ok(None) } From 2833f744022fe0eb6e7223ce0dbfe739ff1af6c7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 10:04:35 +0200 Subject: [PATCH 076/133] fix(engine): dedupe block range requests (#27426) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/engine/tree/src/download.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/engine/tree/src/download.rs b/crates/engine/tree/src/download.rs index 7dd2611f93b..4e9a8f3053d 100644 --- a/crates/engine/tree/src/download.rs +++ b/crates/engine/tree/src/download.rs @@ -127,6 +127,14 @@ where if count == 1 { self.download_full_block(hash, access_lists); } else { + if self.inflight_block_range_requests.iter().any(|request| { + request.start_hash() == hash && + request.count() == count && + request.access_lists() == access_lists + }) { + return + } + trace!( target: "engine::download", ?hash, @@ -371,6 +379,11 @@ where } } + /// Returns whether the request also fetches block access lists. + const fn access_lists(&self) -> bool { + matches!(self, Self::WithAccessLists(_)) + } + /// Advances the download. fn poll( &mut self, @@ -487,6 +500,21 @@ mod tests { }); } + #[tokio::test] + async fn block_downloader_deduplicates_range_requests() { + let TestHarness { mut block_downloader, client } = TestHarness::new(3); + let hash = client.highest_block().unwrap().hash(); + let request = || DownloadRequest::block_range(hash, 2); + + block_downloader.on_action(DownloadAction::Download(request())); + block_downloader.on_action(DownloadAction::Download(request())); + assert_eq!(block_downloader.inflight_block_range_requests.len(), 1); + + block_downloader.on_action(DownloadAction::Download(DownloadRequest::block_range(hash, 3))); + block_downloader.on_action(DownloadAction::Download(request().with_access_lists(true))); + assert_eq!(block_downloader.inflight_block_range_requests.len(), 3); + } + #[tokio::test] async fn block_downloader_set_request() { const TOTAL_BLOCKS: usize = 2; From a1bf317d15e51d5ea725db01f3427ed60694a846 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 09:09:56 +0000 Subject: [PATCH 077/133] fix(net): honor tx propagation policy on connect and requests (#27431) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/net/network/src/transactions/config.rs | 4 +- crates/net/network/src/transactions/mod.rs | 11 ++ crates/net/network/tests/it/txgossip.rs | 149 +++++++++++++++++- 3 files changed, 158 insertions(+), 6 deletions(-) diff --git a/crates/net/network/src/transactions/config.rs b/crates/net/network/src/transactions/config.rs index c8ec184830f..edd23a84755 100644 --- a/crates/net/network/src/transactions/config.rs +++ b/crates/net/network/src/transactions/config.rs @@ -197,7 +197,9 @@ pub trait TransactionPropagationPolicy: { /// Filter a given peer based on the policy. /// - /// This determines whether transactions can be propagated to this peer. + /// This determines whether transactions can be propagated to this peer. Peers that are + /// filtered out also don't receive the pending pool announcement on session establishment and + /// are served empty responses to `GetPooledTransactions` requests. fn can_propagate(&self, peer: &mut PeerMetadata) -> bool; /// A callback on the policy when a new peer session is established. diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 42752eb9e1e..7627e080e00 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1108,6 +1108,12 @@ where return } if let Some(peer) = self.peers.get_mut(&peer_id) { + // don't serve pending transactions to peers the policy doesn't propagate to + if !self.policies.propagation_policy().can_propagate(peer) { + let _ = response.send(Ok(PooledTransactions::default())); + return + } + let transactions = self.pool.get_pooled_transaction_elements( request.0, GetPooledTransactionLimit::ResponseSizeSoftLimit( @@ -1207,6 +1213,11 @@ where return } + // skip peers we should not propagate to + if !self.policies.propagation_policy().can_propagate(peer) { + return + } + // Get transactions to broadcast let pooled_txs = self.pool.pooled_transactions_max( SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE, diff --git a/crates/net/network/tests/it/txgossip.rs b/crates/net/network/tests/it/txgossip.rs index ba5d92b824b..ab1338cc4ab 100644 --- a/crates/net/network/tests/it/txgossip.rs +++ b/crates/net/network/tests/it/txgossip.rs @@ -1,22 +1,26 @@ //! Testing gossiping of transactions. use alloy_consensus::TxLegacy; -use alloy_primitives::{Signature, U256}; +use alloy_primitives::{map::B256Set, Signature, TxHash, U256}; use reth_ethereum_primitives::TransactionSigned; use reth_network::{ - test_utils::{NetworkEventStream, Testnet}, + test_utils::{NetworkEventStream, PeerHandle, Testnet}, transactions::config::{ TransactionIngressPolicy, TransactionPropagationKind, TransactionsManagerConfig, }, NetworkEventListenerProvider, Peers, }; -use reth_network_api::{PeerKind, PeersInfo}; +use reth_network_api::{PeerId, PeerKind, PeersInfo}; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_transaction_pool::{ - test_utils::TransactionGenerator, AddedTransactionOutcome, PoolTransaction, TransactionPool, + test_utils::TransactionGenerator, AddedTransactionOutcome, EthPooledTransaction, + PoolTransaction, TransactionPool, }; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use tokio::join; +/// How long a peer may take to observe a session or an announcement. +const TIMEOUT: Duration = Duration::from_secs(30); + #[tokio::test(flavor = "multi_thread")] async fn test_tx_gossip() { reth_tracing::init_test_tracing(); @@ -191,6 +195,81 @@ async fn test_tx_ingress_policy_trusted_only() { assert!(buff.contains(&outcome_1.hash)); } +#[tokio::test(flavor = "multi_thread")] +async fn test_tx_propagation_policy_trusted_only_on_connect() { + reth_tracing::init_test_tracing(); + + let provider = MockEthProvider::default().with_genesis_block(); + let net = Testnet::create_with(3, provider.clone()).await; + let net = net + .with_eth_pool_config_and_policy(Default::default(), TransactionPropagationKind::Trusted); + let handle = net.spawn(); + + let [peer0, untrusted, trusted] = handle.peers() else { unreachable!() }; + let pool0 = peer0.pool().unwrap(); + + // insert the tx before connecting, so it can only be learned from the announcement of the + // pending pool sent on session establishment + let pending = pool0.add_external_transaction(funded_transaction(&provider)).await.unwrap().hash; + + connect_untrusted_and_trusted(peer0, untrusted, trusted).await; + + // ensure the trusted peer learns about the pending tx + wait_for_announcement(trusted, peer0, pending).await; + + // announcements to a peer are delivered in order, so the untrusted peer receives any + // announcement sent on session establishment before this one + let marker = pool0.add_external_transaction(funded_transaction(&provider)).await.unwrap().hash; + peer0.transactions().unwrap().propagate_hash_to(marker, *untrusted.peer_id()); + + // ensure the untrusted peer never learned about the pending tx + let announced = wait_for_announcement(untrusted, peer0, marker).await; + assert!(!announced.contains(&pending)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_tx_propagation_policy_trusted_only_get_pooled_transactions() { + reth_tracing::init_test_tracing(); + + let provider = MockEthProvider::default().with_genesis_block(); + let net = Testnet::create_with(3, provider.clone()).await; + let net = net + .with_eth_pool_config_and_policy(Default::default(), TransactionPropagationKind::Trusted); + let handle = net.spawn(); + + let [peer0, untrusted, trusted] = handle.peers() else { unreachable!() }; + + let pending = peer0 + .pool() + .unwrap() + .add_external_transaction(funded_transaction(&provider)) + .await + .unwrap() + .hash; + + connect_untrusted_and_trusted(peer0, untrusted, trusted).await; + + // ensure the untrusted peer can't request the pending tx + let txs = untrusted + .transactions() + .unwrap() + .get_pooled_transactions_from(*peer0.peer_id(), vec![pending]) + .await + .unwrap() + .unwrap(); + assert!(txs.is_empty()); + + // ensure the trusted peer can + let txs = trusted + .transactions() + .unwrap() + .get_pooled_transactions_from(*peer0.peer_id(), vec![pending]) + .await + .unwrap() + .unwrap(); + assert_eq!(txs.iter().map(|tx| *tx.hash()).collect::>(), vec![pending]); +} + #[tokio::test(flavor = "multi_thread")] async fn test_4844_tx_gossip_penalization() { reth_tracing::init_test_tracing(); @@ -292,3 +371,63 @@ async fn test_sending_invalid_transactions() { // ensure txs never made it to the pool assert!(tx_listener.try_recv().is_err()); } + +/// Returns a transaction from a new sender that is funded in the provider. +fn funded_transaction(provider: &MockEthProvider) -> EthPooledTransaction { + let tx = TransactionGenerator::new(rand::rng()).gen_eip1559_pooled(); + provider.add_account(tx.sender(), ExtendedAccount::new(0, U256::from(100_000_000))); + tx +} + +/// Connects `peer` to `untrusted` as a basic peer and to `trusted` as a trusted peer, and waits +/// until the transactions managers of all peers handled the new sessions. +async fn connect_untrusted_and_trusted( + peer: &PeerHandle, + untrusted: &PeerHandle, + trusted: &PeerHandle, +) { + peer.network().add_peer(*untrusted.peer_id(), untrusted.local_addr()); + peer.network().add_trusted_peer(*trusted.peer_id(), trusted.local_addr()); + + wait_for_active_peers(peer, &[*untrusted.peer_id(), *trusted.peer_id()]).await; + wait_for_active_peers(untrusted, &[*peer.peer_id()]).await; + wait_for_active_peers(trusted, &[*peer.peer_id()]).await; +} + +/// Waits until the transactions manager of `peer` has active sessions with all `peers`. +async fn wait_for_active_peers(peer: &PeerHandle, peers: &[PeerId]) { + let transactions = peer.transactions().unwrap(); + tokio::time::timeout(TIMEOUT, async { + loop { + let active = transactions.get_active_peers().await.unwrap(); + if peers.iter().all(|peer| active.contains(peer)) { + return + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("timed out waiting for sessions"); +} + +/// Waits until `peer` learned from `announcer` that it has `hash`, and returns all hashes `peer` +/// knows `announcer` has. +async fn wait_for_announcement( + peer: &PeerHandle, + announcer: &PeerHandle, + hash: TxHash, +) -> B256Set { + let transactions = peer.transactions().unwrap(); + tokio::time::timeout(TIMEOUT, async { + loop { + let hashes = + transactions.get_peer_transaction_hashes(*announcer.peer_id()).await.unwrap(); + if hashes.contains(&hash) { + return hashes + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("timed out waiting for announcement") +} From 444e867155c2a7a4c028e479ca7201b0b6871579 Mon Sep 17 00:00:00 2001 From: banteg <4562643+banteg@users.noreply.github.com> Date: Fri, 25 Sep 2026 09:27:42 +0000 Subject: [PATCH 078/133] fix(engine): don't treat stale persisted fcu head as canonical (#27429) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/engine/tree/src/tree/mod.rs | 43 +++--- crates/engine/tree/src/tree/tests.rs | 61 ++++++++- crates/ethereum/node/tests/e2e/forkchoice.rs | 133 ++++++++++++++++++- 3 files changed, 213 insertions(+), 24 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 545ea65e88b..c2cb7dc1bdb 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3362,17 +3362,33 @@ where } /// Attempts to find the header for the given block hash if it is canonical. + /// + /// A persisted header can still be found by hash while its disk reorg is pending, so it is + /// only canonical if it is not above the canonical head and matches the canonical hash at its + /// height. The in-memory canonical hash is preferred over the persisted one. pub fn find_canonical_header( &self, hash: B256, ) -> Result>, ProviderError> { - let mut canonical = self.canonical_in_memory_state.header_by_hash(hash); + if let Some(header) = self.canonical_in_memory_state.header_by_hash(hash) { + return Ok(Some(header)) + } - if canonical.is_none() { - canonical = self.provider.header(hash)?.map(|header| SealedHeader::new(header, hash)); + let Some(header) = self.provider.sealed_header_by_hash(hash)? else { return Ok(None) }; + // Reorged-out headers remain on disk until persistence catches up, so reject old tips above + // the in-memory head and use in-memory hashes to reject old blocks at shared heights. + let number = header.number(); + if number > self.canonical_in_memory_state.get_canonical_block_number() { + return Ok(None) } + let canonical_hash = + if let Some(hash) = self.canonical_in_memory_state.hash_by_number(number) { + Some(hash) + } else { + self.provider.block_hash(number)? + }; - Ok(canonical) + Ok((canonical_hash == Some(hash)).then_some(header)) } /// Checks that nonzero safe and finalized hashes belong to the chain defined by the FCU head. @@ -3386,9 +3402,9 @@ where /// `chain_update` describes a proposed commit or reorg that has not yet been applied. Its new /// blocks and the canonical prefix below its first block form the proposed chain. Without a /// chain update, the proposed head is already canonical, so only canonical blocks through its - /// height are eligible. For a canonical-prefix hash, the current in-memory or persisted - /// canonical hash must match too: a stale persisted header may still be found by hash while - /// disk reorg cleanup is pending. A zero safe or finalized hash leaves that marker unchanged. + /// height are eligible. Canonical-prefix hashes are resolved via + /// [`Self::find_canonical_header`], which rejects stale persisted headers whose disk reorg + /// cleanup is pending. A zero safe or finalized hash leaves that marker unchanged. /// Returns `Ok(false)` for an unknown or off-chain hash and propagates provider errors. /// /// [Engine API forkchoiceUpdated specification]: https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#specification-1 @@ -3419,19 +3435,6 @@ where if header.number() > canonical_head_number { return Ok(false) } - - // A persisted header can still be found by hash while its disk reorg is pending. - // Prefer the in-memory canonical hash at this height over the persisted one. - let canonical_hash = if let Some(hash) = - self.canonical_in_memory_state.hash_by_number(header.number()) - { - Some(hash) - } else { - self.provider.block_hash(header.number())? - }; - if canonical_hash != Some(hash) { - return Ok(false) - } } Ok(true) } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 0932dccf7f0..de102c4710e 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -33,7 +33,9 @@ use reth_ethereum_primitives::{Block, EthPrimitives}; use reth_evm_ethereum::MockEvmConfig; use reth_payload_builder::PayloadServiceCommand; use reth_primitives_traits::Block as _; -use reth_provider::{test_utils::MockEthProvider, BalStoreHandle, InMemoryBalStore, RawBal}; +use reth_provider::{ + test_utils::MockEthProvider, BalStoreHandle, HeaderProvider, InMemoryBalStore, RawBal, +}; use reth_tasks::spawn_os_thread; use reth_trie_common::ComputedTrieData; use revm::state::bal::Bal as RevmBal; @@ -3470,3 +3472,60 @@ fn test_forkchoice_rejects_stale_persisted_prefix_hash() { let state = ForkchoiceState { safe_block_hash: new[0].recovered_block().hash(), ..state }; assert!(test_harness.tree.is_consistent_forkchoice_state(state, Some(&update)).unwrap()); } + +/// Reorgs from a persisted chain to a sibling branch of `sibling_len` blocks without advancing +/// persistence, then asserts that an FCU back to the reorged-out tip makes it canonical again. +async fn assert_fcu_back_to_reorged_out_head_with_pending_disk_reorg(sibling_len: u64) { + reth_tracing::init_test_tracing(); + let mut test_harness = TestHarness::new(MAINNET.clone()); + let old: Vec<_> = test_harness.block_builder.get_executed_blocks(0..4).collect(); + test_harness = test_harness.with_blocks(old.clone()); + let old_tip = old[3].recovered_block().hash(); + + let new: Vec<_> = test_harness + .block_builder + .create_fork(old[0].recovered_block(), sibling_len) + .into_iter() + .map(|block| { + ExecutedBlock::new( + Arc::new(block), + Arc::new(BlockExecutionOutput::default()), + ComputedTrieData::default(), + ) + }) + .collect(); + for block in &new { + test_harness.tree.state.tree_state.insert_executed(block.clone()); + } + let new_tip = new.last().unwrap().recovered_block().hash(); + + let fcu = |head_block_hash| ForkchoiceState { + head_block_hash, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let outcome = test_harness.tree.on_forkchoice_updated(fcu(new_tip), None).unwrap(); + assert_eq!(outcome.outcome.forkchoice_status(), ForkchoiceStatus::Valid); + assert_eq!(test_harness.tree.canonical_in_memory_state.get_canonical_head().hash(), new_tip); + + // Persistence is not advanced, so the reorged-out blocks are still found on disk. + assert!(test_harness.provider.header(old_tip).unwrap().is_some()); + + let outcome = test_harness.tree.on_forkchoice_updated(fcu(old_tip), None).unwrap(); + assert_eq!(outcome.outcome.forkchoice_status(), ForkchoiceStatus::Valid); + assert_eq!(test_harness.tree.state.tree_state.canonical_block_hash(), old_tip); + assert_eq!(test_harness.tree.canonical_in_memory_state.get_canonical_head().hash(), old_tip); +} + +/// A reorged-out head must become canonical again while its disk reorg is still pending. +#[tokio::test] +async fn test_fcu_back_to_reorged_out_head_with_pending_disk_reorg() { + assert_fcu_back_to_reorged_out_head_with_pending_disk_reorg(3).await; +} + +/// Same as above, but the reorged-out head is above the tip of the shorter sibling branch. +#[tokio::test] +async fn test_fcu_back_to_reorged_out_head_above_shorter_branch_with_pending_disk_reorg() { + assert_fcu_back_to_reorged_out_head_with_pending_disk_reorg(1).await; +} diff --git a/crates/ethereum/node/tests/e2e/forkchoice.rs b/crates/ethereum/node/tests/e2e/forkchoice.rs index e9ff2ef457f..6e0e5806922 100644 --- a/crates/ethereum/node/tests/e2e/forkchoice.rs +++ b/crates/ethereum/node/tests/e2e/forkchoice.rs @@ -1,4 +1,4 @@ -//! Tests for atomic forkchoice state updates via the Engine API. +//! Tests for canonical chain and atomic forkchoice state updates via the Engine API. use crate::utils::eth_payload_attributes; use alloy_eips::BlockNumberOrTag; @@ -7,11 +7,12 @@ use alloy_provider::{Provider, ProviderBuilder}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; use jsonrpsee_core::client::Error; use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::E2ETestSetupBuilder; +use reth_e2e_test_utils::{trie::wait_for_persisted_block, E2ETestSetupBuilder}; use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_provider::{DatabaseProviderFactory, HeaderProvider}; use reth_rpc_api::{EngineApiClient, TestingBuildBlockRequestV1}; use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; #[tokio::test] async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { @@ -159,3 +160,129 @@ async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { Ok(()) } + +/// Restoring a persisted head must update canonical state before disk reorg cleanup completes. +#[tokio::test] +async fn fcu_restores_reorged_out_persisted_head() -> eyre::Result<()> { + assert_fcu_restores_reorged_out_persisted_head(3).await +} + +/// A stale persisted head above a shorter sibling branch must also become canonical again. +#[tokio::test] +async fn fcu_restores_reorged_out_persisted_head_above_shorter_branch() -> eyre::Result<()> { + assert_fcu_restores_reorged_out_persisted_head(1).await +} + +/// Holds the database writer lock across both FCUs so disk cleanup cannot win the race. +async fn assert_fcu_restores_reorged_out_persisted_head(sibling_len: u64) -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .build(), + ); + let (mut nodes, _) = + E2ETestSetupBuilder::::new(2, chain_spec, eth_payload_attributes) + .with_connect_nodes(false) + .with_tree_config_modifier(|config| { + config + .with_num_state_masking_blocks(0) + .with_persistence_threshold(0) + .with_memory_block_buffer_target(0) + }) + .with_node_config_modifier(|mut config| { + config.rpc.http_api = + Some(RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing])); + config + }) + .build() + .await?; + let node = nodes.pop().unwrap(); + let producer = nodes.pop().unwrap(); + let genesis = node.block_hash(0); + let engine = node.auth_server_handle().http_client(); + let producer_engine = producer.auth_server_handle().http_client(); + let rpc = ProviderBuilder::new().connect_http(node.rpc_url()); + + // Persist A through block 3, then import a sibling branch without canonicalizing it. + let mut chains = [vec![genesis], vec![genesis]]; + for (branch, length) in [3, sibling_len].into_iter().enumerate() { + for number in 1..=length { + let envelope = producer + .testing_build_block_v1(TestingBuildBlockRequestV1 { + parent_block_hash: *chains[branch].last().unwrap(), + payload_attributes: eth_payload_attributes(number + branch as u64 * 10), + transactions: vec![], + extra_data: None, + }) + .await?; + let payload = envelope.execution_payload; + let hash = payload.payload_inner.payload_inner.block_hash; + for client in [&producer_engine, &engine] { + let status = EngineApiClient::::new_payload_v3( + client, + payload.clone(), + vec![], + B256::ZERO, + ) + .await?; + assert_eq!(status.status, PayloadStatusEnum::Valid); + } + producer.update_forkchoice(genesis, hash).await?; + chains[branch].push(hash); + } + if branch == 0 { + node.update_forkchoice(genesis, chains[0][3]).await?; + wait_for_persisted_block(&node.inner.provider, 3, Duration::from_secs(30)).await?; + } + } + let [a, b] = chains; + + // MDBX allows only one writer. Acquire it off the executor and keep it alive while the + // persistence task attempts to remove A, leaving the old headers visible on disk. + let provider = node.inner.provider.clone(); + let disk_reorg_guard = + tokio::task::spawn_blocking(move || provider.database_provider_rw()).await??; + for head in [*b.last().unwrap(), a[3]] { + let status = tokio::time::timeout( + Duration::from_secs(10), + EngineApiClient::::fork_choice_updated_v3( + &engine, + ForkchoiceState { + head_block_hash: head, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }, + None, + ), + ) + .await??; + assert_eq!(status.payload_status.status, PayloadStatusEnum::Valid); + assert_eq!( + node.inner.provider.database_provider_ro()?.sealed_header(3)?.unwrap().hash(), + a[3], + "The old tip must remain on disk throughout both FCUs" + ); + assert_eq!( + rpc.get_block_by_number(BlockNumberOrTag::Latest).await?.unwrap().header.hash, + head, + "A VALID FCU must make the requested sibling tip canonical" + ); + } + for (number, hash) in a.into_iter().enumerate() { + assert_eq!( + rpc.get_block_by_number(BlockNumberOrTag::Number(number as u64)) + .await? + .unwrap() + .header + .hash, + hash + ); + } + drop(disk_reorg_guard); + + Ok(()) +} From 828b14ccbdaeb18c9b1f10a0975fbc03d3272a63 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 12:52:20 +0200 Subject: [PATCH 079/133] refactor(engine): reuse alloy SSZ engine types (#27435) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../node/src/engine_ssz_containers.rs | 1973 ----------------- crates/ethereum/node/src/engine_ssz_proxy.rs | 54 +- .../ethereum/node/src/engine_ssz_witness.rs | 133 +- crates/ethereum/node/src/lib.rs | 1 - crates/ethereum/node/tests/e2e/eth.rs | 35 +- 5 files changed, 171 insertions(+), 2025 deletions(-) delete mode 100644 crates/ethereum/node/src/engine_ssz_containers.rs diff --git a/crates/ethereum/node/src/engine_ssz_containers.rs b/crates/ethereum/node/src/engine_ssz_containers.rs deleted file mode 100644 index 54fb83c8912..00000000000 --- a/crates/ethereum/node/src/engine_ssz_containers.rs +++ /dev/null @@ -1,1973 +0,0 @@ -//! Experimental Engine API v2 REST-SSZ wire types. -//! -//! These types intentionally live apart from the legacy JSON-RPC Engine API types because their -//! SSZ encodings are not always wire-compatible. This module contains the shared endpoint -//! containers, fork-specific payload containers, blob containers, payload-body containers, and the -//! experimental payload-with-witness response type from the same REST-SSZ model. - -use alloy_eips::{ - eip4844::{Blob, BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1, Bytes48}, - eip4895::Withdrawal, - eip7594::Cell, - eip7685::Requests, -}; -use alloy_primitives::{Address, Bytes, B128, B256, U256}; -use alloy_rpc_types_engine::{ - BlobsBundleV1, BlobsBundleV2, ExecutionPayloadBodyV1 as LegacyExecutionPayloadBodyV1, - ExecutionPayloadBodyV2 as LegacyExecutionPayloadBodyV2, - ExecutionPayloadEnvelopeV2 as LegacyBuiltPayloadShanghai, - ExecutionPayloadEnvelopeV4 as LegacyBuiltPayloadPrague, - ExecutionPayloadEnvelopeV5 as LegacyBuiltPayloadOsaka, - ExecutionPayloadEnvelopeV6 as LegacyBuiltPayloadAmsterdam, ExecutionPayloadFieldV2, - ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4, - ForkchoiceState, ForkchoiceUpdated as LegacyForkchoice, - PayloadAttributes as LegacyPayloadAttributes, PayloadId, PayloadStatus as LegacyPayloadStatus, - PayloadStatusEnum, -}; - -type ErrorBytes = Vec; - -/// Maximum number of blobs in a REST-SSZ blob request or response. -pub const MAX_BLOBS_REQUEST: usize = 128; - -/// Maximum number of payload bodies in a REST-SSZ request or response. -pub const MAX_BODIES_REQUEST: usize = 32; - -/// An Engine API v2 SSZ optional encoded as `List[T, 1]`. -/// -/// This differs from [`Option`]'s `ethereum_ssz` encoding, which uses an SSZ union. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Optional(Vec); - -impl Optional { - /// Creates an absent optional. - pub const fn none() -> Self { - Self(Vec::new()) - } - - /// Creates a present optional. - pub fn some(value: T) -> Self { - Self(vec![value]) - } - - /// Returns the contained value, if present. - pub fn as_ref(&self) -> Option<&T> { - self.0.first() - } - - /// Returns true if no value is present. - pub const fn is_none(&self) -> bool { - self.0.is_empty() - } - - /// Returns true if a value is present. - pub const fn is_some(&self) -> bool { - !self.is_none() - } - - /// Converts into a Rust optional. - pub fn into_option(mut self) -> Option { - self.0.pop() - } -} - -impl From> for Optional { - fn from(value: Option) -> Self { - value.map_or_else(Self::none, Self::some) - } -} - -impl From> for Option { - fn from(value: Optional) -> Self { - value.into_option() - } -} - -impl ssz::Encode for Optional { - fn is_ssz_fixed_len() -> bool { - false - } - - fn ssz_bytes_len(&self) -> usize { - self.0.ssz_bytes_len() - } - - fn ssz_append(&self, buf: &mut Vec) { - self.0.ssz_append(buf); - } -} - -impl ssz::Decode for Optional { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - let values = Vec::::from_ssz_bytes(bytes)?; - if values.len() > 1 { - return Err(ssz::DecodeError::BytesInvalid("optional has more than one value".into())) - } - Ok(Self(values)) - } -} - -/// Engine API v2 REST-SSZ payload status. -/// -/// This is separate from the legacy status because REST-SSZ uses `Optional` fields instead of -/// zero-value sentinels and legacy byte lists. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PayloadStatus { - /// Payload validation status. - pub status: PayloadStatusEnum, - /// Most recent valid block hash. - pub latest_valid_hash: Optional, - /// Optional payload validation error bytes. - pub validation_error: Optional, -} - -const MAX_ERROR_BYTES: usize = 1024; - -const fn status_code(status: &PayloadStatusEnum) -> u8 { - match status { - PayloadStatusEnum::Valid => 0, - PayloadStatusEnum::Invalid { .. } => 1, - PayloadStatusEnum::Syncing => 2, - PayloadStatusEnum::Accepted => 3, - } -} - -fn status_from_code(code: u8) -> Result { - match code { - 0 => Ok(PayloadStatusEnum::Valid), - 1 => Ok(PayloadStatusEnum::Invalid { validation_error: String::new() }), - 2 => Ok(PayloadStatusEnum::Syncing), - 3 => Ok(PayloadStatusEnum::Accepted), - _ => Err(ssz::DecodeError::BytesInvalid("unknown payload status code".into())), - } -} - -fn legacy_validation_error( - status: &PayloadStatusEnum, -) -> Result, ConversionError> { - match status { - PayloadStatusEnum::Invalid { validation_error } => { - let bytes = validation_error.as_bytes().to_vec(); - if bytes.len() > MAX_ERROR_BYTES { - return Err(ConversionError::ErrorBytesTooLong) - } - Ok(Optional::some(bytes)) - } - _ => Ok(Optional::none()), - } -} - -impl ssz::Encode for PayloadStatus { - fn is_ssz_fixed_len() -> bool { - false - } - - fn ssz_bytes_len(&self) -> usize { - 1 + ssz::BYTES_PER_LENGTH_OFFSET * 2 + - self.latest_valid_hash.ssz_bytes_len() + - self.validation_error.ssz_bytes_len() - } - - fn ssz_append(&self, buf: &mut Vec) { - let mut encoder = ssz::SszEncoder::container(buf, 1 + ssz::BYTES_PER_LENGTH_OFFSET * 2); - encoder.append(&status_code(&self.status)); - encoder.append(&self.latest_valid_hash); - encoder.append(&self.validation_error); - encoder.finalize(); - } -} - -impl ssz::Decode for PayloadStatus { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - let mut builder = ssz::SszDecoderBuilder::new(bytes); - builder.register_type::()?; - builder.register_type::>()?; - builder.register_type::>()?; - let mut decoder = builder.build()?; - let mut status = status_from_code(decoder.decode_next()?)?; - let latest_valid_hash = decoder.decode_next()?; - let validation_error: Optional = decoder.decode_next()?; - if let PayloadStatusEnum::Invalid { validation_error: error } = &mut status { - *error = match validation_error.as_ref() { - Some(error) => String::from_utf8(error.clone()) - .map_err(|err| ssz::DecodeError::BytesInvalid(err.to_string()))?, - None => String::new(), - }; - if validation_error.as_ref().is_some_and(|error| error.len() > MAX_ERROR_BYTES) { - return Err(ssz::DecodeError::BytesInvalid( - "payload validation error is too long".into(), - )) - } - } else if validation_error.is_some() { - return Err(ssz::DecodeError::BytesInvalid( - "validation error is only valid for INVALID status".into(), - )); - } - Ok(Self { status, latest_valid_hash, validation_error }) - } -} - -/// Error converting legacy Engine API values into v2 REST-SSZ values. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ConversionError { - /// Payload validation error exceeded the REST-SSZ byte bound. - ErrorBytesTooLong, - /// `ACCEPTED` is not permitted in a forkchoice response. - AcceptedForkchoice, - /// A bounded REST-SSZ list exceeded its maximum length. - TooManyItems { - /// Name of the field that exceeded its bound. - field: &'static str, - /// Maximum permitted item count. - max: usize, - /// Actual item count. - actual: usize, - }, -} - -impl core::fmt::Display for ConversionError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::ErrorBytesTooLong => f.write_str("payload validation error is too long"), - Self::AcceptedForkchoice => { - f.write_str("ACCEPTED is not valid in a forkchoice response") - } - Self::TooManyItems { field, max, actual } => { - write!(f, "too many {field}: expected at most {max}, got {actual}") - } - } - } -} - -impl core::error::Error for ConversionError {} - -impl TryFrom for PayloadStatus { - type Error = ConversionError; - - fn try_from(value: LegacyPayloadStatus) -> Result { - let validation_error = legacy_validation_error(&value.status)?; - Ok(Self { - status: value.status, - latest_valid_hash: value.latest_valid_hash.into(), - validation_error, - }) - } -} - -impl From for LegacyPayloadStatus { - fn from(value: PayloadStatus) -> Self { - let status = match value.status { - PayloadStatusEnum::Invalid { .. } => PayloadStatusEnum::Invalid { - validation_error: value - .validation_error - .as_ref() - .and_then(|error| String::from_utf8(error.clone()).ok()) - .unwrap_or_default(), - }, - status => status, - }; - Self { status, latest_valid_hash: value.latest_valid_hash.into() } - } -} - -/// Engine API v2 REST-SSZ forkchoice update response. -/// -/// The REST response is a container of two variable fields, unlike the legacy fixed payload ID. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ForkchoiceUpdateResponse { - /// Restricted payload status; `ACCEPTED` is invalid here. - pub payload_status: PayloadStatus, - /// Opaque server-assigned payload identifier. - pub payload_id: Optional, -} - -impl ssz::Encode for ForkchoiceUpdateResponse { - fn is_ssz_fixed_len() -> bool { - false - } - - fn ssz_bytes_len(&self) -> usize { - ssz::BYTES_PER_LENGTH_OFFSET * 2 + - self.payload_status.ssz_bytes_len() + - self.payload_id.ssz_bytes_len() - } - - fn ssz_append(&self, buf: &mut Vec) { - let mut encoder = ssz::SszEncoder::container(buf, ssz::BYTES_PER_LENGTH_OFFSET * 2); - encoder.append(&self.payload_status); - encoder.append(&self.payload_id); - encoder.finalize(); - } -} - -impl ssz::Decode for ForkchoiceUpdateResponse { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - let mut builder = ssz::SszDecoderBuilder::new(bytes); - builder.register_type::()?; - builder.register_type::>()?; - let mut decoder = builder.build()?; - let response = - Self { payload_status: decoder.decode_next()?, payload_id: decoder.decode_next()? }; - if matches!(response.payload_status.status, PayloadStatusEnum::Accepted) { - return Err(ssz::DecodeError::BytesInvalid( - "ACCEPTED is not valid in a forkchoice response".into(), - )); - } - Ok(response) - } -} - -impl TryFrom for ForkchoiceUpdateResponse { - type Error = ConversionError; - - fn try_from(value: LegacyForkchoice) -> Result { - let payload_status = PayloadStatus::try_from(value.payload_status)?; - if matches!(payload_status.status, PayloadStatusEnum::Accepted) { - return Err(ConversionError::AcceptedForkchoice) - } - Ok(Self { payload_status, payload_id: value.payload_id.into() }) - } -} - -impl From for LegacyForkchoice { - fn from(value: ForkchoiceUpdateResponse) -> Self { - Self { payload_status: value.payload_status.into(), payload_id: value.payload_id.into() } - } -} - -/// Paris execution payload. -pub type ExecutionPayloadParis = ExecutionPayloadV1; - -/// Shanghai execution payload. -pub type ExecutionPayloadShanghai = ExecutionPayloadV2; - -/// Cancun execution payload. -pub type ExecutionPayloadCancun = ExecutionPayloadV3; - -/// Prague execution payload. -pub type ExecutionPayloadPrague = ExecutionPayloadV3; - -/// Osaka execution payload. -pub type ExecutionPayloadOsaka = ExecutionPayloadV3; - -/// Amsterdam execution payload. -pub type ExecutionPayloadAmsterdam = ExecutionPayloadV4; - -/// Paris payload attributes. -/// -/// Fork-specific attributes keep later-fork fields out of the SSZ body; the legacy type is a -/// permissive superset. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct PayloadAttributesParis { - /// Payload timestamp. - pub timestamp: u64, - /// Previous RANDAO value. - pub prev_randao: B256, - /// Suggested fee recipient. - pub suggested_fee_recipient: Address, -} - -/// Shanghai payload attributes. -/// -/// Fork-specific attributes keep later-fork fields out of the SSZ body; the legacy type is a -/// permissive superset. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct PayloadAttributesShanghai { - /// Payload timestamp. - pub timestamp: u64, - /// Previous RANDAO value. - pub prev_randao: B256, - /// Suggested fee recipient. - pub suggested_fee_recipient: Address, - /// Withdrawals to include in the payload. - pub withdrawals: Vec, -} - -/// Cancun payload attributes. -/// -/// Fork-specific attributes keep later-fork fields out of the SSZ body; the legacy type is a -/// permissive superset. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct PayloadAttributesCancun { - /// Payload timestamp. - pub timestamp: u64, - /// Previous RANDAO value. - pub prev_randao: B256, - /// Suggested fee recipient. - pub suggested_fee_recipient: Address, - /// Withdrawals to include in the payload. - pub withdrawals: Vec, - /// Root of the parent beacon block. - pub parent_beacon_block_root: B256, -} - -/// Prague uses the Cancun payload-attributes schema. -pub type PayloadAttributesPrague = PayloadAttributesCancun; - -/// Osaka uses the Cancun payload-attributes schema. -pub type PayloadAttributesOsaka = PayloadAttributesCancun; - -/// Amsterdam payload attributes. -/// -/// Fork-specific attributes keep the Amsterdam-only fields in their defined SSZ position. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct PayloadAttributesAmsterdam { - /// Payload timestamp. - pub timestamp: u64, - /// Previous RANDAO value. - pub prev_randao: B256, - /// Suggested fee recipient. - pub suggested_fee_recipient: Address, - /// Withdrawals to include in the payload. - pub withdrawals: Vec, - /// Root of the parent beacon block. - pub parent_beacon_block_root: B256, - /// Consensus-layer slot number. - pub slot_number: u64, - /// Target gas limit. - pub target_gas_limit: u64, -} - -/// Error converting legacy cross-fork payload attributes into a fork-specific SSZ container. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum PayloadAttributesConversionError { - /// A field required by the selected fork is absent. - MissingField(&'static str), - /// A field from a later fork is populated and would be lost. - UnexpectedField(&'static str), -} - -impl core::fmt::Display for PayloadAttributesConversionError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::MissingField(field) => { - write!(f, "missing required payload attributes field: {field}") - } - Self::UnexpectedField(field) => { - write!(f, "unexpected later-fork payload attributes field: {field}") - } - } - } -} - -impl core::error::Error for PayloadAttributesConversionError {} - -const fn ensure_absent( - value: &Option, - field: &'static str, -) -> Result<(), PayloadAttributesConversionError> { - if value.is_some() { - Err(PayloadAttributesConversionError::UnexpectedField(field)) - } else { - Ok(()) - } -} - -fn require( - value: Option, - field: &'static str, -) -> Result { - value.ok_or(PayloadAttributesConversionError::MissingField(field)) -} - -impl From for LegacyPayloadAttributes { - fn from(value: PayloadAttributesParis) -> Self { - Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - withdrawals: None, - parent_beacon_block_root: None, - slot_number: None, - ..Default::default() - } - } -} - -impl TryFrom for PayloadAttributesParis { - type Error = PayloadAttributesConversionError; - - fn try_from(value: LegacyPayloadAttributes) -> Result { - ensure_absent(&value.withdrawals, "withdrawals")?; - ensure_absent(&value.parent_beacon_block_root, "parent_beacon_block_root")?; - ensure_absent(&value.slot_number, "slot_number")?; - ensure_absent(&value.target_gas_limit, "target_gas_limit")?; - Ok(Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - }) - } -} - -impl From for LegacyPayloadAttributes { - fn from(value: PayloadAttributesShanghai) -> Self { - Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - withdrawals: Some(value.withdrawals), - parent_beacon_block_root: None, - slot_number: None, - ..Default::default() - } - } -} - -impl TryFrom for PayloadAttributesShanghai { - type Error = PayloadAttributesConversionError; - - fn try_from(value: LegacyPayloadAttributes) -> Result { - ensure_absent(&value.parent_beacon_block_root, "parent_beacon_block_root")?; - ensure_absent(&value.slot_number, "slot_number")?; - ensure_absent(&value.target_gas_limit, "target_gas_limit")?; - Ok(Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - withdrawals: require(value.withdrawals, "withdrawals")?, - }) - } -} - -impl From for LegacyPayloadAttributes { - fn from(value: PayloadAttributesCancun) -> Self { - Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - withdrawals: Some(value.withdrawals), - parent_beacon_block_root: Some(value.parent_beacon_block_root), - slot_number: None, - ..Default::default() - } - } -} - -impl TryFrom for PayloadAttributesCancun { - type Error = PayloadAttributesConversionError; - - fn try_from(value: LegacyPayloadAttributes) -> Result { - ensure_absent(&value.slot_number, "slot_number")?; - ensure_absent(&value.target_gas_limit, "target_gas_limit")?; - Ok(Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - withdrawals: require(value.withdrawals, "withdrawals")?, - parent_beacon_block_root: require( - value.parent_beacon_block_root, - "parent_beacon_block_root", - )?, - }) - } -} - -impl From for LegacyPayloadAttributes { - #[allow(clippy::needless_update)] - fn from(value: PayloadAttributesAmsterdam) -> Self { - Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - withdrawals: Some(value.withdrawals), - parent_beacon_block_root: Some(value.parent_beacon_block_root), - slot_number: Some(value.slot_number), - target_gas_limit: Some(value.target_gas_limit), - ..Default::default() - } - } -} - -impl TryFrom for PayloadAttributesAmsterdam { - type Error = PayloadAttributesConversionError; - - fn try_from(value: LegacyPayloadAttributes) -> Result { - Ok(Self { - timestamp: value.timestamp, - prev_randao: value.prev_randao, - suggested_fee_recipient: value.suggested_fee_recipient, - withdrawals: require(value.withdrawals, "withdrawals")?, - parent_beacon_block_root: require( - value.parent_beacon_block_root, - "parent_beacon_block_root", - )?, - slot_number: require(value.slot_number, "slot_number")?, - target_gas_limit: require(value.target_gas_limit, "target_gas_limit")?, - }) - } -} - -/// This structure maps to the Engine API v2 REST-SSZ payload-build response for Paris. -/// -/// Unlike the legacy `engine_getPayloadV1` response, this includes the expected block value. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BuiltPayloadParis { - /// Execution payload V1. - pub payload: ExecutionPayloadParis, - /// The expected value to be received by the fee recipient in wei. - pub block_value: U256, -} - -/// This structure maps to the Engine API v2 REST-SSZ payload-build response for Shanghai. -/// -/// This follows the legacy `engine_getPayloadV2` payload-build response shape: execution payload -/// plus block value only. `should_override_builder` starts at Cancun. -/// The concrete V2 payload prevents the legacy V1/V2 untagged field from accepting a Paris payload. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BuiltPayloadShanghai { - /// Execution payload V2. - pub payload: ExecutionPayloadShanghai, - /// The expected value to be received by the fee recipient in wei. - pub block_value: U256, -} - -/// Engine API v2 REST-SSZ payload-build response for Cancun. -/// -/// This is wire-compatible with the legacy `engine_getPayloadV3` response envelope. -pub type BuiltPayloadCancun = alloy_rpc_types_engine::ExecutionPayloadEnvelopeV3; - -/// This structure maps to the Engine API v2 REST-SSZ payload-build response for Prague. -/// -/// Unlike the legacy [`alloy_rpc_types_engine::ExecutionPayloadEnvelopeV4`], -/// `execution_requests` precedes `should_override_builder` in the normative SSZ field order. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BuiltPayloadPrague { - /// Execution payload V3. - pub payload: ExecutionPayloadPrague, - /// The expected value to be received by the fee recipient in wei. - pub block_value: U256, - /// The blobs, commitments, and proofs associated with the executed payload. - pub blobs_bundle: BlobsBundleV1, - /// A list of opaque EIP-7685 requests. - pub execution_requests: Requests, - /// A suggestion from the execution layer whether this payload should be used instead of an - /// externally provided one. - pub should_override_builder: bool, -} - -/// This structure maps to the Engine API v2 REST-SSZ payload-build response for Osaka. -/// -/// It is separate from legacy V5 because REST-SSZ places `execution_requests` before the builder -/// override flag. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BuiltPayloadOsaka { - /// Execution payload V3. - pub payload: ExecutionPayloadOsaka, - /// The expected value to be received by the fee recipient in wei. - pub block_value: U256, - /// The blobs, commitments, and EIP-7594 cell proofs associated with the executed payload. - pub blobs_bundle: BlobsBundleV2, - /// A list of opaque EIP-7685 requests. - pub execution_requests: Requests, - /// A suggestion from the execution layer whether this payload should be used instead of an - /// externally provided one. - pub should_override_builder: bool, -} - -/// This structure maps to the Engine API v2 REST-SSZ payload-build response for Amsterdam. -/// -/// It is separate from legacy V6 because REST-SSZ places `execution_requests` before the builder -/// override flag. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BuiltPayloadAmsterdam { - /// Execution payload V4. - pub payload: ExecutionPayloadAmsterdam, - /// The expected value to be received by the fee recipient in wei. - pub block_value: U256, - /// The blobs, commitments, and EIP-7594 cell proofs associated with the executed payload. - pub blobs_bundle: BlobsBundleV2, - /// A list of opaque EIP-7685 requests. - pub execution_requests: Requests, - /// A suggestion from the execution layer whether this payload should be used instead of an - /// externally provided one. - pub should_override_builder: bool, -} - -/// Error converting legacy payload-build envelopes into fork-specific REST-SSZ containers. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum BuiltPayloadConversionError { - /// The legacy envelope carried an execution payload from a different fork. - UnexpectedPayloadFork(&'static str), -} - -impl core::fmt::Display for BuiltPayloadConversionError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::UnexpectedPayloadFork(fork) => { - write!(f, "unexpected execution payload fork: {fork}") - } - } - } -} - -impl core::error::Error for BuiltPayloadConversionError {} - -impl From for LegacyBuiltPayloadShanghai { - fn from(value: BuiltPayloadShanghai) -> Self { - Self { - execution_payload: ExecutionPayloadFieldV2::V2(value.payload), - block_value: value.block_value, - } - } -} - -impl TryFrom for BuiltPayloadShanghai { - type Error = BuiltPayloadConversionError; - - fn try_from(value: LegacyBuiltPayloadShanghai) -> Result { - match value.execution_payload { - ExecutionPayloadFieldV2::V2(payload) => { - Ok(Self { payload, block_value: value.block_value }) - } - ExecutionPayloadFieldV2::V1(_) => { - Err(BuiltPayloadConversionError::UnexpectedPayloadFork("Paris")) - } - } - } -} - -impl From for BuiltPayloadPrague { - fn from(value: LegacyBuiltPayloadPrague) -> Self { - Self { - payload: value.envelope_inner.execution_payload, - block_value: value.envelope_inner.block_value, - blobs_bundle: value.envelope_inner.blobs_bundle, - execution_requests: value.execution_requests, - should_override_builder: value.envelope_inner.should_override_builder, - } - } -} - -impl From for LegacyBuiltPayloadPrague { - fn from(value: BuiltPayloadPrague) -> Self { - Self { - envelope_inner: alloy_rpc_types_engine::ExecutionPayloadEnvelopeV3 { - execution_payload: value.payload, - block_value: value.block_value, - blobs_bundle: value.blobs_bundle, - should_override_builder: value.should_override_builder, - }, - execution_requests: value.execution_requests, - } - } -} - -impl From for BuiltPayloadOsaka { - fn from(value: LegacyBuiltPayloadOsaka) -> Self { - Self { - payload: value.execution_payload, - block_value: value.block_value, - blobs_bundle: value.blobs_bundle, - execution_requests: value.execution_requests, - should_override_builder: value.should_override_builder, - } - } -} - -impl From for LegacyBuiltPayloadOsaka { - fn from(value: BuiltPayloadOsaka) -> Self { - Self { - execution_payload: value.payload, - block_value: value.block_value, - blobs_bundle: value.blobs_bundle, - should_override_builder: value.should_override_builder, - execution_requests: value.execution_requests, - } - } -} - -impl From for BuiltPayloadAmsterdam { - fn from(value: LegacyBuiltPayloadAmsterdam) -> Self { - Self { - payload: value.execution_payload, - block_value: value.block_value, - blobs_bundle: value.blobs_bundle, - execution_requests: value.execution_requests, - should_override_builder: value.should_override_builder, - } - } -} - -impl From for LegacyBuiltPayloadAmsterdam { - fn from(value: BuiltPayloadAmsterdam) -> Self { - Self { - execution_payload: value.payload, - block_value: value.block_value, - blobs_bundle: value.blobs_bundle, - should_override_builder: value.should_override_builder, - execution_requests: value.execution_requests, - } - } -} - -/// REST-SSZ payload-submission request containers. -/// -/// These are distinct from the legacy Engine JSON-RPC get-payload envelopes: submission requests -/// are fork-specific request bodies, while the legacy envelope types mostly model get-payload -/// responses and sometimes carry response-only fields such as block value, blob bundles, builder -/// override hints, or a different field order. -/// -/// Paris payload-submission request. -/// -/// The single-field container is required by REST-SSZ; the legacy endpoint submitted a bare -/// payload. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadEnvelopeParis { - /// Submitted execution payload. - pub payload: ExecutionPayloadParis, -} - -/// Shanghai payload-submission request. -/// -/// The single-field container is required by REST-SSZ and fixes the payload fork at decode time. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadEnvelopeShanghai { - /// Submitted execution payload. - pub payload: ExecutionPayloadShanghai, -} - -/// Cancun payload-submission request. -/// -/// Cancun adds the parent beacon block root to the REST request envelope. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadEnvelopeCancun { - /// Submitted execution payload. - pub payload: ExecutionPayloadCancun, - /// Root of the parent beacon block. - pub parent_beacon_block_root: B256, -} - -/// Prague payload-submission request. -/// -/// Prague adds execution requests to the REST request envelope. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadEnvelopePrague { - /// Submitted execution payload. - pub payload: ExecutionPayloadPrague, - /// Root of the parent beacon block. - pub parent_beacon_block_root: B256, - /// EIP-7685 execution requests. - pub execution_requests: Requests, -} - -/// Osaka payload-submission request. -/// -/// Osaka keeps the REST envelope shape while selecting the Osaka payload schema. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadEnvelopeOsaka { - /// Submitted execution payload. - pub payload: ExecutionPayloadOsaka, - /// Root of the parent beacon block. - pub parent_beacon_block_root: B256, - /// EIP-7685 execution requests. - pub execution_requests: Requests, -} - -/// Amsterdam payload-submission request. -/// -/// Amsterdam selects the V4 payload while retaining the Cancun and Prague envelope fields. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadEnvelopeAmsterdam { - /// Submitted execution payload. - pub payload: ExecutionPayloadAmsterdam, - /// Root of the parent beacon block. - pub parent_beacon_block_root: B256, - /// EIP-7685 execution requests. - pub execution_requests: Requests, -} - -impl From for ExecutionPayloadEnvelopeParis { - fn from(payload: ExecutionPayloadParis) -> Self { - Self { payload } - } -} - -impl From for ExecutionPayloadEnvelopeShanghai { - fn from(payload: ExecutionPayloadShanghai) -> Self { - Self { payload } - } -} - -impl From<(ExecutionPayloadCancun, B256)> for ExecutionPayloadEnvelopeCancun { - fn from((payload, parent_beacon_block_root): (ExecutionPayloadCancun, B256)) -> Self { - Self { payload, parent_beacon_block_root } - } -} - -impl From<(ExecutionPayloadPrague, B256, Requests)> for ExecutionPayloadEnvelopePrague { - fn from( - (payload, parent_beacon_block_root, execution_requests): ( - ExecutionPayloadPrague, - B256, - Requests, - ), - ) -> Self { - Self { payload, parent_beacon_block_root, execution_requests } - } -} - -impl From<(ExecutionPayloadOsaka, B256, Requests)> for ExecutionPayloadEnvelopeOsaka { - fn from( - (payload, parent_beacon_block_root, execution_requests): ( - ExecutionPayloadOsaka, - B256, - Requests, - ), - ) -> Self { - Self { payload, parent_beacon_block_root, execution_requests } - } -} - -impl From<(ExecutionPayloadAmsterdam, B256, Requests)> for ExecutionPayloadEnvelopeAmsterdam { - fn from( - (payload, parent_beacon_block_root, execution_requests): ( - ExecutionPayloadAmsterdam, - B256, - Requests, - ), - ) -> Self { - Self { payload, parent_beacon_block_root, execution_requests } - } -} - -/// Paris forkchoice-update request. -/// -/// REST-SSZ uses an `Optional` field inside one container; legacy FCU used separate RPC -/// parameters and a legacy `Option` encoding. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ForkchoiceUpdateParis { - /// Current forkchoice state. - pub forkchoice_state: ForkchoiceState, - /// Optional Paris payload attributes. - pub payload_attributes: Optional, -} - -/// Shanghai forkchoice-update request. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ForkchoiceUpdateShanghai { - /// Current forkchoice state. - pub forkchoice_state: ForkchoiceState, - /// Optional Shanghai payload attributes. - pub payload_attributes: Optional, -} - -/// Cancun forkchoice-update request. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ForkchoiceUpdateCancun { - /// Current forkchoice state. - pub forkchoice_state: ForkchoiceState, - /// Optional Cancun payload attributes. - pub payload_attributes: Optional, -} - -/// Prague forkchoice-update request. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ForkchoiceUpdatePrague { - /// Current forkchoice state. - pub forkchoice_state: ForkchoiceState, - /// Optional Prague payload attributes. - pub payload_attributes: Optional, -} - -/// Osaka forkchoice-update request. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ForkchoiceUpdateOsaka { - /// Current forkchoice state. - pub forkchoice_state: ForkchoiceState, - /// Optional Osaka payload attributes. - pub payload_attributes: Optional, -} - -/// Amsterdam forkchoice-update request. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ForkchoiceUpdateAmsterdam { - /// Current forkchoice state. - pub forkchoice_state: ForkchoiceState, - /// Optional Amsterdam payload attributes. - pub payload_attributes: Optional, - /// Optional `Bitvector[128]` custody-column selection. - pub custody_columns: Optional, -} - -/// Fork-specific execution payload body for Paris. -/// -/// Paris omits withdrawals entirely; the legacy body keeps them as an optional union field. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadBodyParis { - /// Enveloped encoded transactions. - pub transactions: Vec, -} - -/// Fork-specific execution payload body for Shanghai. -/// -/// Shanghai makes withdrawals a direct field rather than the legacy optional union. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadBodyShanghai { - /// Enveloped encoded transactions. - pub transactions: Vec, - /// Withdrawals included in the block. - pub withdrawals: Vec, -} - -/// Cancun uses the Shanghai execution-payload-body schema. -pub type ExecutionPayloadBodyCancun = ExecutionPayloadBodyShanghai; - -/// Prague uses the Shanghai execution-payload-body schema. -pub type ExecutionPayloadBodyPrague = ExecutionPayloadBodyShanghai; - -/// Osaka uses the Shanghai execution-payload-body schema. -pub type ExecutionPayloadBodyOsaka = ExecutionPayloadBodyShanghai; - -/// Fork-specific execution payload body for Amsterdam. -/// -/// Amsterdam adds the block access list as a direct field rather than a legacy optional field. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionPayloadBodyAmsterdam { - /// Enveloped encoded transactions. - pub transactions: Vec, - /// Withdrawals included in the block. - pub withdrawals: Vec, - /// RLP-encoded EIP-7928 block access list. - pub block_access_list: Bytes, -} - -/// Error converting legacy cross-fork execution payload bodies into fork-specific containers. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ExecutionPayloadBodyConversionError { - /// A field required by the selected fork is absent. - MissingField(&'static str), - /// A field from a later fork is populated and would be lost. - UnexpectedField(&'static str), -} - -impl core::fmt::Display for ExecutionPayloadBodyConversionError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::MissingField(field) => { - write!(f, "missing required execution payload body field: {field}") - } - Self::UnexpectedField(field) => { - write!(f, "unexpected later-fork execution payload body field: {field}") - } - } - } -} - -impl core::error::Error for ExecutionPayloadBodyConversionError {} - -impl From for LegacyExecutionPayloadBodyV1 { - fn from(value: ExecutionPayloadBodyParis) -> Self { - Self { transactions: value.transactions, withdrawals: None } - } -} - -impl TryFrom for ExecutionPayloadBodyParis { - type Error = ExecutionPayloadBodyConversionError; - - fn try_from(value: LegacyExecutionPayloadBodyV1) -> Result { - if value.withdrawals.is_some() { - return Err(ExecutionPayloadBodyConversionError::UnexpectedField("withdrawals")) - } - Ok(Self { transactions: value.transactions }) - } -} - -impl From for LegacyExecutionPayloadBodyV1 { - fn from(value: ExecutionPayloadBodyShanghai) -> Self { - Self { transactions: value.transactions, withdrawals: Some(value.withdrawals) } - } -} - -impl TryFrom for ExecutionPayloadBodyShanghai { - type Error = ExecutionPayloadBodyConversionError; - - fn try_from(value: LegacyExecutionPayloadBodyV1) -> Result { - Ok(Self { - transactions: value.transactions, - withdrawals: value - .withdrawals - .ok_or(ExecutionPayloadBodyConversionError::MissingField("withdrawals"))?, - }) - } -} - -impl From for LegacyExecutionPayloadBodyV2 { - fn from(value: ExecutionPayloadBodyAmsterdam) -> Self { - Self { - transactions: value.transactions, - withdrawals: Some(value.withdrawals), - block_access_list: Some(value.block_access_list), - } - } -} - -impl TryFrom for ExecutionPayloadBodyAmsterdam { - type Error = ExecutionPayloadBodyConversionError; - - fn try_from(value: LegacyExecutionPayloadBodyV2) -> Result { - Ok(Self { - transactions: value.transactions, - withdrawals: value - .withdrawals - .ok_or(ExecutionPayloadBodyConversionError::MissingField("withdrawals"))?, - block_access_list: value - .block_access_list - .ok_or(ExecutionPayloadBodyConversionError::MissingField("block_access_list"))?, - }) - } -} - -/// REST-SSZ historical bodies-by-hash request. -/// -/// This is a single-field container, not a bare SSZ list. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BodiesByHashRequest { - /// Requested block hashes. - pub block_hashes: Vec, -} - -/// Historical body response entry with explicit availability. -/// -/// REST-SSZ uses a boolean availability bit instead of the legacy `Option` union. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BodyEntry { - /// Whether the body is available and belongs to the requested fork. - pub available: bool, - /// Fork-specific body, ignored when `available` is false. - pub body: T, -} - -impl BodyEntry { - /// Creates an available body entry. - pub const fn available(body: T) -> Self { - Self { available: true, body } - } -} - -impl BodyEntry { - /// Creates an unavailable body entry. - pub fn unavailable() -> Self { - Self { available: false, body: T::default() } - } -} - -/// REST-SSZ historical bodies response. -/// -/// The response is a one-field SSZ container around the entries list, not a bare list. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BodiesResponse { - /// Body entries in request or range order. - pub entries: Vec>, -} - -impl BodiesResponse { - /// Creates a response from optional legacy bodies. - /// - /// Missing bodies, or bodies that do not convert to the requested fork container, are encoded - /// as unavailable entries. - pub fn from_optional_bodies( - bodies: Vec>, - convert: impl Fn(LegacyBody) -> Option, - ) -> Self { - let entries = bodies - .into_iter() - .map(|body| match body.and_then(&convert) { - Some(body) => BodyEntry::available(body), - None => BodyEntry::unavailable(), - }) - .collect(); - - Self { entries } - } -} - -/// Paris historical bodies response. -pub type BodiesResponseParis = BodiesResponse; - -/// Shanghai historical bodies response. -pub type BodiesResponseShanghai = BodiesResponse; - -/// Cancun historical bodies response. -pub type BodiesResponseCancun = BodiesResponse; - -/// Prague historical bodies response. -pub type BodiesResponsePrague = BodiesResponse; - -/// Osaka historical bodies response. -pub type BodiesResponseOsaka = BodiesResponse; - -/// Amsterdam historical bodies response. -pub type BodiesResponseAmsterdam = BodiesResponse; - -/// V1-V3 blob request container. -/// -/// This single-field container starts with a four-byte SSZ offset and is not wire-equivalent to a -/// top-level list. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BlobsV1Request { - /// Requested versioned blob hashes. - pub versioned_hashes: Vec, -} - -/// V2 uses the V1 request schema. -pub type BlobsV2Request = BlobsV1Request; - -/// V3 uses the V1 request schema. -pub type BlobsV3Request = BlobsV1Request; - -/// V4 blob request container with a packed 128-bit index bitvector. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BlobsV4Request { - /// Requested versioned blob hashes. - pub versioned_hashes: Vec, - /// Requested cell indices, SSZ `Bitvector[128]`. - pub indices_bitarray: B128, -} - -/// Blob response entry with explicit outer availability. -/// -/// REST-SSZ keeps availability separate from the blob contents instead of using a legacy option. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BlobEntry { - /// Whether the complete blob contents are available. - pub available: bool, - /// Complete contents, or valid zero-valued contents when unavailable. - pub contents: T, -} - -/// Bounded blob response container. -/// -/// The outer container and entry availability match the REST-SSZ blob endpoint contract. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BlobsResponse { - /// One response entry per requested hash. - pub entries: Vec>, -} - -/// V1 whole-blob response. -pub type BlobsV1Response = BlobsResponse; - -/// V2 all-or-nothing cell-proof response. -pub type BlobsV2Response = BlobsResponse; - -/// V3 partial cell-proof response. -pub type BlobsV3Response = BlobsResponse; - -/// V4 partial cell-range response. -pub type BlobsV4Response = BlobsResponse; - -/// Blob cells and proofs with REST-SSZ optional cell positions. -/// -/// This uses [`Optional`] (`List[T, 1]`) for per-cell nullability, not Rust [`Option`]'s SSZ -/// union encoding. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct BlobCellsAndProofs { - /// Requested blob cells. - pub blob_cells: Vec>, - /// KZG proofs for the requested blob cells. - pub proofs: Vec>, -} - -fn zero_blob_v1() -> BlobAndProofV1 { - BlobAndProofV1 { blob: Box::new(Blob::ZERO), proof: Bytes48::ZERO } -} - -fn zero_blob_v2() -> BlobAndProofV2 { - BlobAndProofV2 { blob: Box::new(Blob::ZERO), proofs: Vec::new() } -} - -impl TryFrom>> for BlobsV1Response { - type Error = ConversionError; - - fn try_from(value: Vec>) -> Result { - if value.len() > MAX_BLOBS_REQUEST { - return Err(ConversionError::TooManyItems { - field: "blobs", - max: MAX_BLOBS_REQUEST, - actual: value.len(), - }) - } - - let entries = value - .into_iter() - .map(|value| match value { - Some(contents) => BlobEntry { available: true, contents }, - None => BlobEntry { available: false, contents: zero_blob_v1() }, - }) - .collect(); - Ok(Self { entries }) - } -} - -impl TryFrom> for BlobsV2Response { - type Error = ConversionError; - - fn try_from(value: Vec) -> Result { - if value.len() > MAX_BLOBS_REQUEST { - return Err(ConversionError::TooManyItems { - field: "blobs", - max: MAX_BLOBS_REQUEST, - actual: value.len(), - }) - } - - let entries = - value.into_iter().map(|contents| BlobEntry { available: true, contents }).collect(); - Ok(Self { entries }) - } -} - -impl TryFrom>> for BlobsV3Response { - type Error = ConversionError; - - fn try_from(value: Vec>) -> Result { - if value.len() > MAX_BLOBS_REQUEST { - return Err(ConversionError::TooManyItems { - field: "blobs", - max: MAX_BLOBS_REQUEST, - actual: value.len(), - }) - } - - let entries = value - .into_iter() - .map(|value| match value { - Some(contents) => BlobEntry { available: true, contents }, - None => BlobEntry { available: false, contents: zero_blob_v2() }, - }) - .collect(); - Ok(Self { entries }) - } -} - -impl TryFrom>> for BlobsV4Response { - type Error = ConversionError; - - fn try_from(value: Vec>) -> Result { - if value.len() > MAX_BLOBS_REQUEST { - return Err(ConversionError::TooManyItems { - field: "blobs", - max: MAX_BLOBS_REQUEST, - actual: value.len(), - }) - } - - let entries = value - .into_iter() - .map(|value| match value { - Some(contents) => BlobEntry { - available: true, - contents: BlobCellsAndProofs { - blob_cells: contents.blob_cells.into_iter().map(Optional::from).collect(), - proofs: contents.proofs.into_iter().map(Optional::from).collect(), - }, - }, - None => BlobEntry { available: false, contents: BlobCellsAndProofs::default() }, - }) - .collect(); - Ok(Self { entries }) - } -} - -/// A trie-node byte list in an [`ExecutionWitnessV1`]. -pub type WitnessNodeV1 = Bytes; - -/// A contract-code byte list in an [`ExecutionWitnessV1`]. -pub type WitnessCodeV1 = Bytes; - -/// An RLP-encoded header byte list in an [`ExecutionWitnessV1`]. -pub type WitnessHeaderV1 = Bytes; - -/// Canonical execution witness for `POST /payloads/witness`. -/// -/// `state` and `codes` are produced in lexicographic ascending byte order. `headers` are -/// RLP-encoded and ordered by ascending block number; consecutive headers must be parent-linked. -/// These ordering rules are producer-side requirements from the execution-specs witness builder. -/// -/// This is a REST-SSZ wire container, not the JSON-RPC debug witness shape. -#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] -pub struct ExecutionWitnessV1 { - /// Hashed trie-node preimages required during execution and state-root recomputation. - pub state: Vec, - /// Contract bytecode preimages required from the pre-state. - pub codes: Vec, - /// RLP-encoded ancestor headers used for pre-state and `BLOCKHASH` correctness proofs. - pub headers: Vec, -} - -/// Canonical execution witness for `POST /payloads/witness`. -pub type ExecutionWitness = ExecutionWitnessV1; - -/// REST-SSZ response for `POST /payloads/witness`. -/// -/// The witness uses the Engine REST-SSZ `Optional[T]` encoding from execution-apis and is present -/// only when the payload status is `VALID`. A `VALID` status without a witness means the parent -/// state was not yet available through the provider (the parent is only known to the engine -/// tree); resubmitting the payload once forkchoice has made the parent canonical yields it. -#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode)] -pub struct PayloadStatusWithWitness { - /// Result of processing the submitted payload. - pub payload_status: PayloadStatus, - /// Execution witness produced for a valid payload. - pub witness: Optional, -} - -impl PayloadStatusWithWitness { - /// Creates a response, converting the witness into the REST-SSZ `Optional[T]` representation. - pub fn new(payload_status: PayloadStatus, witness: Option) -> Self { - let witness = match &payload_status.status { - PayloadStatusEnum::Valid => witness.into(), - _ => Optional::none(), - }; - Self { payload_status, witness } - } -} - -/// Backwards-compatible alias for the experimental witness response name. -pub type NewPayloadWithWitnessResponseV1 = PayloadStatusWithWitness; - -impl ssz::Decode for PayloadStatusWithWitness { - fn is_ssz_fixed_len() -> bool { - false - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - let mut builder = ssz::SszDecoderBuilder::new(bytes); - builder.register_type::()?; - builder.register_type::>()?; - let mut decoder = builder.build()?; - let response = - Self { payload_status: decoder.decode_next()?, witness: decoder.decode_next()? }; - if response.witness.is_some() && - !matches!(response.payload_status.status, PayloadStatusEnum::Valid) - { - return Err(ssz::DecodeError::BytesInvalid( - "execution witness is only valid for VALID payload status".into(), - )) - } - Ok(response) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_eips::eip4895::Withdrawal; - use alloy_primitives::{Address, Bloom, Bytes}; - use ssz::{Decode, Encode}; - - fn payload_v1() -> ExecutionPayloadV1 { - ExecutionPayloadV1 { - parent_hash: B256::repeat_byte(1), - fee_recipient: Address::repeat_byte(2), - state_root: B256::repeat_byte(3), - receipts_root: B256::repeat_byte(4), - logs_bloom: Bloom::repeat_byte(5), - prev_randao: B256::repeat_byte(6), - block_number: 7, - gas_limit: 8, - gas_used: 9, - timestamp: 10, - extra_data: Bytes::from_static(&[11, 12]), - base_fee_per_gas: U256::from(13), - block_hash: B256::repeat_byte(14), - transactions: vec![Bytes::from_static(&[15, 16])], - } - } - - fn payload_v2() -> ExecutionPayloadV2 { - ExecutionPayloadV2 { payload_inner: payload_v1(), withdrawals: vec![Withdrawal::default()] } - } - - fn payload_v3() -> ExecutionPayloadV3 { - ExecutionPayloadV3 { payload_inner: payload_v2(), blob_gas_used: 17, excess_blob_gas: 18 } - } - - fn payload_v4() -> ExecutionPayloadV4 { - ExecutionPayloadV4 { - payload_inner: payload_v3(), - block_access_list: Bytes::from_static(&[19, 20]), - slot_number: 21, - } - } - - fn attributes_cancun() -> PayloadAttributesCancun { - PayloadAttributesCancun { - timestamp: 1, - prev_randao: B256::repeat_byte(2), - suggested_fee_recipient: Address::repeat_byte(3), - withdrawals: vec![Withdrawal::default()], - parent_beacon_block_root: B256::repeat_byte(4), - } - } - - fn state() -> ForkchoiceState { - ForkchoiceState { - head_block_hash: B256::repeat_byte(1), - safe_block_hash: B256::repeat_byte(2), - finalized_block_hash: B256::repeat_byte(3), - } - } - - fn assert_roundtrip(value: &T) - where - T: Encode + Decode + PartialEq + core::fmt::Debug, - { - assert_eq!(T::from_ssz_bytes(&value.as_ssz_bytes()).unwrap(), *value); - } - - #[test] - fn execution_payload_envelopes_roundtrip() { - assert_roundtrip(&ExecutionPayloadEnvelopeParis { payload: payload_v1() }); - assert_roundtrip(&ExecutionPayloadEnvelopeShanghai { payload: payload_v2() }); - assert_roundtrip(&ExecutionPayloadEnvelopeCancun { - payload: payload_v3(), - parent_beacon_block_root: B256::repeat_byte(1), - }); - assert_roundtrip(&ExecutionPayloadEnvelopePrague { - payload: payload_v3(), - parent_beacon_block_root: B256::repeat_byte(1), - execution_requests: Requests::from_requests([Bytes::from_static(&[2, 3])]), - }); - assert_roundtrip(&ExecutionPayloadEnvelopeOsaka { - payload: payload_v3(), - parent_beacon_block_root: B256::repeat_byte(1), - execution_requests: Requests::from_requests([Bytes::from_static(&[2, 3])]), - }); - assert_roundtrip(&ExecutionPayloadEnvelopeAmsterdam { - payload: payload_v4(), - parent_beacon_block_root: B256::repeat_byte(1), - execution_requests: Requests::from_requests([Bytes::from_static(&[2, 3])]), - }); - } - - #[test] - fn paris_submission_is_a_single_field_container() { - let payload = payload_v1(); - let payload_bytes = payload.as_ssz_bytes(); - let envelope = ExecutionPayloadEnvelopeParis { payload }; - let encoded = envelope.as_ssz_bytes(); - assert_eq!(&encoded[..4], &4u32.to_le_bytes()); - assert_eq!(&encoded[4..], payload_bytes); - } - - #[test] - fn built_payloads_roundtrip() { - assert_roundtrip(&BuiltPayloadParis { payload: payload_v1(), block_value: U256::from(1) }); - assert_roundtrip(&BuiltPayloadShanghai { - payload: payload_v2(), - block_value: U256::from(1), - }); - assert_roundtrip(&BuiltPayloadCancun { - execution_payload: payload_v3(), - block_value: U256::from(1), - blobs_bundle: BlobsBundleV1::empty(), - should_override_builder: true, - }); - assert_roundtrip(&BuiltPayloadPrague { - payload: payload_v3(), - block_value: U256::from(1), - blobs_bundle: BlobsBundleV1::empty(), - execution_requests: Requests::from_requests([Bytes::from_static(&[2, 3])]), - should_override_builder: true, - }); - assert_roundtrip(&BuiltPayloadOsaka { - payload: payload_v3(), - block_value: U256::from(1), - blobs_bundle: BlobsBundleV2::empty(), - execution_requests: Requests::from_requests([Bytes::from_static(&[2, 3])]), - should_override_builder: true, - }); - assert_roundtrip(&BuiltPayloadAmsterdam { - payload: payload_v4(), - block_value: U256::from(1), - blobs_bundle: BlobsBundleV2::empty(), - execution_requests: Requests::from_requests([Bytes::from_static(&[2, 3])]), - should_override_builder: true, - }); - } - - #[test] - fn shanghai_built_payload_has_no_builder_override() { - let payload = payload_v2(); - let payload_len = payload.ssz_bytes_len(); - let value = BuiltPayloadShanghai { payload, block_value: U256::from(1) }; - let encoded = value.as_ssz_bytes(); - - assert_eq!(&encoded[..4], &36u32.to_le_bytes()); - assert_eq!(encoded.len(), 36 + payload_len); - } - - #[test] - fn legacy_built_payload_conversions_preserve_fields() { - let shanghai = BuiltPayloadShanghai { payload: payload_v2(), block_value: U256::from(1) }; - let legacy = LegacyBuiltPayloadShanghai::from(shanghai.clone()); - assert_eq!(BuiltPayloadShanghai::try_from(legacy).unwrap(), shanghai); - - let prague = BuiltPayloadPrague { - payload: payload_v3(), - block_value: U256::from(2), - blobs_bundle: BlobsBundleV1::empty(), - execution_requests: Requests::from_requests([Bytes::from_static(&[3, 4])]), - should_override_builder: true, - }; - let legacy = LegacyBuiltPayloadPrague::from(prague.clone()); - assert_eq!(BuiltPayloadPrague::from(legacy), prague); - - let osaka = BuiltPayloadOsaka { - payload: payload_v3(), - block_value: U256::from(5), - blobs_bundle: BlobsBundleV2::empty(), - execution_requests: Requests::from_requests([Bytes::from_static(&[6, 7])]), - should_override_builder: true, - }; - let legacy = LegacyBuiltPayloadOsaka::from(osaka.clone()); - assert_eq!(BuiltPayloadOsaka::from(legacy), osaka); - - let amsterdam = BuiltPayloadAmsterdam { - payload: payload_v4(), - block_value: U256::from(8), - blobs_bundle: BlobsBundleV2::empty(), - execution_requests: Requests::from_requests([Bytes::from_static(&[9, 10])]), - should_override_builder: true, - }; - let legacy = LegacyBuiltPayloadAmsterdam::from(amsterdam.clone()); - assert_eq!(BuiltPayloadAmsterdam::from(legacy), amsterdam); - } - - #[test] - fn legacy_shanghai_built_payload_rejects_paris_payload() { - let legacy = LegacyBuiltPayloadShanghai { - execution_payload: ExecutionPayloadFieldV2::V1(payload_v1()), - block_value: U256::from(1), - }; - assert_eq!( - BuiltPayloadShanghai::try_from(legacy), - Err(BuiltPayloadConversionError::UnexpectedPayloadFork("Paris")) - ); - } - - #[test] - fn prague_requests_precede_should_override_builder() { - let value = BuiltPayloadPrague { - payload: payload_v3(), - block_value: U256::from(1), - blobs_bundle: BlobsBundleV1::empty(), - execution_requests: Requests::from_requests([Bytes::from_static(&[0xaa, 0xbb])]), - should_override_builder: true, - }; - let encoded = value.as_ssz_bytes(); - assert_eq!(encoded[48], 1); - assert_eq!(&encoded[encoded.len() - 2..], &[0xaa, 0xbb]); - } - - #[test] - fn forkchoice_updates_roundtrip() { - let paris = PayloadAttributesParis { - timestamp: 1, - prev_randao: B256::repeat_byte(2), - suggested_fee_recipient: Address::repeat_byte(3), - }; - let shanghai = PayloadAttributesShanghai { - timestamp: 1, - prev_randao: B256::repeat_byte(2), - suggested_fee_recipient: Address::repeat_byte(3), - withdrawals: vec![Withdrawal::default()], - }; - let amsterdam = PayloadAttributesAmsterdam { - timestamp: 1, - prev_randao: B256::repeat_byte(2), - suggested_fee_recipient: Address::repeat_byte(3), - withdrawals: vec![Withdrawal::default()], - parent_beacon_block_root: B256::repeat_byte(4), - slot_number: 5, - target_gas_limit: 6, - }; - assert_roundtrip(&ForkchoiceUpdateParis { - forkchoice_state: state(), - payload_attributes: Optional::some(paris), - }); - assert_roundtrip(&ForkchoiceUpdateShanghai { - forkchoice_state: state(), - payload_attributes: Optional::some(shanghai), - }); - assert_roundtrip(&ForkchoiceUpdateCancun { - forkchoice_state: state(), - payload_attributes: Optional::some(attributes_cancun()), - }); - assert_roundtrip(&ForkchoiceUpdatePrague { - forkchoice_state: state(), - payload_attributes: Optional::some(attributes_cancun()), - }); - assert_roundtrip(&ForkchoiceUpdateOsaka { - forkchoice_state: state(), - payload_attributes: Optional::some(attributes_cancun()), - }); - assert_roundtrip(&ForkchoiceUpdateAmsterdam { - forkchoice_state: state(), - payload_attributes: Optional::some(amsterdam), - custody_columns: Optional::some(B128::repeat_byte(0xa5)), - }); - } - - #[test] - fn payload_attributes_legacy_conversions_preserve_fork_shape() { - let cancun = attributes_cancun(); - let legacy = LegacyPayloadAttributes::from(cancun.clone()); - assert_eq!(PayloadAttributesCancun::try_from(legacy).unwrap(), cancun); - - let amsterdam = PayloadAttributesAmsterdam { - timestamp: 1, - prev_randao: B256::repeat_byte(2), - suggested_fee_recipient: Address::repeat_byte(3), - withdrawals: vec![Withdrawal::default()], - parent_beacon_block_root: B256::repeat_byte(4), - slot_number: 5, - target_gas_limit: 6, - }; - let legacy = LegacyPayloadAttributes::from(amsterdam.clone()); - assert_eq!(PayloadAttributesAmsterdam::try_from(legacy).unwrap(), amsterdam); - } - - #[test] - fn payload_attributes_legacy_conversions_reject_loss() { - let mut legacy = LegacyPayloadAttributes::default(); - assert_eq!( - PayloadAttributesShanghai::try_from(legacy.clone()), - Err(PayloadAttributesConversionError::MissingField("withdrawals")) - ); - - legacy.withdrawals = Some(vec![]); - legacy.parent_beacon_block_root = Some(B256::ZERO); - assert_eq!( - PayloadAttributesShanghai::try_from(legacy), - Err(PayloadAttributesConversionError::UnexpectedField("parent_beacon_block_root")) - ); - } - - #[test] - fn every_payload_status_roundtrips() { - for status in [ - PayloadStatusEnum::Valid, - PayloadStatusEnum::Invalid { validation_error: "invalid".into() }, - PayloadStatusEnum::Syncing, - PayloadStatusEnum::Accepted, - ] { - let validation_error = legacy_validation_error(&status).unwrap(); - let value = PayloadStatus { - status, - latest_valid_hash: Optional::some(B256::ZERO), - validation_error, - }; - assert_eq!(PayloadStatus::from_ssz_bytes(&value.as_ssz_bytes()).unwrap(), value); - } - } - - #[test] - fn payload_status_preserves_absent_invalid_validation_error() { - let mut bytes = Vec::new(); - let mut encoder = ssz::SszEncoder::container(&mut bytes, 9); - encoder.append(&1u8); - encoder.append(&Optional::::none()); - encoder.append(&Optional::::none()); - encoder.finalize(); - - let decoded = PayloadStatus::from_ssz_bytes(&bytes).unwrap(); - assert!(decoded.validation_error.is_none()); - assert_eq!(decoded.as_ssz_bytes(), bytes); - } - - #[test] - fn payload_status_rejects_non_invalid_validation_error() { - let mut bytes = Vec::new(); - let mut encoder = ssz::SszEncoder::container(&mut bytes, 9); - encoder.append(&0u8); - encoder.append(&Optional::::none()); - encoder.append(&Optional::some(Vec::::new())); - encoder.finalize(); - assert!(PayloadStatus::from_ssz_bytes(&bytes).is_err()); - } - - #[test] - fn payload_status_legacy_conversion_rejects_oversized_error() { - assert!(PayloadStatus::try_from(LegacyPayloadStatus { - status: PayloadStatusEnum::Invalid { validation_error: "x".repeat(1025) }, - latest_valid_hash: None, - }) - .is_err()); - } - - #[test] - fn forkchoice_response_distinguishes_absent_and_zero_payload_id() { - let status = PayloadStatus { - status: PayloadStatusEnum::Valid, - latest_valid_hash: Optional::none(), - validation_error: Optional::none(), - }; - let none = ForkchoiceUpdateResponse { - payload_status: status.clone(), - payload_id: Optional::none(), - }; - let zero = ForkchoiceUpdateResponse { - payload_status: status, - payload_id: Optional::some(PayloadId::default()), - }; - - assert_ne!(none.as_ssz_bytes(), zero.as_ssz_bytes()); - assert_roundtrip(&none); - assert_roundtrip(&zero); - } - - #[test] - fn forkchoice_conversion_rejects_accepted() { - let legacy = LegacyForkchoice::from_status(PayloadStatusEnum::Accepted); - assert_eq!( - ForkchoiceUpdateResponse::try_from(legacy), - Err(ConversionError::AcceptedForkchoice) - ); - } - - fn blob_v2(byte: u8) -> BlobAndProofV2 { - BlobAndProofV2 { - blob: Box::new(Blob::repeat_byte(byte)), - proofs: vec![Bytes48::repeat_byte(byte)], - } - } - - #[test] - fn blob_requests_are_single_field_containers() { - let request = BlobsV1Request { versioned_hashes: vec![B256::repeat_byte(0x42)] }; - let encoded = request.as_ssz_bytes(); - - assert_eq!(&encoded[..4], &4u32.to_le_bytes()); - assert_eq!(&encoded[4..], B256::repeat_byte(0x42).as_slice()); - assert_eq!(BlobsV1Request::from_ssz_bytes(&encoded).unwrap(), request); - - let _: BlobsV2Request = BlobsV2Request::from_ssz_bytes(&encoded).unwrap(); - let _: BlobsV3Request = BlobsV3Request::from_ssz_bytes(&encoded).unwrap(); - } - - #[test] - fn blob_v4_request_roundtrips_bitvector() { - let request = BlobsV4Request { - versioned_hashes: vec![B256::repeat_byte(0x11)], - indices_bitarray: B128::repeat_byte(0xa5), - }; - - assert_roundtrip(&request); - } - - #[test] - fn blob_response_conversions_preserve_availability_and_order() { - let v1 = BlobsV1Response::try_from(vec![None]).unwrap(); - assert!(!v1.entries[0].available); - assert_eq!(v1.entries[0].contents, zero_blob_v1()); - - let v2 = BlobsV2Response::try_from(vec![blob_v2(1), blob_v2(2)]).unwrap(); - assert!(v2.entries.iter().all(|entry| entry.available)); - - let v3 = BlobsV3Response::try_from(vec![Some(blob_v2(1)), None, Some(blob_v2(3))]).unwrap(); - assert_eq!( - v3.entries.iter().map(|entry| entry.available).collect::>(), - [true, false, true] - ); - assert_eq!(v3.entries[2].contents.blob.as_slice(), Blob::repeat_byte(3).as_slice()); - - let legacy_partial = BlobCellsAndProofsV1 { - blob_cells: vec![Some(Cell::repeat_byte(1)), None], - proofs: vec![Some(Bytes48::repeat_byte(2)), None], - }; - let v4 = BlobsV4Response::try_from(vec![None, Some(legacy_partial)]).unwrap(); - assert!(!v4.entries[0].available); - assert!(v4.entries[1].available); - assert!(v4.entries[1].contents.blob_cells[0].is_some()); - assert!(v4.entries[1].contents.proofs[1].is_none()); - } - - #[test] - fn blob_cells_and_proofs_uses_rest_optional() { - let value = BlobCellsAndProofs { - blob_cells: vec![Optional::some(Cell::repeat_byte(1))], - proofs: vec![Optional::some(Bytes48::repeat_byte(2))], - }; - let encoded = value.as_ssz_bytes(); - - assert_eq!(BlobCellsAndProofs::from_ssz_bytes(&encoded).unwrap(), value); - assert!(!encoded[8..].starts_with(&[1, 0, 0, 0])); - } - - #[test] - fn payload_body_requests_are_single_field_containers() { - let request = BodiesByHashRequest { block_hashes: vec![B256::repeat_byte(0x33)] }; - let encoded = request.as_ssz_bytes(); - - assert_eq!(&encoded[..4], &4u32.to_le_bytes()); - assert_eq!(&encoded[4..], B256::repeat_byte(0x33).as_slice()); - assert_eq!(BodiesByHashRequest::from_ssz_bytes(&encoded).unwrap(), request); - } - - #[test] - fn payload_body_responses_preserve_availability() { - let legacy = LegacyExecutionPayloadBodyV1 { - transactions: vec![Bytes::from_static(&[1, 2, 3])], - withdrawals: Some(vec![Withdrawal::default()]), - }; - let response = - BodiesResponseShanghai::from_optional_bodies(vec![Some(legacy), None], |body| { - ExecutionPayloadBodyShanghai::try_from(body).ok() - }); - - assert!(response.entries[0].available); - assert!(!response.entries[1].available); - assert_roundtrip(&response); - } - - #[test] - fn witness_response_roundtrips_when_status_is_valid() { - let payload_status = PayloadStatus { - status: PayloadStatusEnum::Valid, - latest_valid_hash: Optional::none(), - validation_error: Optional::none(), - }; - let witness = ExecutionWitnessV1 { - state: vec![vec![1, 2, 3].into()], - codes: vec![vec![4, 5].into()], - headers: vec![vec![6].into()], - }; - let response = PayloadStatusWithWitness::new(payload_status, Some(witness)); - - assert_roundtrip(&response); - } - - #[test] - fn witness_response_omits_witness_for_non_valid_status() { - let payload_status = PayloadStatus { - status: PayloadStatusEnum::Syncing, - latest_valid_hash: Optional::none(), - validation_error: Optional::none(), - }; - let response = - PayloadStatusWithWitness::new(payload_status, Some(ExecutionWitnessV1::default())); - - assert!(response.witness.is_none()); - assert_roundtrip(&response); - } - - #[test] - fn optional_rejects_more_than_one_value() { - assert!(Optional::::from_ssz_bytes(&[0; 64]).is_err()); - } -} diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index 26260293a0d..06bda66b1bb 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -4,8 +4,14 @@ //! //! [EIP-8178]: https://eips.ethereum.org/EIPS/eip-8178 -use crate::{ - engine_ssz_containers::{ +use crate::engine_ssz_witness::{ + EngineSszWitness, EngineSszWitnessError, PayloadStatusWithWitness, +}; +use alloy_consensus::{Transaction, TxEnvelope}; +use alloy_eips::{eip2718::Decodable2718, eip7685::Requests}; +use alloy_primitives::{Bytes, B128, B256}; +use alloy_rpc_types_engine::{ + ssz_engine_types::{ BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, BlobsV4Response, BodiesByHashRequest, BodiesResponse, BuiltPayloadAmsterdam, BuiltPayloadOsaka, BuiltPayloadParis, BuiltPayloadPrague, BuiltPayloadShanghai, @@ -16,17 +22,11 @@ use crate::{ ForkchoiceUpdateAmsterdam, ForkchoiceUpdateCancun, ForkchoiceUpdateOsaka, ForkchoiceUpdateParis, ForkchoiceUpdatePrague, ForkchoiceUpdateResponse, ForkchoiceUpdateShanghai, Optional, PayloadStatus as EngineSszPayloadStatus, - PayloadStatusWithWitness, + PayloadStatusKind, MAX_BLOBS_REQUEST, MAX_BODIES_REQUEST, }, - engine_ssz_witness::{EngineSszWitness, EngineSszWitnessError}, -}; -use alloy_consensus::{Transaction, TxEnvelope}; -use alloy_eips::{eip2718::Decodable2718, eip7685::Requests}; -use alloy_primitives::{Bytes, B128, B256}; -use alloy_rpc_types_engine::{ CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadBodyV1, ExecutionPayloadFieldV2, ExecutionPayloadSidecar, ForkchoiceState, PayloadAttributes, - PayloadId, PayloadStatusEnum, PraguePayloadFields, + PayloadId, PraguePayloadFields, }; use futures::future::{BoxFuture, Either}; use http_body_util::{BodyExt, LengthLimitError, Limited}; @@ -65,9 +65,7 @@ const STATUS_INTERNAL_SERVER_ERROR: u16 = 500; const STATUS_SERVICE_UNAVAILABLE: u16 = 503; const STATUS_UNSUPPORTED_MEDIA_TYPE: u16 = 415; -const MAX_BLOB_LIMIT: usize = crate::engine_ssz_containers::MAX_BLOBS_REQUEST; -const MAX_BLOB_REQUEST_BYTES: usize = 4 + 16 + MAX_BLOB_LIMIT * 32; -const MAX_BODIES_REQUEST: usize = crate::engine_ssz_containers::MAX_BODIES_REQUEST; +const MAX_BLOB_REQUEST_BYTES: usize = 4 + 16 + MAX_BLOBS_REQUEST * 32; const MAX_BODIES_REQUEST_BYTES: usize = 4 + MAX_BODIES_REQUEST * 32; const MAX_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; const PROBLEM_JSON: &str = "application/problem+json"; @@ -407,7 +405,7 @@ where Err(response) => return response, }; let witness = match status.status { - PayloadStatusEnum::Valid => match witness_handler.generate_witness(payload).await { + PayloadStatusKind::Valid => match witness_handler.generate_witness(payload).await { Ok(witness) => Some(witness), // The block is valid but its parent is only known to the engine tree. The // status stays authoritative; resubmitting once forkchoice has made the parent @@ -540,7 +538,7 @@ where Ok(request) => request, Err(_) => return problem_response(STATUS_BAD_REQUEST, "ssz-decode-error", None), }; - if request.versioned_hashes.len() > MAX_BLOB_LIMIT { + if request.versioned_hashes.len() > MAX_BLOBS_REQUEST { return problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None) } return blob_response::( @@ -551,7 +549,7 @@ where Ok(request) => request, Err(_) => return problem_response(STATUS_BAD_REQUEST, "ssz-decode-error", None), }; - if request.versioned_hashes.len() > MAX_BLOB_LIMIT { + if request.versioned_hashes.len() > MAX_BLOBS_REQUEST { return problem_response(STATUS_PAYLOAD_TOO_LARGE, "request-too-large", None) } let hashes = request.versioned_hashes; @@ -797,7 +795,7 @@ fn handle_capabilities(witness_enabled: bool) -> HttpResponse { "unscoped_endpoints": ["capabilities", "identity"], "limits": { "bodies.max_count": MAX_BODIES_REQUEST, - "blobs.max_versioned_hashes": MAX_BLOB_LIMIT, + "blobs.max_versioned_hashes": MAX_BLOBS_REQUEST, "payload.max_bytes": MAX_PAYLOAD_BYTES, }, })) @@ -923,11 +921,11 @@ fn payload_bodies_response( convert: impl Fn(LegacyBody) -> Option, fork: EngineSszFork, chain_spec: &impl EthereumHardforks, -) -> Result, EngineApiError> +) -> Result, HttpResponse> where ForkBody: Default + ssz::Encode + ssz::Decode, { - let bodies = response?; + let bodies = response.map_err(engine_error_response)?; let bodies = bodies .into_iter() .map(|body| { @@ -935,7 +933,9 @@ where .map(|(_, body)| body) }) .collect(); - Ok(BodiesResponse::from_optional_bodies(bodies, convert)) + BodiesResponse::from_optional_bodies(bodies, convert).map_err(|error| { + problem_response(STATUS_INTERNAL_SERVER_ERROR, "internal", Some(error.to_string())) + }) } fn payload_bodies_http_response( @@ -949,7 +949,7 @@ where { match payload_bodies_response(response, convert, fork, chain_spec) { Ok(response) => ssz_response(response), - Err(err) => engine_error_response(err), + Err(response) => response, } } @@ -1255,6 +1255,10 @@ fn problem_response( #[cfg(test)] mod tests { use super::*; + use alloy_rpc_types_engine::ssz_engine_types::{ + PayloadAttributesAmsterdam, PayloadAttributesCancun, PayloadAttributesParis, + PayloadAttributesShanghai, + }; use ssz::Encode; #[tokio::test] @@ -1273,7 +1277,7 @@ mod tests { _: ExecutionData, ) -> BoxFuture< 'static, - Result, + Result, > { Box::pin(async { Ok(Default::default()) }) } @@ -1375,9 +1379,6 @@ mod tests { #[test] fn forkchoice_withdrawal_bounds() { - use crate::engine_ssz_containers::{ - PayloadAttributesAmsterdam, PayloadAttributesCancun, PayloadAttributesShanghai, - }; for count in [16, 17] { let withdrawals = vec![Default::default(); count]; let state = ForkchoiceState::default(); @@ -1421,7 +1422,6 @@ mod tests { #[test] fn paris_forkchoice_uses_fixed_size_optional_attributes() { - use crate::engine_ssz_containers::PayloadAttributesParis; let request = ForkchoiceUpdateParis { forkchoice_state: ForkchoiceState::default(), payload_attributes: Optional::some(PayloadAttributesParis { @@ -1698,7 +1698,7 @@ mod tests { safe_block_hash: B256::ZERO, finalized_block_hash: B256::ZERO, }; - let attrs = crate::engine_ssz_containers::PayloadAttributesCancun { + let attrs = PayloadAttributesCancun { timestamp: 1, prev_randao: B256::with_last_byte(2), suggested_fee_recipient: Default::default(), diff --git a/crates/ethereum/node/src/engine_ssz_witness.rs b/crates/ethereum/node/src/engine_ssz_witness.rs index 457daa8eac2..c8b331ac40d 100644 --- a/crates/ethereum/node/src/engine_ssz_witness.rs +++ b/crates/ethereum/node/src/engine_ssz_witness.rs @@ -1,8 +1,10 @@ -//! Execution witness generation for the SSZ Engine API extension. +//! Execution witness generation and wire types for the SSZ Engine API extension. -use crate::engine_ssz_containers::ExecutionWitnessV1; -use alloy_primitives::B256; -use alloy_rpc_types_engine::ExecutionData; +use alloy_primitives::{Bytes, B256}; +use alloy_rpc_types_engine::{ + ssz_engine_types::{Optional, PayloadStatus, PayloadStatusKind}, + ExecutionData, +}; use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; use reth_evm::{execute::Executor, ConfigureEvm}; use reth_primitives_traits::{AlloyBlockHeader, Block}; @@ -168,3 +170,126 @@ impl std::error::Error for EngineSszWitnessError { } } } + +/// A trie-node byte list in an [`ExecutionWitnessV1`]. +pub type WitnessNodeV1 = Bytes; + +/// A contract-code byte list in an [`ExecutionWitnessV1`]. +pub type WitnessCodeV1 = Bytes; + +/// An RLP-encoded header byte list in an [`ExecutionWitnessV1`]. +pub type WitnessHeaderV1 = Bytes; + +/// Canonical execution witness for `POST /payloads/witness`. +/// +/// `state` and `codes` are produced in lexicographic ascending byte order. `headers` are +/// RLP-encoded and ordered by ascending block number; consecutive headers must be parent-linked. +/// These ordering rules are producer-side requirements from the execution-specs witness builder. +/// +/// This is a REST-SSZ wire container, not the JSON-RPC debug witness shape. +#[derive(Clone, Debug, Default, PartialEq, Eq, ssz_derive::Encode, ssz_derive::Decode)] +pub struct ExecutionWitnessV1 { + /// Hashed trie-node preimages required during execution and state-root recomputation. + pub state: Vec, + /// Contract bytecode preimages required from the pre-state. + pub codes: Vec, + /// RLP-encoded ancestor headers used for pre-state and `BLOCKHASH` correctness proofs. + pub headers: Vec, +} + +/// Canonical execution witness for `POST /payloads/witness`. +pub type ExecutionWitness = ExecutionWitnessV1; + +/// REST-SSZ response for `POST /payloads/witness`. +/// +/// The witness uses the Engine REST-SSZ `Optional[T]` encoding from execution-apis and is present +/// only when the payload status is `VALID`. A `VALID` status without a witness means the parent +/// state was not yet available through the provider (the parent is only known to the engine +/// tree); resubmitting the payload once forkchoice has made the parent canonical yields it. +#[derive(Clone, Debug, PartialEq, Eq, ssz_derive::Encode)] +pub struct PayloadStatusWithWitness { + /// Result of processing the submitted payload. + pub payload_status: PayloadStatus, + /// Execution witness produced for a valid payload. + pub witness: Optional, +} + +impl PayloadStatusWithWitness { + /// Creates a response, converting the witness into the REST-SSZ `Optional[T]` representation. + pub fn new(payload_status: PayloadStatus, witness: Option) -> Self { + let witness = match payload_status.status { + PayloadStatusKind::Valid => witness.into(), + _ => Optional::none(), + }; + Self { payload_status, witness } + } +} + +/// Backwards-compatible alias for the experimental witness response name. +pub type NewPayloadWithWitnessResponseV1 = PayloadStatusWithWitness; + +impl ssz::Decode for PayloadStatusWithWitness { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let mut builder = ssz::SszDecoderBuilder::new(bytes); + builder.register_type::()?; + builder.register_type::>()?; + let mut decoder = builder.build()?; + let response = + Self { payload_status: decoder.decode_next()?, witness: decoder.decode_next()? }; + if response.witness.is_some() && response.payload_status.status != PayloadStatusKind::Valid + { + return Err(ssz::DecodeError::BytesInvalid( + "execution witness is only valid for VALID payload status".into(), + )) + } + Ok(response) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ssz::{Decode, Encode}; + + fn assert_roundtrip(value: &T) + where + T: Encode + Decode + PartialEq + core::fmt::Debug, + { + assert_eq!(T::from_ssz_bytes(&value.as_ssz_bytes()).unwrap(), *value); + } + + #[test] + fn witness_response_roundtrips_when_status_is_valid() { + let payload_status = PayloadStatus { + status: PayloadStatusKind::Valid, + latest_valid_hash: Optional::none(), + validation_error: Optional::none(), + }; + let witness = ExecutionWitnessV1 { + state: vec![vec![1, 2, 3].into()], + codes: vec![vec![4, 5].into()], + headers: vec![vec![6].into()], + }; + let response = PayloadStatusWithWitness::new(payload_status, Some(witness)); + + assert_roundtrip(&response); + } + + #[test] + fn witness_response_omits_witness_for_non_valid_status() { + let payload_status = PayloadStatus { + status: PayloadStatusKind::Syncing, + latest_valid_hash: Optional::none(), + validation_error: Optional::none(), + }; + let response = + PayloadStatusWithWitness::new(payload_status, Some(ExecutionWitnessV1::default())); + + assert!(response.witness.is_none()); + assert_roundtrip(&response); + } +} diff --git a/crates/ethereum/node/src/lib.rs b/crates/ethereum/node/src/lib.rs index be55029642d..e2e93efc81e 100644 --- a/crates/ethereum/node/src/lib.rs +++ b/crates/ethereum/node/src/lib.rs @@ -31,6 +31,5 @@ pub mod payload; pub mod engine; pub use engine::EthereumEngineValidator; -pub mod engine_ssz_containers; pub mod engine_ssz_proxy; pub mod engine_ssz_witness; diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index 728ba82b06e..3f30a025a11 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -3,8 +3,15 @@ use crate::utils::{ }; use alloy_eips::eip7685::RequestsOrHash; use alloy_genesis::Genesis; -use alloy_primitives::{Address, B256}; +use alloy_primitives::{Address, Bytes, B256}; use alloy_rpc_types_engine::{ + ssz_engine_types::{ + BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, + BlobsV4Response, BodiesByHashRequest, BodiesResponsePrague, BuiltPayloadPrague, + ExecutionPayloadEnvelopeAmsterdam, ExecutionPayloadEnvelopePrague, ExecutionPayloadPrague, + ForkchoiceUpdateCancun, ForkchoiceUpdateResponse as SszForkchoiceUpdateResponse, Optional, + PayloadAttributesCancun, PayloadStatus as SszPayloadStatus, PayloadStatusKind, + }, ClientVersionV1, ForkchoiceState, PayloadAttributes, PayloadStatusEnum, }; use jsonrpsee_core::client::ClientT; @@ -20,13 +27,7 @@ use reth_node_core::{ version::{version_metadata, CLIENT_CODE}, }; use reth_node_ethereum::{ - engine_ssz_containers::{ - BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, - BlobsV4Response, BodiesByHashRequest, BodiesResponsePrague, - ExecutionPayloadEnvelopeAmsterdam, ForkchoiceUpdateResponse as SszForkchoiceUpdateResponse, - PayloadStatus as SszPayloadStatus, PayloadStatusWithWitness, - }, - EthereumAddOns, EthereumNode, + engine_ssz_witness::PayloadStatusWithWitness, EthereumAddOns, EthereumNode, }; use reth_provider::{BlockNumReader, StateProviderFactory}; use reth_rpc_api::TestingBuildBlockRequestV1; @@ -419,7 +420,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { assert_eq!(new_payload_response.status(), reqwest::StatusCode::OK); let status = SszPayloadStatus::from_ssz_bytes(&new_payload_response.bytes().await?).unwrap(); - assert_eq!(status.status, PayloadStatusEnum::Valid); + assert_eq!(status.status, PayloadStatusKind::Valid); let fcu_response = client .post(format!("{auth_url}{ENGINE_FORKCHOICE_ROUTE}")) @@ -465,7 +466,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { assert!(!blobs.entries.last().unwrap().available); let fcu = SszForkchoiceUpdateResponse::from_ssz_bytes(&fcu_response.bytes().await?).unwrap(); - assert_eq!(fcu.payload_status.status, PayloadStatusEnum::Valid); + assert_eq!(fcu.payload_status.status, PayloadStatusKind::Valid); node.wait_block(1, block_hash, false).await?; @@ -639,7 +640,7 @@ async fn test_engine_ssz_proxy_returns_canonical_witness() -> eyre::Result<()> { let bytes = response.bytes().await?; assert_eq!(status, reqwest::StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let response = PayloadStatusWithWitness::from_ssz_bytes(&bytes).unwrap(); - assert_eq!(response.payload_status.status, PayloadStatusEnum::Valid); + assert_eq!(response.payload_status.status, PayloadStatusKind::Valid); let witness = response.witness.as_ref().expect("valid payload includes a witness"); assert!(!witness.state.is_empty()); assert!(witness.state.windows(2).all(|pair| pair[0] < pair[1])); @@ -659,7 +660,7 @@ async fn test_engine_ssz_proxy_returns_canonical_witness() -> eyre::Result<()> { .await?; assert_eq!(response.status(), reqwest::StatusCode::OK); let response = PayloadStatusWithWitness::from_ssz_bytes(&response.bytes().await?).unwrap(); - assert!(matches!(response.payload_status.status, PayloadStatusEnum::Invalid { .. })); + assert_eq!(response.payload_status.status, PayloadStatusKind::Invalid); assert!(response.witness.is_none()); invalid.payload.payload_inner.payload_inner.payload_inner.transactions = vec![alloy_primitives::Bytes::from_static(&[2])]; @@ -777,12 +778,6 @@ async fn test_sparse_trie_reuse_across_blocks() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_request_validation() -> eyre::Result<()> { - use alloy_primitives::Bytes; - use reth_node_ethereum::engine_ssz_containers::{ - BuiltPayloadPrague, ExecutionPayloadEnvelopePrague, ExecutionPayloadPrague, - ForkchoiceUpdateCancun, Optional, PayloadAttributesCancun, - }; - let chain = Arc::new( ChainSpecBuilder::default() .chain(MAINNET.chain) @@ -837,7 +832,7 @@ async fn test_engine_ssz_request_validation() -> eyre::Result<()> { assert_eq!(response.status(), reqwest::StatusCode::OK); let fcu = SszForkchoiceUpdateResponse::from_ssz_bytes(&response.bytes().await?).unwrap(); - assert!(matches!(fcu.payload_status.status, PayloadStatusEnum::Valid)); + assert_eq!(fcu.payload_status.status, PayloadStatusKind::Valid); let id = fcu.payload_id.into_option().unwrap(); let response = client .get(format!("{url}{ENGINE_PAYLOADS_ROUTE}/{id}")) @@ -1002,7 +997,7 @@ async fn test_engine_ssz_witness_omitted_without_provider_parent_state() -> eyre .await?; assert_eq!(response.status(), reqwest::StatusCode::OK); let response = PayloadStatusWithWitness::from_ssz_bytes(&response.bytes().await?).unwrap(); - assert_eq!(response.payload_status.status, PayloadStatusEnum::Valid); + assert_eq!(response.payload_status.status, PayloadStatusKind::Valid); assert!(response.witness.is_none()); Ok(()) } From b4b2e8d1938fcf73b1658309118c3472d914b82a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 10:50:49 +0000 Subject: [PATCH 080/133] perf(engine): skip BAL bytecode prefetch (#27416) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../src/tree/payload_processor/bal_prewarm_pool.rs | 14 +++++--------- .../tree/src/tree/payload_processor/prewarm.rs | 5 ++--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs index d7e0d0cab93..b46e51b3940 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal_prewarm_pool.rs @@ -17,8 +17,8 @@ use tracing::trace; /// generic over the provider factory; each worker builds its own per block. pub type BuildProviderFn = dyn Fn() -> ProviderResult + Send + Sync; -/// A single warm request: a whole account (basic account + its bytecode) followed by a batch of -/// its storage slots, or a batch of storage slots on their own. +/// A single warm request: an account and a batch of its storage slots, or a batch of storage slots +/// on their own. enum PrewarmTarget { Account(Address, Box<[StorageKey]>), Storage(Address, Box<[StorageKey]>), @@ -86,7 +86,7 @@ impl BalPrewarmPool { } } - /// Fire-and-forget: warm an account (basic account + bytecode) and its storage slots. + /// Fire-and-forget: warm an account and its storage slots. /// /// The slots are dispatched in `WARM_BATCH_SIZE` chunks that are distributed independently, /// so a single account with a large read-set does not serialize onto one worker; @@ -180,12 +180,7 @@ fn prewarm_loop(rx: crossbeam_channel::Receiver) { let Some(provider) = provider.as_ref() else { continue }; match target { PrewarmTarget::Account(addr, slots) => { - if let Ok(Some(account)) = provider.basic_account(&addr) && - let Some(code_hash) = account.bytecode_hash && - code_hash != alloy_consensus::constants::KECCAK_EMPTY - { - let _ = provider.bytecode_by_hash(&code_hash); - } + let _ = provider.basic_account(&addr); for &slot in &slots { let _ = provider.storage(addr, slot); } @@ -204,6 +199,7 @@ fn prewarm_loop(rx: crossbeam_channel::Receiver) { } } } + struct SendOnDrop { sender: Option>, } diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 23b04f83c99..6d920cbacc4 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -428,9 +428,8 @@ where // - dispatch_bal_batch_io is false // - execution cache is not disabled // - // we launch prewarming sequence of the BAL read set here. The BAL read-set consists - // of the accounts, their code if present, and declared storages (both storage_reads - // and storage_changes). + // we launch prewarming of the BAL accounts and declared storages (both storage_reads + // and storage_changes). Bytecode is loaded on demand during execution. // // This runs side-by-side with the parallel transaction execution reducing the time it // spends blocking on the data. From 218657e03c66124f62a9189d5ea7887e666d5195 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 13:38:01 +0200 Subject: [PATCH 081/133] fix(e2e): send ReorgTo fcu to its target (#27438) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/e2e-test-utils/src/testsuite/README.md | 3 +- .../src/testsuite/actions/produce_blocks.rs | 93 +++++++++++-------- .../src/testsuite/actions/reorg.rs | 27 +++--- .../tests/e2e-testsuite/main.rs | 4 +- .../engine/tree/tests/e2e-testsuite/main.rs | 38 ++++++-- 5 files changed, 99 insertions(+), 66 deletions(-) diff --git a/crates/e2e-test-utils/src/testsuite/README.md b/crates/e2e-test-utils/src/testsuite/README.md index 6e9f789b67f..51ac9e9712c 100644 --- a/crates/e2e-test-utils/src/testsuite/README.md +++ b/crates/e2e-test-utils/src/testsuite/README.md @@ -312,7 +312,8 @@ let action = ValidateFork::new(fork_base_number); #### Reorg Actions ##### `ReorgTo` -Performs a reorg by setting a new head block as canonical. +Performs a reorg by broadcasting a forkchoice update with the target block as head to all nodes. +The finalized block is left unchanged, so a target below it is rejected as a too deep reorg. ```rust use reth_e2e_test_utils::testsuite::actions::ReorgTo; diff --git a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs index 02a19982e7d..6475566afd1 100644 --- a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs +++ b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs @@ -410,47 +410,7 @@ where current_head_block.header.hash }; - let fork_choice_state = ForkchoiceState { - head_block_hash: head_hash, - safe_block_hash: head_hash, - // Making a block canonical does not imply finality: tests advance the finalized - // block explicitly via `FinalizeBlock`, and a finalized tip would reject any - // later forkchoice update below it as a too deep reorg. - finalized_block_hash: B256::ZERO, - }; - debug!( - "Broadcasting forkchoice update to {} clients. Head: {:?}", - env.node_clients.len(), - fork_choice_state.head_block_hash - ); - - for (idx, client) in env.node_clients.iter().enumerate() { - match EngineApiClient::::fork_choice_updated_v3( - &client.engine.http_client(), - fork_choice_state, - None, - ) - .await - { - Ok(resp) => { - debug!( - "Client {}: Forkchoice update status: {:?}", - idx, resp.payload_status.status - ); - // validate that the forkchoice update was accepted - validate_fcu_response(&resp, &format!("Client {idx}"))?; - } - Err(err) => { - return Err(eyre::eyre!( - "Client {}: Failed to broadcast forkchoice: {:?}", - idx, - err - )); - } - } - } - debug!("Forkchoice update broadcasted successfully"); - Ok(()) + broadcast_forkchoice(env, head_hash).await }) } } @@ -1166,3 +1126,54 @@ where }) } } + +/// Broadcasts a forkchoice update to all clients, using `head_hash` as the head and safe block. +pub(super) async fn broadcast_forkchoice( + env: &Environment, + head_hash: B256, +) -> Result<()> +where + Engine: EngineTypes, +{ + let fork_choice_state = ForkchoiceState { + head_block_hash: head_hash, + safe_block_hash: head_hash, + // Making a block canonical does not imply finality: tests advance the finalized block + // explicitly via `FinalizeBlock`, and a finalized tip would reject any later forkchoice + // update below it as a too deep reorg. + finalized_block_hash: B256::ZERO, + }; + debug!( + "Broadcasting forkchoice update to {} clients. Head: {:?}", + env.node_clients.len(), + fork_choice_state.head_block_hash + ); + + for (idx, client) in env.node_clients.iter().enumerate() { + match EngineApiClient::::fork_choice_updated_v3( + &client.engine.http_client(), + fork_choice_state, + None, + ) + .await + { + Ok(resp) => { + debug!( + "Client {}: Forkchoice update status: {:?}", + idx, resp.payload_status.status + ); + // validate that the forkchoice update was accepted + validate_fcu_response(&resp, &format!("Client {idx}"))?; + } + Err(err) => { + return Err(eyre::eyre!( + "Client {}: Failed to broadcast forkchoice: {:?}", + idx, + err + )); + } + } + } + debug!("Forkchoice update broadcasted successfully"); + Ok(()) +} diff --git a/crates/e2e-test-utils/src/testsuite/actions/reorg.rs b/crates/e2e-test-utils/src/testsuite/actions/reorg.rs index 337734c3282..2b2adec8660 100644 --- a/crates/e2e-test-utils/src/testsuite/actions/reorg.rs +++ b/crates/e2e-test-utils/src/testsuite/actions/reorg.rs @@ -1,14 +1,14 @@ //! Reorg actions for the e2e testing framework. use crate::testsuite::{ - actions::{produce_blocks::BroadcastLatestForkchoice, Action, Sequence}, + actions::{produce_blocks::broadcast_forkchoice, Action}, BlockInfo, Environment, }; use alloy_primitives::B256; -use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; +use alloy_rpc_types_engine::ForkchoiceState; use eyre::Result; use futures_util::future::BoxFuture; -use reth_node_api::{EngineTypes, PayloadTypes}; +use reth_node_api::EngineTypes; use std::marker::PhantomData; use tracing::debug; @@ -21,7 +21,11 @@ pub enum ReorgTarget { Tag(String), } -/// Action that performs a reorg by setting a new head block as canonical +/// Action that performs a reorg by setting a new head block as canonical. +/// +/// Points the active node's environment at the target block and broadcasts a forkchoice update +/// with the target as head to all clients. The update leaves the finalized block unchanged, so a +/// target below it is rejected as a too deep reorg. #[derive(Debug)] pub struct ReorgTo { /// Target for the reorg operation @@ -44,10 +48,7 @@ impl ReorgTo { impl Action for ReorgTo where - Engine: EngineTypes + PayloadTypes, - Engine::PayloadAttributes: From + Clone, - Engine::ExecutionPayloadEnvelopeV3: - Into, + Engine: EngineTypes, { fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { Box::pin(async move { @@ -67,12 +68,12 @@ where } }; - let mut sequence = Sequence::new(vec![ - Box::new(SetReorgTarget::new(target_block_info)), - Box::new(BroadcastLatestForkchoice::default()), - ]); + let mut set_target = SetReorgTarget::new(target_block_info); + set_target.execute(env).await?; - sequence.execute(env).await + // Target the reorg block itself rather than the latest built payload, which can be a + // descendant of the target or belong to another fork. + broadcast_forkchoice(env, target_block_info.hash).await }) } } diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index f3ac2f95892..089cea386b8 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -294,8 +294,10 @@ async fn test_testsuite_deep_reorg() -> Result<()> { .with_action(CreateFork::::new(1, 1)) .with_action(CaptureBlock::new("blockA_height2")) .with_action(MakeCanonical::new()) - // receive newPayload with block hash B and height 2 + // send forkchoiceUpdated back to block 1. Building block A finalized block 1, and a + // canonical ancestor at the finalized block is accepted without unwinding the chain. .with_action(ReorgTo::::new_from_tag("block1")) + // receive newPayload with block hash B and height 2 .with_action(CreateFork::::new(1, 1)) .with_action(CaptureBlock::new("blockB_height2")) // receive forkchoiceUpdated with block hash B as head diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index 6ffd2624802..e41a5030b1a 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -252,7 +252,15 @@ async fn test_engine_tree_valid_forks_with_older_canonical_head_e2e() -> Result< .with_action(ProduceBlocks::::new(5)) .with_action(CaptureBlock::new("fork_point")) .with_action(MakeCanonical::new()) - // revert to old head to simulate scenario where canonical head is older + // block production finalizes the produced blocks, so re-establish finality at the old + // head: an FCU below the finalized block is rejected as a too deep reorg. + .with_action( + FinalizeBlock::::new(BlockReference::Tag("old_head".to_string())) + .with_head(BlockReference::Tag("fork_point".to_string())), + ) + // send an FCU back to the old head to simulate a CL with an older view of the canonical + // head. The old head is a canonical ancestor, so the engine accepts the FCU without + // unwinding the canonical chain. .with_action(ReorgTo::::new_from_tag("old_head")) // create first competing chain (chain A) from fork point with 10 blocks .with_action(CreateFork::::new_from_tag("fork_point", 10)) @@ -289,22 +297,31 @@ async fn test_engine_tree_valid_and_invalid_forks_with_older_canonical_head_e2e( .with_action(ProduceBlocks::::new(5)) .with_action(CaptureBlock::new("fork_point")) .with_action(MakeCanonical::new()) - // revert to old head to simulate older canonical head scenario + // block production finalizes the produced blocks, so re-establish finality at the old + // head: an FCU below the finalized block is rejected as a too deep reorg. + .with_action( + FinalizeBlock::::new(BlockReference::Tag("old_head".to_string())) + .with_head(BlockReference::Tag("fork_point".to_string())), + ) + // send an FCU back to the old head to simulate a CL with an older view of the canonical + // head. The old head is a canonical ancestor, so the engine accepts the FCU without + // unwinding the canonical chain. .with_action(ReorgTo::::new_from_tag("old_head")) // create chain B (the valid chain) from fork point with 10 blocks .with_action(CreateFork::::new_from_tag("fork_point", 10)) .with_action(CaptureBlock::new("chain_b_tip")) // make chain B canonical via FCU - this becomes the valid chain .with_action(ReorgTo::::new_from_tag("chain_b_tip")) - // create chain A (competing chain) - first produce valid blocks, then test invalid - // scenario - .with_action(ReorgTo::::new_from_tag("fork_point")) // producing chain B finalized its blocks, so re-establish finality at the fork point - // before building below it again + // before the FCU back to it and building below chain B again. .with_action( FinalizeBlock::::new(BlockReference::Tag("fork_point".to_string())) .with_head(BlockReference::Tag("chain_b_tip".to_string())), ) + // create chain A (competing chain) from the fork point. The fork point is a canonical + // ancestor of chain B, so the FCU to it is accepted without unwinding chain B. First + // produce valid blocks, then test invalid scenario. + .with_action(ReorgTo::::new_from_tag("fork_point")) .with_action(ProduceBlocks::::new(10)) .with_action(CaptureBlock::new("chain_a_tip")) // test that FCU to chain A tip returns VALID status (it's a valid competing chain) @@ -421,13 +438,14 @@ async fn test_engine_tree_fcu_extends_canon_chain_e2e() -> Result<()> { // create and make canonical a base chain with 1 block .with_action(ProduceBlocks::::new(1)) .with_action(MakeCanonical::new()) - // extend the chain with 10 more blocks (total 11 blocks: 0-10) + // extend the chain with 10 more blocks (blocks 2-11). Building each block makes its + // parent canonical, so the canonical head is block 10. .with_action(ProduceBlocks::::new(10)) - // capture block 6 as our intermediate target (from 0-indexed, this is block 6) + // capture the chain tip (block 11) as the target. .with_action(CaptureBlock::new("target_block")) - // make the intermediate target canonical via FCU + // extend the canonical chain to the target via FCU. .with_action(ReorgTo::::new_from_tag("target_block")) - // now make the chain tip canonical via FCU + // repeat the FCU to the chain tip, which is already canonical. .with_action(MakeCanonical::new()); test.run::().await?; From f50fa4221f919bad85946bdf35ee281e29bcbe69 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 13:38:56 +0200 Subject: [PATCH 082/133] fix(scripts): add patch section in patch-alloy (#27440) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- scripts/patch-alloy.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/patch-alloy.sh b/scripts/patch-alloy.sh index abe22b12811..dd4442fe654 100755 --- a/scripts/patch-alloy.sh +++ b/scripts/patch-alloy.sh @@ -29,7 +29,7 @@ while [[ $# -gt 0 ]]; do echo "" echo "Options:" echo " --alloy Patch alloy-rs/alloy crates" - echo " --evm Patch alloy-rs/evm crates (alloy-evm, alloy-op-evm)" + echo " --evm Patch alloy-rs/evm crates (alloy-evm)" exit 0 ;; *) @@ -46,9 +46,23 @@ fi CARGO_TOML="${CARGO_TOML:-Cargo.toml}" +# Patches are appended to the end of the file, so they only land in `[patch.crates-io]` if that is +# the last section. Add the section if it is missing, and refuse to append anywhere else. +ADD_PATCH_SECTION="" +if ! grep -q '^\[patch\.crates-io\]' "$CARGO_TOML"; then + ADD_PATCH_SECTION=1 +elif ! grep '^\[' "$CARGO_TOML" | tail -n 1 | grep -q '^\[patch\.crates-io\]'; then + echo "Error: [patch.crates-io] must be the last section in $CARGO_TOML" + exit 1 +fi + echo "Patching $CARGO_TOML..." echo "" >> "$CARGO_TOML" +if [[ -n "$ADD_PATCH_SECTION" ]]; then + echo "[patch.crates-io]" >> "$CARGO_TOML" +fi + if [[ -n "$ALLOY_BRANCH" ]]; then echo "Patching alloy-rs/alloy with branch: $ALLOY_BRANCH" cat >> "$CARGO_TOML" << EOF @@ -87,7 +101,6 @@ if [[ -n "$ALLOY_EVM_BRANCH" ]]; then echo "Patching alloy-rs/evm with branch: $ALLOY_EVM_BRANCH" cat >> "$CARGO_TOML" << EOF alloy-evm = { git = "https://github.com/alloy-rs/evm", branch = "$ALLOY_EVM_BRANCH" } -alloy-op-evm = { git = "https://github.com/alloy-rs/evm", branch = "$ALLOY_EVM_BRANCH" } EOF fi From f9fea9db61293cb4fe59b51656001f859ef9d731 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 13:55:27 +0200 Subject: [PATCH 083/133] test(engine): exercise on-disk reorg in e2e test (#27441) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/e2e-test-utils/src/node.rs | 13 +- crates/e2e-test-utils/src/testsuite/mod.rs | 28 +++- .../engine/tree/tests/e2e-testsuite/main.rs | 126 +++++++++++++++++- 3 files changed, 156 insertions(+), 11 deletions(-) diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 21cf7b90c1b..bb7054ec6f0 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -14,14 +14,14 @@ use reth_node_builder::{rpc::RethRpcAddOns, FullNode, NodeTypes}; use reth_payload_primitives::BuiltPayload; use reth_provider::{ - BlockReader, BlockReaderIdExt, CanonStateNotificationStream, CanonStateSubscriptions, - HeaderProvider, StageCheckpointReader, + BlockNumReader, BlockReader, BlockReaderIdExt, CanonStateNotificationStream, + CanonStateSubscriptions, DatabaseProviderFactory, HeaderProvider, StageCheckpointReader, }; use reth_rpc_api::TestingBuildBlockRequestV1; use reth_rpc_builder::auth::AuthServerHandle; use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt}; use reth_stages_types::StageId; -use std::pin::Pin; +use std::{pin::Pin, sync::Arc}; use tokio_stream::StreamExt; use url::Url; @@ -334,7 +334,8 @@ where /// /// This helper method extracts the necessary handles and creates a client /// that can interact with both the regular RPC and Engine API endpoints. - /// It automatically includes the beacon engine handle for direct consensus engine interaction. + /// It automatically includes the beacon engine handle for direct consensus engine interaction + /// and read-only access to the node's database. pub fn to_node_client(&self) -> eyre::Result> { let rpc = self .rpc_client() @@ -346,6 +347,10 @@ where let mut client = crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle); client.payload_builder = Some(self.inner.payload_builder_handle.clone()); + let provider = self.inner.provider.clone(); + client.database = Some(Arc::new(move || { + provider.database_provider_ro().map(|db| Box::new(db) as Box) + })); Ok(client) } diff --git a/crates/e2e-test-utils/src/testsuite/mod.rs b/crates/e2e-test-utils/src/testsuite/mod.rs index 59041a0ec81..5bf829fd050 100644 --- a/crates/e2e-test-utils/src/testsuite/mod.rs +++ b/crates/e2e-test-utils/src/testsuite/mod.rs @@ -16,6 +16,7 @@ use crate::testsuite::setup::Setup; use alloy_provider::{Provider, ProviderBuilder}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; use reth_engine_primitives::ConsensusEngineHandle; +use reth_provider::{BlockNumReader, ProviderResult}; use reth_rpc_builder::auth::AuthServerHandle; use std::sync::Arc; use url::Url; @@ -36,6 +37,8 @@ where pub(crate) payload_builder: Option>, /// Alloy provider for interacting with the node provider: Arc, + /// Opens read-only views of the node's database, if the node runs in-process. + pub(crate) database: Option, } impl NodeClient @@ -46,7 +49,14 @@ where pub fn new(rpc: HttpClient, engine: AuthServerHandle, url: Url) -> Self { let provider = Arc::new(ProviderBuilder::new().connect_http(url)) as Arc; - Self { rpc, engine, beacon_engine_handle: None, payload_builder: None, provider } + Self { + rpc, + engine, + beacon_engine_handle: None, + payload_builder: None, + provider, + database: None, + } } /// Instantiates a new [`NodeClient`] with the given handles, RPC URL, and beacon engine handle @@ -64,6 +74,7 @@ where beacon_engine_handle: Some(beacon_engine_handle), payload_builder: None, provider, + database: None, } } @@ -92,6 +103,16 @@ where pub async fn is_ready(&self) -> bool { self.get_block_by_number(alloy_eips::BlockNumberOrTag::Latest).await.is_ok() } + + /// Opens a read-only view of the node's database. + /// + /// Unlike the RPC endpoints, the view only contains blocks that were persisted to disk, not + /// canonical blocks that the engine still holds in memory. + pub fn database_provider_ro(&self) -> Result> { + let open = + self.database.as_ref().ok_or_else(|| eyre::eyre!("Node database is not accessible"))?; + Ok(open()?) + } } impl std::fmt::Debug for NodeClient @@ -104,10 +125,15 @@ where .field("engine", &self.engine) .field("beacon_engine_handle", &self.beacon_engine_handle.is_some()) .field("provider", &"") + .field("database", &self.database.is_some()) .finish() } } +/// Opens a read-only view of a node's database. +pub(crate) type DatabaseOpener = + Arc ProviderResult> + Send + Sync>; + /// Represents complete block information. #[derive(Debug, Clone, Copy)] pub struct BlockInfo { diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index e41a5030b1a..37f34d995d0 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -26,7 +26,11 @@ use reth_e2e_test_utils::{ use reth_engine_tree::tree::TreeConfig; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_node_ethereum::EthereumNode; -use std::sync::Arc; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::time::sleep; /// Waits for in-flight persistence by resubmitting the latest built payload to a node. #[derive(Debug)] @@ -155,6 +159,103 @@ impl Action for AssertBlockHasTransactions { } } +/// Waits until a node's database holds its canonical chain up to a tagged block and no further. +/// +/// The database is read directly rather than through the in-memory canonical state, so this fails +/// if the blocks were never persisted or if a reorged-out branch was left on disk. +#[derive(Debug)] +struct WaitForPersistedChain { + node_idx: usize, + tip: String, + unwound: Vec, +} + +impl WaitForPersistedChain { + /// How long to wait for the tagged block to become the persisted tip. + const TIMEOUT: Duration = Duration::from_secs(30); + + fn new(node_idx: usize, tip: impl Into) -> Self { + Self { node_idx, tip: tip.into(), unwound: Vec::new() } + } + + /// Also requires the tagged block, which was persisted earlier, to be removed from disk. + fn with_unwound(mut self, tag: impl Into) -> Self { + self.unwound.push(tag.into()); + self + } +} + +impl Action for WaitForPersistedChain { + fn execute<'a>( + &'a mut self, + env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let tagged = |tag: &String| { + env.block_registry + .get(tag) + .map(|(block, _)| *block) + .ok_or_else(|| eyre::eyre!("Block tag '{tag}' not found in registry")) + }; + let tip = tagged(&self.tip)?; + let unwound = self.unwound.iter().map(tagged).collect::>>()?; + let node = env + .node_clients + .get(self.node_idx) + .ok_or_else(|| eyre::eyre!("Node index {} out of bounds", self.node_idx))?; + + let start = Instant::now(); + loop { + let persisted = node.database_provider_ro()?.chain_info()?; + if persisted.best_number == tip.number && persisted.best_hash == tip.hash { + break + } + eyre::ensure!( + start.elapsed() < Self::TIMEOUT, + "Timed out waiting for block {} ({}) to be persisted, database tip is block {} ({})", + tip.number, + tip.hash, + persisted.best_number, + persisted.best_hash + ); + sleep(Duration::from_millis(50)).await; + } + + let mut canonical = Vec::new(); + for number in 0..=tip.number { + let block = node + .get_block_by_number(alloy_eips::BlockNumberOrTag::Number(number)) + .await? + .ok_or_else(|| eyre::eyre!("Canonical block {number} not found"))?; + canonical.push(block.header.hash); + } + + let database = node.database_provider_ro()?; + let persisted = database.canonical_hashes_range(0, tip.number + 1)?; + eyre::ensure!( + persisted == canonical, + "Persisted chain {persisted:?} does not match canonical chain {canonical:?}" + ); + // Unwinding must also prune the static file headers above the new tip. + let last_header = database.last_block_number()?; + eyre::ensure!( + last_header == tip.number, + "Static file headers end at block {last_header}, expected block {}", + tip.number + ); + for block in unwound { + eyre::ensure!( + database.block_number(block.hash)?.is_none(), + "Block {} ({}) is still persisted", + block.number, + block.hash + ); + } + Ok(()) + }) + } +} + /// Creates the standard setup for engine tree e2e tests. fn default_engine_tree_setup() -> Setup { Setup::default() @@ -666,7 +767,11 @@ async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> { /// /// Uses unconnected nodes so fork blocks can be produced independently on Node 1 and then /// sent to Node 0 via newPayload only (no FCU), keeping Node 0's persisted chain intact -/// until the final `ReorgTo` triggers `find_disk_reorg`. +/// until the final forkchoice update triggers `find_disk_reorg`. +/// +/// Persistence starts once more than 7 canonical blocks are held in memory. Without a memory +/// block buffer, each cycle persists up to the canonical head, so the persisted ranges only depend +/// on the chain lengths and not on how fast persistence keeps up with block production. fn disk_reorg_setup(storage_v2: bool) -> Setup { let mut setup = Setup::default() .with_chain_spec(Arc::new( @@ -686,6 +791,7 @@ fn disk_reorg_setup(storage_v2: bool) -> Setup { TreeConfig::default() .with_num_state_masking_blocks(0) .with_persistence_threshold(7) + .with_memory_block_buffer_target(0) .with_has_enough_parallelism(true), ); if storage_v2 { @@ -697,18 +803,23 @@ fn disk_reorg_setup(storage_v2: bool) -> Setup { /// Builds a disk-level reorg test scenario. /// /// 1. Both nodes receive 3 shared blocks -/// 2. Node 0 extends to 10 blocks locally (persisted to disk) +/// 2. Node 0 extends to 8 blocks locally, which persists blocks 1-8 to its disk /// 3. Node 1 builds an 8-block fork from block 3 (its canonical head) /// 4. Fork blocks are sent to Node 0 via newPayload (no FCU, old chain stays on disk) -/// 5. FCU to fork tip on Node 0 triggers `find_disk_reorg` → `RemoveBlocksAbove(3)` +/// 5. FCU to the fork tip on Node 0 makes `find_disk_reorg` detect the persisted main chain blocks +/// 4-8, so `RemoveBlocksAbove(3)` unwinds them +/// 6. Node 0 then persists fork blocks 4-11 fn disk_reorg_test(storage_v2: bool) -> TestBuilder { TestBuilder::new() .with_setup(disk_reorg_setup(storage_v2)) .with_action(SelectActiveNode::new(0)) .with_action(ProduceBlocks::::new(3)) .with_action(MakeCanonical::new()) - .with_action(ProduceBlocksLocally::::new(7)) + .with_action(ProduceBlocksLocally::::new(5)) .with_action(MakeCanonical::with_active_node()) + .with_action(CaptureBlock::new("main_tip")) + // the 8th in-memory canonical block triggers persistence of the whole main chain. + .with_action(WaitForPersistedChain::new(0, "main_tip")) .with_action(SelectActiveNode::new(1)) .with_action(SetForkBase::new(3)) .with_action(ProduceBlocksLocally::::new(8)) @@ -730,12 +841,15 @@ fn disk_reorg_test(storage_v2: bool) -> TestBuilder { .with_expected_status(PayloadStatusEnum::Valid) .with_node_idx(0), ) + // the persisted main chain above the fork base is unwound and the 8 fork blocks, which + // exceed the persistence threshold, replace it on disk. + .with_action(WaitForPersistedChain::new(0, "fork_tip").with_unwound("main_tip")) } /// Verifies disk-level reorg in v1 (plain key) storage mode. /// /// Confirms `find_disk_reorg()` detects persisted blocks on the wrong fork and calls -/// `RemoveBlocksAbove` to truncate, then re-persists the correct fork chain. +/// `RemoveBlocksAbove` to truncate them from disk, then persists the new canonical fork. #[tokio::test] async fn test_engine_tree_disk_reorg_v1_e2e() -> Result<()> { reth_tracing::init_test_tracing(); From 25139370a22c843bf323bcf363a07305fe5bb28f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:48:40 +0000 Subject: [PATCH 084/133] chore(deps): bump the cargo-weekly group across 1 directory with 10 updates (#27402) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Matthias Seitz --- Cargo.lock | 210 +++++++++++++++++++++++++++++------------------------ 1 file changed, 114 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b4c6c9b7f65..12d9c094972 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,9 +106,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy-chains" -version = "0.2.37" +version = "0.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5fdcfed8f106be3df944054aaa42bc13ae103a3ac8a9f4b08d4f053e3a743f8" +checksum = "c1c4f6f6f020f38a4ef079428b5c7009e913bf8102e4d65a90c6cb2cf93800eb" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -189,9 +189,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201b9e973fe90b2effd9ab356d4f2a46ab56046ba9d46f163367553e72c045b0" +checksum = "14b3b9990889736af898bd4e51f7d0c0228d6a9b9c3a5fecb3e6b25a7abc791c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -370,9 +370,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dba4e59c3581a39e03e0b0b4a46ee9c41315b5d843b1e903271952b32bedb7c" +checksum = "858acd7fdad1e4a7057fd9c1b39c1f2cd6bcd57ebf3c56e2853c02ead049e816" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -458,9 +458,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce7b00f0cb42c66ec353076ded1dff1fbf818f6e0e26c40c8a8456c04483fca4" +checksum = "d5e9dbd49258ac3ab893a481d46be29b58be7f734dcac46cd80b6b13ee36566c" dependencies = [ "alloy-rlp", "arbitrary", @@ -821,9 +821,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64558980fb038cd34b4285ec2b36a2a8bd8d4ddd13b3f6e42d97cb5ee29938e" +checksum = "60dd79f578c3912f1fc2a150dbeb8110b8cfb976c60f98ebf6de9d5da5965a2b" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffb0e793abdbaea9d01259493c8272c3af295d03389e70a2acdf53b57ad1edf" +checksum = "9edb8520f2f94275e1caa73c85207dcc78ec402a9a4c429240f9a0783a8f16c0" dependencies = [ "alloy-sol-macro-input", "const-hex", @@ -853,9 +853,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c2c0ec8425d9663dac939ba75750f17d2933d2f020814b4f8fde4a60da6d26" +checksum = "66af2d9344882172993be5f5cbfd349fdfa52cb548f7af8715b446fb35ef6001" dependencies = [ "const-hex", "dunce", @@ -869,9 +869,9 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b37db6a7ad8170596345f864522f789cc7af093f516d6cdf34bbdcfba8adab" +checksum = "77c2a8abc3334044013b23d5e438cc5dc802e5544cc42b713d0840d5185967cc" dependencies = [ "serde", "winnow 1.0.1", @@ -879,9 +879,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f40e33a0f588dde548c3b6767a71e61b593421a8c1b253b9cf1441dc7cf7ab5" +checksum = "3e6f87cf007caa54e95455875fbd25879b008d0d9c7c340a2c258b3a3400a28c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -1051,7 +1051,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1062,7 +1062,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1495,9 +1495,9 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-compression" -version = "0.4.44" +version = "0.4.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844" +checksum = "fb61aea1a7def73ee7c350a184f0e70b32c182344e2e75bf70c9b621b83417fd" dependencies = [ "compression-codecs", "compression-core", @@ -1713,7 +1713,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1764,9 +1764,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "arbitrary", "serde_core", @@ -1834,7 +1834,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6339a700715bda376f5ea65c76e8fe8fc880930d8b0638cea68e7f3da6538e0a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "boa_interner", "boa_macros", "boa_string", @@ -1851,7 +1851,7 @@ checksum = "1521be326f8a5c8887e95d4ce7f002917a002a23f7b93b9a6a2bf50ed4157824" dependencies = [ "aligned-vec", "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "boa_ast", "boa_gc", "boa_interner", @@ -1943,7 +1943,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd957fa9fa93e3a001a8aba5a5cd40c2bbfde486378be4c4b472fd304aaddb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "boa_ast", "boa_interner", "boa_macros", @@ -2197,9 +2197,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.5" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" dependencies = [ "find-msvc-tools", "jobserver", @@ -2224,9 +2224,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" @@ -2309,9 +2309,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -2319,9 +2319,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -2331,9 +2331,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", @@ -2527,16 +2527,16 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.39" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df" +checksum = "bef16c47ba2797aa6a909cc37d39911f3a6743811fe7408ac0b0cc0276b656e9" dependencies = [ "brotli", "compression-core", "flate2", "memchr", - "zstd", - "zstd-safe", + "zstd 0.14.0", + "zstd-safe 8.0.0", ] [[package]] @@ -2776,7 +2776,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "crossterm_winapi", "derive_more", "document-features", @@ -3027,7 +3027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.118", + "syn 1.0.109", ] [[package]] @@ -3512,7 +3512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4037,9 +4037,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" [[package]] name = "fixed-cache" @@ -4379,7 +4379,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "libgit2-sys", "log", @@ -5129,7 +5129,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7decbc9dfa45a4a827a6ff7b822c113b1285678a937e84213417d4ca8a095782" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "inkwell_internals", "libc", "llvm-sys", @@ -5153,7 +5153,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd5b3eaf1a28b758ac0faa5a4254e8ab2705605496f1b1f3fbbc3988ad73d199" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "inotify-sys", "libc", ] @@ -5255,7 +5255,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5845,7 +5845,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libc", "plain", "redox_syscall 0.7.4", @@ -5885,7 +5885,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f50e8f47623268b5407192d26876c4d7f89d686ca130fdc53bced4814cd29f8" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -6309,7 +6309,7 @@ version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -6349,7 +6349,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "fsevent-sys", "inotify", "kqueue", @@ -6367,7 +6367,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -6385,7 +6385,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6531,7 +6531,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -7241,7 +7241,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "flate2", "procfs-core", @@ -7254,7 +7254,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "chrono", "hex", ] @@ -7267,7 +7267,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.13.1", + "bitflags 2.13.2", "num-traits", "rand 0.9.4", "rand_chacha 0.9.0", @@ -7425,7 +7425,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7581,7 +7581,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbb175c433c8e28a809d1f5773a2ae96e68c0ce40db865cbab1020bf33ae479c" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "compact_str", "hashbrown 0.17.0", "itertools 0.14.0", @@ -7614,7 +7614,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66e3d19bcc9130ca376277d93b60767ff121ace3be06f5f95f81dd68956407d1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "hashbrown 0.17.0", "indoc", "instability", @@ -7634,7 +7634,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -7669,7 +7669,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -7678,7 +7678,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -7759,11 +7759,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-core", @@ -7856,7 +7856,7 @@ dependencies = [ "tokio", "toml", "tracing", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -8081,7 +8081,7 @@ dependencies = [ "toml", "tracing", "url", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -9258,7 +9258,7 @@ dependencies = [ name = "reth-libmdbx" version = "2.5.2" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "crossbeam-queue", "dashmap", @@ -9479,7 +9479,7 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tracing", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -10380,7 +10380,7 @@ dependencies = [ "tower", "tower-http", "tracing", - "zstd", + "zstd 0.13.3", ] [[package]] @@ -10798,7 +10798,7 @@ dependencies = [ "aquamarine", "assert_matches", "auto_impl", - "bitflags 2.13.1", + "bitflags 2.13.2", "codspeed-criterion-compat", "futures", "futures-util", @@ -10992,7 +10992,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "253f7d5772f3c8d0dcc3fdac398db6bca4f6f6e602d240c3bc47338dde0910dd" dependencies = [ - "zstd", + "zstd 0.13.3", ] [[package]] @@ -11205,7 +11205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b2d2ec0be925c4c87ff4afae33af4b9f693d9b011f74d6b4d52267a62df0c49" dependencies = [ "alloy-eip7928", - "bitflags 2.13.1", + "bitflags 2.13.2", "nonmax", "revm-bytecode", "revm-primitives", @@ -11268,7 +11268,7 @@ version = "0.1.0" source = "git+https://github.com/paradigmxyz/revmc?branch=main#87ea260a9bd25759beab64753d2f36227891ca4a" dependencies = [ "alloy-primitives", - "bitflags 2.13.1", + "bitflags 2.13.2", "bitvec", "derive_more", "either", @@ -11569,11 +11569,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11653,7 +11653,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11840,7 +11840,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -12287,7 +12287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12442,9 +12442,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6415502cd1e9ed58b3ceb415164b812d5572757b1a6f0e280ae806723c1fab" +checksum = "eb6a2e3c7f7a3e4e83d1752cec5d1e357ced0cf96e85419b6a07f227db3def3a" dependencies = [ "paste", "proc-macro2", @@ -12492,7 +12492,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -12546,7 +12546,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12686,7 +12686,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d2e834949be5111506bb252643498af1514f600d9e1dceedaa42afae155b67f" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", "log", @@ -13030,7 +13030,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -13705,7 +13705,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "hashbrown 0.15.5", "indexmap 2.14.2", "semver 1.0.28", @@ -13819,7 +13819,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -14284,7 +14284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.13.1", + "bitflags 2.13.2", "indexmap 2.14.2", "log", "serde", @@ -14513,7 +14513,16 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.2.4", +] + +[[package]] +name = "zstd" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +dependencies = [ + "zstd-safe 8.0.0", ] [[package]] @@ -14525,11 +14534,20 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", From fa5f941c1c077765d13cb09238eb6e4b477fd37e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 14:24:36 +0200 Subject: [PATCH 085/133] test(engine): persist blocks in v1/v2 e2e twins (#27436) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../engine/tree/tests/e2e-testsuite/main.rs | 73 +++++++++++++++---- 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index 37f34d995d0..77a3b8af47b 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -275,21 +275,39 @@ fn default_engine_tree_setup() -> Setup { .with_tree_config(TreeConfig::default().with_has_enough_parallelism(true)) } -/// Creates a v2 storage mode setup for engine tree e2e tests. +/// Creates a v1 storage mode setup that persists every canonical block. +/// +/// With a persistence threshold and memory block buffer target of zero and no state masking, the +/// engine persists every canonical block up to the head together with its state and trie updates, +/// so even short chains exercise the storage writes and unwinds instead of staying in memory. +fn v1_engine_tree_setup() -> Setup { + default_engine_tree_setup().with_tree_config( + TreeConfig::default() + .with_num_state_masking_blocks(0) + .with_persistence_threshold(0) + .with_memory_block_buffer_target(0) + .with_has_enough_parallelism(true), + ) +} + +/// Creates a v2 storage mode setup that persists every canonical block. /// /// v2 mode uses keccak256-hashed slot keys in static file changesets and rocksdb history /// instead of plain keys in MDBX. fn v2_engine_tree_setup() -> Setup { - default_engine_tree_setup().with_storage_v2() + v1_engine_tree_setup().with_storage_v2() } /// Test that verifies forkchoice update and canonical chain insertion functionality. +/// +/// Every canonical block is persisted, exercising `save_blocks` with plain-key changesets and +/// history indices in MDBX. #[tokio::test] async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> { reth_tracing::init_test_tracing(); let test = TestBuilder::new() - .with_setup(default_engine_tree_setup()) + .with_setup(v1_engine_tree_setup()) // produce one block .with_action(ProduceBlocks::::new(1)) // make it canonical via forkchoice update @@ -297,7 +315,10 @@ async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> { // extend with 3 more blocks .with_action(ProduceBlocks::::new(3)) // make the latest block canonical - .with_action(MakeCanonical::new()); + .with_action(MakeCanonical::new()) + .with_action(CaptureBlock::new("canonical_tip")) + // every canonical block reaches disk. + .with_action(WaitForPersistedChain::new(0, "canonical_tip")); test.run::().await?; @@ -305,29 +326,37 @@ async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> { } /// Test that verifies forkchoice update with a reorg where all blocks are already available. +/// +/// The main chain is persisted before the fork becomes canonical, so the reorg unwinds the +/// persisted main chain blocks above the fork base and persists the fork instead. #[tokio::test] async fn test_engine_tree_fcu_reorg_with_all_blocks_e2e() -> Result<()> { reth_tracing::init_test_tracing(); let test = TestBuilder::new() - .with_setup(default_engine_tree_setup()) - // create a main chain with 5 blocks (blocks 0-4) + .with_setup(v1_engine_tree_setup()) + // create a main chain with 5 blocks (blocks 1-5). .with_action(ProduceBlocks::::new(2)) .with_action(CaptureBlock::new("fork_base")) .with_action(ProduceBlocks::::new(3)) .with_action(CaptureBlock::new("main_tip")) .with_action(MakeCanonical::new()) + // the main chain blocks above the fork base reach disk before the reorg. + .with_action(WaitForPersistedChain::new(0, "main_tip")) // block production finalizes the produced blocks, so re-establish finality at the fork // base: building below the finalized block is rejected as a too deep reorg .with_action( FinalizeBlock::::new(BlockReference::Tag("fork_base".to_string())) .with_head(BlockReference::Tag("main_tip".to_string())), ) - // create a fork from block 2 with 3 additional blocks + // create a fork from block 2 with 3 additional blocks. Building on the first fork block + // makes it canonical, which unwinds the persisted main chain blocks above the fork base. .with_action(CreateFork::::new_from_tag("fork_base", 3)) .with_action(CaptureBlock::new("fork_tip")) // perform FCU to the fork tip - this should make the fork canonical - .with_action(ReorgTo::::new_from_tag("fork_tip")); + .with_action(ReorgTo::::new_from_tag("fork_tip")) + // the fork replaces the unwound main chain blocks on disk. + .with_action(WaitForPersistedChain::new(0, "fork_tip").with_unwound("main_tip")); test.run::().await?; @@ -529,13 +558,14 @@ async fn test_engine_tree_buffered_blocks_are_eventually_connected_e2e() -> Resu /// /// This test creates a longer chain of blocks, then uses forkchoice updates to make /// different parts of the chain canonical in sequence, verifying that FCU properly -/// advances the canonical head when all blocks are already available. +/// advances the canonical head when all blocks are already available. Every canonical block +/// is persisted as the head advances. #[tokio::test] async fn test_engine_tree_fcu_extends_canon_chain_e2e() -> Result<()> { reth_tracing::init_test_tracing(); let test = TestBuilder::new() - .with_setup(default_engine_tree_setup()) + .with_setup(v1_engine_tree_setup()) // create and make canonical a base chain with 1 block .with_action(ProduceBlocks::::new(1)) .with_action(MakeCanonical::new()) @@ -547,7 +577,9 @@ async fn test_engine_tree_fcu_extends_canon_chain_e2e() -> Result<()> { // extend the canonical chain to the target via FCU. .with_action(ReorgTo::::new_from_tag("target_block")) // repeat the FCU to the chain tip, which is already canonical. - .with_action(MakeCanonical::new()); + .with_action(MakeCanonical::new()) + // every canonical block reaches disk. + .with_action(WaitForPersistedChain::new(0, "target_block")); test.run::().await?; @@ -698,7 +730,8 @@ async fn test_engine_tree_pipeline_sync_catches_up_masked_state_e2e() -> Result< /// v2 variant: Verifies forkchoice update and canonical chain insertion in v2 storage mode. /// -/// Exercises the full `save_blocks` → `write_state` → static file changeset path with hashed keys. +/// Every canonical block is persisted, exercising `save_blocks` with hashed-key changesets in +/// static files and history indices in rocksdb. #[tokio::test] async fn test_engine_tree_fcu_canon_chain_insertion_v2_e2e() -> Result<()> { reth_tracing::init_test_tracing(); @@ -708,7 +741,9 @@ async fn test_engine_tree_fcu_canon_chain_insertion_v2_e2e() -> Result<()> { .with_action(ProduceBlocks::::new(1)) .with_action(MakeCanonical::new()) .with_action(ProduceBlocks::::new(3)) - .with_action(MakeCanonical::new()); + .with_action(MakeCanonical::new()) + .with_action(CaptureBlock::new("canonical_tip")) + .with_action(WaitForPersistedChain::new(0, "canonical_tip")); test.run::().await?; @@ -717,7 +752,8 @@ async fn test_engine_tree_fcu_canon_chain_insertion_v2_e2e() -> Result<()> { /// v2 variant: Verifies forkchoice update with a reorg where all blocks are already available. /// -/// Exercises `write_state_reverts` path with hashed changeset keys during CL-driven reorgs. +/// The reorg unwinds the persisted main chain blocks above the fork base from the hashed-key +/// static file changesets and rocksdb history indices, then persists the fork instead. #[tokio::test] async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> { reth_tracing::init_test_tracing(); @@ -729,6 +765,7 @@ async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> { .with_action(ProduceBlocks::::new(3)) .with_action(CaptureBlock::new("main_tip")) .with_action(MakeCanonical::new()) + .with_action(WaitForPersistedChain::new(0, "main_tip")) // block production finalizes the produced blocks, so re-establish finality at the fork // base: building below the finalized block is rejected as a too deep reorg .with_action( @@ -737,7 +774,8 @@ async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> { ) .with_action(CreateFork::::new_from_tag("fork_base", 3)) .with_action(CaptureBlock::new("fork_tip")) - .with_action(ReorgTo::::new_from_tag("fork_tip")); + .with_action(ReorgTo::::new_from_tag("fork_tip")) + .with_action(WaitForPersistedChain::new(0, "fork_tip").with_unwound("main_tip")); test.run::().await?; @@ -745,6 +783,8 @@ async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> { } /// v2 variant: Verifies progressive canonical chain extension in v2 storage mode. +/// +/// Every canonical block is persisted as the head advances. #[tokio::test] async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> { reth_tracing::init_test_tracing(); @@ -756,7 +796,8 @@ async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> { .with_action(ProduceBlocks::::new(10)) .with_action(CaptureBlock::new("target_block")) .with_action(ReorgTo::::new_from_tag("target_block")) - .with_action(MakeCanonical::new()); + .with_action(MakeCanonical::new()) + .with_action(WaitForPersistedChain::new(0, "target_block")); test.run::().await?; From 3f9dfc4250bb87760a1cf89f47d7fb86d814cfe0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 15:12:24 +0200 Subject: [PATCH 086/133] test(e2e): make storage mode explicit (#27434) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/e2e-test-utils/src/setup_builder.rs | 23 +++++++----- crates/e2e-test-utils/src/testsuite/setup.rs | 28 +++++++-------- .../tests/e2e-testsuite/main.rs | 36 +++++++++++++++++++ crates/e2e-test-utils/tests/rocksdb/main.rs | 20 +++++------ .../engine/tree/tests/e2e-testsuite/main.rs | 33 +++++++++-------- 5 files changed, 91 insertions(+), 49 deletions(-) diff --git a/crates/e2e-test-utils/src/setup_builder.rs b/crates/e2e-test-utils/src/setup_builder.rs index 1bbc9a01c15..b4ccbf1f2f5 100644 --- a/crates/e2e-test-utils/src/setup_builder.rs +++ b/crates/e2e-test-utils/src/setup_builder.rs @@ -10,7 +10,7 @@ use reth_node_builder::{ EngineNodeLauncher, NodeBuilder, NodeConfig, NodeHandle, NodeTypes, NodeTypesWithDBAdapter, PayloadTypes, }; -use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs, StorageArgs}; use reth_primitives_traits::AlloyBlockHeader; use reth_provider::providers::BlockchainProvider; use reth_rpc_server_types::RpcModuleSelection; @@ -45,6 +45,7 @@ where connect_nodes: bool, tree_config_modifier: Option, node_config_modifier: Option>, + storage_v2: bool, } impl E2ETestSetupBuilder @@ -65,6 +66,7 @@ where connect_nodes: true, tree_config_modifier: None, node_config_modifier: None, + storage_v2: StorageArgs::default().v2, } } @@ -101,13 +103,14 @@ where self.with_node_config_modifier(move |config| config.with_pruning(pruning.clone())) } - /// Enables v2 storage defaults (`--storage.v2`), routing tx hashes, history - /// indices, etc. to `RocksDB` and changesets/senders to static files. - pub fn with_storage_v2(self) -> Self { - self.with_node_config_modifier(|mut config| { - config.storage.v2 = true; - config - }) + /// Sets whether nodes use the v2 storage layout (`--storage.v2`), which routes tx hashes, + /// history indices, etc. to `RocksDB` and changesets/senders to static files. + /// + /// Defaults to the node's `--storage.v2` default. The node config modifier runs afterwards + /// and can still override it. + pub const fn with_storage_v2(mut self, storage_v2: bool) -> Self { + self.storage_v2 = storage_v2; + self } /// Builds and launches the test nodes. @@ -144,7 +147,8 @@ where .with_unused_ports() .with_http() .with_http_api(RpcModuleSelection::All), - ); + ) + .with_storage(StorageArgs { v2: self.storage_v2 }); // Apply node config modifier if present let node_config = if let Some(modifier) = &self.node_config_modifier { @@ -219,6 +223,7 @@ where .field("connect_nodes", &self.connect_nodes) .field("tree_config_modifier", &self.tree_config_modifier.as_ref().map(|_| "")) .field("node_config_modifier", &self.node_config_modifier.as_ref().map(|_| "")) + .field("storage_v2", &self.storage_v2) .finish_non_exhaustive() } } diff --git a/crates/e2e-test-utils/src/testsuite/setup.rs b/crates/e2e-test-utils/src/testsuite/setup.rs index 79f4a86d0e8..a04ecfbdec2 100644 --- a/crates/e2e-test-utils/src/testsuite/setup.rs +++ b/crates/e2e-test-utils/src/testsuite/setup.rs @@ -9,7 +9,7 @@ use reth_chainspec::ChainSpec; use reth_ethereum_primitives::Block; use reth_network_p2p::sync::{NetworkSyncUpdater, SyncState}; use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes, TreeConfig}; -use reth_node_core::primitives::RecoveredBlock; +use reth_node_core::{args::StorageArgs, primitives::RecoveredBlock}; use revm::state::EvmState; use std::{marker::PhantomData, path::Path, sync::Arc}; use tokio::{ @@ -37,7 +37,9 @@ pub struct Setup { shutdown_tx: Option>, /// Is this setup in dev mode pub is_dev: bool, - /// Whether to use v2 storage mode (hashed keys, static file changesets, rocksdb history) + /// Whether to use v2 storage mode (hashed keys, static file changesets, rocksdb history). + /// + /// Defaults to the node's `--storage.v2` default. Not applied to chains imported from RLP. pub storage_v2: bool, /// Tracks instance generic. _phantom: PhantomData, @@ -59,7 +61,7 @@ impl Default for Setup { tree_config: TreeConfig::default(), shutdown_tx: None, is_dev: true, - storage_v2: false, + storage_v2: StorageArgs::default().v2, _phantom: Default::default(), import_result_holder: None, import_rlp_path: None, @@ -128,9 +130,9 @@ where self } - /// Enable v2 storage mode (hashed keys, static file changesets, rocksdb history) - pub const fn with_storage_v2(mut self) -> Self { - self.storage_v2 = true; + /// Set whether to use v2 storage mode (hashed keys, static file changesets, rocksdb history) + pub const fn with_storage_v2(mut self, storage_v2: bool) -> Self { + self.storage_v2 = storage_v2; self } @@ -202,13 +204,12 @@ where self.shutdown_tx = Some(shutdown_tx); let is_dev = self.is_dev; - let storage_v2 = self.storage_v2; let node_count = self.network.node_count; let tree_config = self.tree_config.clone(); let attributes_generator = Self::create_static_attributes_generator::(); - let mut builder = E2ETestSetupBuilder::::new( + let result = E2ETestSetupBuilder::::new( node_count, Arc::::new((*chain_spec).clone().into()), attributes_generator, @@ -217,13 +218,10 @@ where tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size()) }) .with_node_config_modifier(move |config| config.set_dev(is_dev)) - .with_connect_nodes(self.network.connect_nodes); - - if storage_v2 { - builder = builder.with_storage_v2(); - } - - let result = builder.build().await; + .with_storage_v2(self.storage_v2) + .with_connect_nodes(self.network.connect_nodes) + .build() + .await; let mut node_clients = Vec::new(); match result { diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index 089cea386b8..d182dc92cd0 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -19,6 +19,7 @@ use reth_e2e_test_utils::{ }; use reth_node_api::TreeConfig; use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_provider::{DatabaseProviderFactory, MetadataProvider, StorageSettings}; use std::sync::Arc; use tempfile::TempDir; use tracing::debug; @@ -392,3 +393,38 @@ async fn test_setup_builder_with_custom_tree_config() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_setup_builder_with_storage_v2() -> Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + ); + + for storage_v2 in [false, true] { + // A node config modifier set afterwards must not reset the storage mode. + let (nodes, _wallet) = + E2ETestSetupBuilder::::new(1, chain_spec.clone(), |_| { + PayloadAttributes::default() + }) + .with_storage_v2(storage_v2) + .with_node_config_modifier(|config| config.set_dev(true)) + .build() + .await?; + + let provider = nodes[0].inner.provider.database_provider_ro()?; + assert_eq!(provider.storage_settings()?, Some(StorageSettings { storage_v2 })); + } + + Ok(()) +} diff --git a/crates/e2e-test-utils/tests/rocksdb/main.rs b/crates/e2e-test-utils/tests/rocksdb/main.rs index dc55ae8430e..11283fd1c2f 100644 --- a/crates/e2e-test-utils/tests/rocksdb/main.rs +++ b/crates/e2e-test-utils/tests/rocksdb/main.rs @@ -104,7 +104,7 @@ async fn test_rocksdb_node_startup() -> Result<()> { let (nodes, _wallet) = E2ETestSetupBuilder::::new(1, chain_spec, test_attributes_generator) - .with_storage_v2() + .with_storage_v2(true) .build() .await?; @@ -132,7 +132,7 @@ async fn test_rocksdb_block_mining() -> Result<()> { let (mut nodes, _wallet) = E2ETestSetupBuilder::::new(1, chain_spec, test_attributes_generator) - .with_storage_v2() + .with_storage_v2(true) .build() .await?; @@ -189,7 +189,7 @@ async fn test_rocksdb_transaction_queries() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) @@ -258,7 +258,7 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) @@ -328,7 +328,7 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) @@ -415,7 +415,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) @@ -481,7 +481,7 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) @@ -606,7 +606,7 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) @@ -755,7 +755,7 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) @@ -854,7 +854,7 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { chain_spec.clone(), test_attributes_generator, ) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index 77a3b8af47b..68d901f54a1 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -257,6 +257,8 @@ impl Action for WaitForPersistedChain { } /// Creates the standard setup for engine tree e2e tests. +/// +/// Uses the node's default storage mode. fn default_engine_tree_setup() -> Setup { Setup::default() .with_chain_spec(Arc::new( @@ -277,11 +279,13 @@ fn default_engine_tree_setup() -> Setup { /// Creates a v1 storage mode setup that persists every canonical block. /// +/// v1 mode keeps all data in MDBX, using plain state tables and plain-key changesets and history. +/// /// With a persistence threshold and memory block buffer target of zero and no state masking, the /// engine persists every canonical block up to the head together with its state and trie updates, /// so even short chains exercise the storage writes and unwinds instead of staying in memory. fn v1_engine_tree_setup() -> Setup { - default_engine_tree_setup().with_tree_config( + default_engine_tree_setup().with_storage_v2(false).with_tree_config( TreeConfig::default() .with_num_state_masking_blocks(0) .with_persistence_threshold(0) @@ -295,15 +299,15 @@ fn v1_engine_tree_setup() -> Setup { /// v2 mode uses keccak256-hashed slot keys in static file changesets and rocksdb history /// instead of plain keys in MDBX. fn v2_engine_tree_setup() -> Setup { - v1_engine_tree_setup().with_storage_v2() + v1_engine_tree_setup().with_storage_v2(true) } -/// Test that verifies forkchoice update and canonical chain insertion functionality. +/// Test that verifies forkchoice update and canonical chain insertion in v1 storage mode. /// /// Every canonical block is persisted, exercising `save_blocks` with plain-key changesets and /// history indices in MDBX. #[tokio::test] -async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> { +async fn test_engine_tree_fcu_canon_chain_insertion_v1_e2e() -> Result<()> { reth_tracing::init_test_tracing(); let test = TestBuilder::new() @@ -325,12 +329,13 @@ async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> { Ok(()) } -/// Test that verifies forkchoice update with a reorg where all blocks are already available. +/// Test that verifies forkchoice update with a reorg where all blocks are already available, in +/// v1 storage mode. /// /// The main chain is persisted before the fork becomes canonical, so the reorg unwinds the /// persisted main chain blocks above the fork base and persists the fork instead. #[tokio::test] -async fn test_engine_tree_fcu_reorg_with_all_blocks_e2e() -> Result<()> { +async fn test_engine_tree_fcu_reorg_with_all_blocks_v1_e2e() -> Result<()> { reth_tracing::init_test_tracing(); let test = TestBuilder::new() @@ -554,14 +559,15 @@ async fn test_engine_tree_buffered_blocks_are_eventually_connected_e2e() -> Resu Ok(()) } -/// Test that verifies forkchoice updates can extend the canonical chain progressively. +/// Test that verifies forkchoice updates can extend the canonical chain progressively in v1 +/// storage mode. /// /// This test creates a longer chain of blocks, then uses forkchoice updates to make /// different parts of the chain canonical in sequence, verifying that FCU properly /// advances the canonical head when all blocks are already available. Every canonical block /// is persisted as the head advances. #[tokio::test] -async fn test_engine_tree_fcu_extends_canon_chain_e2e() -> Result<()> { +async fn test_engine_tree_fcu_extends_canon_chain_v1_e2e() -> Result<()> { reth_tracing::init_test_tracing(); let test = TestBuilder::new() @@ -677,7 +683,7 @@ async fn test_engine_tree_pipeline_sync_catches_up_masked_state_e2e() -> Result< .build(), )) .with_network(NetworkSetup::multi_node(2)) - .with_storage_v2() + .with_storage_v2(true) .with_tree_config( TreeConfig::default() .with_num_state_masking_blocks(0) @@ -814,7 +820,7 @@ async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> { /// block buffer, each cycle persists up to the canonical head, so the persisted ranges only depend /// on the chain lengths and not on how fast persistence keeps up with block production. fn disk_reorg_setup(storage_v2: bool) -> Setup { - let mut setup = Setup::default() + Setup::default() .with_chain_spec(Arc::new( ChainSpecBuilder::default() .chain(MAINNET.chain) @@ -828,17 +834,14 @@ fn disk_reorg_setup(storage_v2: bool) -> Setup { .build(), )) .with_network(NetworkSetup::multi_node_unconnected(2)) + .with_storage_v2(storage_v2) .with_tree_config( TreeConfig::default() .with_num_state_masking_blocks(0) .with_persistence_threshold(7) .with_memory_block_buffer_target(0) .with_has_enough_parallelism(true), - ); - if storage_v2 { - setup = setup.with_storage_v2(); - } - setup + ) } /// Builds a disk-level reorg test scenario. From 915ff944f25e84bd9ce8faa1b28f05db5004c7ff Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 13:27:38 +0000 Subject: [PATCH 087/133] perf(rpc): cache validated V2 builder blobs (#27413) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc/src/validation.rs | 17 +- crates/rpc/rpc/src/validation/blob_cache.rs | 208 ++++++++++++++++++++ 2 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 crates/rpc/rpc/src/validation/blob_cache.rs diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index b9e087ce041..8ea1e0ba920 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -1,3 +1,4 @@ +use self::blob_cache::BlobValidationCache; use alloy_consensus::{ BlobTransactionValidationError, BlockHeader, EnvKzgSettings, Transaction, TxReceipt, }; @@ -47,6 +48,8 @@ use std::sync::Arc; use tokio::sync::{oneshot, RwLock}; use tracing::warn; +mod blob_cache; + /// The type that implements the `validation` rpc namespace trait #[derive(Clone, Debug, derive_more::Deref)] pub struct ValidationApi { @@ -84,6 +87,7 @@ where disallow, validation_window, cached_state: Default::default(), + validated_blobs: Default::default(), task_spawner, sender_recovery_cache, metrics: Default::default(), @@ -373,16 +377,15 @@ where } /// Validates the given [`BlobsBundleV2`] and returns versioned hashes for blobs. + /// + /// Exact blob, commitment, and cell-proof matches from recent submissions reuse KZG + /// validation. The resulting hashes are still checked against the block's EIP-4844 + /// transactions during payload validation. pub fn validate_blobs_bundle_v2( &self, blobs_bundle: BlobsBundleV2, ) -> Result, ValidationApiError> { - let versioned_hashes = blobs_bundle.versioned_hashes(); - let sidecar = - blobs_bundle.try_into_sidecar().map_err(|_| ValidationApiError::InvalidBlobsBundle)?; - - sidecar.validate(&versioned_hashes, EnvKzgSettings::default().get())?; - Ok(versioned_hashes) + self.validated_blobs.validate(blobs_bundle) } /// Converts the payload into a block and recovers the transaction senders. @@ -676,6 +679,8 @@ pub struct ValidationApiInner { /// latest head block state. Uses async `RwLock` to safely handle concurrent validation /// requests. cached_state: RwLock<(B256, CachedReads)>, + /// Recently validated blob, commitment, and cell-proof tuples shared by V2 submissions. + validated_blobs: BlobValidationCache, /// Task spawner for blocking operations task_spawner: Runtime, /// Cache of recovered transaction senders shared with transaction ingress and payload diff --git a/crates/rpc/rpc/src/validation/blob_cache.rs b/crates/rpc/rpc/src/validation/blob_cache.rs new file mode 100644 index 00000000000..e71bf6bf32d --- /dev/null +++ b/crates/rpc/rpc/src/validation/blob_cache.rs @@ -0,0 +1,208 @@ +//! Small cache of successfully validated builder-submission blobs. +//! +//! A blob versioned hash identifies its KZG commitment, not the blob bytes. EIP-7594 validation +//! verifies that the blob's cells and proofs match that commitment. This check is independent of +//! the block containing the blob, so a successful validation can be reused across submissions, +//! even if the original submission later fails a separate block check. +//! +//! Cache hits require an exact match of the blob, commitment, and all cell proofs. Every submission +//! still checks bundle lengths and derives versioned hashes in bundle order. Payload validation +//! then compares those hashes with the EIP-4844 transactions in the submitted block; the cache +//! only skips repeated KZG verification. + +use super::ValidationApiError; +use alloy_consensus::EnvKzgSettings; +use alloy_eips::eip7594::{BlobTransactionSidecarEip7594, CELLS_PER_EXT_BLOB}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::BlobsBundleV2; +use parking_lot::Mutex; +use std::collections::VecDeque; + +/// Maximum number of individual blobs retained across competing submissions. +const VALIDATED_BLOB_CACHE_CAPACITY: usize = 15; + +/// Reuses KZG validation only for exact V2 blob sidecar matches. +#[derive(Debug, Default)] +pub(super) struct BlobValidationCache { + /// Every entry contains exactly one validated blob. + entries: Mutex>, +} + +impl BlobValidationCache { + /// Validates a V2 bundle, verifying only blobs absent from the cache. + pub(super) fn validate( + &self, + mut bundle: BlobsBundleV2, + ) -> Result, ValidationApiError> { + if bundle.blobs.len() != bundle.commitments.len() || + bundle.blobs.len().checked_mul(CELLS_PER_EXT_BLOB) != Some(bundle.proofs.len()) + { + return Err(ValidationApiError::InvalidBlobsBundle) + } + if bundle.blobs.is_empty() { + return Ok(Vec::new()) + } + + let versioned_hashes = bundle.versioned_hashes(); + let hits = { + let entries = self.entries.lock(); + (0..bundle.blobs.len()) + .map(|index| { + let proofs = &bundle.proofs + [index * CELLS_PER_EXT_BLOB..(index + 1) * CELLS_PER_EXT_BLOB]; + entries.iter().any(|cached| { + cached.commitments[0] == bundle.commitments[index] && + cached.cell_proofs == proofs && + cached.blobs[0] == bundle.blobs[index] + }) + }) + .collect::>() + }; + if hits.iter().all(|hit| *hit) { + return Ok(versioned_hashes) + } + + let mut cached = BlobsBundleV2::empty(); + let mut missing = BlobsBundleV2::empty(); + for hit in hits { + let partition = if hit { &mut cached } else { &mut missing }; + // The checked lengths keep each blob, commitment, and proof group aligned. + partition.blobs.extend(bundle.blobs.drain(..1)); + partition.commitments.extend(bundle.commitments.drain(..1)); + partition.proofs.extend(bundle.proofs.drain(..CELLS_PER_EXT_BLOB)); + } + drop(cached); + + let sidecar = + missing.try_into_sidecar().map_err(|_| ValidationApiError::InvalidBlobsBundle)?; + let missing_hashes = sidecar.versioned_hashes().collect::>(); + sidecar.validate(&missing_hashes, EnvKzgSettings::default().get())?; + + for index in 0..sidecar.blobs.len() { + let proofs = + &sidecar.cell_proofs[index * CELLS_PER_EXT_BLOB..(index + 1) * CELLS_PER_EXT_BLOB]; + self.insert(BlobTransactionSidecarEip7594::new( + vec![sidecar.blobs[index]], + vec![sidecar.commitments[index]], + proofs.to_vec(), + )); + } + + Ok(versioned_hashes) + } + + fn insert(&self, sidecar: BlobTransactionSidecarEip7594) { + let mut entries = self.entries.lock(); + if entries + .iter() + .any(|cached| cached.commitments == sidecar.commitments && cached == &sidecar) + { + return; + } + if entries.len() == VALIDATED_BLOB_CACHE_CAPACITY { + entries.pop_front(); + } + entries.push_back(sidecar); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::{Blob, BlobTransactionSidecar, Bytes48, SidecarBuilder, SimpleCoder}; + use alloy_rpc_types_engine::BlobsBundleV1; + + fn valid_v1_bundle(data: &[u8]) -> BlobsBundleV1 { + let mut builder = SidecarBuilder::::new(); + builder.ingest(data); + BlobsBundleV1::new([builder.build::().unwrap()]) + } + + fn valid_v2_bundle(data: &[u8]) -> BlobsBundleV2 { + valid_v1_bundle(data).try_into_v2().unwrap() + } + + #[test] + fn mixed_bundle_validates_only_missing_blobs() { + let cache = BlobValidationCache::default(); + let mut bundle = valid_v2_bundle(b"first blob"); + let second = valid_v2_bundle(b"second blob"); + cache.validate(valid_v2_bundle(b"first blob")).unwrap(); + + bundle.blobs.extend(second.blobs); + bundle.commitments.extend(second.commitments); + bundle.proofs.extend(second.proofs); + assert_eq!(cache.validate(bundle.clone()).unwrap(), bundle.versioned_hashes()); + assert_eq!(cache.entries.lock().len(), 2); + } + + #[test] + fn v2_reuses_only_exact_validated_blobs() { + let cache = BlobValidationCache::default(); + let hashes = valid_v2_bundle(b"peer das blob").versioned_hashes(); + + assert_eq!(cache.validate(valid_v2_bundle(b"peer das blob")).unwrap(), hashes); + assert_eq!(cache.validate(valid_v2_bundle(b"peer das blob")).unwrap(), hashes); + assert_eq!(cache.entries.lock().len(), 1); + + let mut changed_blob = valid_v2_bundle(b"peer das blob"); + changed_blob.blobs[0].0[0] ^= 1; + assert!(cache.validate(changed_blob).is_err()); + + let mut changed_proof = valid_v2_bundle(b"peer das blob"); + changed_proof.proofs[0].0[0] ^= 1; + assert!(cache.validate(changed_proof).is_err()); + assert_eq!(cache.entries.lock().len(), 1); + } + + #[test] + fn validates_uncached_blob_even_when_another_hits() { + let cache = BlobValidationCache::default(); + let mut bundle = valid_v2_bundle(b"first blob"); + let mut second = valid_v2_bundle(b"second blob"); + cache.validate(valid_v2_bundle(b"first blob")).unwrap(); + + second.proofs[0].0[0] ^= 1; + bundle.blobs.extend(second.blobs); + bundle.commitments.extend(second.commitments); + bundle.proofs.extend(second.proofs); + assert!(cache.validate(bundle).is_err()); + assert_eq!(cache.entries.lock().len(), 1); + } + + #[test] + fn rejects_incomplete_bundles_without_changing_cache() { + let cache = BlobValidationCache::default(); + let mut v2 = valid_v2_bundle(b"blob"); + v2.proofs.pop(); + assert!(matches!(cache.validate(v2), Err(ValidationApiError::InvalidBlobsBundle))); + assert!(cache.entries.lock().is_empty()); + } + + #[test] + fn empty_bundle_still_checks_lengths() { + let cache = BlobValidationCache::default(); + assert!(cache.validate(BlobsBundleV2::empty()).unwrap().is_empty()); + + let mut malformed = BlobsBundleV2::empty(); + malformed.proofs.push(Bytes48::ZERO); + assert!(matches!(cache.validate(malformed), Err(ValidationApiError::InvalidBlobsBundle))); + } + + #[test] + fn retains_only_fifteen_recent_blobs() { + let cache = BlobValidationCache::default(); + let blob = Blob::default(); + for index in 0..=VALIDATED_BLOB_CACHE_CAPACITY { + cache.insert(BlobTransactionSidecarEip7594::new( + vec![blob], + vec![Bytes48::repeat_byte(index as u8)], + vec![Bytes48::ZERO; CELLS_PER_EXT_BLOB], + )); + } + + let entries = cache.entries.lock(); + assert_eq!(entries.len(), VALIDATED_BLOB_CACHE_CAPACITY); + assert_eq!(entries.front().unwrap().commitments[0], Bytes48::repeat_byte(1)); + } +} From ec64b19f9870a7c413e953654e052a1c71c11746 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 13:49:59 +0000 Subject: [PATCH 088/133] refactor: init account literals with defaults (#27443) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../execution-cache/src/cached_state.rs | 5 +-- .../engine/invalid-block-hooks/src/witness.rs | 5 +-- .../src/tree/payload_processor/bal/execute.rs | 23 ++++------- .../tree/src/tree/payload_processor/mod.rs | 2 +- .../src/tree/payload_processor/prewarm.rs | 2 + .../tree/src/tree/state_root_strategy/mod.rs | 2 +- .../tree/state_root_strategy/sparse_trie.rs | 6 ++- crates/ethereum/evm/tests/execute.rs | 9 ++--- .../execution-types/src/execution_outcome.rs | 4 +- crates/net/downloaders/src/snap/mod.rs | 9 +---- crates/net/downloaders/src/snap/storage.rs | 4 +- crates/net/network/src/eth_requests.rs | 12 ++++-- crates/net/network/tests/it/snap/mod.rs | 16 ++++---- .../src/segments/user/account_history.rs | 2 +- .../src/segments/user/storage_history.rs | 2 +- crates/revm/src/database.rs | 31 +++++++++++---- crates/snap-sync/src/catch_up/apply.rs | 2 +- crates/snap-sync/src/test_utils.rs | 11 ++---- .../stages/stages/src/stages/execution/mod.rs | 35 ++++++----------- .../stages/src/stages/hashing_account.rs | 2 +- crates/stages/stages/src/stages/mod.rs | 8 +--- .../storage/db/src/implementation/mdbx/mod.rs | 2 + .../src/providers/blockchain_provider.rs | 2 +- .../provider/src/providers/consistent.rs | 6 +-- .../src/providers/database/provider.rs | 13 +++---- .../provider/src/providers/static_file/mod.rs | 10 ++--- .../src/providers/static_file/writer_tests.rs | 2 +- .../storage/provider/src/test_utils/mock.rs | 2 +- crates/storage/provider/src/writer/mod.rs | 3 +- crates/storage/storage-overlay/src/builder.rs | 7 ++-- crates/trie/common/src/account.rs | 9 +++-- crates/trie/common/src/constants.rs | 2 + crates/trie/common/src/hashed_state.rs | 12 +++--- crates/trie/common/src/proofs.rs | 4 +- crates/trie/db/src/state.rs | 4 +- crates/trie/db/tests/proof.rs | 5 +-- crates/trie/db/tests/trie.rs | 38 ++++++++++++------- crates/trie/trie/src/verify.rs | 4 ++ testing/ef-tests/src/models.rs | 2 +- testing/testing-utils/src/generators.rs | 2 +- 40 files changed, 164 insertions(+), 157 deletions(-) diff --git a/crates/engine/execution-cache/src/cached_state.rs b/crates/engine/execution-cache/src/cached_state.rs index efb05a40217..1783bd07f15 100644 --- a/crates/engine/execution-cache/src/cached_state.rs +++ b/crates/engine/execution-cache/src/cached_state.rs @@ -1442,11 +1442,10 @@ mod tests { Address::random(), BundleAccount::new( Some(AccountInfo { - balance: U256::ZERO, nonce: 1, code_hash: B256::random(), // Non-empty code hash code: None, - account_id: None, + ..Default::default() }), None, // Destroyed, so no current info Default::default(), @@ -1491,7 +1490,7 @@ mod tests { nonce: 1, code_hash: alloy_primitives::KECCAK256_EMPTY, // Empty code hash = EOA code: None, - account_id: None, + ..Default::default() }), None, // Destroyed Default::default(), diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index a0d79857aef..811d8543cb0 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -455,14 +455,13 @@ mod tests { nonce: account.nonce, code_hash: account.bytecode_hash.unwrap_or_default(), code: None, - account_id: None, + ..Default::default() }), original_info: (i == 0).then(|| AccountInfo { balance: account.balance.checked_div(U256::from(2)).unwrap_or(U256::ZERO), - nonce: 0, code_hash: account.bytecode_hash.unwrap_or_default(), code: None, - account_id: None, + ..Default::default() }), storage, status: AccountStatus::default(), diff --git a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs index 41063f32ae3..90cca4e3b73 100644 --- a/crates/engine/tree/src/tree/payload_processor/bal/execute.rs +++ b/crates/engine/tree/src/tree/payload_processor/bal/execute.rs @@ -410,31 +410,28 @@ mod tests { db.insert_account_info( BEACON_ROOTS_ADDRESS, AccountInfo { - balance: U256::ZERO, nonce: 1, code_hash: keccak256(BEACON_ROOTS_CODE.clone()), code: Some(Bytecode::new_raw(BEACON_ROOTS_CODE.clone())), - account_id: None, + ..Default::default() }, ); db.insert_account_info( WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, AccountInfo { - balance: U256::ZERO, nonce: 1, code_hash: keccak256(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone()), code: Some(Bytecode::new_raw(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone())), - account_id: None, + ..Default::default() }, ); db.insert_account_info( HISTORY_STORAGE_ADDRESS, AccountInfo { - balance: U256::ZERO, nonce: 1, code_hash: keccak256(HISTORY_STORAGE_CODE.clone()), code: Some(Bytecode::new_raw(HISTORY_STORAGE_CODE.clone())), - account_id: None, + ..Default::default() }, ); db @@ -619,7 +616,7 @@ mod tests { fn insert_funded(db: &mut CacheDB, addr: alloy_primitives::Address, balance: U256) { db.insert_account_info( addr, - AccountInfo { nonce: 0, balance, code_hash: B256::ZERO, code: None, account_id: None }, + AccountInfo { balance, code_hash: B256::ZERO, code: None, ..Default::default() }, ); } @@ -729,21 +726,19 @@ mod tests { db.insert_account_info( alice, AccountInfo { - nonce: 0, balance: sender_balance, code_hash: B256::ZERO, code: None, - account_id: None, + ..Default::default() }, ); db.insert_account_info( bob, AccountInfo { - nonce: 0, balance: sender_balance, code_hash: B256::ZERO, code: None, - account_id: None, + ..Default::default() }, ); db @@ -1310,10 +1305,9 @@ mod tests { revert_contract, AccountInfo { nonce: 1, - balance: U256::ZERO, code_hash, code: Some(Bytecode::new_raw(revert_code)), - account_id: None, + ..Default::default() }, ); @@ -1367,10 +1361,9 @@ mod tests { sstore_contract, AccountInfo { nonce: 1, - balance: U256::ZERO, code_hash, code: Some(Bytecode::new_raw(sstore_code)), - account_id: None, + ..Default::default() }, ); diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 2fd848295aa..f3170406683 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -1037,7 +1037,7 @@ mod tests { nonce: 7, code_hash: KECCAK_EMPTY, code: None, - account_id: None, + ..Default::default() }, ) .build(); diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 6d920cbacc4..c4a289d6a7d 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -1171,6 +1171,7 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn bal_account_uses_existing_fields_only_when_missing() { let changes = AccountChanges::new(address!("0000000000000000000000000000000000000001")) .with_balance_change(BalanceChange::new(BlockAccessIndex::new(1), U256::from(10))); @@ -1181,6 +1182,7 @@ mod tests { balance: U256::from(1), nonce: 3, bytecode_hash: Some(B256::repeat_byte(0xaa)), + ..Default::default() }; account.apply_bal_info(info); diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index cb9be5c5516..aefd61b06b1 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -1392,7 +1392,7 @@ mod tests { nonce: rng.random::(), code_hash: KECCAK_EMPTY, code: Some(Default::default()), - account_id: None, + ..Default::default() }; account.storage = storage; account.status = AccountStatus::Touched; diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 566df3e58be..16f0d8bf257 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1788,7 +1788,7 @@ mod tests { let mut hashed_state = HashedPostState::default(); hashed_state.accounts.insert( address, - Some(Account { balance: U256::from(100), nonce: 1, bytecode_hash: None }), + Some(Account { balance: U256::from(100), nonce: 1, ..Default::default() }), ); let mut storage = reth_trie::HashedStorage::default(); storage.storage.insert(slot, value); @@ -1851,12 +1851,14 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn test_encode_account_leaf_value_non_empty_account_is_rlp() { let storage_root = B256::from([0x99; 32]); let account = Some(Account { nonce: 7, balance: U256::from(42), bytecode_hash: Some(B256::from([0xAA; 32])), + ..Default::default() }); let mut account_rlp_buf = vec![0x00, 0x01]; @@ -2252,7 +2254,7 @@ mod tests { let account = Account { nonce: u64::from(index) + 1, balance: U256::from(index), - bytecode_hash: None, + ..Default::default() }; let storage = (0..4u8) .map(|slot| { diff --git a/crates/ethereum/evm/tests/execute.rs b/crates/ethereum/evm/tests/execute.rs index 4c09cd7b037..03b466943a1 100644 --- a/crates/ethereum/evm/tests/execute.rs +++ b/crates/ethereum/evm/tests/execute.rs @@ -34,11 +34,10 @@ fn create_database_with_beacon_root_contract() -> CacheDB { let mut db = CacheDB::new(Default::default()); let beacon_root_contract_account = AccountInfo { - balance: U256::ZERO, code_hash: keccak256(BEACON_ROOTS_CODE.clone()), nonce: 1, code: Some(Bytecode::new_raw(BEACON_ROOTS_CODE.clone())), - account_id: None, + ..Default::default() }; db.insert_account_info(BEACON_ROOTS_ADDRESS, beacon_root_contract_account); @@ -51,10 +50,9 @@ fn create_database_with_withdrawal_requests_contract() -> CacheDB { let withdrawal_requests_contract_account = AccountInfo { nonce: 1, - balance: U256::ZERO, code_hash: keccak256(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone()), code: Some(Bytecode::new_raw(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone())), - account_id: None, + ..Default::default() }; db.insert_account_info( @@ -337,11 +335,10 @@ fn create_database_with_block_hashes(latest_block: u64) -> CacheDB { } let blockhashes_contract_account = AccountInfo { - balance: U256::ZERO, code_hash: keccak256(HISTORY_STORAGE_CODE.clone()), code: Some(Bytecode::new_raw(HISTORY_STORAGE_CODE.clone())), nonce: 1, - account_id: None, + ..Default::default() }; db.insert_account_info(HISTORY_STORAGE_ADDRESS, blockhashes_contract_account); diff --git a/crates/evm/execution-types/src/execution_outcome.rs b/crates/evm/execution-types/src/execution_outcome.rs index 3c7fc2c642b..dbc47168465 100644 --- a/crates/evm/execution-types/src/execution_outcome.rs +++ b/crates/evm/execution-types/src/execution_outcome.rs @@ -941,14 +941,14 @@ mod tests { balance: U256::from(100), code_hash: B256::ZERO, code: None, - account_id: None, + ..Default::default() }; let account_info2 = AccountInfo { nonce: 2, balance: U256::from(200), code_hash: B256::ZERO, code: None, - account_id: None, + ..Default::default() }; // Set up the bundle state with these accounts diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index 28eeedea617..9deac39b784 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -335,7 +335,7 @@ fn verify_proof( #[cfg(test)] mod tests { use super::{request::MAX_RETRIES, test_utils::TestSnapClient, *}; - use alloy_primitives::{Bytes, KECCAK256_EMPTY, U256}; + use alloy_primitives::{Bytes, U256}; use reth_eth_wire_types::snap::ByteCodesMessage; use reth_network_p2p::{error::PeerRequestResult, priority::Priority}; use reth_network_peers::WithPeerId; @@ -349,12 +349,7 @@ mod tests { } fn account(nonce: u64) -> TrieAccount { - TrieAccount { - nonce, - balance: U256::from(1), - storage_root: EMPTY_ROOT_HASH, - code_hash: KECCAK256_EMPTY, - } + TrieAccount { nonce, balance: U256::from(1), ..Default::default() } } fn root(accounts: &[(B256, TrieAccount)]) -> B256 { diff --git a/crates/net/downloaders/src/snap/storage.rs b/crates/net/downloaders/src/snap/storage.rs index 50fd1f9528b..6b810820bd4 100644 --- a/crates/net/downloaders/src/snap/storage.rs +++ b/crates/net/downloaders/src/snap/storage.rs @@ -452,7 +452,7 @@ mod tests { super::{request::MAX_RETRIES, test_utils::TestSnapClient, VerifiedAccountRange}, *, }; - use alloy_primitives::{Bytes, KECCAK256_EMPTY}; + use alloy_primitives::Bytes; use reth_network_p2p::{error::PeerRequestResult, priority::Priority}; use reth_network_peers::WithPeerId; use reth_trie_common::{ @@ -500,7 +500,7 @@ mod tests { } fn account(storage_root: B256) -> TrieAccount { - TrieAccount { nonce: 1, balance: U256::from(2), storage_root, code_hash: KECCAK256_EMPTY } + TrieAccount { nonce: 1, balance: U256::from(2), storage_root, ..Default::default() } } fn wire_slots(slots: &[(B256, U256)]) -> Vec { diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 3a51bbae4c6..b549b31720e 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -1225,6 +1225,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::needless_update)] async fn snap_account_range_response_encodes_accounts_and_proof() { let provider = MockEthProvider::default(); let first_hash = B256::repeat_byte(0x01); @@ -1236,9 +1237,14 @@ mod tests { vec![ ( first_hash, - Account { nonce: 1, balance: U256::from(2), bytecode_hash: Some(code_hash) }, + Account { + nonce: 1, + balance: U256::from(2), + bytecode_hash: Some(code_hash), + ..Default::default() + }, ), - (second_hash, Account { nonce: 3, balance: U256::from(4), bytecode_hash: None }), + (second_hash, Account { nonce: 3, balance: U256::from(4), ..Default::default() }), ], // A hash-limit stop (not an exhausted trie) so a boundary proof is still expected, // matching the mocked `proof` below. @@ -1293,7 +1299,7 @@ mod tests { let provider = MockEthProvider::default(); let hash = B256::repeat_byte(0x01); provider.set_snap_account_range( - vec![(hash, Account { nonce: 1, balance: U256::from(2), bytecode_hash: None })], + vec![(hash, Account { nonce: 1, balance: U256::from(2), ..Default::default() })], RangeEnd::Exhausted, ); provider.set_snap_storage_root(hash, EMPTY_ROOT_HASH); diff --git a/crates/net/network/tests/it/snap/mod.rs b/crates/net/network/tests/it/snap/mod.rs index b84d8d482ad..61597cf8cb2 100644 --- a/crates/net/network/tests/it/snap/mod.rs +++ b/crates/net/network/tests/it/snap/mod.rs @@ -222,7 +222,7 @@ async fn account_range_roundtrip_carries_slim_encoding_and_proof() { let factory = genesis_provider_factory(); let accounts: Vec<(Address, Account)> = (0..5u64) .map(|nonce| { - (Address::random(), Account { nonce, balance: U256::from(nonce), bytecode_hash: None }) + (Address::random(), Account { nonce, balance: U256::from(nonce), ..Default::default() }) }) .collect(); @@ -296,7 +296,7 @@ async fn account_range_bounded_by_response_bytes_excludes_trailing_account() { let factory = genesis_provider_factory(); let accounts: Vec<(Address, Account)> = (0..5u64) .map(|nonce| { - (Address::random(), Account { nonce, balance: U256::from(nonce), bytecode_hash: None }) + (Address::random(), Account { nonce, balance: U256::from(nonce), ..Default::default() }) }) .collect(); @@ -360,7 +360,7 @@ async fn storage_range_roundtrip_carries_rlp_values_and_proof() { let factory = genesis_provider_factory(); let address = Address::random(); - let account = Account { nonce: 1, balance: U256::from(1), bytecode_hash: None }; + let account = Account { nonce: 1, balance: U256::from(1), ..Default::default() }; let slots: Vec = (0..6u8) .map(|i| StorageEntry { key: B256::with_last_byte(i), value: U256::from(i as u64 + 1) }) .collect(); @@ -430,7 +430,7 @@ async fn storage_range_empty_window_returns_boundary_slot() { let factory = genesis_provider_factory(); let address = Address::random(); - let account = Account { nonce: 1, balance: U256::from(1), bytecode_hash: None }; + let account = Account { nonce: 1, balance: U256::from(1), ..Default::default() }; let slots: Vec = (0..4u8) .map(|i| StorageEntry { key: B256::with_last_byte(i), value: U256::from(i as u64 + 1) }) .collect(); @@ -497,9 +497,9 @@ async fn storage_ranges_multi_account_bounds_only_first_account() { let factory = genesis_provider_factory(); let (address_a, account_a) = - (Address::random(), Account { nonce: 1, balance: U256::from(1), bytecode_hash: None }); + (Address::random(), Account { nonce: 1, balance: U256::from(1), ..Default::default() }); let (address_b, account_b) = - (Address::random(), Account { nonce: 2, balance: U256::from(2), bytecode_hash: None }); + (Address::random(), Account { nonce: 2, balance: U256::from(2), ..Default::default() }); // 2 slots for A (fits fully in the byte budget below), 5 for B (doesn't). let slots_a: Vec = (0..2u8) .map(|i| StorageEntry { key: B256::with_last_byte(i), value: U256::from(i as u64 + 1) }) @@ -607,7 +607,7 @@ async fn retained_and_expired_account_range_requests_resolve_without_hanging() { // Real trie root of `expired_account` alone, committed to block 0. let expired_account = - (Address::random(), Account { nonce: 3, balance: U256::from(3), bytecode_hash: None }); + (Address::random(), Account { nonce: 3, balance: U256::from(3), ..Default::default() }); { let provider_rw = factory.provider_rw().unwrap(); provider_rw @@ -619,7 +619,7 @@ async fn retained_and_expired_account_range_requests_resolve_without_hanging() { // Real trie root of `expired_account` + `retained_account` together, committed to block 64. let retained_account = - (Address::random(), Account { nonce: 7, balance: U256::from(7), bytecode_hash: None }); + (Address::random(), Account { nonce: 7, balance: U256::from(7), ..Default::default() }); { let provider_rw = factory.provider_rw().unwrap(); provider_rw diff --git a/crates/prune/prune/src/segments/user/account_history.rs b/crates/prune/prune/src/segments/user/account_history.rs index acd0c312fb0..e1c24267951 100644 --- a/crates/prune/prune/src/segments/user/account_history.rs +++ b/crates/prune/prune/src/segments/user/account_history.rs @@ -652,7 +652,7 @@ mod tests { let addr4 = Address::with_last_byte(4); let addr5 = Address::with_last_byte(5); - let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None }; + let account = Account { nonce: 1, balance: U256::from(100), ..Default::default() }; // Build changesets: blocks 0-4 have 1 change each, block 5 has 4 changes, block 6 has 1 let changesets: Vec = vec![ diff --git a/crates/prune/prune/src/segments/user/storage_history.rs b/crates/prune/prune/src/segments/user/storage_history.rs index 06a8522ea39..b2062d3d819 100644 --- a/crates/prune/prune/src/segments/user/storage_history.rs +++ b/crates/prune/prune/src/segments/user/storage_history.rs @@ -559,7 +559,7 @@ mod tests { let addr1 = Address::with_last_byte(1); let addr2 = Address::with_last_byte(2); - let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None }; + let account = Account { nonce: 1, balance: U256::from(100), ..Default::default() }; // Create storage entries let storage_entry = |key: u8| reth_primitives_traits::StorageEntry { diff --git a/crates/revm/src/database.rs b/crates/revm/src/database.rs index 496336933b2..d05f37620a7 100644 --- a/crates/revm/src/database.rs +++ b/crates/revm/src/database.rs @@ -260,18 +260,19 @@ mod tests { balance: U256::from(42), code_hash: KECCAK_EMPTY, code: None, - account_id: None, + ..Default::default() }; let db = CountingDatabaseRef::new(address, Some(account), Bytecode::default()); let provider = DatabaseStateProvider::new(db); assert_eq!( provider.basic_account(&address).unwrap(), - Some(Account { nonce: 7, balance: U256::from(42), bytecode_hash: None }) + Some(Account { nonce: 7, balance: U256::from(42), ..Default::default() }) ); } #[test] + #[allow(clippy::needless_update)] fn database_state_provider_maps_code_hash_and_bytecode() { let address = Address::repeat_byte(0x01); let code_hash = B256::repeat_byte(0x42); @@ -281,14 +282,19 @@ mod tests { balance: U256::from(42), code_hash, code: Some(bytecode.clone()), - account_id: None, + ..Default::default() }; let db = CountingDatabaseRef::new(address, Some(account), bytecode.clone()); let provider = DatabaseStateProvider::new(db); assert_eq!( provider.basic_account(&address).unwrap(), - Some(Account { nonce: 7, balance: U256::from(42), bytecode_hash: Some(code_hash) }) + Some(Account { + nonce: 7, + balance: U256::from(42), + bytecode_hash: Some(code_hash), + ..Default::default() + }) ); assert_eq!( provider.bytecode_by_hash(&code_hash).unwrap(), @@ -331,6 +337,7 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn database_state_provider_uses_cached_reads() { let address = Address::repeat_byte(0x01); let code_hash = B256::repeat_byte(0x42); @@ -340,7 +347,7 @@ mod tests { balance: U256::from(42), code_hash, code: Some(bytecode.clone()), - account_id: None, + ..Default::default() }; let db = CountingDatabaseRef::new(address, Some(account), bytecode.clone()); let account_reads = db.account_reads.clone(); @@ -350,11 +357,21 @@ mod tests { assert_eq!( provider.basic_account(&address).unwrap(), - Some(Account { nonce: 7, balance: U256::from(42), bytecode_hash: Some(code_hash) }) + Some(Account { + nonce: 7, + balance: U256::from(42), + bytecode_hash: Some(code_hash), + ..Default::default() + }) ); assert_eq!( provider.basic_account(&address).unwrap(), - Some(Account { nonce: 7, balance: U256::from(42), bytecode_hash: Some(code_hash) }) + Some(Account { + nonce: 7, + balance: U256::from(42), + bytecode_hash: Some(code_hash), + ..Default::default() + }) ); assert_eq!(account_reads.load(Ordering::Relaxed), 1); diff --git a/crates/snap-sync/src/catch_up/apply.rs b/crates/snap-sync/src/catch_up/apply.rs index 32d8807c97c..f694118458a 100644 --- a/crates/snap-sync/src/catch_up/apply.rs +++ b/crates/snap-sync/src/catch_up/apply.rs @@ -189,7 +189,7 @@ mod tests { let update = apply(&changes, DownloadedAccount::Absent); assert_eq!( update.state.accounts[&keccak256(ACCOUNT)], - Some(Account { balance: U256::from(20), nonce: 0, bytecode_hash: None }) + Some(Account { balance: U256::from(20), ..Default::default() }) ); } diff --git a/crates/snap-sync/src/test_utils.rs b/crates/snap-sync/src/test_utils.rs index 3b53edfc689..02454668ee8 100644 --- a/crates/snap-sync/src/test_utils.rs +++ b/crates/snap-sync/src/test_utils.rs @@ -7,7 +7,7 @@ use crate::{ use alloy_consensus::Header; use alloy_eip7928::{compute_block_access_list_hash, AccountChanges}; use alloy_eips::{eip7928::bal::Bal, BlockNumHash}; -use alloy_primitives::{Bytes, B256, KECCAK256_EMPTY, U256}; +use alloy_primitives::{Bytes, B256, U256}; use futures::future::{ready, Ready}; use reth_db_api::{ cursor::{DbCursorRO, DbDupCursorRO}, @@ -44,7 +44,7 @@ use reth_tasks::Runtime; use reth_trie_common::{ proof::ProofRetainer, root::{state_root_unsorted, storage_root_unsorted}, - HashBuilder, Nibbles, TrieAccount, EMPTY_ROOT_HASH, + HashBuilder, Nibbles, TrieAccount, }; use std::{ collections::VecDeque, @@ -127,12 +127,7 @@ pub(crate) fn key(value: u64) -> B256 { /// An account without storage or code, distinguished by its nonce. pub(crate) fn account(nonce: u64) -> TrieAccount { - TrieAccount { - nonce, - balance: U256::from(1), - storage_root: EMPTY_ROOT_HASH, - code_hash: KECCAK256_EMPTY, - } + TrieAccount { nonce, balance: U256::from(1), ..Default::default() } } /// Root of the account trie holding `accounts`. diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index 990f893ecd5..482738e3ca2 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -1044,14 +1044,11 @@ mod tests { db_tx .put::( acc1, - Account { nonce: 0, balance: U256::ZERO, bytecode_hash: Some(code_hash) }, + Account { bytecode_hash: Some(code_hash), ..Default::default() }, ) .unwrap(); db_tx - .put::( - acc2, - Account { nonce: 0, balance, bytecode_hash: None }, - ) + .put::(acc2, Account { balance, ..Default::default() }) .unwrap(); db_tx.put::(code_hash, Bytecode::new_raw(code.to_vec().into())).unwrap(); provider.commit().unwrap(); @@ -1104,18 +1101,15 @@ mod tests { // check post state let account1 = address!("0x1000000000000000000000000000000000000000"); let account1_info = - Account { balance: U256::ZERO, nonce: 0x00, bytecode_hash: Some(code_hash) }; + Account { bytecode_hash: Some(code_hash), ..Default::default() }; let account2 = address!("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba"); - let account2_info = Account { - balance: U256::from(0x1bc16d674ece94bau128), - nonce: 0x00, - bytecode_hash: None, - }; + let account2_info = + Account { balance: U256::from(0x1bc16d674ece94bau128), ..Default::default() }; let account3 = address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"); let account3_info = Account { balance: U256::from(0x3635c9adc5de996b46u128), nonce: 0x01, - bytecode_hash: None, + ..Default::default() }; // assert accounts @@ -1188,9 +1182,9 @@ mod tests { let db_tx = provider.tx_ref(); let acc1 = address!("0x1000000000000000000000000000000000000000"); - let acc1_info = Account { nonce: 0, balance: U256::ZERO, bytecode_hash: Some(code_hash) }; + let acc1_info = Account { bytecode_hash: Some(code_hash), ..Default::default() }; let acc2 = address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"); - let acc2_info = Account { nonce: 0, balance, bytecode_hash: None }; + let acc2_info = Account { balance, ..Default::default() }; db_tx.put::(acc1, acc1_info).unwrap(); db_tx.put::(acc2, acc2_info).unwrap(); @@ -1368,9 +1362,8 @@ mod tests { let code_hash = keccak256(code); // pre state - let caller_info = Account { nonce: 0, balance, bytecode_hash: None }; - let destroyed_info = - Account { nonce: 0, balance: U256::ZERO, bytecode_hash: Some(code_hash) }; + let caller_info = Account { balance, ..Default::default() }; + let destroyed_info = Account { bytecode_hash: Some(code_hash), ..Default::default() }; // set account let provider = test_db.factory.provider_rw().unwrap(); @@ -1425,18 +1418,14 @@ mod tests { vec![ ( beneficiary_address, - Account { - nonce: 0, - balance: U256::from(0x1bc16d674eca30a0u64), - bytecode_hash: None - } + Account { balance: U256::from(0x1bc16d674eca30a0u64), ..Default::default() } ), ( caller_address, Account { nonce: 1, balance: U256::from(0xde0b6b3a761cf60u64), - bytecode_hash: None + ..Default::default() } ) ] diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index d060a67b3ea..d1d350cb35a 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -463,7 +463,7 @@ mod tests { let old_acc = Account { nonce: nonce - 1, balance: balance - U256::from(1), - bytecode_hash: None, + ..Default::default() }; let hashed_addr = keccak256(address); if let Some((_, acc)) = hashed_acc_cursor.seek_exact(hashed_addr)? { diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index bd9daeeed34..cfac4cf0b32 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -125,18 +125,14 @@ mod tests { .tx_ref() .put::( address!("0x1000000000000000000000000000000000000000"), - Account { nonce: 0, balance: U256::ZERO, bytecode_hash: Some(code_hash) }, + Account { bytecode_hash: Some(code_hash), ..Default::default() }, ) .unwrap(); provider_rw .tx_ref() .put::( address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"), - Account { - nonce: 0, - balance: U256::from(0x3635c9adc5dea00000u128), - bytecode_hash: None, - }, + Account { balance: U256::from(0x3635c9adc5dea00000u128), ..Default::default() }, ) .unwrap(); provider_rw diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 6585cd1f9bf..20a3d92a9ce 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -1411,6 +1411,7 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn db_closure_put_get() { let tempdir = TempDir::new().expect(ERROR_TEMPDIR); let path = tempdir.path(); @@ -1419,6 +1420,7 @@ mod tests { nonce: 18446744073709551615, bytecode_hash: Some(B256::random()), balance: U256::MAX, + ..Default::default() }; let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047") .expect(ERROR_ETH_ADDRESS); diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 4bbee9ba45e..23077ae21db 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -3046,7 +3046,7 @@ mod tests { } fn random_account(nonce: u64) -> (Address, Account) { - (Address::random(), Account { nonce, balance: U256::from(nonce), bytecode_hash: None }) + (Address::random(), Account { nonce, balance: U256::from(nonce), ..Default::default() }) } /// [`BlockchainProvider::new`] needs a genesis header to initialize its chain tracker. diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index f951c5f902a..adce953c744 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -1826,7 +1826,7 @@ mod tests { let account = reth_primitives_traits::Account { nonce: 1, balance: U256::from(1000), - bytecode_hash: None, + ..Default::default() }; let slot = U256::from(0x42); let slot_b256 = B256::from(slot); @@ -1909,7 +1909,7 @@ mod tests { let account = reth_primitives_traits::Account { nonce: 1, balance: U256::from(1000), - bytecode_hash: None, + ..Default::default() }; let slot = U256::from(0x42); @@ -2008,7 +2008,7 @@ mod tests { let account = reth_primitives_traits::Account { nonce: 1, balance: U256::from(1000), - bytecode_hash: None, + ..Default::default() }; let slot = U256::from(0x42); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 53ea7f2e54d..7edd4facdbb 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -5183,7 +5183,7 @@ mod tests { .tx .cursor_write::() .unwrap() - .upsert(address, &Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None }) + .upsert(address, &Account::default()) .unwrap(); provider_rw.commit().unwrap(); } @@ -5196,8 +5196,8 @@ mod tests { state_init.insert( address, ( - Some(Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None }), - Some(Account { nonce: 1, balance: U256::ZERO, bytecode_hash: None }), + Some(Account::default()), + Some(Account { nonce: 1, ..Default::default() }), storage_map, ), ); @@ -5207,7 +5207,7 @@ mod tests { block_reverts.insert( address, ( - Some(Some(Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None })), + Some(Some(Account::default())), vec![StorageEntry { key: slot_key, value: U256::ZERO }], ), ); @@ -5800,10 +5800,7 @@ mod tests { .tx .cursor_write::() .unwrap() - .upsert( - hashed_address, - &Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None }, - ) + .upsert(hashed_address, &Account::default()) .unwrap(); provider_rw.commit().unwrap(); } diff --git a/crates/storage/provider/src/providers/static_file/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs index d06393f1f1c..2fac861f3c4 100644 --- a/crates/storage/provider/src/providers/static_file/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -727,7 +727,7 @@ mod tests { info: Some(Account { nonce: block_num, balance: U256::from(block_num * 1000), - bytecode_hash: None, + ..Default::default() }), }) .collect() @@ -814,7 +814,7 @@ mod tests { .append_account_changeset( vec![AccountBeforeTx { address: other_address, - info: Some(Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None }), + info: Some(Account::default()), }], 1, ) @@ -828,7 +828,7 @@ mod tests { info: Some(Account { nonce: 1, balance: U256::from(1000), - bytecode_hash: None, + ..Default::default() }), }], 2, @@ -910,7 +910,7 @@ mod tests { info: Some(Account { nonce: block_num, balance: U256::from(block_num * 1000 + i as u64), - bytecode_hash: None, + ..Default::default() }), }); } @@ -1027,7 +1027,7 @@ mod tests { info: Some(Account { nonce: 1, balance: U256::from(1000), - bytecode_hash: None, + ..Default::default() }), }) .collect(); diff --git a/crates/storage/provider/src/providers/static_file/writer_tests.rs b/crates/storage/provider/src/providers/static_file/writer_tests.rs index c8c0d645f46..f85201df2cd 100644 --- a/crates/storage/provider/src/providers/static_file/writer_tests.rs +++ b/crates/storage/provider/src/providers/static_file/writer_tests.rs @@ -46,7 +46,7 @@ mod tests { info: Some(Account { nonce: block_num, balance: U256::from(block_num * 1000 + i as u64), - bytecode_hash: None, + ..Default::default() }), } }) diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 2640f4a775c..d370fda62a9 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -473,7 +473,7 @@ impl ExtendedAccount { /// Create new instance of extended account pub fn new(nonce: u64, balance: U256) -> Self { Self { - account: Account { nonce, balance, bytecode_hash: None }, + account: Account { nonce, balance, ..Default::default() }, bytecode: None, storage: Default::default(), } diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index b5a313c4448..cda90df6f70 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -891,7 +891,8 @@ mod tests { type PreState = BTreeMap)>; let mut prestate: PreState = (0..10) .map(|key| { - let account = Account { nonce: 1, balance: U256::from(key), bytecode_hash: None }; + let account = + Account { nonce: 1, balance: U256::from(key), ..Default::default() }; let storage = (1..11).map(|key| (B256::with_last_byte(key), U256::from(key))).collect(); (Address::with_last_byte(key), (account, storage)) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index c7234cacec6..615c730e1c6 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -968,6 +968,7 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn execution_overlay_extends_bundle_state_without_account_ids() { let address = Address::with_last_byte(1); let slot = U256::from(2); @@ -980,6 +981,7 @@ mod tests { code_hash, code: Some(code.clone()), account_id: AccountId::new(6), + ..Default::default() }; let state = BundleState::builder(0..=0) .state_present_account_info(address, account.clone()) @@ -1036,10 +1038,7 @@ mod tests { let mut overlay = ExecutionOverlay::default(); overlay.block_hashes.push(first_block); - overlay.accounts.insert( - address, - Some(AccountInfo { nonce: 1, account_id: None, ..Default::default() }), - ); + overlay.accounts.insert(address, Some(AccountInfo { nonce: 1, ..Default::default() })); overlay.accounts.insert(retained_address, Some(AccountInfo::default())); overlay.storage.entry(address).or_default().insert(slot, U256::from(9)); overlay.storage.entry(address).or_default().insert(retained_slot, U256::from(10)); diff --git a/crates/trie/common/src/account.rs b/crates/trie/common/src/account.rs index b8a86907a1e..382157e4f92 100644 --- a/crates/trie/common/src/account.rs +++ b/crates/trie/common/src/account.rs @@ -31,6 +31,7 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn test_from_genesis_account_with_values() { // Create a GenesisAccount with specific values let mut storage = BTreeMap::new(); @@ -41,7 +42,7 @@ mod tests { balance: U256::from(1000), code: Some(Bytes::from(vec![0x60, 0x61])), storage: Some(storage), - private_key: None, + ..Default::default() }; // Convert the GenesisAccount to a TrieAccount @@ -63,7 +64,8 @@ mod tests { Account { nonce: 10, balance: U256::from(1000), - bytecode_hash: Some(keccak256([0x60, 0x61])) + bytecode_hash: Some(keccak256([0x60, 0x61])), + ..Default::default() } .into_trie_account(expected_storage_root), trie_account @@ -78,9 +80,8 @@ mod tests { let genesis_account = GenesisAccount { nonce: Some(3), balance: U256::from(300), - code: None, storage: Some(storage), - private_key: None, + ..Default::default() }; // Convert the GenesisAccount to a TrieAccount diff --git a/crates/trie/common/src/constants.rs b/crates/trie/common/src/constants.rs index 471b8bd9dcc..3977ef5dd85 100644 --- a/crates/trie/common/src/constants.rs +++ b/crates/trie/common/src/constants.rs @@ -10,12 +10,14 @@ mod tests { use alloy_rlp::Encodable; #[test] + #[allow(clippy::needless_update)] fn account_rlp_max_size() { let account = TrieAccount { nonce: u64::MAX, balance: U256::MAX, storage_root: B256::from_slice(&[u8::MAX; 32]), code_hash: B256::from_slice(&[u8::MAX; 32]), + ..Default::default() }; let mut encoded = Vec::new(); account.encode(&mut encoded); diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index 3a8176464e2..a154be9c279 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -851,7 +851,7 @@ mod tests { nonce: 42, code_hash: B256::random(), code: Some(Bytecode::new_raw(Bytes::from(vec![1, 2]))), - account_id: None, + ..Default::default() }; let mut storage = StorageWithOriginalValues::default(); @@ -957,7 +957,7 @@ mod tests { nonce: 1, code_hash: B256::random(), code: None, - account_id: None, + ..Default::default() }; // Create hashed accounts with addresses. @@ -1275,7 +1275,7 @@ mod tests { #[test] fn test_hashed_post_state_sorted_disjointed_merge_batch() { fn account(nonce: u64) -> Account { - Account { nonce, balance: U256::ZERO, bytecode_hash: None } + Account { nonce, ..Default::default() } } let kept_account = B256::with_last_byte(1); @@ -1358,7 +1358,7 @@ mod tests { #[test] fn test_hashed_post_state_sorted_disjointed_merge_batch_removes_overlapping_batch_key() { fn account(nonce: u64) -> Account { - Account { nonce, balance: U256::ZERO, bytecode_hash: None } + Account { nonce, ..Default::default() } } let overlapping_account = B256::with_last_byte(21); @@ -1384,7 +1384,7 @@ mod tests { #[test] fn test_hashed_post_state_sorted_disjointed_merge_batch_keeps_equal_overlaps() { fn account(nonce: u64) -> Account { - Account { nonce, balance: U256::ZERO, bytecode_hash: None } + Account { nonce, ..Default::default() } } let address = B256::with_last_byte(21); @@ -1542,7 +1542,7 @@ mod tests { let state = HashedPostState { accounts: B256Map::from_iter([ - (addr1, Some(Account { nonce: 1, balance: U256::from(100), bytecode_hash: None })), + (addr1, Some(Account { nonce: 1, balance: U256::from(100), ..Default::default() })), (addr2, None), (addr3, Some(Account::default())), ]), diff --git a/crates/trie/common/src/proofs.rs b/crates/trie/common/src/proofs.rs index 99233b668cf..95e7ff7c123 100644 --- a/crates/trie/common/src/proofs.rs +++ b/crates/trie/common/src/proofs.rs @@ -1564,7 +1564,7 @@ mod tests { address: Address::random(), info: Some( // non-empty account - Account { nonce: 100, balance: U256::ZERO, bytecode_hash: Some(KECCAK_EMPTY) }, + Account { nonce: 100, bytecode_hash: Some(KECCAK_EMPTY), ..Default::default() }, ), proof: vec![], storage_root: B256::ZERO, @@ -1627,6 +1627,7 @@ mod tests { #[test] #[cfg(feature = "eip1186")] + #[allow(clippy::needless_update)] fn into_eip1186_response_zero_empty_account() { // Non-existent account (info = None) let acc = AccountProof { @@ -1659,6 +1660,7 @@ mod tests { nonce: 42, balance: U256::from(100), bytecode_hash: Some(KECCAK_EMPTY), + ..Default::default() }), proof: vec![], storage_root: B256::random(), diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index d35c9b7d008..2c985cca0d8 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -102,7 +102,7 @@ pub trait DatabaseStateRoot<'a, TX>: Sized { /// let mut hashed_state = HashedPostState::default(); /// hashed_state.accounts.insert( /// [0x11; 32].into(), - /// Some(Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }), + /// Some(Account { nonce: 1, balance: U256::from(10), ..Default::default() }), /// ); /// /// // Calculate the state root @@ -384,7 +384,7 @@ mod tests { let mut hashed_state = HashedPostState::default(); hashed_state.accounts.insert( B256::from(U256::from(1)), - Some(Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }), + Some(Account { nonce: 1, balance: U256::from(10), ..Default::default() }), ); hashed_state.accounts.insert(B256::from(U256::from(2)), None); hashed_state.storages.insert( diff --git a/crates/trie/db/tests/proof.rs b/crates/trie/db/tests/proof.rs index de03f30f6db..0136a11a895 100644 --- a/crates/trie/db/tests/proof.rs +++ b/crates/trie/db/tests/proof.rs @@ -286,9 +286,8 @@ fn holesky_deposit_contract_proof() { let expected = AccountProof { address: target, info: Some(Account { - balance: U256::ZERO, - nonce: 0, - bytecode_hash: Some(b256!("0x2034f79e0e33b0ae6bef948532021baceb116adf2616478703bec6b17329f1cc")) + bytecode_hash: Some(b256!("0x2034f79e0e33b0ae6bef948532021baceb116adf2616478703bec6b17329f1cc")), + ..Default::default() }), storage_root: b256!("0x556a482068355939c95a3412bdb21213a301483edb1b64402fb66ac9f3583599"), proof: convert_to_proof([ diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index 75af2b41ca8..bc670160298 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -157,21 +157,19 @@ fn arbitrary_storage_root() { #[test] // This ensures we don't add empty accounts to the trie +#[allow(clippy::needless_update)] fn test_empty_account() { let state: State = BTreeMap::from([ ( Address::random(), ( - Account { nonce: 0, balance: U256::from(0), bytecode_hash: None }, + Account { balance: U256::from(0), ..Default::default() }, BTreeMap::from([(B256::with_last_byte(0x4), U256::from(12))]), ), ), ( Address::random(), - ( - Account { nonce: 0, balance: U256::from(0), bytecode_hash: None }, - BTreeMap::default(), - ), + (Account { balance: U256::from(0), ..Default::default() }, BTreeMap::default()), ), ( Address::random(), @@ -180,6 +178,7 @@ fn test_empty_account() { nonce: 155, balance: U256::from(414241124u32), bytecode_hash: Some(keccak256("test")), + ..Default::default() }, BTreeMap::from([ (B256::ZERO, U256::from(3)), @@ -193,6 +192,7 @@ fn test_empty_account() { #[test] // This ensures we return an empty root when there are no storage entries +#[allow(clippy::needless_update)] fn test_empty_storage_root() { let factory = create_test_provider_factory(); let tx = factory.provider_rw().unwrap(); @@ -203,6 +203,7 @@ fn test_empty_storage_root() { nonce: 155, balance: U256::from(414241124u32), bytecode_hash: Some(keccak256(code)), + ..Default::default() }; insert_account(tx.tx_ref(), address, account, &Default::default()); tx.commit().unwrap(); @@ -360,6 +361,7 @@ fn destroyed_account_storage_emits_node_removals() { #[test] // This ensures that the walker goes over all the storage slots +#[allow(clippy::needless_update)] fn test_storage_root() { let factory = create_test_provider_factory(); let tx = factory.provider_rw().unwrap(); @@ -373,6 +375,7 @@ fn test_storage_root() { nonce: 155, balance: U256::from(414241124u32), bytecode_hash: Some(keccak256(code)), + ..Default::default() }; insert_account(tx.tx_ref(), address, account, &storage); @@ -526,7 +529,7 @@ fn account_and_storage_trie() { // Insert first account let key1 = b256!("0xb000000000000000000000000000000000000000000000000000000000000000"); - let account1 = Account { nonce: 0, balance: U256::from(3).mul(ether), bytecode_hash: None }; + let account1 = Account { balance: U256::from(3).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key1, &account1).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key1), &encode_account(account1, None)); @@ -535,7 +538,7 @@ fn account_and_storage_trie() { let key2 = keccak256(address2); assert_eq!(key2[0], 0xB0); assert_eq!(key2[1], 0x40); - let account2 = Account { nonce: 0, balance: ether, ..Default::default() }; + let account2 = Account { balance: ether, ..Default::default() }; hashed_account_cursor.upsert(key2, &account2).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key2), &encode_account(account2, None)); @@ -545,8 +548,11 @@ fn account_and_storage_trie() { assert_eq!(key3[0], 0xB0); assert_eq!(key3[1], 0x41); let code_hash = b256!("0x5be74cad16203c4905c068b012a2e9fb6d19d036c410f16fd177f337541440dd"); - let account3 = - Account { nonce: 0, balance: U256::from(2).mul(ether), bytecode_hash: Some(code_hash) }; + let account3 = Account { + balance: U256::from(2).mul(ether), + bytecode_hash: Some(code_hash), + ..Default::default() + }; hashed_account_cursor.upsert(key3, &account3).unwrap(); for (hashed_slot, value) in storage { if hashed_storage_cursor @@ -565,17 +571,17 @@ fn account_and_storage_trie() { .add_leaf(Nibbles::unpack(key3), &encode_account(account3, Some(account3_storage_root))); let key4a = b256!("0xB1A0000000000000000000000000000000000000000000000000000000000000"); - let account4a = Account { nonce: 0, balance: U256::from(4).mul(ether), ..Default::default() }; + let account4a = Account { balance: U256::from(4).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key4a, &account4a).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key4a), &encode_account(account4a, None)); let key5 = b256!("0xB310000000000000000000000000000000000000000000000000000000000000"); - let account5 = Account { nonce: 0, balance: U256::from(8).mul(ether), ..Default::default() }; + let account5 = Account { balance: U256::from(8).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key5, &account5).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key5), &encode_account(account5, None)); let key6 = b256!("0xB340000000000000000000000000000000000000000000000000000000000000"); - let account6 = Account { nonce: 0, balance: U256::from(1).mul(ether), ..Default::default() }; + let account6 = Account { balance: U256::from(1).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key6, &account6).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key6), &encode_account(account6, None)); @@ -629,7 +635,7 @@ fn account_and_storage_trie() { let address4b = address!("0x4f61f2d5ebd991b85aa1677db97307caf5215c91"); let key4b = keccak256(address4b); assert_eq!(key4b.0[0], key4a.0[0]); - let account4b = Account { nonce: 0, balance: U256::from(5).mul(ether), bytecode_hash: None }; + let account4b = Account { balance: U256::from(5).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key4b, &account4b).unwrap(); let mut prefix_set = PrefixSetMut::default(); @@ -915,7 +921,11 @@ fn extension_node_storage_trie( fn extension_node_trie( tx: &DatabaseProviderRW>, N>, ) -> B256 { - let a = Account { nonce: 0, balance: U256::from(1u64), bytecode_hash: Some(B256::random()) }; + let a = Account { + balance: U256::from(1u64), + bytecode_hash: Some(B256::random()), + ..Default::default() + }; let val = encode_account(a, None); let mut hashed_accounts = tx.tx_ref().cursor_write::().unwrap(); diff --git a/crates/trie/trie/src/verify.rs b/crates/trie/trie/src/verify.rs index b8aeb276eeb..fcf751bb5a3 100644 --- a/crates/trie/trie/src/verify.rs +++ b/crates/trie/trie/src/verify.rs @@ -531,6 +531,7 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn test_state_root_branch_nodes_iter_basic() { // Simple test with a few accounts and storage let mut accounts = BTreeMap::new(); @@ -544,6 +545,7 @@ mod tests { nonce: 1, balance: U256::from(1000), bytecode_hash: Some(keccak256(b"code1")), + ..Default::default() }, ); @@ -599,6 +601,7 @@ mod tests { } #[test] + #[allow(clippy::needless_update)] fn test_state_root_branch_nodes_iter_multiple_accounts() { // Test with multiple accounts to verify ordering let mut accounts = BTreeMap::new(); @@ -613,6 +616,7 @@ mod tests { nonce: i as u64, balance: U256::from(i as u64 * 1000), bytecode_hash: (i == 2).then(|| keccak256([i])), + ..Default::default() }, ); diff --git a/testing/ef-tests/src/models.rs b/testing/ef-tests/src/models.rs index 06f5b6b678c..a2cb4f78679 100644 --- a/testing/ef-tests/src/models.rs +++ b/testing/ef-tests/src/models.rs @@ -180,7 +180,7 @@ impl State { nonce: Some(account.nonce.try_into().unwrap()), code: Some(account.code).filter(|c| !c.is_empty()), storage: Some(storage), - private_key: None, + ..Default::default() }; (address, account) }) diff --git a/testing/testing-utils/src/generators.rs b/testing/testing-utils/src/generators.rs index f61cded5244..3e601f1d101 100644 --- a/testing/testing-utils/src/generators.rs +++ b/testing/testing-utils/src/generators.rs @@ -421,7 +421,7 @@ pub fn random_eoa_account(rng: &mut R) -> (Address, Account) { let balance = U256::from(rng.random::()); let addr = Address::random(); - (addr, Account { nonce, balance, bytecode_hash: None }) + (addr, Account { nonce, balance, ..Default::default() }) } /// Generate random Externally Owned Accounts From 654ed5088cc75864cd49711fd89bbf2368cc5c70 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 16:11:46 +0200 Subject: [PATCH 089/133] test(e2e): honor storage_v2 in RLP import setup (#27444) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/e2e-test-utils/src/setup_import.rs | 104 ++++++++++++------- crates/e2e-test-utils/src/testsuite/setup.rs | 3 +- 2 files changed, 66 insertions(+), 41 deletions(-) diff --git a/crates/e2e-test-utils/src/setup_import.rs b/crates/e2e-test-utils/src/setup_import.rs index f30e0a6c9f7..775ec1e2b3d 100644 --- a/crates/e2e-test-utils/src/setup_import.rs +++ b/crates/e2e-test-utils/src/setup_import.rs @@ -8,7 +8,7 @@ use reth_config::Config; use reth_db::DatabaseEnv; use reth_node_api::{NodeTypesWithDBAdapter, TreeConfig}; use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; -use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs, StorageArgs}; use reth_node_ethereum::EthereumNode; use reth_provider::{ providers::BlockchainProvider, DatabaseProviderFactory, ProviderFactory, StageCheckpointReader, @@ -54,10 +54,14 @@ impl std::fmt::Debug for ChainImportResult { /// It uses `NoopConsensus` during import to bypass validation checks like gas limit constraints, /// which allows importing test chains that may not strictly conform to mainnet consensus rules. The /// nodes themselves still run with proper consensus when started. +/// +/// `storage_v2` selects the storage layout (`--storage.v2`) for both the imported database and the +/// launched nodes. pub async fn setup_engine_with_chain_import( num_nodes: usize, chain_spec: Arc, is_dev: bool, + storage_v2: bool, tree_config: TreeConfig, rlp_path: &Path, attributes_generator: impl Fn(u64) -> PayloadAttributes + Send + Sync + Copy + 'static, @@ -87,7 +91,8 @@ pub async fn setup_engine_with_chain_import( .with_http() .with_http_api(RpcModuleSelection::All), ) - .set_dev(is_dev); + .set_dev(is_dev) + .with_storage(StorageArgs { v2: storage_v2 }); // Set the datadir node_config.datadir.datadir = @@ -125,8 +130,12 @@ pub async fn setup_engine_with_chain_import( reth_tasks::Runtime::test(), )?; - // Initialize genesis if needed - reth_db_common::init::init_genesis(&provider_factory)?; + // Initialize genesis with the node's storage settings, which the launched node then reads + // back from the database. + reth_db_common::init::init_genesis_with_settings( + &provider_factory, + node_config.storage_settings(), + )?; // Import the chain data // Use no_state to skip state validation for test chains @@ -277,6 +286,7 @@ mod tests { use reth_primitives_traits::SealedBlock; use reth_provider::{ test_utils::MockNodeTypesWithDB, BlockHashReader, BlockNumReader, BlockReaderIdExt, + MetadataProvider, StorageSettings, }; use std::path::PathBuf; @@ -558,43 +568,57 @@ mod tests { std::fs::write(&fcu_path, create_fcu_json(tip).to_string()) .expect("Failed to write FCU data"); - // Setup nodes with imported chain - let result = setup_engine_with_chain_import( - 1, - chain_spec, - false, - TreeConfig::default(), - &rlp_path, - |_| PayloadAttributes::default(), - ) - .await - .expect("Failed to setup nodes with chain import"); - - // Load and apply forkchoice state - let fcu_state = load_forkchoice_state(&fcu_path).expect("Failed to load forkchoice state"); - - let node = &result.nodes[0]; - - // Send forkchoice update to make the imported chain canonical - node.update_forkchoice(fcu_state.finalized_block_hash, fcu_state.head_block_hash) + for storage_v2 in [false, true] { + // Setup nodes with imported chain + let result = setup_engine_with_chain_import( + 1, + chain_spec.clone(), + false, + storage_v2, + TreeConfig::default(), + &rlp_path, + |_| PayloadAttributes::default(), + ) .await - .expect("Failed to update forkchoice"); - - // Wait for the node to sync to the head - node.sync_to(fcu_state.head_block_hash).await.expect("Failed to sync to head"); + .expect("Failed to setup nodes with chain import"); + + // Load and apply forkchoice state + let fcu_state = + load_forkchoice_state(&fcu_path).expect("Failed to load forkchoice state"); + + let node = &result.nodes[0]; + + // The imported database and the node use the requested storage layout. + let settings = node + .inner + .provider + .database_provider_ro() + .expect("Failed to open database provider") + .storage_settings() + .expect("Failed to read storage settings"); + assert_eq!(settings, Some(StorageSettings { storage_v2 })); + + // Send forkchoice update to make the imported chain canonical + node.update_forkchoice(fcu_state.finalized_block_hash, fcu_state.head_block_hash) + .await + .expect("Failed to update forkchoice"); + + // Wait for the node to sync to the head + node.sync_to(fcu_state.head_block_hash).await.expect("Failed to sync to head"); + + // Verify the chain tip + let latest = node + .inner + .provider + .sealed_header_by_id(alloy_eips::BlockId::latest()) + .expect("Failed to get latest header") + .expect("No latest header found"); - // Verify the chain tip - let latest = node - .inner - .provider - .sealed_header_by_id(alloy_eips::BlockId::latest()) - .expect("Failed to get latest header") - .expect("No latest header found"); - - assert_eq!( - latest.hash(), - fcu_state.head_block_hash, - "Chain tip does not match expected head" - ); + assert_eq!( + latest.hash(), + fcu_state.head_block_hash, + "Chain tip does not match expected head" + ); + } } } diff --git a/crates/e2e-test-utils/src/testsuite/setup.rs b/crates/e2e-test-utils/src/testsuite/setup.rs index a04ecfbdec2..28a8407ccb9 100644 --- a/crates/e2e-test-utils/src/testsuite/setup.rs +++ b/crates/e2e-test-utils/src/testsuite/setup.rs @@ -39,7 +39,7 @@ pub struct Setup { pub is_dev: bool, /// Whether to use v2 storage mode (hashed keys, static file changesets, rocksdb history). /// - /// Defaults to the node's `--storage.v2` default. Not applied to chains imported from RLP. + /// Defaults to the node's `--storage.v2` default. pub storage_v2: bool, /// Tracks instance generic. _phantom: PhantomData, @@ -275,6 +275,7 @@ where self.network.node_count, chain_spec, self.is_dev, + self.storage_v2, self.tree_config.clone(), rlp_path, attributes_generator, From 8ff3237697fe8fb330d5c00d24901c9b6232563c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 14:01:32 +0000 Subject: [PATCH 090/133] revert: restore RocksDB BAL storage (#26789) --- crates/cli/commands/src/common.rs | 16 ++++--- crates/engine/tree/src/persistence.rs | 37 ++++++++++++++++ crates/node/builder/src/launch/common.rs | 19 ++++---- crates/storage/provider/src/bal/rocksdb.rs | 14 ++---- .../provider/src/providers/rocksdb/metrics.rs | 2 + .../src/providers/rocksdb/provider.rs | 43 +++++++++++-------- 6 files changed, 86 insertions(+), 45 deletions(-) diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 2748ba5ec9c..77cd552633f 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -27,8 +27,7 @@ use reth_provider::{ BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider, StaticFileProviderBuilder, }, - BalConfig, BalStoreHandle, InMemoryBalStore, ProviderFactory, StaticFileProviderFactory, - StorageSettings, + BalStoreHandle, ProviderFactory, RocksDBBalStore, StaticFileProviderFactory, StorageSettings, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; @@ -196,11 +195,14 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let balstore_cache_size = - self.db.balstore_cache_size.unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + let bal_store = self + .db + .balstore_cache_size + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::>::new( db, self.chain.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 3c6261945c1..f466dad914e 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -562,6 +562,43 @@ mod tests { assert_eq!(result.last_state_trie_block, result.last_block); } + #[test] + fn test_save_blocks_flushes_bal_store() { + use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; + + reth_tracing::init_test_tracing(); + let provider = create_test_provider_factory(); + let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); + let provider = provider.with_bal_store(bal_store); + init_genesis(&provider).unwrap(); + + let mut test_block_builder = TestBlockBuilder::eth(); + let executed = test_block_builder.get_executed_block_with_number(1, B256::random()); + let num_hash = executed.recovered_block().num_hash(); + let raw_bal = Bytes::from_static(&[0xc0]); + + provider.bal_store().insert(num_hash, RawBal::new(raw_bal.clone())).unwrap(); + + let (_finished_exex_height_tx, finished_exex_height_rx) = + tokio::sync::watch::channel(FinishedExExHeight::NoExExs); + let pruner = + Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); + let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); + let handle = PersistenceHandle::::spawn_service( + provider.clone(), + pruner, + sync_metrics_tx, + ); + let (tx, rx) = crossbeam_channel::bounded(1); + + handle.save_blocks(full_save_input(vec![executed]), tx).unwrap(); + + let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); + assert_eq!(result.last_block, num_hash); + let persisted_store = RocksDBBalStore::new(provider.rocksdb_provider()); + assert_eq!(persisted_store.get_by_hash(num_hash.hash).unwrap(), Some(raw_bal)); + } + #[test] fn test_save_blocks_multiple_blocks() { reth_tracing::init_test_tracing(); diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 6f54bfca5a3..1a7146c96c5 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -71,10 +71,10 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, - DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, - ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, - StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, + BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, + MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, + RocksDBBalStore, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, + StaticFileProviderFactory, StorageSettingsCache, }; use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; @@ -522,14 +522,15 @@ where .build()? }; - let balstore_cache_size = self + let bal_store = self .node_config() .db .balstore_cache_size - .unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index 34494ba3a34..8944ec0a014 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -333,20 +333,12 @@ struct RocksDBBalEntry { #[cfg(test)] mod tests { use super::*; - use crate::providers::{RocksDBBuilder, RocksDBProvider}; + use crate::providers::RocksDBBuilder; use alloy_primitives::B256; - fn test_rocksdb(dir: &tempfile::TempDir) -> RocksDBProvider { - RocksDBBuilder::new(dir.path()) - .with_table::() - .with_table::() - .build() - .unwrap() - } - fn test_store() -> (tempfile::TempDir, RocksDBBalStore) { let dir = tempfile::tempdir().unwrap(); - let rocksdb = test_rocksdb(&dir); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); (dir, RocksDBBalStore::new(rocksdb)) } @@ -435,7 +427,7 @@ mod tests { #[test] fn configured_buffer_retention_distance_is_used() { let dir = tempfile::tempdir().unwrap(); - let rocksdb = test_rocksdb(&dir); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); let store = RocksDBBalStore::with_buffer_retention_distance(rocksdb, 64); let old = NumHash::new(1, B256::with_last_byte(1)); let tip = NumHash::new(34, B256::with_last_byte(2)); diff --git a/crates/storage/provider/src/providers/rocksdb/metrics.rs b/crates/storage/provider/src/providers/rocksdb/metrics.rs index 3971ed978ba..c22edf65176 100644 --- a/crates/storage/provider/src/providers/rocksdb/metrics.rs +++ b/crates/storage/provider/src/providers/rocksdb/metrics.rs @@ -8,6 +8,8 @@ use strum::{EnumIter, IntoEnumIterator}; pub(super) const ROCKSDB_TABLES: &[&str] = &[ Tables::TransactionHashNumbers.name(), + Tables::BlockAccessLists.name(), + Tables::BlockAccessListBlockNumbers.name(), Tables::StoragesHistory.name(), Tables::AccountsHistory.name(), ]; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 3fd66f00067..7a401053195 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -333,10 +333,14 @@ impl RocksDBBuilder { /// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping /// - [`tables::AccountsHistory`] - Account history index /// - [`tables::StoragesHistory`] - Storage history index + /// - [`tables::BlockAccessLists`] - Block access list payloads + /// - [`tables::BlockAccessListBlockNumbers`] - Block access list hash index pub fn with_default_tables(self) -> Self { self.with_table::() .with_table::() .with_table::() + .with_table::() + .with_table::() } /// Enables metrics. @@ -3166,22 +3170,25 @@ mod tests { provider.put::(key.clone(), &value).unwrap(); assert!(provider.get::(key).unwrap().is_some()); - drop(provider); - - let column_families = DB::list_cf(&Options::default(), temp_dir.path()).unwrap(); - assert!(!column_families.iter().any(|name| name == tables::BlockAccessLists::NAME)); - assert!(!column_families - .iter() - .any(|name| name == tables::BlockAccessListBlockNumbers::NAME)); + let bal_key = + reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); + let bal_value = + reth_db_api::models::StoredBlockAccessList::new(Bytes::from_static(&[0xc0])); + provider.put::(bal_key, &bal_value).unwrap(); + assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); + provider + .put::(bal_key.hash(), &bal_key.number()) + .unwrap(); + assert_eq!( + provider.get::(bal_key.hash()).unwrap(), + Some(bal_key.number()) + ); } #[test] fn block_access_lists_store_large_payloads_in_blob_files() { let temp_dir = TempDir::new().unwrap(); - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_table::() - .build() - .unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); let bal_key = reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ @@ -3237,12 +3244,7 @@ mod tests { 1 ])); - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_default_tables() - .with_table::() - .with_table::() - .build() - .unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); provider.put::(bal_key, &bal_value).unwrap(); provider .put::(bal_key.hash(), &bal_key.number()) @@ -3250,7 +3252,12 @@ mod tests { provider.flush(&[tables::BlockAccessLists::NAME]).unwrap(); drop(provider); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_table::() + .with_table::() + .with_table::() + .build() + .unwrap(); assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); assert_eq!( provider.get::(bal_key.hash()).unwrap(), From bc7776a13bebb40905ef3c90c6e5fb85b3c0ad4f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 14:15:00 +0000 Subject: [PATCH 091/133] perf: enable sender recovery cache by default (#27447) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .github/workflows/docker.yml | 1 - Dockerfile.depot | 2 -- crates/ethereum/node/tests/e2e/rpc.rs | 8 ------ crates/node/core/src/args/engine.rs | 37 +++++++++++++++++++++----- docs/vocs/docs/pages/cli/reth/node.mdx | 8 ++++-- 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 812dca24d9c..0a3cb4a646c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -110,7 +110,6 @@ jobs: echo "ethereum_set<> "$GITHUB_OUTPUT" diff --git a/Dockerfile.depot b/Dockerfile.depot index a99641d3469..47d293a9404 100644 --- a/Dockerfile.depot +++ b/Dockerfile.depot @@ -77,8 +77,6 @@ ARG BINARY=reth # Runtime defaults ARG RETH_ENGINE_TXPOOL_PREWARMING=false ENV RETH_ENGINE_TXPOOL_PREWARMING=$RETH_ENGINE_TXPOOL_PREWARMING -ARG RETH_ENGINE_SENDER_RECOVERY_CACHE=false -ENV RETH_ENGINE_SENDER_RECOVERY_CACHE=$RETH_ENGINE_SENDER_RECOVERY_CACHE # Install runtime dependencies RUN apt-get update && \ diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index b8e520c1ef4..38380eb0108 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -73,10 +73,6 @@ async fn test_rpc_shares_sender_recovery_cache_with_execution() -> eyre::Result< let chain_spec = Arc::new(ChainSpecBuilder::mainnet().cancun_activated().build()); let (mut nodes, _) = E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .with_node_config_modifier(|mut config| { - config.engine.sender_recovery_cache_enabled = true; - config - }) .build() .await?; let node = nodes.pop().unwrap(); @@ -601,10 +597,6 @@ async fn test_flashbots_validate_uses_shared_sender_recovery_cache() -> eyre::Re let (mut nodes, wallet) = E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .with_node_config_modifier(|mut config| { - config.engine.sender_recovery_cache_enabled = true; - config - }) .build() .await?; let mut node = nodes.pop().unwrap(); diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index 6b6358b0e52..0441a5e850c 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -279,7 +279,7 @@ impl Default for DefaultEngineValues { state_cache_disabled: false, prewarming_disabled: false, txpool_prewarming_enabled: false, - sender_recovery_cache_enabled: false, + sender_recovery_cache_enabled: true, state_provider_metrics: false, cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, state_root_task_compare_updates: false, @@ -405,12 +405,15 @@ pub struct EngineArgs { )] pub txpool_prewarming_enabled: bool, - /// Enable caching recovered transaction senders across transaction ingress and payload - /// execution. + /// Cache recovered transaction senders across transaction ingress and payload execution. + /// + /// Enabled by default, use `--engine.sender-recovery-cache=false` to disable it. #[arg( long = "engine.sender-recovery-cache", env = "RETH_ENGINE_SENDER_RECOVERY_CACHE", - default_value_t = DefaultEngineValues::get_global().sender_recovery_cache_enabled + default_value_t = DefaultEngineValues::get_global().sender_recovery_cache_enabled, + num_args = 0..=1, + default_missing_value = "true", )] pub sender_recovery_cache_enabled: bool, @@ -904,14 +907,36 @@ mod tests { } #[test] - fn sender_recovery_cache_is_disabled_by_default_and_can_be_enabled() { + fn sender_recovery_cache_is_enabled_by_default_and_can_be_disabled() { let args = CommandParser::::parse_from(["reth"]).args; - assert!(!args.sender_recovery_cache_enabled); + assert!(args.sender_recovery_cache_enabled); let args = CommandParser::::parse_from(["reth", "--engine.sender-recovery-cache"]) .args; assert!(args.sender_recovery_cache_enabled); + + let args = CommandParser::::parse_from([ + "reth", + "--engine.sender-recovery-cache=true", + ]) + .args; + assert!(args.sender_recovery_cache_enabled); + + let args = CommandParser::::parse_from([ + "reth", + "--engine.sender-recovery-cache=false", + ]) + .args; + assert!(!args.sender_recovery_cache_enabled); + + let args = CommandParser::::parse_from([ + "reth", + "--engine.sender-recovery-cache", + "false", + ]) + .args; + assert!(!args.sender_recovery_cache_enabled); } #[test] diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index b1c063a0bdb..29c42d9925f 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1077,10 +1077,14 @@ Engine: [env: RETH_ENGINE_TXPOOL_PREWARMING=] - --engine.sender-recovery-cache - Enable caching recovered transaction senders across transaction ingress and payload execution + --engine.sender-recovery-cache [] + Cache recovered transaction senders across transaction ingress and payload execution. + + Enabled by default, use `--engine.sender-recovery-cache=false` to disable it. [env: RETH_ENGINE_SENDER_RECOVERY_CACHE=] + [default: true] + [possible values: true, false] --engine.state-provider-metrics Enable state provider latency metrics. This allows the engine to collect and report stats about how long state provider calls took during execution, but this does introduce slight overhead to state provider calls From eb03d80dddbf671e256ab235095ad9f3ccfa9923 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 17:20:39 +0200 Subject: [PATCH 092/133] chore(deps): bump revm-inspectors to 0.44 (#27448) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 35 +++++++++++++++++++---------------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12d9c094972..7a8898a3c9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "alloy-eip2930" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9441120fa82df73e8959ae0e4ab8ade03de2aaae61be313fbf5746277847ce25" +checksum = "e64579d931b3f8eacc7c9ab0b220e87e9c4816e5c724ede1947b55c2f8e92ae5" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -1051,7 +1051,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1062,7 +1062,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3027,7 +3027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.118", ] [[package]] @@ -3512,7 +3512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5255,7 +5255,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6385,7 +6385,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7425,7 +7425,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -11129,10 +11129,11 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b609cb78fcb3a8c10f3190847c9bb0223423cf46e2de30a61ea9e37a3e6736" +checksum = "8c1f17ebaa6be220728837731a3c87d1a542bf31a68138e9364a3a33b149415c" dependencies = [ + "alloy-eip2930", "alloy-primitives", "alloy-rpc-types-eth", "alloy-rpc-types-trace", @@ -11141,7 +11142,9 @@ dependencies = [ "boa_engine", "boa_gc", "colorchoice", + "num-bigint", "revm", + "ruint", "serde", "serde_json", "thiserror 2.0.20", @@ -11573,7 +11576,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11653,7 +11656,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12287,7 +12290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12546,7 +12549,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -13819,7 +13822,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a5a5544d01f..ce3344448cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -429,7 +429,7 @@ reth-zstd-compressors = { version = "0.8.1", default-features = false } # revm revm = { version = "43.0.3", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } -revm-inspectors = "0.43.0" +revm-inspectors = "0.44.0" # eth alloy-dyn-abi = "1.6.1" From 6d117042f524f9893d147cb21d752a106d6d77d7 Mon Sep 17 00:00:00 2001 From: banteg <4562643+banteg@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:54:58 +0000 Subject: [PATCH 093/133] fix(rpc): populate VM bytecode in block replay traces (#27213) Signed-off-by: banteg <4562643+banteg@users.noreply.github.com> Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/trace.rs | 139 +++++++++++++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 10 deletions(-) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index eee781fef8e..4aa98ac83fe 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -34,7 +34,7 @@ use revm::DatabaseCommit; use revm_inspectors::{ opcode::OpcodeGasInspector, storage::StorageInspector, - tracing::{parity::populate_state_diff, TracingInspector, TracingInspectorConfig}, + tracing::{TracingInspector, TracingInspectorConfig}, }; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -573,17 +573,16 @@ where None, TracingInspectorConfig::from_parity_config(&trace_types), move |tx_info, mut ctx| { - let mut full_trace = ctx + let full_trace = ctx .take_inspector() .into_parity_builder() - .into_trace_results(&ctx.result, &trace_types); - - // If statediffs were requested, populate them with the account balance and - // nonce from pre-state - if let Some(ref mut state_diff) = full_trace.state_diff { - populate_state_diff(state_diff, &ctx.db, ctx.state.iter()) - .map_err(Eth::Error::from_eth_err)?; - } + .into_trace_results_with_state_parts( + &ctx.result, + ctx.state, + &trace_types, + &ctx.db, + ) + .map_err(Eth::Error::from_eth_err)?; let trace = TraceResultsWithTransactionHash { transaction_hash: tx_info.hash.expect("tx hash is set"), @@ -1314,4 +1313,124 @@ mod tests { assert_eq!(trace_order(&paginated), vec![(1, None, true)]); } + + #[tokio::test] + async fn replay_block_vmtrace_includes_root_and_callcode_bytecode() { + use crate::EthApiBuilder; + use alloy_consensus::{Header, TxLegacy}; + use alloy_primitives::{hex, Signature, TxKind}; + use reth_chain_state::CanonStateNotification; + use reth_ethereum_primitives::{Block, BlockBody, TransactionSigned}; + use reth_evm_ethereum::EthEvmConfig; + use reth_execution_types::{Chain, ExecutionOutcome}; + use reth_network_api::noop::NoopNetwork; + use reth_primitives_traits::{RecoveredBlock, SignerRecoverable}; + use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; + use reth_rpc_eth_types::cache::cache_new_blocks_task; + use reth_transaction_pool::test_utils::testing_pool; + + let provider = MockEthProvider::default(); + let target = Address::with_last_byte(0x42); + let child = Address::with_last_byte(0x43); + // Increment slot zero, CALLCODE the child, then return the incremented value. + let root_code: Bytes = + hex!("60005460010160005560006000600060006000604361fffff25060005460005260206000f3") + .into(); + let child_code: Bytes = hex!("60025000").into(); + provider.add_account( + target, + ExtendedAccount::new(0, U256::ZERO).with_bytecode(root_code.clone()), + ); + provider.add_account( + child, + ExtendedAccount::new(0, U256::ZERO).with_bytecode(child_code.clone()), + ); + let mut transactions = Vec::new(); + for nonce in 0..2 { + let tx = TransactionSigned::new_unhashed( + TxLegacy { + nonce, + gas_limit: 500_000, + to: TxKind::Call(target), + ..Default::default() + } + .into(), + Signature::test_signature(), + ); + provider.add_account( + tx.recover_signer().unwrap(), + ExtendedAccount::new(nonce, U256::from(1_000_000)), + ); + transactions.push(tx); + } + let parent = Header { gas_limit: 30_000_000, ..Default::default() }; + let parent_hash = parent.hash_slow(); + provider.add_header(parent_hash, parent); + let block = Block { + header: Header { parent_hash, number: 1, gas_limit: 30_000_000, ..Default::default() }, + body: BlockBody { transactions, ..Default::default() }, + }; + let block_hash = block.header.hash_slow(); + let senders = + block.body.transactions.iter().map(|tx| tx.recover_signer().unwrap()).collect(); + provider.add_block(block_hash, block.clone()); + let recovered = RecoveredBlock::new_unhashed(block, senders); + let eth_api = EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build(); + // Seed the RPC cache: MockEthProvider does not implement recovered_block. + cache_new_blocks_task( + eth_api.cache().clone(), + futures::stream::iter([CanonStateNotification::Commit { + new: Arc::new(Chain::new( + [recovered], + ExecutionOutcome { + receipts: vec![vec![]], + first_block: 1, + ..Default::default() + }, + Default::default(), + )), + }]), + ) + .await; + let api = TraceApi::new(eth_api, BlockingTaskGuard::new(1), EthConfig::default()); + for types in [ + HashSet::from_iter([TraceType::VmTrace]), + HashSet::from_iter([TraceType::VmTrace, TraceType::Trace, TraceType::StateDiff]), + HashSet::from_iter([TraceType::Trace, TraceType::StateDiff]), + ] { + let traces = api + .replay_block_transactions(block_hash.into(), types.clone()) + .await + .unwrap() + .unwrap(); + assert_eq!(traces.len(), 2); + for (index, trace) in traces.into_iter().enumerate() { + assert_eq!( + trace.full_trace.output.as_ref(), + U256::from(index + 1).to_be_bytes::<32>() + ); + if types.contains(&TraceType::VmTrace) { + let vm = trace.full_trace.vm_trace.as_ref().unwrap(); + assert_eq!(vm.code, root_code); + assert_eq!( + vm.ops.iter().find_map(|op| op.sub.as_ref()).unwrap().code, + child_code + ); + } + let individual = api + .replay_transaction(trace.transaction_hash, types.clone()) + .await + .unwrap() + .unwrap(); + assert_eq!(individual.transaction_hash, trace.transaction_hash); + assert_eq!(trace.full_trace, individual.full_trace); + } + } + } } From 118e1bdfe04557de527f8dffc4b4d7a71398515e Mon Sep 17 00:00:00 2001 From: banteg <4562643+banteg@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:00:50 +0000 Subject: [PATCH 094/133] fix(rpc): correct Otterscan block and transaction responses (#27217) Signed-off-by: banteg <4562643+banteg@users.noreply.github.com> Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/rpc/rpc-api/src/otterscan.rs | 21 +- crates/rpc/rpc/src/otterscan.rs | 727 +++++++++++++++++++++---- docs/vocs/docs/pages/jsonrpc/intro.mdx | 2 +- docs/vocs/docs/pages/jsonrpc/ots.mdx | 86 +++ docs/vocs/sidebar.ts | 4 + 5 files changed, 725 insertions(+), 115 deletions(-) create mode 100644 docs/vocs/docs/pages/jsonrpc/ots.mdx diff --git a/crates/rpc/rpc-api/src/otterscan.rs b/crates/rpc/rpc-api/src/otterscan.rs index c8651e608f5..d515f91ae4c 100644 --- a/crates/rpc/rpc-api/src/otterscan.rs +++ b/crates/rpc/rpc-api/src/otterscan.rs @@ -7,7 +7,10 @@ use alloy_rpc_types_trace::otterscan::{ }; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; -/// Otterscan rpc interface. +/// Otterscan RPC interface. +/// +/// Reth implements a subset of the API. In particular, address history search is unimplemented +/// even though `getApiLevel` returns 8. Historical queries require the relevant state and history. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "ots"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "ots"))] pub trait Otterscan { @@ -28,9 +31,8 @@ pub trait Otterscan { #[method(name = "hasCode")] async fn has_code(&self, address: Address, block_id: Option) -> RpcResult; - /// Very simple API versioning scheme. Every time we add a new capability, the number is - /// incremented. This allows for Otterscan to check if the node contains all API it - /// needs. + /// Returns API level 8 for frontend compatibility, not a guarantee of complete support. + /// In particular, both address history search methods remain unimplemented. #[method(name = "getApiLevel")] async fn get_api_level(&self) -> RpcResult; @@ -39,6 +41,8 @@ pub trait Otterscan { async fn get_internal_operations(&self, tx_hash: TxHash) -> RpcResult>; /// Given a transaction hash, returns its raw revert reason. + /// Known transactions without revert data return empty bytes; unknown transactions return + /// `None`. #[method(name = "getTransactionError")] async fn get_transaction_error(&self, tx_hash: TxHash) -> RpcResult>; @@ -61,6 +65,7 @@ pub trait Otterscan { async fn get_block_details_by_hash(&self, block_hash: B256) -> RpcResult>; /// Get paginated transactions for a certain block. Also remove some verbose fields like logs. + /// Page zero selects the block's last transactions; each page retains ascending block order. #[method(name = "getBlockTransactions")] async fn get_block_transactions( &self, @@ -70,6 +75,9 @@ pub trait Otterscan { ) -> RpcResult>; /// Gets paginated inbound/outbound transaction calls for a certain address. + /// + /// Unimplemented: returns JSON-RPC error -32603 with message "unimplemented". + /// See . #[method(name = "searchTransactionsBefore")] async fn search_transactions_before( &self, @@ -79,6 +87,9 @@ pub trait Otterscan { ) -> RpcResult; /// Gets paginated inbound/outbound transaction calls for a certain address. + /// + /// Unimplemented: returns JSON-RPC error -32603 with message "unimplemented". + /// See . #[method(name = "searchTransactionsAfter")] async fn search_transactions_after( &self, @@ -96,6 +107,8 @@ pub trait Otterscan { ) -> RpcResult>; /// Gets the transaction hash and the address who created a contract. + /// Requires historical state. Code-presence binary search is unreliable for destroyed and + /// redeployed contracts, and cannot identify genesis allocations or EIP-7702 delegations. #[method(name = "getContractCreator")] async fn get_contract_creator(&self, address: Address) -> RpcResult>; } diff --git a/crates/rpc/rpc/src/otterscan.rs b/crates/rpc/rpc/src/otterscan.rs index 7d43540b7fe..54355746a60 100644 --- a/crates/rpc/rpc/src/otterscan.rs +++ b/crates/rpc/rpc/src/otterscan.rs @@ -1,18 +1,20 @@ use alloy_consensus::{BlockHeader, Typed2718}; use alloy_eips::{eip1898::LenientBlockNumberOrTag, BlockId}; -use alloy_network::{ReceiptResponse, TransactionResponse}; +use alloy_evm::block::calc::{base_block_reward, block_reward, ommer_reward}; +use alloy_network::{primitives::HeaderResponse, ReceiptResponse, TransactionResponse}; use alloy_primitives::{Address, Bytes, TxHash, B256, U256}; use alloy_rpc_types_eth::{BlockTransactions, TransactionReceipt}; use alloy_rpc_types_trace::{ otterscan::{ - BlockDetails, ContractCreator, InternalOperation, OperationType, OtsBlockTransactions, + BlockDetails, ContractCreator, InternalIssuance, InternalOperation, OtsBlockTransactions, OtsReceipt, OtsTransactionReceipt, TraceEntry, TransactionsWithReceipts, }, - parity::{Action, CreateAction, CreateOutput, TraceOutput}, + parity::{Action, CreateAction, CreateOutput, LocalizedTransactionTrace, TraceOutput}, }; use async_trait::async_trait; use jsonrpsee::{core::RpcResult, types::ErrorObjectOwned}; -use reth_primitives_traits::TxTy; +use reth_chainspec::ChainSpecProvider; +use reth_primitives_traits::{BlockBody, TxTy}; use reth_rpc_api::{EthApiServer, OtterscanServer}; use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ @@ -23,8 +25,11 @@ use reth_rpc_eth_types::{utils::binary_search, EthApiError}; use reth_rpc_server_types::result::internal_rpc_err; use revm::context_interface::result::ExecutionResult; use revm_inspectors::{ - tracing::{types::CallTraceNode, TracingInspectorConfig}, - transfer::{TransferInspector, TransferKind}, + otterscan::InternalOperationsInspector, + tracing::{ + types::{CallKind, CallTraceNode}, + TracingInspectorConfig, + }, }; const API_LEVEL: u64 = 8; @@ -44,23 +49,51 @@ impl OtterscanApi { impl OtterscanApi where - Eth: FullEthApiTypes, + Eth: FullEthApiTypes + TraceExt, { /// Constructs a `BlockDetails` from a block and its receipts. - fn block_details( + async fn block_details( &self, block: RpcBlock, receipts: Vec>, ) -> RpcResult>> { - // blob fee is burnt, so we don't need to calculate it + // Execution fees include the base fee; blob fees are not part of this field. let total_fees = receipts .iter() .map(|receipt| { - (receipt.gas_used() as u128).saturating_mul(receipt.effective_gas_price()) + U256::from(receipt.gas_used()) * U256::from(receipt.effective_gas_price()) }) - .sum::(); + .sum::(); - Ok(BlockDetails::new(block, Default::default(), U256::from(total_fees))) + let chain_spec = self.eth.provider().chain_spec(); + let reward = if block.header.number() == 0 { + None + } else { + base_block_reward(&chain_spec, block.header.number()) + }; + let issuance = if let Some(reward) = reward { + let recovered = if block.uncles.is_empty() { + None + } else { + Some( + self.eth + .recovered_block(block.header.hash().into()) + .await + .map_err(Into::::into)? + .ok_or(EthApiError::HeaderNotFound(block.header.hash().into()))?, + ) + }; + let ommers = + recovered.as_ref().and_then(|block| block.body().ommers()).unwrap_or_default(); + calculate_issuance( + reward, + block.header.number(), + ommers.iter().map(BlockHeader::number), + ) + } else { + InternalIssuance::default() + }; + Ok(BlockDetails::new(block, issuance, total_fees)) } } @@ -99,79 +132,37 @@ where /// Handler for `ots_getInternalOperations` async fn get_internal_operations(&self, tx_hash: TxHash) -> RpcResult> { - let internal_operations = self - .eth + self.eth .spawn_trace_transaction_in_block_with_inspector( tx_hash, - TransferInspector::new(false), - |_tx_info, inspector, _, _| Ok(inspector.into_transfers()), + InternalOperationsInspector::default(), + |_tx_info, inspector, _, _| Ok(inspector.into_operations()), ) .await - .map_err(Into::into)? - .map(|transfer_operations| { - transfer_operations - .iter() - .map(|op| InternalOperation { - from: op.from, - to: op.to, - value: op.value, - r#type: match op.kind { - TransferKind::Call => OperationType::OpTransfer, - TransferKind::Create => OperationType::OpCreate, - TransferKind::Create2 => OperationType::OpCreate2, - TransferKind::SelfDestruct => OperationType::OpSelfDestruct, - }, - }) - .collect::>() - }) - .unwrap_or_default(); - Ok(internal_operations) + .map_err(Into::into) + .map(Option::unwrap_or_default) } /// Handler for `ots_getTransactionError` async fn get_transaction_error(&self, tx_hash: TxHash) -> RpcResult> { - let maybe_revert = self - .eth - .spawn_replay_transaction(tx_hash, |_tx_info, res, _| match res.result { - ExecutionResult::Revert { output, .. } => Ok(Some(output)), - _ => Ok(None), - }) + self.eth + .spawn_replay_transaction(tx_hash, |_tx_info, res, _| Ok(transaction_error(res.result))) .await - .map(Option::flatten) - .map_err(Into::into)?; - Ok(maybe_revert) + .map_err(Into::into) } /// Handler for `ots_traceTransaction` async fn trace_transaction(&self, tx_hash: TxHash) -> RpcResult>> { - let traces = self - .eth + self.eth .spawn_trace_transaction_in_block( tx_hash, TracingInspectorConfig::default_parity(), - move |_tx_info, inspector, _, _| Ok(inspector.into_traces().into_nodes()), + |_tx_info, inspector, _, _| { + Ok(otterscan_traces(inspector.into_traces().into_nodes())) + }, ) .await - .map_err(Into::into)? - .map(|traces| { - traces - .into_iter() - .map(|CallTraceNode { trace, .. }| TraceEntry { - r#type: if trace.is_selfdestruct() { - "SELFDESTRUCT".to_string() - } else { - trace.kind.to_string() - }, - depth: trace.depth as u32, - from: trace.caller, - to: trace.address, - value: Some(trace.value), - input: trace.data, - output: trace.output, - }) - .collect::>() - }); - Ok(traces) + .map_err(Into::into) } /// Handler for `ots_getBlockDetails` @@ -180,7 +171,7 @@ where block_number: LenientBlockNumberOrTag, ) -> RpcResult>> { let block_number = block_number.into_inner(); - let block = self.eth.block_by_number(block_number, true); + let block = self.eth.block_by_number(block_number, false); let block_id = block_number.into(); let receipts = self.eth.block_receipts(block_id); let (block, receipts) = futures::try_join!(block, receipts)?; @@ -188,6 +179,7 @@ where block.ok_or(EthApiError::HeaderNotFound(block_id))?, receipts.ok_or(EthApiError::ReceiptsNotFound(block_id))?, ) + .await } /// Handler for `ots_getBlockDetailsByHash` @@ -195,7 +187,7 @@ where &self, block_hash: B256, ) -> RpcResult>> { - let block = self.eth.block_by_hash(block_hash, true); + let block = self.eth.block_by_hash(block_hash, false); let block_id = block_hash.into(); let receipts = self.eth.block_receipts(block_id); let (block, receipts) = futures::try_join!(block, receipts)?; @@ -203,6 +195,7 @@ where block.ok_or(EthApiError::HeaderNotFound(block_id))?, receipts.ok_or(EthApiError::ReceiptsNotFound(block_id))?, ) + .await } /// Handler for `ots_getBlockTransactions` @@ -349,67 +342,581 @@ where ) .await .map_err(Into::into)? - .map(|traces| { - traces - .into_iter() - .flatten() - .map(|tx_trace| { - let trace = tx_trace.trace; - Ok(match (trace.action, trace.result, trace.error) { - ( - Action::Create(CreateAction { from: creator, .. }), - Some(TraceOutput::Create(CreateOutput { - address: contract, .. - })), - None, - ) if contract == address => Some(ContractCreator { - hash: tx_trace - .transaction_hash - .ok_or(EthApiError::TransactionNotFound)?, - creator, - }), - _ => None, - }) - }) - .filter_map(Result::transpose) - .collect::, EthApiError>>() - }) + .map(|traces| find_contract_creator(address, traces)) .transpose()?; - // A contract maybe created and then destroyed in multiple transactions, here we - // return the first found transaction, this behavior is consistent with etherscan's - let found = traces.and_then(|traces| traces.first().copied()); - Ok(found) + // Code-presence search assumes a single deployment. It cannot reliably identify + // the first deployment of contracts that were destroyed and recreated. + Ok(traces.flatten()) + } +} + +/// Finds a creation that was not rolled back by its own frame or an enclosing frame. +fn find_contract_creator( + address: Address, + transactions: Vec>, +) -> Result, EthApiError> { + for traces in transactions { + let mut reverted_path: Option> = None; + for tx_trace in traces { + let trace = tx_trace.trace; + // Parity traces are in preorder, so a failed frame precedes its entire subtree. + if reverted_path.as_ref().is_some_and(|path| trace.trace_address.starts_with(path)) { + continue + } + reverted_path = None; + if trace.error.is_some() { + reverted_path = Some(trace.trace_address); + continue + } + if let ( + Action::Create(CreateAction { from: creator, .. }), + Some(TraceOutput::Create(CreateOutput { address: contract, .. })), + ) = (trace.action, trace.result) && + contract == address + { + return Ok(Some(ContractCreator { + hash: tx_trace.transaction_hash.ok_or(EthApiError::TransactionNotFound)?, + creator, + })) + } + } } + Ok(None) } /// Returns the transaction slice for an Otterscan block page. /// -/// Otterscan paginates in block order, so page `0` corresponds to the first transactions in the -/// block instead of the tail. -fn block_transaction_page_range( +/// Pages are selected from the end of the block, retaining block order within each page. +/// The frontend reverses each page and uses this ordering for transaction-index links. +const fn block_transaction_page_range( tx_len: usize, page_number: usize, page_size: usize, ) -> std::ops::Range { - let page_start = page_number.saturating_mul(page_size).min(tx_len); - let page_end = page_start.saturating_add(page_size).min(tx_len); + let page_end = tx_len.saturating_sub(page_number.saturating_mul(page_size)); + let page_start = page_end.saturating_sub(page_size); page_start..page_end } +/// Rewards issued to the miner (including ommer inclusion) and to the ommers themselves. +fn calculate_issuance( + reward: u128, + number: u64, + ommer_numbers: impl ExactSizeIterator, +) -> InternalIssuance { + let block_reward = U256::from(block_reward(reward, ommer_numbers.len())); + let uncle_reward = + ommer_numbers.map(|ommer| U256::from(ommer_reward(reward, number, ommer))).sum::(); + InternalIssuance { block_reward, uncle_reward, issuance: block_reward + uncle_reward } +} + +fn transaction_error(result: ExecutionResult) -> Bytes { + match result { + ExecutionResult::Revert { output, .. } => output, + _ => Bytes::new(), + } +} + +/// A self-destruct is a separate operation at the end of its enclosing call, after any children. +fn otterscan_traces(nodes: Vec) -> Vec { + let mut entries = Vec::with_capacity(nodes.len()); + let mut selfdestructs: Vec = Vec::new(); + for CallTraceNode { trace, .. } in nodes { + // The arena is in call-entry order. Emit completed calls' self-destructs before + // entering a sibling or returning to an ancestor. + while selfdestructs.last().is_some_and(|entry| entry.depth > trace.depth as u32) { + entries.push(selfdestructs.pop().expect("checked above")); + } + if let Some(to) = trace.selfdestruct_refund_target { + selfdestructs.push(TraceEntry { + r#type: "SELFDESTRUCT".to_string(), + depth: trace.depth as u32 + 1, + from: trace.selfdestruct_address.unwrap_or(trace.address), + to, + value: trace.selfdestruct_transferred_value, + input: Bytes::new(), + output: Bytes::new(), + }); + } + entries.push(TraceEntry { + r#type: trace.kind.to_string(), + depth: trace.depth as u32, + from: trace.caller, + to: trace.address, + value: (!matches!(trace.kind, CallKind::StaticCall | CallKind::DelegateCall)) + .then_some(trace.value), + input: trace.data, + output: trace.output, + }); + } + entries.extend(selfdestructs.into_iter().rev()); + entries +} + #[cfg(test)] mod tests { - use super::block_transaction_page_range; + use super::*; + use alloy_consensus::{constants::ETH_TO_WEI, Header}; + use alloy_primitives::{hex, TxKind}; + use alloy_rpc_types_trace::parity::TransactionTrace; + use reth_chainspec::MAINNET; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_provider::test_utils::MockEthProvider; + use reth_transaction_pool::test_utils::testing_pool; + use revm::{ + context::TxEnv, + context_interface::result::{HaltReason, Output, SuccessReason}, + database::InMemoryDB, + inspector::InspectorEvmTr, + primitives::hardfork::SpecId, + state::{AccountInfo, Bytecode}, + Context, InspectEvm, MainBuilder, MainContext, + }; + use revm_inspectors::tracing::{types::CallTrace, TracingInspector}; + + #[test] + fn block_transaction_pages_match_frontend_index_navigation() { + // The frontend requests floor((count - 1 - index) / size), then reverses the page. + for count in [0, 1, 10, 25, 51] { + for size in [1, 10, 25] { + for index in 0..count { + let reverse_index = count - 1 - index; + let page = block_transaction_page_range(count, reverse_index / size, size); + let displayed = page.rev().collect::>(); + assert_eq!(displayed[reverse_index % size], index); + } + } + } + assert_eq!(block_transaction_page_range(25, 0, 10), 15..25); + assert_eq!(block_transaction_page_range(25, 1, 10), 5..15); + assert_eq!(block_transaction_page_range(25, 2, 10), 0..5); + assert_eq!(block_transaction_page_range(25, 3, 10), 0..0); + assert_eq!(block_transaction_page_range(25, usize::MAX, 10), 0..0); + assert_eq!(block_transaction_page_range(25, 0, usize::MAX), 0..25); + assert_eq!(block_transaction_page_range(25, 1, usize::MAX), 0..0); + assert_eq!(block_transaction_page_range(25, 0, 0), 25..25); + } + + #[test] + fn transaction_error_returns_only_revert_data() { + let success: ExecutionResult = ExecutionResult::Success { + reason: SuccessReason::Return, + gas: Default::default(), + logs: vec![], + output: Output::Call(Bytes::from_static(b"successful return data")), + }; + let halt = ExecutionResult::Halt { + reason: HaltReason::OutOfGas(revm::context_interface::result::OutOfGasError::Basic), + gas: Default::default(), + logs: vec![], + }; + for result in [ + success, + halt, + ExecutionResult::Revert { gas: Default::default(), logs: vec![], output: Bytes::new() }, + ] { + assert_eq!(serde_json::to_value(Some(transaction_error(result))).unwrap(), "0x"); + } + let output = Bytes::from_static(b"revert data"); + assert_eq!( + transaction_error::(ExecutionResult::Revert { + gas: Default::default(), + logs: vec![], + output: output.clone() + }), + output + ); + } + + async fn cache_empty_block( + cache: reth_rpc_eth_types::EthStateCache, + block: reth_ethereum_primitives::Block, + ) { + // MockEthProvider does not implement recovered_block; seed the normal RPC cache. + let outcome = reth_execution_types::ExecutionOutcome::new( + Default::default(), + vec![vec![]], + block.header.number, + vec![], + ); + let chain = reth_execution_types::Chain::new( + [reth_primitives_traits::RecoveredBlock::new_unhashed(block, vec![])], + outcome, + Default::default(), + ); + reth_rpc_eth_types::cache::cache_new_blocks_task( + cache, + futures::stream::iter([reth_chain_state::CanonStateNotification::Commit { + new: std::sync::Arc::new(chain), + }]), + ) + .await; + } + + #[tokio::test] + async fn block_details_issuance_follows_chain_forks() { + for (spec, number, reward) in [ + (0, 0), + (1, 5), + (4_369_999, 5), + (4_370_000, 3), + (7_279_999, 3), + (7_280_000, 2), + (15_537_393, 2), + (15_537_394, 0), + ] + .into_iter() + .map(|(number, reward)| (MAINNET.clone(), number, reward)) + .chain(std::iter::once(( + std::sync::Arc::new( + reth_chainspec::ChainSpecBuilder::mainnet().paris_activated().build(), + ), + 1, + 0, + ))) { + let provider = MockEthProvider::default().with_chain_spec((*spec).clone()); + let header = Header { number, ..Default::default() }; + let hash = header.hash_slow(); + let block = reth_ethereum_primitives::Block { header, body: Default::default() }; + provider.add_block(hash, block.clone()); + provider.add_receipts(number, vec![]); + let api = OtterscanApi::new( + crate::eth::EthApiBuilder::new( + provider, + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(spec), + ) + .build(), + ); + cache_empty_block(api.eth.cache().clone(), block).await; + let by_number = api + .get_block_details(alloy_eips::BlockNumberOrTag::Number(number).into()) + .await + .unwrap(); + let by_hash = api.get_block_details_by_hash(hash).await.unwrap(); + assert_eq!( + by_number.issuance.block_reward, + U256::from(reward * ETH_TO_WEI), + "block {number}" + ); + assert_eq!(by_number.issuance.uncle_reward, U256::ZERO); + assert_eq!(by_number.issuance.issuance, by_number.issuance.block_reward); + assert_eq!(by_number.issuance, by_hash.issuance); + } + } + + #[tokio::test] + async fn block_details_loads_ommer_headers() { + let provider = MockEthProvider::default(); + let block = reth_ethereum_primitives::Block { + header: Header { number: 126, ..Default::default() }, + body: alloy_consensus::BlockBody { + ommers: vec![Header { number: 123, ..Default::default() }], + ..Default::default() + }, + }; + let hash = block.header.hash_slow(); + provider.add_block(hash, block.clone()); + provider.add_receipts(126, vec![]); + let api = OtterscanApi::new( + crate::eth::EthApiBuilder::new( + provider, + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(MAINNET.clone()), + ) + .build(), + ); + cache_empty_block(api.eth.cache().clone(), block).await; + let expected = calculate_issuance(5 * ETH_TO_WEI, 126, [123].into_iter()); + assert_eq!( + api.get_block_details(alloy_eips::BlockNumberOrTag::Number(126).into()) + .await + .unwrap() + .issuance, + expected + ); + assert_eq!(api.get_block_details_by_hash(hash).await.unwrap().issuance, expected); + } + + #[tokio::test] + async fn unknown_transactions_and_unimplemented_history_remain_distinct() { + let api = OtterscanApi::new( + crate::eth::EthApiBuilder::new( + MockEthProvider::default(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(MAINNET.clone()), + ) + .build(), + ); + assert_eq!(api.get_api_level().await.unwrap(), 8); + assert_eq!(api.get_transaction_error(B256::ZERO).await.unwrap(), None); + assert_eq!(api.trace_transaction(B256::ZERO).await.unwrap(), None); + assert!(api.get_internal_operations(B256::ZERO).await.unwrap().is_empty()); + assert!(api + .get_block_details(alloy_eips::BlockNumberOrTag::Number(1).into()) + .await + .is_err()); + assert!(api + .get_block_transactions(alloy_eips::BlockNumberOrTag::Number(1).into(), 0, 10) + .await + .is_err()); + for error in [ + api.search_transactions_before( + Address::ZERO, + alloy_eips::BlockNumberOrTag::Number(0).into(), + 10, + ) + .await + .unwrap_err(), + api.search_transactions_after( + Address::ZERO, + alloy_eips::BlockNumberOrTag::Number(0).into(), + 10, + ) + .await + .unwrap_err(), + ] { + assert_eq!(error.code(), -32603); + assert_eq!(error.message(), "unimplemented"); + } + } + + #[tokio::test] + async fn execution_fees_use_full_u256_width() { + let api = OtterscanApi::new( + crate::eth::EthApiBuilder::new( + MockEthProvider::default(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(MAINNET.clone()), + ) + .build(), + ); + let receipt = TransactionReceipt { + inner: alloy_consensus::ReceiptEnvelope::Legacy( + alloy_consensus::Receipt { + status: true.into(), + cumulative_gas_used: 2, + logs: vec![], + } + .with_bloom(), + ), + transaction_hash: B256::ZERO, + transaction_index: Some(0), + block_hash: Some(B256::ZERO), + block_number: Some(0), + gas_used: 2, + effective_gas_price: u128::MAX, + blob_gas_used: None, + blob_gas_price: None, + from: Address::ZERO, + to: None, + contract_address: None, + }; + let details = + api.block_details(Default::default(), vec![receipt.clone(), receipt]).await.unwrap(); + assert_eq!(details.total_fees, U256::from(u128::MAX) * U256::from(4)); + } + + #[test] + fn issuance_includes_ommer_and_inclusion_rewards() { + // A Frontier reward with an ommer three blocks behind. + let issuance = calculate_issuance(5 * ETH_TO_WEI, 126, [123].into_iter()); + assert_eq!(issuance.block_reward, U256::from(5_156_250_000_000_000_000u128)); + assert_eq!(issuance.uncle_reward, U256::from(3_125_000_000_000_000_000u128)); + assert_eq!(issuance.issuance, U256::from(8_281_250_000_000_000_000u128)); + let issuance = calculate_issuance(2 * ETH_TO_WEI, 100, [99, 94].into_iter()); + assert_eq!(issuance.block_reward, U256::from(2_125_000_000_000_000_000u128)); + assert_eq!(issuance.uncle_reward, U256::from(2_250_000_000_000_000_000u128)); + } + + fn execute(code: Bytes, spec: SpecId) -> (ExecutionResult, Vec) { + let contract = Address::repeat_byte(0x11); + let mut db = InMemoryDB::default(); + db.insert_account_info( + Address::ZERO, + AccountInfo { balance: U256::from(ETH_TO_WEI), ..Default::default() }, + ); + db.insert_account_info( + contract, + AccountInfo { + balance: U256::from(100), + code: Some(Bytecode::new_legacy(code)), + ..Default::default() + }, + ); + let mut evm = Context::mainnet() + .modify_cfg_chained(|cfg| cfg.spec = spec) + .with_db(db) + .build_mainnet_with_inspector(TracingInspector::new( + TracingInspectorConfig::default_parity(), + )); + let result = evm + .inspect_tx(TxEnv { + kind: TxKind::Call(contract), + value: U256::from(7), + gas_limit: 1_000_000, + ..Default::default() + }) + .unwrap(); + let (_, inspector) = evm.ctx_inspector(); + (result.result, otterscan_traces(inspector.traces().nodes().to_vec())) + } + + #[test] + fn selfdestruct_preserves_enclosing_call_and_beneficiary() { + for spec in [SpecId::SHANGHAI, SpecId::CANCUN] { + let (result, traces) = execute(hex!("6022ff").into(), spec); + assert!(result.is_success()); + assert_eq!(traces.len(), 2); + assert_eq!(traces[0].r#type, "CALL"); + assert_eq!(traces[0].depth, 0); + assert_eq!(traces[0].value, Some(U256::from(7))); + assert_eq!(traces[1].r#type, "SELFDESTRUCT"); + assert_eq!(traces[1].depth, 1); + assert_eq!(traces[1].from, Address::repeat_byte(0x11)); + assert_eq!(traces[1].to, Address::with_last_byte(0x22)); + assert_eq!(traces[1].value, Some(U256::from(107))); + } + } + + #[test] + fn selfdestruct_after_a_child_call_uses_enclosing_depth() { + // Call another account, then destroy the root contract. The destruction is a sibling + // of that call, not its child. + let (result, traces) = + execute(hex!("60006000600060006000603361fffff1506022ff").into(), SpecId::CANCUN); + assert!(result.is_success()); + assert_eq!(traces.len(), 3); + assert_eq!(traces[1].r#type, "CALL"); + assert_eq!(traces[1].depth, 1); + assert_eq!(traces[2].r#type, "SELFDESTRUCT"); + assert_eq!(traces[2].depth, 1); + assert_eq!(traces[2].from, Address::repeat_byte(0x11)); + assert_eq!(traces[2].to, Address::with_last_byte(0x22)); + } + + #[test] + fn selfdestructs_follow_children_and_precede_siblings() { + let nodes = [0, 1, 2, 1] + .into_iter() + .enumerate() + .map(|(idx, depth)| CallTraceNode { + trace: CallTrace { + depth, + address: Address::with_last_byte(idx as u8), + selfdestruct_address: (idx < 3).then_some(Address::with_last_byte(idx as u8)), + selfdestruct_refund_target: (idx < 3).then_some(Address::with_last_byte(0xff)), + selfdestruct_transferred_value: (idx < 3).then_some(U256::ZERO), + ..Default::default() + }, + ..Default::default() + }) + .collect(); + let traces = otterscan_traces(nodes); + let order = + traces.iter().map(|trace| (trace.r#type.as_str(), trace.depth)).collect::>(); + assert_eq!( + order, + [ + ("CALL", 0), + ("CALL", 1), + ("CALL", 2), + ("SELFDESTRUCT", 3), + ("SELFDESTRUCT", 2), + ("CALL", 1), + ("SELFDESTRUCT", 1) + ] + ); + assert_eq!( + traces + .iter() + .filter(|trace| trace.r#type == "SELFDESTRUCT") + .map(|trace| trace.from) + .collect::>(), + [Address::with_last_byte(2), Address::with_last_byte(1), Address::ZERO] + ); + } + + #[test] + fn precompile_calls_remain_in_transaction_traces() { + let (result, traces) = + execute(hex!("60006000600060006000600461fffff15000").into(), SpecId::CANCUN); + assert!(result.is_success()); + assert_eq!(traces.len(), 2); + assert_eq!(traces[1].to, Address::with_last_byte(4)); + } #[test] - fn block_transaction_page_range_returns_first_pages_in_block_order() { - assert_eq!(block_transaction_page_range(25, 0, 10), 0..10); - assert_eq!(block_transaction_page_range(25, 1, 10), 10..20); + fn contract_creator_ignores_reverted_ancestors() { + let contract = Address::repeat_byte(0x11); + let creator = Address::repeat_byte(0x22); + let localize = |trace, hash| LocalizedTransactionTrace { + trace, + transaction_hash: Some(hash), + block_hash: None, + block_number: Some(1), + transaction_position: None, + }; + let creation = TransactionTrace { + action: Action::Create(CreateAction { from: creator, ..Default::default() }), + result: Some(TraceOutput::Create(CreateOutput { + address: contract, + code: Bytes::new(), + gas_used: 0, + })), + trace_address: vec![0, 0], + ..Default::default() + }; + for path in [vec![], vec![0]] { + let failed = TransactionTrace { + trace_address: path, + error: Some("Reverted".into()), + ..Default::default() + }; + let reverted = + vec![localize(failed, B256::ZERO), localize(creation.clone(), B256::ZERO)]; + assert_eq!(find_contract_creator(contract, vec![reverted.clone()]).unwrap(), None); + let successful = vec![localize(creation.clone(), B256::repeat_byte(1))]; + assert_eq!( + find_contract_creator(contract, vec![reverted, successful]).unwrap(), + Some(ContractCreator { creator, hash: B256::repeat_byte(1) }) + ); + } + // A reverted sibling does not invalidate a subsequent successful creation. + let failed = TransactionTrace { + trace_address: vec![0], + error: Some("Reverted".into()), + ..Default::default() + }; + let successful = TransactionTrace { trace_address: vec![1], ..creation }; + assert!(find_contract_creator( + contract, + vec![vec![localize(failed, B256::ZERO), localize(successful, B256::ZERO)]] + ) + .unwrap() + .is_some()); } #[test] - fn block_transaction_page_range_caps_pages_at_the_block_length() { - assert_eq!(block_transaction_page_range(25, 2, 10), 20..25); - assert_eq!(block_transaction_page_range(25, 3, 10), 25..25); + fn static_and_delegate_calls_have_no_value() { + // STATICCALL, DELEGATECALL and CALLCODE to 0x22 do not transfer ETH. + let (result, traces) = execute( + hex!("6000600060006000602261fffffa506000600060006000602261fffff45060006000600060006001602261fffff25000").into(), + SpecId::CANCUN, + ); + assert!(result.is_success()); + assert_eq!(traces.len(), 4); + assert_eq!(traces[1].r#type, "STATICCALL"); + assert_eq!(traces[1].value, None); + assert_eq!(traces[2].r#type, "DELEGATECALL"); + assert_eq!(traces[2].value, None); + assert_eq!(traces[3].r#type, "CALLCODE"); + assert_eq!(traces[3].value, Some(U256::from(1))); } } diff --git a/docs/vocs/docs/pages/jsonrpc/intro.mdx b/docs/vocs/docs/pages/jsonrpc/intro.mdx index 78a0aae04c0..ecdb4ac555d 100644 --- a/docs/vocs/docs/pages/jsonrpc/intro.mdx +++ b/docs/vocs/docs/pages/jsonrpc/intro.mdx @@ -27,7 +27,7 @@ The methods are grouped into namespaces, which are listed below: | [`admin`](/jsonrpc/admin) | The `admin` API allows you to configure your node. | **Yes** | | [`rpc`](/jsonrpc/rpc) | The `rpc` API provides information about the RPC server and its modules. | No | | [`reth`](/jsonrpc/reth) | The `reth` API provides reth-specific methods like balance changes and chain notifications. | No | -| `ots` | The `ots` API provides Otterscan-compatible methods for block exploration. | No | +| [`ots`](/jsonrpc/ots) | The `ots` API provides partial Otterscan support; address history search is unimplemented. | No | | `flashbots` | The `flashbots` API provides block submission validation methods for builders. | No | | `miner` | The `miner` API allows you to configure miner/builder settings like extra data and gas limits. | **Yes** | | `mev` | The `mev` API provides MEV bundle submission and simulation methods. | No | diff --git a/docs/vocs/docs/pages/jsonrpc/ots.mdx b/docs/vocs/docs/pages/jsonrpc/ots.mdx new file mode 100644 index 00000000000..ba6f6b30d07 --- /dev/null +++ b/docs/vocs/docs/pages/jsonrpc/ots.mdx @@ -0,0 +1,86 @@ +# `ots` namespace + +Reth implements part of the [Otterscan API](https://docs.otterscan.io/api-docs/ots-api). +Enable the namespace with `--http.api eth,ots` (or include `ots` in your existing +HTTP or WebSocket API selection). + +`ots_getApiLevel` returns **8**, but this does **not** guarantee complete support +for API level 8 or for every feature of the Otterscan frontend. + +## Supported methods + +| Method | Behavior | +| --- | --- | +| `ots_getApiLevel` | Returns `8`. | +| `ots_getHeaderByNumber` | Returns a header, or `null` for an unknown block. Also exposed as `erigon_getHeaderByNumber` for frontend compatibility. | +| `ots_hasCode` | Checks code at the requested block; defaults to latest state. | +| `ots_getTransactionError` | Returns raw revert bytes. A known transaction with no revert data (including success and exceptional halt) returns `"0x"`. An unknown transaction returns `null`. Check the receipt status to distinguish success from failure. | +| `ots_traceTransaction` | Returns call, creation and self-destruct entries, or `null` for an unknown transaction. Includes precompile calls. | +| `ots_getInternalOperations` | Returns internal nonzero-value calls, creations (including zero endowment), and self-destructs. Excludes the transaction's top-level call or creation. An unknown transaction returns an empty array. | +| `ots_getBlockDetails` | Returns block metadata, execution fees, and issuance using the chain's Ethereum hardfork schedule. | +| `ots_getBlockDetailsByHash` | The same block details, selected by hash. | +| `ots_getBlockTransactions` | Returns paginated transactions and matching receipts. | +| `ots_getTransactionBySenderAndNonce` | Searches canonical transaction history for a sender and nonce; excludes pending transactions. Returns `null` if not found. | +| `ots_getContractCreator` | Searches historical code and traces for a creation transaction, subject to the limitations below. Returns `null` if no creator is found. | + +Block details and block transaction methods return an RPC error when the block +or required receipts are unavailable. They do not return an empty block. + +Block issuance includes the proof-of-work miner reward, the miner's ommer +inclusion rewards, and the ommers' rewards. It is zero for genesis and from the +Paris (Merge) activation configured in the chain specification. It does not +include consensus-layer issuance or withdrawals. `totalFees` sums execution gas +used times effective gas price, including the base fee; it excludes blob fees +and is not the miner's fee income. + +Block transaction pages are zero-indexed **from the end of the block**, with +transactions and receipts in ascending block order within each page. For 25 +transactions and a page size of 10, pages contain indices 15–24, 5–14, and 0–4. +Out-of-range pages and a page size of zero return empty lists. The original +block transaction count is retained. Transaction input is capped to the first +four bytes; receipt logs and log blooms are `null`. + +Tracing reports attempted operations, including operations reverted by an +enclosing call. These entries are not a list of committed balance changes. +`SELFDESTRUCT` is represented separately from the call that executes it; after +Cancun it does not necessarily delete the contract. `STATICCALL` and +`DELEGATECALL` trace entries have a `null` value. + +## Unimplemented address history search + +`ots_searchTransactionsBefore` and `ots_searchTransactionsAfter` are registered +but **unimplemented**. Both return this JSON-RPC error: + +```json +{"code": -32603, "message": "unimplemented"} +``` + +Enabling `ots` or running an archive node does not enable these endpoints. +Otterscan address transaction history therefore remains unavailable through +these methods. Implementing them requires efficient address/call history +indexing or another bounded search strategy; replaying arbitrary chain history +per request is not a practical substitute. + +Earlier work is recorded in [closed issue #13499](https://github.com/paradigmxyz/reth/issues/13499). +The [closed prototype PR #13621](https://github.com/paradigmxyz/reth/pull/13621) +contains earlier implementation and indexing discussion. Cross-client response +and specification alignment is discussed in +[Otterscan #1081](https://github.com/otterscan/otterscan/issues/1081). + +## History and state requirements + +Historical code queries need state for the selected block. Tracing methods need +the transaction, its block, and state sufficient to replay it, including preceding +transactions when necessary. Pruned state, receipts, bodies, or transaction +lookup data can prevent these methods from answering older queries. Archive +state alone does not restore other pruned history. Replays and historical +searches can be expensive. + +Contract creator lookup currently binary-searches code presence, assuming one +transition from absent to present. Destruction and redeployment violate that +assumption, so the result is not a reliable first-deployment record for such +contracts. Contracts without code at latest state return `null`; genesis +allocations and code set through EIP-7702 delegation have no corresponding +`CREATE`/`CREATE2` transaction and may also return `null`. A complete solution +needs creation history rather than code-presence binary search; address history +work above is a related indexing follow-up. diff --git a/docs/vocs/sidebar.ts b/docs/vocs/sidebar.ts index 37c5d56dee2..4fbd5fc7df2 100644 --- a/docs/vocs/sidebar.ts +++ b/docs/vocs/sidebar.ts @@ -238,6 +238,10 @@ export const sidebar: SidebarItem[] = [ text: "trace", link: "/jsonrpc/trace" }, + { + text: "ots", + link: "/jsonrpc/ots" + }, { text: "admin", link: "/jsonrpc/admin" From 3f984abf389d8068319729d7a8d8b04a37104bff Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 16:53:25 +0000 Subject: [PATCH 095/133] refactor(e2e): simplify test setup and add helpers (#27432) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 3 +- .../debug-client/src/providers/etherscan.rs | 17 +- crates/e2e-test-utils/Cargo.toml | 1 + crates/e2e-test-utils/src/chain_spec.rs | 86 +++ crates/e2e-test-utils/src/lib.rs | 150 +---- crates/e2e-test-utils/src/node.rs | 326 ++++++--- crates/e2e-test-utils/src/receipt.rs | 44 ++ crates/e2e-test-utils/src/rpc.rs | 40 +- crates/e2e-test-utils/src/setup_builder.rs | 509 ++++++++++---- crates/e2e-test-utils/src/setup_import.rs | 607 ++++++----------- crates/e2e-test-utils/src/testsuite/mod.rs | 3 +- crates/e2e-test-utils/src/testsuite/setup.rs | 52 +- crates/e2e-test-utils/src/transaction.rs | 6 + crates/e2e-test-utils/src/wait.rs | 33 + crates/e2e-test-utils/src/wallet.rs | 145 +++- .../tests/e2e-testsuite/main.rs | 152 +---- crates/e2e-test-utils/tests/rocksdb/main.rs | 413 +++++------- crates/engine/tree/Cargo.toml | 1 - .../e2e-testsuite/fcu_finalized_blocks.rs | 31 +- .../engine/tree/tests/e2e-testsuite/main.rs | 73 +-- .../ethereum/node/tests/assets/genesis.json | 1 - crates/ethereum/node/tests/e2e/blobs.rs | 77 +-- .../ethereum/node/tests/e2e/custom_genesis.rs | 68 +- crates/ethereum/node/tests/e2e/dev.rs | 74 ++- crates/ethereum/node/tests/e2e/eip6780.rs | 57 +- crates/ethereum/node/tests/e2e/eth.rs | 620 +++++------------- crates/ethereum/node/tests/e2e/forkchoice.rs | 95 ++- .../node/tests/e2e/invalid_payload.rs | 93 +-- crates/ethereum/node/tests/e2e/p2p.rs | 168 +---- crates/ethereum/node/tests/e2e/pool.rs | 169 ++--- crates/ethereum/node/tests/e2e/prestate.rs | 23 +- crates/ethereum/node/tests/e2e/rpc.rs | 332 ++-------- .../ethereum/node/tests/e2e/selfdestruct.rs | 120 +--- crates/ethereum/node/tests/e2e/simulate.rs | 221 +------ crates/ethereum/node/tests/e2e/utils.rs | 96 ++- crates/ethereum/node/tests/it/testing.rs | 34 +- examples/exex-test/Cargo.toml | 1 - examples/exex-test/src/main.rs | 26 +- 38 files changed, 1993 insertions(+), 2974 deletions(-) create mode 100644 crates/e2e-test-utils/src/chain_spec.rs create mode 100644 crates/e2e-test-utils/src/receipt.rs create mode 100644 crates/e2e-test-utils/src/wait.rs delete mode 100644 crates/ethereum/node/tests/assets/genesis.json diff --git a/Cargo.lock b/Cargo.lock index 7a8898a3c9a..0c7e38c492c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3809,7 +3809,6 @@ dependencies = [ "futures-util", "reth-e2e-test-utils", "reth-ethereum", - "serde_json", "tokio", ] @@ -8453,6 +8452,7 @@ version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", + "alloy-genesis", "alloy-network", "alloy-primitives", "alloy-provider", @@ -8644,7 +8644,6 @@ dependencies = [ "reth-trie-sparse", "revm", "schnellru", - "serde_json", "thiserror 2.0.20", "tokio", "tracing", diff --git a/crates/consensus/debug-client/src/providers/etherscan.rs b/crates/consensus/debug-client/src/providers/etherscan.rs index fba239d7dda..2671624851e 100644 --- a/crates/consensus/debug-client/src/providers/etherscan.rs +++ b/crates/consensus/debug-client/src/providers/etherscan.rs @@ -9,7 +9,7 @@ use std::{sync::Arc, time::Duration}; use tokio::{sync::mpsc, time::interval}; /// Block provider that fetches new blocks from Etherscan API. -#[derive(derive_more::Debug, Clone)] +#[derive(derive_more::Debug)] pub struct EtherscanBlockProvider { http_client: Client, base_url: String, @@ -88,6 +88,21 @@ where } } +// Implemented manually so cloning doesn't require `RpcBlock: Clone` and `ExecutionData: Clone`, +// which are only used by the shared conversion function. +impl Clone for EtherscanBlockProvider { + fn clone(&self) -> Self { + Self { + http_client: self.http_client.clone(), + base_url: self.base_url.clone(), + api_key: self.api_key.clone(), + chain_id: self.chain_id, + interval: self.interval, + convert: Arc::clone(&self.convert), + } + } +} + impl PayloadProvider for EtherscanBlockProvider where RpcBlock: Serialize + DeserializeOwned + 'static, diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index b337377f522..4fdd0e9b7d4 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -53,6 +53,7 @@ url.workspace = true # ethereum alloy-primitives.workspace = true alloy-eips.workspace = true +alloy-genesis.workspace = true alloy-rlp.workspace = true alloy-signer.workspace = true alloy-signer-local = { workspace = true, features = ["mnemonic"] } diff --git a/crates/e2e-test-utils/src/chain_spec.rs b/crates/e2e-test-utils/src/chain_spec.rs new file mode 100644 index 00000000000..ac603d43d69 --- /dev/null +++ b/crates/e2e-test-utils/src/chain_spec.rs @@ -0,0 +1,86 @@ +//! Chain specifications and payload attributes for test nodes. + +use alloy_genesis::Genesis; +use alloy_primitives::{Address, B256}; +use alloy_rpc_types_engine::PayloadAttributes; +use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, EthereumHardforks, MAINNET}; +use std::sync::Arc; + +/// Returns the genesis of the test chain. +/// +/// The genesis funds the first 20 accounts derived from the test mnemonic, see +/// [`Wallet`](crate::wallet::Wallet). +pub fn test_genesis() -> Genesis { + serde_json::from_str(include_str!("testsuite/assets/genesis.json")) + .expect("failed to parse test genesis") +} + +/// Returns a [`ChainSpecBuilder`] for the mainnet chain id with the [`test_genesis`] and no +/// hardforks activated. +/// +/// Use this to schedule hardforks at non-genesis timestamps, otherwise prefer +/// [`test_chain_spec`]. +pub fn test_chain_spec_builder() -> ChainSpecBuilder { + ChainSpecBuilder::default().chain(MAINNET.chain).genesis(test_genesis()) +} + +/// Returns the test chain spec with every hardfork up to and including `fork` active at genesis. +/// +/// # Panics +/// +/// If `fork` can not be activated at genesis via [`ChainSpecBuilder`]. +pub fn test_chain_spec(fork: EthereumHardfork) -> Arc { + let builder = test_chain_spec_builder(); + let builder = match fork { + EthereumHardfork::Frontier => builder.frontier_activated(), + EthereumHardfork::Homestead => builder.homestead_activated(), + EthereumHardfork::Dao => builder.dao_activated(), + EthereumHardfork::Tangerine => builder.tangerine_whistle_activated(), + EthereumHardfork::SpuriousDragon => builder.spurious_dragon_activated(), + EthereumHardfork::Byzantium => builder.byzantium_activated(), + EthereumHardfork::Constantinople => builder.constantinople_activated(), + EthereumHardfork::Petersburg => builder.petersburg_activated(), + EthereumHardfork::Istanbul => builder.istanbul_activated(), + EthereumHardfork::MuirGlacier => builder.muirglacier_activated(), + EthereumHardfork::Berlin => builder.berlin_activated(), + EthereumHardfork::London => builder.london_activated(), + EthereumHardfork::ArrowGlacier => builder.arrowglacier_activated(), + EthereumHardfork::GrayGlacier => builder.grayglacier_activated(), + EthereumHardfork::Paris => builder.paris_activated(), + EthereumHardfork::Shanghai => builder.shanghai_activated(), + EthereumHardfork::Cancun => builder.cancun_activated(), + EthereumHardfork::Prague => builder.prague_activated(), + EthereumHardfork::Osaka => builder.osaka_activated(), + EthereumHardfork::Amsterdam => builder.amsterdam_activated(), + EthereumHardfork::Bogota => builder.bogota_activated(), + fork => panic!("{fork} can not be activated at genesis"), + }; + Arc::new(builder.build()) +} + +/// Creates Ethereum [`PayloadAttributes`] that are valid for the hardforks active in `chain_spec` +/// at `timestamp`. +/// +/// Withdrawals are set once Shanghai is active, the parent beacon block root once Cancun is +/// active, and a slot number once Amsterdam is active. The payload builder requires a slot number +/// for EIP-7843; tests use the timestamp as a deterministic dummy slot because the exact beacon +/// slot is irrelevant for local e2e payloads. +/// +/// This is the default attributes generator of [`E2ETestSetupBuilder`](crate::E2ETestSetupBuilder), +/// so the same test works against any hardfork schedule, including forks activating mid-test. +pub fn eth_payload_attributes( + chain_spec: &C, + timestamp: u64, +) -> PayloadAttributes { + PayloadAttributes { + timestamp, + prev_randao: B256::ZERO, + suggested_fee_recipient: Address::ZERO, + withdrawals: chain_spec.is_shanghai_active_at_timestamp(timestamp).then(Vec::new), + parent_beacon_block_root: chain_spec + .is_cancun_active_at_timestamp(timestamp) + .then_some(B256::ZERO), + slot_number: chain_spec.is_amsterdam_active_at_timestamp(timestamp).then_some(timestamp), + ..Default::default() + } +} diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 4507c295188..d1631f71891 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -1,20 +1,16 @@ //! Utilities for end-to-end tests. -use alloy_primitives::{Address, B256}; use alloy_rpc_types_engine::PayloadAttributes; use node::NodeTestContext; -use reth_chainspec::{ChainSpec, EthereumHardfork}; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; use reth_network_api::test_utils::PeersHandleProvider; use reth_node_builder::{ components::NodeComponentsBuilder, rpc::{EngineValidatorAddOn, RethRpcAddOns}, - FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypes, NodeTypesWithDBAdapter, - PayloadTypes, + FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypesWithDBAdapter, PayloadTypes, }; use reth_provider::providers::{BlockchainProvider, NodeTypesForProvider}; use std::sync::Arc; -use wallet::Wallet; /// Wrapper type to create test nodes pub mod node; @@ -44,110 +40,18 @@ pub mod test_rlp_utils; /// Helpers for verifying the persisted state and trie representation pub mod trie; -/// Builder for configuring test node setups -mod setup_builder; -pub use setup_builder::E2ETestSetupBuilder; - -/// Creates the initial setup with `num_nodes` started and interconnected. -pub async fn setup( - num_nodes: usize, - chain_spec: Arc, - is_dev: bool, - attributes_generator: impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadAttributes - + Send - + Sync - + Copy - + 'static, -) -> eyre::Result<(Vec>, Wallet)> -where - N: NodeBuilderHelper, -{ - E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) - .with_node_config_modifier(move |config| config.set_dev(is_dev)) - .build() - .await -} +pub mod receipt; -/// Creates the initial setup with `num_nodes` started and interconnected. -pub async fn setup_engine( - num_nodes: usize, - chain_spec: Arc, - is_dev: bool, - tree_config: reth_node_api::TreeConfig, - attributes_generator: impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadAttributes - + Send - + Sync - + Copy - + 'static, -) -> eyre::Result<( - Vec>>>, - Wallet, -)> -where - N: NodeBuilderHelper, -{ - setup_engine_with_connection::( - num_nodes, - chain_spec, - is_dev, - tree_config, - attributes_generator, - true, - ) - .await -} +pub mod wait; -/// Creates the initial setup with `num_nodes` started and optionally interconnected. -pub async fn setup_engine_with_connection( - num_nodes: usize, - chain_spec: Arc, - is_dev: bool, - tree_config: reth_node_api::TreeConfig, - attributes_generator: impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadAttributes - + Send - + Sync - + Copy - + 'static, - connect_nodes: bool, -) -> eyre::Result<( - Vec>>>, - Wallet, -)> -where - N: NodeBuilderHelper, -{ - E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) - .with_tree_config_modifier(move |base| { - // Apply caller's tree_config but preserve the small cache size from base - tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size()) - }) - .with_node_config_modifier(move |config| config.set_dev(is_dev)) - .with_connect_nodes(connect_nodes) - .build() - .await -} +mod chain_spec; +pub use chain_spec::{ + eth_payload_attributes, test_chain_spec, test_chain_spec_builder, test_genesis, +}; -/// Creates Ethereum [`PayloadAttributes`] valid for the given hardfork, allowing the same test -/// to be run against multiple hardfork targets. -/// -/// Withdrawals are set once Shanghai is active, the parent beacon block root once Cancun is -/// active, and a slot number once Amsterdam is active. The payload builder requires a slot number -/// for EIP-7843; tests use the timestamp as a deterministic dummy slot because the exact beacon -/// slot is irrelevant for local e2e payloads. -pub fn eth_payload_attributes_for_fork( - fork: EthereumHardfork, - timestamp: u64, -) -> PayloadAttributes { - PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: (fork >= EthereumHardfork::Shanghai).then(Vec::new), - parent_beacon_block_root: (fork >= EthereumHardfork::Cancun).then_some(B256::ZERO), - slot_number: (fork >= EthereumHardfork::Amsterdam).then_some(timestamp), - ..Default::default() - } -} +/// Builder for configuring test node setups +mod setup_builder; +pub use setup_builder::{E2ETestSetupBuilder, E2ETestSetupExt}; // Type aliases @@ -168,26 +72,18 @@ pub type Adapter>> = NodeTestContext, >>::AddOns>; -/// Helper trait to simplify bounds when calling setup functions. +/// Helper trait to simplify the bounds of nodes launched by [`E2ETestSetupBuilder`]. pub trait NodeBuilderHelper where Self: Default + NodeTypesForProvider>> + Node< - TmpNodeAdapter>>, + TmpNodeAdapter, ComponentsBuilder: NodeComponentsBuilder< - TmpNodeAdapter>>, - Components: NodeComponents< - TmpNodeAdapter>>, - Network: PeersHandleProvider, - >, + TmpNodeAdapter, + Components: NodeComponents, Network: PeersHandleProvider>, >, - AddOns: RethRpcAddOns< - Adapter>>, - > + EngineValidatorAddOn< - Adapter>>, - >, - ChainSpec: From + Clone, + AddOns: RethRpcAddOns> + EngineValidatorAddOn>, >, { } @@ -196,20 +92,12 @@ impl NodeBuilderHelper for T where Self: Default + NodeTypesForProvider>> + Node< - TmpNodeAdapter>>, + TmpNodeAdapter, ComponentsBuilder: NodeComponentsBuilder< - TmpNodeAdapter>>, - Components: NodeComponents< - TmpNodeAdapter>>, - Network: PeersHandleProvider, - >, - >, - AddOns: RethRpcAddOns< - Adapter>>, - > + EngineValidatorAddOn< - Adapter>>, + TmpNodeAdapter, + Components: NodeComponents, Network: PeersHandleProvider>, >, - ChainSpec: From + Clone, + AddOns: RethRpcAddOns> + EngineValidatorAddOn>, > { } diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index bb7054ec6f0..b26bb343641 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -1,17 +1,26 @@ -use crate::{network::NetworkTestContext, payload::PayloadTestContext, rpc::RpcTestContext}; +use crate::{ + network::NetworkTestContext, + payload::PayloadTestContext, + rpc::RpcTestContext, + wait::{poll_until, WAIT_TIMEOUT}, +}; use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_eips::BlockId; +use alloy_network::{Ethereum, IntoWallet}; use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B256}; -use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV5, ForkchoiceState}; +use alloy_provider::{ + fillers::{FillProvider, RecommendedFillers, TxFiller}, + Provider, ProviderBuilder, RootProvider, +}; +use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV5, ForkchoiceState, ForkchoiceUpdated}; use alloy_rpc_types_eth::BlockNumberOrTag; -use eyre::Ok; +use eyre::{ensure, eyre, Ok}; use futures_util::Future; use jsonrpsee::{core::client::ClientT, http_client::HttpClient}; use reth_chainspec::EthereumHardforks; use reth_network_api::test_utils::PeersHandleProvider; use reth_node_api::{Block, BlockBody, BlockTy, FullNodeComponents, PayloadTypes, PrimitivesTy}; use reth_node_builder::{rpc::RethRpcAddOns, FullNode, NodeTypes}; - use reth_payload_primitives::BuiltPayload; use reth_provider::{ BlockNumReader, BlockReader, BlockReaderIdExt, CanonStateNotificationStream, @@ -21,10 +30,17 @@ use reth_rpc_api::TestingBuildBlockRequestV1; use reth_rpc_builder::auth::AuthServerHandle; use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt}; use reth_stages_types::StageId; -use std::{pin::Pin, sync::Arc}; +use std::{pin::Pin, sync::Arc, time::Duration}; use tokio_stream::StreamExt; use url::Url; +/// Interval at which the alloy providers of [`NodeTestContext`] poll the node, e.g. for receipts +/// of pending transactions. +/// +/// Much shorter than alloy's default for local nodes, since test nodes build blocks on demand or +/// with short dev block times. +pub const RPC_PROVIDER_POLL_INTERVAL: Duration = Duration::from_millis(10); + /// A helper struct to handle node actions #[expect(missing_debug_implementations)] pub struct NodeTestContext @@ -53,17 +69,25 @@ where AddOns: RethRpcAddOns, { /// Creates a new test node + /// + /// Payload timestamps start after the default Cancun timestamp of the payload context, or after + /// the timestamp of the latest block if it is newer, e.g. for a custom genesis or an imported + /// chain. pub async fn new( node: FullNode, attributes_generator: impl Fn(u64) -> Payload::PayloadAttributes + Send + Sync + 'static, ) -> eyre::Result { + let mut payload = + PayloadTestContext::new(node.payload_builder_handle.clone(), attributes_generator) + .await?; + if let Some(latest) = + node.provider.sealed_header_by_number_or_tag(BlockNumberOrTag::Latest)? + { + payload.timestamp = payload.timestamp.max(latest.timestamp()); + } Ok(Self { inner: node.clone(), - payload: PayloadTestContext::new( - node.payload_builder_handle.clone(), - attributes_generator, - ) - .await?, + payload, network: NetworkTestContext::new(node.network.clone()), rpc: RpcTestContext { inner: node.add_ons_handle.rpc_registry }, canonical_stream: node.provider.canonical_state_stream(), @@ -103,6 +127,30 @@ where Ok(chain) } + /// Injects the raw transaction into the pool and advances the node one block, returning the + /// transaction hash and the built payload. + /// + /// Returns an error if the transaction is not included in the block. + pub async fn inject_and_advance( + &mut self, + raw_tx: Bytes, + ) -> eyre::Result<(B256, Payload::BuiltPayload)> + where + AddOns::EthApi: EthApiSpec>> + + EthTransactions + + TraceExt, + { + let tx_hash = self.rpc.inject_tx(raw_tx).await?; + let payload = self.advance_block().await?; + let block = payload.block(); + ensure!( + block.body().transactions().iter().any(|tx| *tx.tx_hash() == tx_hash), + "transaction {tx_hash} was not included in block {}", + block.number() + ); + Ok((tx_hash, payload)) + } + /// Returns the current forkchoice state of the node. pub fn current_forkchoice_state(&self) -> eyre::Result { let latest_header = @@ -171,82 +219,108 @@ where } /// Waits for block to be available on node. + /// + /// Returns an error if the block is not available within [`WAIT_TIMEOUT`]. pub async fn wait_block( &self, number: BlockNumber, expected_block_hash: BlockHash, wait_finish_checkpoint: bool, ) -> eyre::Result<()> { - let mut check = !wait_finish_checkpoint; - loop { - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - - if !check && - wait_finish_checkpoint && - let Some(checkpoint) = - self.inner.provider.get_stage_checkpoint(StageId::Finish)? && - checkpoint.block_number >= number + let provider = &self.inner.provider; + poll_until(format_args!("block {number}"), move || async move { + if wait_finish_checkpoint && + provider + .get_stage_checkpoint(StageId::Finish)? + .is_none_or(|checkpoint| checkpoint.block_number < number) { - check = true + return Ok(None) } - - if check { - if let Some(latest_header) = self.inner.provider.header_by_number(number)? { - assert_eq!(latest_header.hash_slow(), expected_block_hash); - break - } - assert!( + let Some(header) = provider.header_by_number(number)? else { + ensure!( !wait_finish_checkpoint, - "Finish checkpoint matches, but could not fetch block." + "Finish checkpoint matches, but could not fetch block {number}" ); - } - } - Ok(()) + return Ok(None) + }; + let hash = header.hash_slow(); + ensure!( + hash == expected_block_hash, + "block {number} is {hash}, expected {expected_block_hash}" + ); + Ok(Some(())) + }) + .await } - /// Waits for the node to unwind to the given block number + /// Waits for the node to unwind to the given block number. + /// + /// Returns an error if the node does not unwind within [`WAIT_TIMEOUT`]. pub async fn wait_unwind(&self, number: BlockNumber) -> eyre::Result<()> { - loop { - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - if let Some(checkpoint) = self.inner.provider.get_stage_checkpoint(StageId::Headers)? && - checkpoint.block_number == number - { - break - } - } - Ok(()) + let provider = &self.inner.provider; + poll_until(format_args!("unwind to block {number}"), move || async move { + let checkpoint = provider.get_stage_checkpoint(StageId::Headers)?; + Ok(checkpoint.is_some_and(|checkpoint| checkpoint.block_number == number).then_some(())) + }) + .await } - /// Asserts that a new block has been added to the blockchain - /// and the tx has been included in the block. + /// Waits until `condition` holds for the transaction pool of the node. /// - /// Does NOT work for pipeline since there's no stream notification! + /// Returns an error if the condition does not hold within [`WAIT_TIMEOUT`]. + pub async fn wait_for_pool( + &self, + mut condition: impl FnMut(&Node::Pool) -> bool, + ) -> eyre::Result<()> { + let pool = &self.inner.pool; + poll_until("transaction pool condition", move || { + let ready = condition(pool); + async move { Ok(ready.then_some(())) } + }) + .await + } + + /// Asserts that a new block has been added to the blockchain and the tx has been included in + /// the block, at any position. + /// + /// Does NOT work for pipeline since there's no stream notification! Returns an error if the + /// block is not committed within [`WAIT_TIMEOUT`]. pub async fn assert_new_block( &mut self, tip_tx_hash: B256, block_hash: B256, block_number: BlockNumber, ) -> eyre::Result<()> { - // get head block from notifications stream and verify the tx has been pushed to the - // pool is actually present in the canonical block - let head = self.canonical_stream.next().await.unwrap(); - let tx = head.tip().body().transactions().first(); - assert_eq!(tx.unwrap().tx_hash().as_slice(), tip_tx_hash.as_slice()); - - loop { - // wait for the block to commit - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - if let Some(latest_block) = - self.inner.provider.block_by_number_or_tag(BlockNumberOrTag::Latest)? && - latest_block.header().number() == block_number - { - // make sure the block hash we submitted via FCU engine api is the new latest - // block using an RPC call - assert_eq!(latest_block.header().hash_slow(), block_hash); - break + // get head block from notifications stream and verify the tx has been pushed to the pool is + // actually present in the canonical block + let head = tokio::time::timeout(WAIT_TIMEOUT, self.canonical_stream.next()) + .await + .map_err(|_| eyre!("timed out waiting for block {block_number}"))? + .ok_or_else(|| eyre!("canonical state stream closed"))?; + ensure!( + head.tip().body().transactions().iter().any(|tx| *tx.tx_hash() == tip_tx_hash), + "transaction {tip_tx_hash} is not included in block {}", + head.tip().number() + ); + + // wait for the block to commit and make sure the block hash we submitted via FCU engine + // api is the new latest block + let provider = &self.inner.provider; + poll_until(format_args!("block {block_number} to be committed"), move || async move { + let Some(latest) = provider.block_by_number_or_tag(BlockNumberOrTag::Latest)? else { + return Ok(None) + }; + if latest.header().number() != block_number { + return Ok(None) } - } - Ok(()) + let hash = latest.header().hash_slow(); + ensure!( + hash == block_hash, + "latest block {block_number} is {hash}, expected {block_hash}" + ); + Ok(Some(())) + }) + .await } /// Gets block hash by number. @@ -260,32 +334,41 @@ where } /// Sends FCU and waits for the node to sync to the given block. + /// + /// Returns an error if the node does not sync within [`WAIT_TIMEOUT`]. pub async fn sync_to(&self, block: BlockHash) -> eyre::Result<()> { - let start = std::time::Instant::now(); - - while self - .inner - .provider - .sealed_header_by_id(BlockId::Number(BlockNumberOrTag::Latest))? - .is_none_or(|h| h.hash() != block) - { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - self.update_forkchoice(block, block).await?; - - assert!(start.elapsed() <= std::time::Duration::from_secs(40), "timed out"); - } + let sync = async { + while self + .inner + .provider + .sealed_header_by_id(BlockId::Number(BlockNumberOrTag::Latest))? + .is_none_or(|h| h.hash() != block) + { + tokio::time::sleep(Duration::from_millis(100)).await; + self.update_forkchoice(block, block).await?; + } + Ok(()) + }; + tokio::time::timeout(WAIT_TIMEOUT, sync) + .await + .map_err(|_| eyre!("timed out syncing to block {block}"))??; // Hack to make sure that all components have time to process canonical state update. // Otherwise, this might result in e.g "nonce too low" errors when advancing chain further, // making tests flaky. - tokio::time::sleep(std::time::Duration::from_millis(1000)).await; + tokio::time::sleep(Duration::from_millis(1000)).await; Ok(()) } - /// Sends a forkchoice update message to the engine. - pub async fn update_forkchoice(&self, current_head: B256, new_head: B256) -> eyre::Result<()> { - self.inner + /// Sends a forkchoice update message to the engine and returns its response. + pub async fn update_forkchoice( + &self, + current_head: B256, + new_head: B256, + ) -> eyre::Result { + Ok(self + .inner .add_ons_handle .beacon_engine_handle .fork_choice_updated( @@ -296,13 +379,15 @@ where }, None, ) - .await?; - - Ok(()) + .await?) } - /// Sends forkchoice update to the engine api with a zero finalized hash - pub async fn update_optimistic_forkchoice(&self, hash: B256) -> eyre::Result<()> { + /// Sends forkchoice update to the engine api with a zero finalized hash and returns its + /// response. + pub async fn update_optimistic_forkchoice( + &self, + hash: B256, + ) -> eyre::Result { self.update_forkchoice(B256::ZERO, hash).await } @@ -325,6 +410,77 @@ where self.inner.rpc_server_handle().http_client() } + /// Returns an alloy provider with the recommended fillers connected to the HTTP RPC server. + /// + /// # Panics + /// + /// If the HTTP RPC server is disabled. + pub fn rpc_provider( + &self, + ) -> FillProvider + use, RootProvider> { + self.rpc_provider_for::() + } + + /// Returns an alloy provider with the recommended fillers and the given wallet connected to + /// the HTTP RPC server. + /// + /// Transactions sent via the provider are signed by the wallet, e.g. a + /// [`PrivateKeySigner`](alloy_signer_local::PrivateKeySigner) of the test + /// [`Wallet`](crate::wallet::Wallet). + /// + /// # Panics + /// + /// If the HTTP RPC server is disabled. + pub fn rpc_provider_with_wallet( + &self, + wallet: W, + ) -> FillProvider + use, RootProvider> + where + W: IntoWallet, + { + self.rpc_provider_with_wallet_for::(wallet) + } + + /// Returns an alloy provider for the network `Net` with its recommended fillers connected to + /// the HTTP RPC server, polling every [`RPC_PROVIDER_POLL_INTERVAL`]. + /// + /// This is [`Self::rpc_provider`] for nodes whose RPC types differ from Ethereum's. + /// + /// # Panics + /// + /// If the HTTP RPC server is disabled. + pub fn rpc_provider_for( + &self, + ) -> FillProvider + use, RootProvider, Net> + { + let provider = ProviderBuilder::new_with_network::().connect_http(self.rpc_url()); + provider.client().set_poll_interval(RPC_PROVIDER_POLL_INTERVAL); + provider + } + + /// Returns an alloy provider for the network `Net` with its recommended fillers and the given + /// wallet connected to the HTTP RPC server, polling every [`RPC_PROVIDER_POLL_INTERVAL`]. + /// + /// This is [`Self::rpc_provider_with_wallet`] for nodes whose RPC types differ from + /// Ethereum's. + /// + /// # Panics + /// + /// If the HTTP RPC server is disabled. + pub fn rpc_provider_with_wallet_for( + &self, + wallet: W, + ) -> FillProvider + use, RootProvider, Net> + where + Net: RecommendedFillers, + W: IntoWallet, + { + let provider = + ProviderBuilder::new_with_network::().wallet(wallet).connect_http(self.rpc_url()); + provider.client().set_poll_interval(RPC_PROVIDER_POLL_INTERVAL); + provider + } + /// Returns an Engine API client. pub fn auth_server_handle(&self) -> AuthServerHandle { self.inner.auth_server_handle().clone() diff --git a/crates/e2e-test-utils/src/receipt.rs b/crates/e2e-test-utils/src/receipt.rs new file mode 100644 index 00000000000..7462104c8e8 --- /dev/null +++ b/crates/e2e-test-utils/src/receipt.rs @@ -0,0 +1,44 @@ +//! Helpers for awaiting the receipts of transactions sent through alloy providers. + +use alloy_network::{Network, ReceiptResponse}; +use alloy_provider::PendingTransactionBuilder; +use eyre::ensure; +use std::future::Future; + +/// Extension trait for transactions sent through an alloy provider. +pub trait PendingTransactionExt { + /// Waits for the receipt of the transaction and returns it, failing if the transaction + /// reverted. + /// + /// Like [`PendingTransactionBuilder::get_receipt`], this waits until the transaction is + /// included, so the block containing it must be produced, e.g. by + /// [`NodeTestContext::advance_block`](crate::node::NodeTestContext::advance_block) or + /// [dev mining](crate::E2ETestSetupBuilder::with_dev_mining). + fn successful_receipt(self) -> impl Future> + Send; +} + +impl PendingTransactionExt for PendingTransactionBuilder { + async fn successful_receipt(self) -> eyre::Result { + let receipt = self.get_receipt().await?; + ensure!( + receipt.status(), + "transaction {} reverted after using {} gas", + receipt.transaction_hash(), + receipt.gas_used() + ); + Ok(receipt) + } +} + +/// Waits for the receipts of all transactions in order, failing if any of them reverted. +/// +/// See [`PendingTransactionExt::successful_receipt`]. +pub async fn await_successful_receipts( + txs: impl IntoIterator>, +) -> eyre::Result> { + let mut receipts = Vec::new(); + for tx in txs { + receipts.push(tx.successful_receipt().await?); + } + Ok(receipts) +} diff --git a/crates/e2e-test-utils/src/rpc.rs b/crates/e2e-test-utils/src/rpc.rs index ff030c390b9..bc40b9235a9 100644 --- a/crates/e2e-test-utils/src/rpc.rs +++ b/crates/e2e-test-utils/src/rpc.rs @@ -8,8 +8,8 @@ use reth_node_builder::{rpc::RpcRegistry, NodeTypes}; use reth_provider::BlockReader; use reth_rpc_api::DebugApiServer; use reth_rpc_eth_api::{ - helpers::{EthApiSpec, EthTransactions, TraceExt}, - EthApiTypes, + helpers::{EthApiSpec, EthTransactions, LoadReceipt, TraceExt}, + EthApiTypes, RpcReceipt, }; #[expect(missing_debug_implementations)] @@ -30,13 +30,41 @@ where eth_api.send_raw_transaction(raw_tx).await } - /// Retrieves a transaction envelope by its hash + /// Retrieves an Ethereum transaction envelope by its hash, including its blob sidecar if the + /// transaction is still pooled. pub async fn envelope_by_hash( &self, hash: B256, ) -> eyre::Result>> { - let tx = self.inner.debug_api().raw_transaction(hash).await?.unwrap(); - let tx = tx.to_vec(); - Ok(EthereumTxEnvelope::decode_2718(&mut tx.as_ref()).unwrap()) + self.decoded_transaction_by_hash(hash).await + } + + /// Retrieves the raw transaction with the given hash via `debug_getRawTransaction` and decodes + /// it as `T`. + /// + /// Returns an error if the node does not know the transaction. + pub async fn decoded_transaction_by_hash( + &self, + hash: B256, + ) -> eyre::Result { + let tx = self + .inner + .debug_api() + .raw_transaction(hash) + .await? + .ok_or_else(|| eyre::eyre!("transaction {hash} not found"))?; + Ok(T::decode_2718_exact(&tx)?) + } + + /// Returns the RPC receipt of the transaction with the given hash, if it is included in a + /// canonical block. + pub async fn transaction_receipt( + &self, + hash: B256, + ) -> Result>, EthApi::Error> + where + EthApi: LoadReceipt + 'static, + { + self.inner.eth_api().transaction_receipt(hash).await } } diff --git a/crates/e2e-test-utils/src/setup_builder.rs b/crates/e2e-test-utils/src/setup_builder.rs index b4ccbf1f2f5..d891e4eb183 100644 --- a/crates/e2e-test-utils/src/setup_builder.rs +++ b/crates/e2e-test-utils/src/setup_builder.rs @@ -1,229 +1,452 @@ -//! Builder for configuring and creating test node setups. +//! Builder for configuring and launching test node setups. //! //! This module provides a flexible builder API for setting up test nodes with custom //! configurations through closures that modify `NodeConfig` and `TreeConfig`. -use crate::{node::NodeTestContext, wallet::Wallet, NodeBuilderHelper, NodeHelperType, TmpDB}; -use futures_util::future::TryJoinAll; -use reth_chainspec::EthChainSpec; +use crate::{ + eth_payload_attributes, node::NodeTestContext, test_chain_spec, wallet::Wallet, Adapter, + NodeBuilderHelper, NodeHelperType, TmpNodeAdapter, +}; +use eyre::ensure; +use futures_util::future::{BoxFuture, TryJoinAll}; +use reth_chainspec::{ChainSpec, EthChainSpec, EthereumHardfork}; +use reth_node_api::{PayloadAttrTy, TreeConfig}; use reth_node_builder::{ - EngineNodeLauncher, NodeBuilder, NodeConfig, NodeHandle, NodeTypes, NodeTypesWithDBAdapter, - PayloadTypes, + DebugNode, DebugNodeLauncher, EngineNodeLauncher, Node, NodeBuilder, NodeBuilderWithComponents, + NodeConfig, NodeHandle, +}; +use reth_node_core::{ + args::{DatadirArgs, DiscoveryArgs, NetworkArgs, PruningArgs, RpcServerArgs, StorageArgs}, + dirs::{ChainPath, DataDirPath, MaybePlatformPath}, }; -use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs, StorageArgs}; use reth_primitives_traits::AlloyBlockHeader; use reth_provider::providers::BlockchainProvider; use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::Runtime; -use std::sync::Arc; +use std::{path::PathBuf, sync::Arc, time::Duration}; use tracing::{span, Instrument, Level}; -/// Type alias for tree config modifier closure -type TreeConfigModifier = - Box reth_node_api::TreeConfig + Send + Sync>; - -/// Type alias for node config modifier closure -type NodeConfigModifier = Box) -> NodeConfig + Send + Sync>; - -/// Builder for configuring and creating test node setups. +/// Builder for configuring and launching test node setups. /// -/// This builder allows customizing test node configurations through closures that -/// modify `NodeConfig` and `TreeConfig`. It avoids code duplication by centralizing -/// the node creation logic. -pub struct E2ETestSetupBuilder -where - N: NodeBuilderHelper, - F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadAttributes - + Send - + Sync - + Copy - + 'static, -{ +/// By default, the nodes: +/// - run on a shared [`Runtime::test`] runtime, +/// - build payloads with [`eth_payload_attributes`] for the hardforks active in the chain spec, +/// - have discovery disabled, use unused ports and serve all RPC modules except `testing` over +/// HTTP, +/// - report an idle sync state from startup, so they gossip transactions before their first block, +/// - are connected to each other. +/// +/// Once launched, each node receives a forkchoice update that makes genesis the head, safe and +/// finalized block, unless [dev mining](Self::with_dev_mining) is enabled. +/// +/// Configuration and tree configuration modifiers are applied in the order they are added. +/// +/// Use [`E2ETestSetupExt::test_setup_for`] to set up nodes on the [`test_chain_spec`] at a +/// hardfork, or [`E2ETestSetupExt::test_setup`] for any other chain spec: +/// +/// ```ignore +/// let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) +/// .with_tree_config_modifier(|config| config.with_persistence_threshold(0)) +/// .build_single() +/// .await?; +/// ``` +pub struct E2ETestSetupBuilder { num_nodes: usize, chain_spec: Arc, - attributes_generator: F, + runtime: Option, + attributes_generator: Option>, connect_nodes: bool, - tree_config_modifier: Option, - node_config_modifier: Option>, + tree_config_modifiers: Vec, + node_config_modifiers: Vec>, storage_v2: bool, + dev_launcher: Option>, + dev_payload_attributes: Option>, } -impl E2ETestSetupBuilder -where - N: NodeBuilderHelper, - F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadAttributes - + Send - + Sync - + Copy - + 'static, -{ - /// Creates a new builder with the required parameters. - pub fn new(num_nodes: usize, chain_spec: Arc, attributes_generator: F) -> Self { +impl E2ETestSetupBuilder { + /// Creates a new builder for `num_nodes` nodes of the given chain. + pub fn new(num_nodes: usize, chain_spec: Arc) -> Self { Self { num_nodes, chain_spec, - attributes_generator, + runtime: None, + attributes_generator: None, connect_nodes: true, - tree_config_modifier: None, - node_config_modifier: None, + tree_config_modifiers: Vec::new(), + node_config_modifiers: Vec::new(), storage_v2: StorageArgs::default().v2, + dev_launcher: None, + dev_payload_attributes: None, } } + /// Sets the number of nodes to launch. + pub const fn with_num_nodes(mut self, num_nodes: usize) -> Self { + self.num_nodes = num_nodes; + self + } + + /// Launches the nodes on the given runtime instead of a new [`Runtime::test`]. + /// + /// This lets multiple setups, or other components of a test, share the same tokio handle and + /// rayon pools. Note that the tasks of the nodes are only shut down once all handles to the + /// runtime are dropped, not when the nodes are dropped. + pub fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + + /// Sets the generator for the payload attributes of the payloads built by the test nodes. + /// + /// The generator is called with the timestamp of the next payload. Defaults to + /// [`eth_payload_attributes`] for the chain spec of the setup. + pub fn with_attributes_generator(mut self, generator: G) -> Self + where + G: Fn(u64) -> PayloadAttrTy + Send + Sync + 'static, + { + self.attributes_generator = Some(Arc::new(generator)); + self + } + /// Sets whether nodes should be interconnected (default: true). pub const fn with_connect_nodes(mut self, connect_nodes: bool) -> Self { self.connect_nodes = connect_nodes; self } - /// Sets a modifier function for the tree configuration. + /// Adds a modifier for the tree configuration. /// - /// The closure receives the base tree config and returns a modified version. + /// The closure receives the current tree config and returns a modified version. The base + /// config is the default config with a small cross block cache. pub fn with_tree_config_modifier(mut self, modifier: G) -> Self where - G: Fn(reth_node_api::TreeConfig) -> reth_node_api::TreeConfig + Send + Sync + 'static, + G: Fn(TreeConfig) -> TreeConfig + Send + Sync + 'static, { - self.tree_config_modifier = Some(Box::new(modifier)); + self.tree_config_modifiers.push(Box::new(modifier)); self } - /// Sets a modifier function for the node configuration. + /// Adds a modifier for the node configuration. /// - /// The closure receives the base node config and returns a modified version. + /// The closure receives the current node config and returns a modified version. pub fn with_node_config_modifier(mut self, modifier: G) -> Self where G: Fn(NodeConfig) -> NodeConfig + Send + Sync + 'static, { - self.node_config_modifier = Some(Box::new(modifier)); + self.node_config_modifiers.push(Box::new(modifier)); self } + /// Adds a modifier for the RPC server arguments. + /// + /// The closure receives the current arguments, which serve all modules except `testing` over + /// HTTP on an unused port. + pub fn with_rpc_modifier(self, modifier: G) -> Self + where + G: Fn(RpcServerArgs) -> RpcServerArgs + Send + Sync + 'static, + { + self.with_node_config_modifier(move |mut config| { + config.rpc = modifier(config.rpc); + config + }) + } + /// Sets the pruning arguments for the test nodes. - pub fn with_pruning(self, pruning: reth_node_core::args::PruningArgs) -> Self { + pub fn with_pruning(self, pruning: PruningArgs) -> Self { self.with_node_config_modifier(move |config| config.with_pruning(pruning.clone())) } /// Sets whether nodes use the v2 storage layout (`--storage.v2`), which routes tx hashes, /// history indices, etc. to `RocksDB` and changesets/senders to static files. /// - /// Defaults to the node's `--storage.v2` default. The node config modifier runs afterwards - /// and can still override it. + /// Defaults to the node's `--storage.v2` default. Node config modifiers run afterwards and can + /// still override it. pub const fn with_storage_v2(mut self, storage_v2: bool) -> Self { self.storage_v2 = storage_v2; self } - /// Builds and launches the test nodes. - pub async fn build( - self, - ) -> eyre::Result<( - Vec>>>, - Wallet, - )> { - let runtime = Runtime::test(); - - let network_config = NetworkArgs { - discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, - ..NetworkArgs::default() - }; + /// Sets whether the nodes run in dev mode (`--dev`). + /// + /// Unlike [dev mining](Self::with_dev_mining), this only sets the dev flag and does not start a + /// local miner. + pub fn with_dev_mode(self, dev: bool) -> Self { + self.with_node_config_modifier(move |config| config.set_dev(dev)) + } + + /// Launches the node in dev mode with a local miner that builds a block every `block_time`, or + /// as soon as a transaction is pending if `None`. + /// + /// Dev mining is limited to a single node, since every node would mine its own chain. The local + /// miner drives forkchoice and payload building, so the node does not receive the initial + /// forkchoice update to genesis, and the block producing helpers of [`NodeTestContext`], e.g. + /// [`NodeTestContext::advance_block`] and [`NodeTestContext::update_forkchoice`], must not be + /// used. Use [`Self::map_dev_payload_attributes`] to customize the payload attributes of the + /// mined blocks. + pub fn with_dev_mining(mut self, block_time: Option) -> Self + where + N: DebugNode>, + { + self.dev_launcher = Some(|args| Box::pin(launch_dev_node::(args))); + self.with_node_config_modifier(move |mut config| { + config.dev.dev = true; + config.dev.block_time = block_time; + config + }) + } + + /// Maps the payload attributes of the blocks built by the local miner of + /// [dev mining](Self::with_dev_mining) nodes, e.g. to set the fee recipient. + /// + /// Mappers are applied in the order they are added. Has no effect unless dev mining is + /// enabled. + pub fn map_dev_payload_attributes(mut self, map: G) -> Self + where + G: Fn(PayloadAttrTy) -> PayloadAttrTy + Send + Sync + 'static, + { + self.dev_payload_attributes = Some(match self.dev_payload_attributes.take() { + Some(prev) => Arc::new(move |attributes| map(prev(attributes))), + None => Arc::new(map), + }); + self + } - // Apply tree config modifier if present, with test-appropriate defaults - let base_tree_config = - reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024); - let tree_config = if let Some(modifier) = self.tree_config_modifier { - modifier(base_tree_config) - } else { - base_tree_config - }; + /// Builds and launches the test nodes. + pub async fn build(self) -> eyre::Result<(Vec>, Wallet)> { + ensure!( + self.dev_launcher.is_none() || self.num_nodes == 1, + "dev mining requires a single node setup, got {} nodes", + self.num_nodes + ); + let dev_mining = self.dev_launcher.is_some(); + let launch = self.dev_launcher.unwrap_or(|args| Box::pin(launch_test_node::(args))); + let runtime = self.runtime.unwrap_or_else(Runtime::test); + let attributes_generator = self.attributes_generator.unwrap_or_else(|| { + let chain_spec = self.chain_spec.clone(); + Arc::new(move |timestamp| eth_payload_attributes(&chain_spec, timestamp).into()) + }); + let tree_config = self + .tree_config_modifiers + .iter() + .fold(test_tree_config(), |config, modifier| modifier(config)); let mut nodes = (0..self.num_nodes) .map(async |idx| { - // Create base node config - let base_config = NodeConfig::new(self.chain_spec.clone()) - .with_network(network_config.clone()) - .with_unused_ports() - .with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All), - ) - .with_storage(StorageArgs { v2: self.storage_v2 }); - - // Apply node config modifier if present - let node_config = if let Some(modifier) = &self.node_config_modifier { - modifier(base_config) - } else { - base_config - }; - - let span = span!(Level::INFO, "node", idx); - let node = N::default(); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) - .testing_node(runtime.clone()) - .with_types_and_provider::>() - .with_components(node.components_builder()) - .with_add_ons(node.add_ons()) - .launch_with_fn(|builder| { - let launcher = EngineNodeLauncher::new( - builder.task_executor().clone(), - builder.config().datadir(), - tree_config.clone(), - ); - builder.launch_with(launcher) - }) - .instrument(span) - .await?; - - let node = NodeTestContext::new(node, self.attributes_generator).await?; - let genesis_number = self.chain_spec.genesis_header().number(); - let genesis = node.block_hash(genesis_number); - node.update_forkchoice(genesis, genesis).await?; + let node_config = self.node_config_modifiers.iter().fold( + test_node_config(self.chain_spec.clone()) + .with_storage(StorageArgs { v2: self.storage_v2 }), + |config, modifier| modifier(config), + ); + // The local miner of dev nodes drives forkchoice, unless a modifier disabled dev + // mode. + let mines = dev_mining && node_config.dev.dev; + let node = launch(LaunchArgs { + node_config, + runtime: runtime.clone(), + tree_config: tree_config.clone(), + datadir: reth_db::test_utils::tempdir_path(), + attributes_generator: attributes_generator.clone(), + dev_payload_attributes: self.dev_payload_attributes.clone(), + }) + .instrument(span!(Level::INFO, "node", idx)) + .await?; + + if !mines { + let genesis = node.block_hash(self.chain_spec.genesis_header().number()); + node.update_forkchoice(genesis, genesis).await?; + } eyre::Ok(node) }) .collect::>() .await?; - for idx in 0..self.num_nodes { - let (prev, current) = nodes.split_at_mut(idx); - let current = current.first_mut().unwrap(); - // Connect nodes if requested - if self.connect_nodes { - if let Some(prev_idx) = idx.checked_sub(1) { - prev[prev_idx].connect(current).await; - } + if self.connect_nodes { + for idx in 1..self.num_nodes { + let (prev, current) = nodes.split_at_mut(idx); + prev[idx - 1].connect(&mut current[0]).await; + } - // Connect last node with the first if there are more than two - if idx + 1 == self.num_nodes && - self.num_nodes > 2 && - let Some(first) = prev.first_mut() - { - current.connect(first).await; - } + // Connect the last node with the first if there are more than two. + if self.num_nodes > 2 { + let (first, rest) = nodes.split_at_mut(1); + rest.last_mut().unwrap().connect(&mut first[0]).await; } } Ok((nodes, Wallet::default().with_chain_id(self.chain_spec.chain().into()))) } + + /// Builds and launches a single test node. + /// + /// Returns an error if the builder was not configured with exactly one node. + pub async fn build_single(self) -> eyre::Result<(NodeHelperType, Wallet)> { + ensure!(self.num_nodes == 1, "expected a single node setup, got {} nodes", self.num_nodes); + let (mut nodes, wallet) = self.build().await?; + Ok((nodes.pop().expect("one node was launched"), wallet)) + } } -impl std::fmt::Debug for E2ETestSetupBuilder -where - N: NodeBuilderHelper, - F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadAttributes - + Send - + Sync - + Copy - + 'static, -{ +impl std::fmt::Debug for E2ETestSetupBuilder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("E2ETestSetupBuilder") .field("num_nodes", &self.num_nodes) + .field("runtime", &self.runtime) .field("connect_nodes", &self.connect_nodes) - .field("tree_config_modifier", &self.tree_config_modifier.as_ref().map(|_| "")) - .field("node_config_modifier", &self.node_config_modifier.as_ref().map(|_| "")) + .field("tree_config_modifiers", &self.tree_config_modifiers.len()) + .field("node_config_modifiers", &self.node_config_modifiers.len()) .field("storage_v2", &self.storage_v2) + .field("dev_mining", &self.dev_launcher.is_some()) .finish_non_exhaustive() } } + +/// Extension trait to create an [`E2ETestSetupBuilder`] from a node type. +pub trait E2ETestSetupExt: NodeBuilderHelper { + /// Returns an [`E2ETestSetupBuilder`] for `num_nodes` nodes of this type. + fn test_setup(num_nodes: usize, chain_spec: Arc) -> E2ETestSetupBuilder { + E2ETestSetupBuilder::new(num_nodes, chain_spec) + } + + /// Returns an [`E2ETestSetupBuilder`] for a single node of this type on the + /// [`test_chain_spec`] with every hardfork up to and including `fork` active at genesis. + /// + /// Use [`E2ETestSetupBuilder::with_num_nodes`] to launch more nodes. + fn test_setup_for(fork: EthereumHardfork) -> E2ETestSetupBuilder + where + Self::ChainSpec: From, + { + let chain_spec = Arc::unwrap_or_clone(test_chain_spec(fork)); + E2ETestSetupBuilder::new(1, Arc::new(chain_spec.into())) + } +} + +impl E2ETestSetupExt for N {} + +/// Closure that modifies the tree configuration of the test nodes. +type TreeConfigModifier = Box TreeConfig + Send + Sync>; + +/// Closure that modifies the node configuration of each test node. +type NodeConfigModifier = Box) -> NodeConfig + Send + Sync>; + +/// Closure that generates the payload attributes for a given timestamp. +pub(crate) type AttributesGenerator = Arc PayloadAttrTy + Send + Sync>; + +/// Closure that maps payload attributes. +type PayloadAttributesMapper = Arc) -> PayloadAttrTy + Send + Sync>; + +/// Builder of a test node that is ready to be launched. +type TestNodeBuilder = NodeBuilderWithComponents< + TmpNodeAdapter, + >>::ComponentsBuilder, + >>::AddOns, +>; + +/// Function that launches a single test node. +type NodeLauncher = fn(LaunchArgs) -> BoxFuture<'static, eyre::Result>>; + +/// Arguments for launching a single test node. +pub(crate) struct LaunchArgs { + /// The node configuration. + pub(crate) node_config: NodeConfig, + /// The runtime to launch the node on. + pub(crate) runtime: Runtime, + /// The engine tree configuration. + pub(crate) tree_config: TreeConfig, + /// The datadir of the node. + pub(crate) datadir: PathBuf, + /// Generator for the payload attributes of the payloads built by the test context. + pub(crate) attributes_generator: AttributesGenerator, + /// Mapper for the payload attributes of the local miner in dev mode. + pub(crate) dev_payload_attributes: Option>, +} + +/// Returns the base tree configuration of test nodes. +pub(crate) fn test_tree_config() -> TreeConfig { + TreeConfig::default().with_cross_block_cache_size(1024 * 1024) +} + +/// Returns the base configuration of a test node. +/// +/// Discovery is disabled, all ports are unused, all RPC modules except `testing` are served over +/// HTTP and the node reports an idle sync state from startup. +pub(crate) fn test_node_config(chain_spec: Arc) -> NodeConfig { + let mut config = NodeConfig::new(chain_spec) + .with_network(NetworkArgs { + discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, + ..NetworkArgs::default() + }) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ); + // Nodes otherwise report that they are syncing until their first canonical block, which + // e.g. stops transaction gossip. + config.debug.startup_sync_state_idle = true; + config +} + +/// Launches a test node with the engine launcher. +pub(crate) async fn launch_test_node( + args: LaunchArgs, +) -> eyre::Result> { + let LaunchArgs { node_config, runtime, tree_config, datadir, attributes_generator, .. } = args; + let (builder, datadir) = test_node_builder::(node_config, datadir); + let NodeHandle { node, node_exit_future: _ } = + builder.launch_with(EngineNodeLauncher::new(runtime, datadir, tree_config)).await?; + + NodeTestContext::new(node, move |timestamp| attributes_generator(timestamp)).await +} + +/// Launches a test node with the debug launcher, which runs a local miner in dev mode. +async fn launch_dev_node(args: LaunchArgs) -> eyre::Result> +where + N: NodeBuilderHelper + DebugNode>, +{ + let LaunchArgs { + node_config, + runtime, + tree_config, + datadir, + attributes_generator, + dev_payload_attributes, + } = args; + let (builder, datadir) = test_node_builder::(node_config, datadir); + let launch = builder.launch_with(DebugNodeLauncher::new(EngineNodeLauncher::new( + runtime, + datadir, + tree_config, + ))); + let launch = match dev_payload_attributes { + Some(map) => launch.map_debug_payload_attributes(move |attributes| map(attributes)), + None => launch, + }; + let NodeHandle { node, node_exit_future: _ } = launch.await?; + + NodeTestContext::new(node, move |timestamp| attributes_generator(timestamp)).await +} + +/// Returns the builder of a test node with a temporary database in `datadir`, and the resolved +/// datadir of the node. +/// +/// The datadir is removed when the node is dropped. +fn test_node_builder( + node_config: NodeConfig, + datadir: PathBuf, +) -> (TestNodeBuilder, ChainPath) { + let datadir_args = + DatadirArgs { datadir: MaybePlatformPath::from(datadir), ..node_config.datadir.clone() }; + let node_config = node_config.with_datadir_args(datadir_args); + let datadir = node_config.datadir(); + let database = reth_db::test_utils::create_test_rw_db_with_datadir(datadir.data_dir()); + let node = N::default(); + let builder = NodeBuilder::new(node_config) + .with_database(database) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(node.add_ons()); + (builder, datadir) +} diff --git a/crates/e2e-test-utils/src/setup_import.rs b/crates/e2e-test-utils/src/setup_import.rs index 775ec1e2b3d..60c4d4aa5c3 100644 --- a/crates/e2e-test-utils/src/setup_import.rs +++ b/crates/e2e-test-utils/src/setup_import.rs @@ -1,24 +1,28 @@ //! Setup utilities for importing RLP chain data before starting nodes. -use crate::{node::NodeTestContext, NodeHelperType, Wallet}; -use alloy_rpc_types_engine::PayloadAttributes; +use crate::{ + eth_payload_attributes, + setup_builder::{launch_test_node, test_node_config, LaunchArgs}, + wallet::Wallet, + NodeHelperType, +}; +use eyre::WrapErr; use reth_chainspec::ChainSpec; -use reth_cli_commands::import_core::{import_blocks_from_file, ImportConfig}; +use reth_cli_commands::import_core::{import_blocks_from_file, ImportConfig, ImportResult}; use reth_config::Config; use reth_db::DatabaseEnv; use reth_node_api::{NodeTypesWithDBAdapter, TreeConfig}; -use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; -use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs, StorageArgs}; +use reth_node_core::args::StorageArgs; use reth_node_ethereum::EthereumNode; use reth_provider::{ - providers::BlockchainProvider, DatabaseProviderFactory, ProviderFactory, StageCheckpointReader, - StaticFileProviderFactory, + providers::{RocksDBProvider, StaticFileProvider}, + DatabaseProviderFactory, ProviderFactory, StageCheckpointReader, StorageSettings, }; -use reth_rpc_server_types::RpcModuleSelection; use reth_stages_types::StageId; +use reth_tasks::Runtime; use std::{path::Path, sync::Arc}; use tempfile::TempDir; -use tracing::{debug, info, span, Level}; +use tracing::{debug, info, span, Instrument, Level}; /// Setup result containing nodes and temporary directories that must be kept alive pub struct ChainImportResult { @@ -48,15 +52,16 @@ impl std::fmt::Debug for ChainImportResult { /// 3. Starts the nodes with the pre-populated database /// 4. Returns the running nodes ready for testing /// +/// The import and all nodes share a single [`Runtime::test`] runtime, and the nodes build payloads +/// with [`eth_payload_attributes`] for the given chain spec. `storage_v2` selects the storage +/// layout (`--storage.v2`) for both the imported database and the launched nodes. +/// /// Note: This function is currently specific to `EthereumNode` because the import process /// uses Ethereum-specific consensus and block format. It can be made generic in the future /// by abstracting the import process. /// It uses `NoopConsensus` during import to bypass validation checks like gas limit constraints, /// which allows importing test chains that may not strictly conform to mainnet consensus rules. The /// nodes themselves still run with proper consensus when started. -/// -/// `storage_v2` selects the storage layout (`--storage.v2`) for both the imported database and the -/// launched nodes. pub async fn setup_engine_with_chain_import( num_nodes: usize, chain_spec: Arc, @@ -64,190 +69,61 @@ pub async fn setup_engine_with_chain_import( storage_v2: bool, tree_config: TreeConfig, rlp_path: &Path, - attributes_generator: impl Fn(u64) -> PayloadAttributes + Send + Sync + Copy + 'static, ) -> eyre::Result { - let runtime = reth_tasks::Runtime::test(); - - let network_config = NetworkArgs { - discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, - ..NetworkArgs::default() - }; + let runtime = Runtime::test(); - // Create nodes with imported data - let mut nodes: Vec> = Vec::with_capacity(num_nodes); - let mut temp_dirs = Vec::with_capacity(num_nodes); // Keep temp dirs alive + // Create nodes with imported data. + let mut nodes = Vec::with_capacity(num_nodes); + // Keep the temp dirs alive for the lifetime of the nodes. + let mut temp_dirs = Vec::with_capacity(num_nodes); for idx in 0..num_nodes { - // Create a temporary datadir for this node + // Create a temporary datadir for this node. let temp_dir = TempDir::new()?; let datadir = temp_dir.path().to_path_buf(); - - let mut node_config = NodeConfig::new(chain_spec.clone()) - .with_network(network_config.clone()) - .with_unused_ports() - .with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All), - ) - .set_dev(is_dev) - .with_storage(StorageArgs { v2: storage_v2 }); - - // Set the datadir - node_config.datadir.datadir = - reth_node_core::dirs::MaybePlatformPath::from(datadir.clone()); debug!(target: "e2e::import", "Node {idx} datadir: {datadir:?}"); let span = span!(Level::INFO, "node", idx); - let _enter = span.enter(); - - // First, import the chain data into this datadir - info!(target: "test", "Importing chain data from {:?} for node {} into {:?}", rlp_path, idx, datadir); - - // Create database path and static files path - let db_path = datadir.join("db"); - let static_files_path = datadir.join("static_files"); - let rocksdb_dir_path = datadir.join("rocksdb"); - - // Initialize the database using init_db (same as CLI import command) - let db_args = reth_node_core::args::DatabaseArgs::default().database_args(); - let db = reth_db::init_db(&db_path, db_args)?; + let node_config = test_node_config(chain_spec.clone()) + .set_dev(is_dev) + .with_storage(StorageArgs { v2: storage_v2 }); - // Create a provider factory with the initialized database (use regular DB, not - // TempDatabase) We need to specify the node types properly for the adapter - let provider_factory = - ProviderFactory::>::new( - db.clone(), - chain_spec.clone(), - reth_provider::providers::StaticFileProvider::read_write( - static_files_path.clone(), - )?, - reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path) - .with_default_tables() - .build() - .unwrap(), - reth_tasks::Runtime::test(), - )?; - - // Initialize genesis with the node's storage settings, which the launched node then reads - // back from the database. - reth_db_common::init::init_genesis_with_settings( - &provider_factory, + // First, import the chain data into this datadir. + import_chain( + &datadir, + chain_spec.clone(), node_config.storage_settings(), - )?; - - // Import the chain data - // Use no_state to skip state validation for test chains - let import_config = ImportConfig::default(); - let config = Config::default(); - - // Create EVM and consensus for Ethereum - let evm_config = reth_node_ethereum::EthEvmConfig::new(chain_spec.clone()); - // Use NoopConsensus to skip gas limit validation for test imports - let consensus = reth_consensus::noop::NoopConsensus::arc(); - - let result = import_blocks_from_file( rlp_path, - import_config, - provider_factory.clone(), - &config, - evm_config, - consensus, runtime.clone(), ) - .await?; - - info!( - target: "test", - "Imported {} blocks and {} transactions for node {}", - result.total_imported_blocks, - result.total_imported_txns, - idx - ); - - debug!(target: "e2e::import", - "Import result for node {}: decoded {} blocks, imported {} blocks, complete: {}", - idx, - result.total_decoded_blocks, - result.total_imported_blocks, - result.is_complete() - ); - - if result.total_decoded_blocks != result.total_imported_blocks { - debug!(target: "e2e::import", - "Import block count mismatch: decoded {} != imported {}", - result.total_decoded_blocks, result.total_imported_blocks - ); - return Err(eyre::eyre!("Chain import block count mismatch for node {}", idx)); - } - - if result.total_decoded_txns != result.total_imported_txns { - debug!(target: "e2e::import", - "Import transaction count mismatch: decoded {} != imported {}", - result.total_decoded_txns, result.total_imported_txns - ); - return Err(eyre::eyre!("Chain import transaction count mismatch for node {}", idx)); - } - - // Verify the database was properly initialized by checking stage checkpoints - { - let provider = provider_factory.database_provider_ro()?; - let headers_checkpoint = provider.get_stage_checkpoint(StageId::Headers)?; - if headers_checkpoint.is_none() { - return Err(eyre::eyre!("Headers stage checkpoint is missing after import!")); - } - debug!(target: "e2e::import", "Headers stage checkpoint after import: {headers_checkpoint:?}"); - drop(provider); - } - - // IMPORTANT: We need to properly flush and close the static files provider - // The static files provider may have open file handles that need to be closed - // before we can reopen the database in the node launcher - { - let static_file_provider = provider_factory.static_file_provider(); - // This will ensure all static file writers are properly closed - drop(static_file_provider); - } - - // Close all database handles to release locks before launching the node - drop(provider_factory); - drop(db); - - // Give the OS a moment to release file locks - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + .instrument(span.clone()) + .await + .wrap_err_with(|| format!("chain import failed for node {idx}"))?; - // Now launch the node with the pre-populated datadir + // Now launch the node with the pre-populated datadir. debug!(target: "e2e::import", "Launching node with datadir: {:?}", datadir); - // Use the testing_node_with_datadir method which properly handles opening existing - // databases - let node = EthereumNode::default(); - - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node_with_datadir(runtime.clone(), datadir.clone()) - .with_types_and_provider::>() - .with_components(node.components_builder()) - .with_add_ons(node.add_ons()) - .launch_with_fn(|builder| { - let launcher = EngineNodeLauncher::new( - builder.task_executor().clone(), - builder.config().datadir(), - tree_config.clone(), - ); - builder.launch_with(launcher) - }) - .await?; - - let node_ctx = NodeTestContext::new(node, attributes_generator).await?; - - nodes.push(node_ctx); - temp_dirs.push(temp_dir); // Keep temp dir alive + let chain_spec_for_attributes = chain_spec.clone(); + let node = launch_test_node::(LaunchArgs { + node_config, + runtime: runtime.clone(), + tree_config: tree_config.clone(), + datadir, + attributes_generator: Arc::new(move |timestamp| { + eth_payload_attributes(&chain_spec_for_attributes, timestamp) + }), + dev_payload_attributes: None, + }) + .instrument(span) + .await?; + + nodes.push(node); + temp_dirs.push(temp_dir); } Ok(ChainImportResult { nodes, - wallet: crate::Wallet::default().with_chain_id(chain_spec.chain.id()), + wallet: Wallet::default().with_chain_id(chain_spec.chain.id()), _temp_dirs: temp_dirs, }) } @@ -275,183 +151,110 @@ pub fn load_forkchoice_state(path: &Path) -> eyre::Result, + storage_settings: StorageSettings, + rlp_path: &Path, + runtime: Runtime, +) -> eyre::Result { + info!(target: "test", "Importing chain data from {:?} into {:?}", rlp_path, datadir); + + // Initialize the database using init_db, same as the CLI import command. + let db_args = reth_node_core::args::DatabaseArgs::default().database_args(); + let db = reth_db::init_db(datadir.join("db"), db_args)?; + + // Create a provider factory with the initialized database, not a `TempDatabase`. + let provider_factory = + ProviderFactory::>::new( + db.clone(), + chain_spec.clone(), + StaticFileProvider::read_write(datadir.join("static_files"))?, + RocksDBProvider::builder(datadir.join("rocksdb")).with_default_tables().build()?, + runtime.clone(), + )?; + + // Initialize genesis with the storage settings of the node that later opens the database. + reth_db_common::init::init_genesis_with_settings(&provider_factory, storage_settings)?; + + // Use NoopConsensus to skip gas limit validation for test imports. + let result = import_blocks_from_file( + rlp_path, + ImportConfig::default(), + provider_factory.clone(), + &Config::default(), + reth_node_ethereum::EthEvmConfig::new(chain_spec), + reth_consensus::noop::NoopConsensus::arc(), + runtime, + ) + .await?; + + info!( + target: "test", + "Imported {} blocks and {} transactions", + result.total_imported_blocks, + result.total_imported_txns, + ); + + debug!(target: "e2e::import", + "Import result: decoded {} blocks, imported {} blocks, complete: {}", + result.total_decoded_blocks, + result.total_imported_blocks, + result.is_complete() + ); + + eyre::ensure!( + result.total_decoded_blocks == result.total_imported_blocks, + "block count mismatch: decoded {} != imported {}", + result.total_decoded_blocks, + result.total_imported_blocks + ); + eyre::ensure!( + result.total_decoded_txns == result.total_imported_txns, + "transaction count mismatch: decoded {} != imported {}", + result.total_decoded_txns, + result.total_imported_txns + ); + + // Verify the database was properly initialized by checking stage checkpoints. + let headers_checkpoint = + provider_factory.database_provider_ro()?.get_stage_checkpoint(StageId::Headers)?; + eyre::ensure!(headers_checkpoint.is_some(), "Headers stage checkpoint is missing after import"); + debug!(target: "e2e::import", "Headers stage checkpoint after import: {headers_checkpoint:?}"); + + // Close all database handles to release locks before launching the node. + drop(provider_factory); + drop(db); + + // The header and body downloader tasks spawned on the runtime hold provider factory clones + // and only exit once they are polled after the pipeline was dropped, so yield long enough for + // them to release the database. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + Ok(result) +} + #[cfg(test)] mod tests { use super::*; - use crate::test_rlp_utils::{create_fcu_json, generate_test_blocks, write_blocks_to_rlp}; - use alloy_rpc_types_engine::PayloadAttributes; - use reth_chainspec::{ChainSpecBuilder, MAINNET}; + use crate::{ + test_chain_spec, + test_rlp_utils::{create_fcu_json, generate_test_blocks, write_blocks_to_rlp}, + }; + use reth_chainspec::EthereumHardfork; use reth_db::mdbx::DatabaseArguments; use reth_ethereum_primitives::Block; use reth_primitives_traits::SealedBlock; - use reth_provider::{ - test_utils::MockNodeTypesWithDB, BlockHashReader, BlockNumReader, BlockReaderIdExt, - MetadataProvider, StorageSettings, - }; + use reth_provider::{BlockHashReader, BlockNumReader, BlockReaderIdExt, MetadataProvider}; use std::path::PathBuf; - #[tokio::test] - async fn test_stage_checkpoints_persistence() { - // This test specifically verifies that stage checkpoints are persisted correctly - // when reopening the database - reth_tracing::init_test_tracing(); - - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!("testsuite/assets/genesis.json")).unwrap(), - ) - .london_activated() - .shanghai_activated() - .build(), - ); - - // Generate test blocks - let test_blocks = generate_test_blocks(&chain_spec, 5); - - // Create temporary files for RLP data - let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); - let rlp_path = temp_dir.path().join("test_chain.rlp"); - write_blocks_to_rlp(&test_blocks, &rlp_path).expect("Failed to write RLP data"); - - // Create a persistent datadir that won't be deleted - let datadir = temp_dir.path().join("datadir"); - std::fs::create_dir_all(&datadir).unwrap(); - let db_path = datadir.join("db"); - let static_files_path = datadir.join("static_files"); - let rocksdb_dir_path = datadir.join("rocksdb"); - - // Import the chain - { - let db_args = reth_node_core::args::DatabaseArgs::default().database_args(); - let db = reth_db::init_db(&db_path, db_args).unwrap(); - - let provider_factory: ProviderFactory< - NodeTypesWithDBAdapter, - > = ProviderFactory::new( - db.clone(), - chain_spec.clone(), - reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone()) - .unwrap(), - reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path.clone()) - .with_default_tables() - .build() - .unwrap(), - reth_tasks::Runtime::test(), - ) - .expect("failed to create provider factory"); - - // Initialize genesis - reth_db_common::init::init_genesis(&provider_factory).unwrap(); - - // Import the chain data - let import_config = ImportConfig::default(); - let config = Config::default(); - let evm_config = reth_node_ethereum::EthEvmConfig::new(chain_spec.clone()); - // Use NoopConsensus to skip gas limit validation for test imports - let consensus = reth_consensus::noop::NoopConsensus::arc(); - let runtime = reth_tasks::Runtime::test(); - - let result = import_blocks_from_file( - &rlp_path, - import_config, - provider_factory.clone(), - &config, - evm_config, - consensus, - runtime, - ) - .await - .unwrap(); - - assert_eq!(result.total_decoded_blocks, 5); - assert_eq!(result.total_imported_blocks, 5); - - // Verify stage checkpoints exist - let provider = provider_factory.database_provider_ro().unwrap(); - let headers_checkpoint = provider.get_stage_checkpoint(StageId::Headers).unwrap(); - assert!(headers_checkpoint.is_some(), "Headers checkpoint should exist after import"); - assert_eq!( - headers_checkpoint.unwrap().block_number, - 5, - "Headers checkpoint should be at block 5" - ); - drop(provider); - - // Properly close static files to release all file handles - let static_file_provider = provider_factory.static_file_provider(); - drop(static_file_provider); - - drop(provider_factory); - drop(db); - } - - // Give the OS a moment to release file locks - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - - // Now reopen the database and verify checkpoints are still there - { - let db = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap(); - - let provider_factory: ProviderFactory< - NodeTypesWithDBAdapter, - > = ProviderFactory::new( - db, - chain_spec.clone(), - reth_provider::providers::StaticFileProvider::read_only(static_files_path).unwrap(), - reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path) - .with_default_tables() - .build() - .unwrap(), - reth_tasks::Runtime::test(), - ) - .expect("failed to create provider factory"); - - let provider = provider_factory.database_provider_ro().unwrap(); - - // Check that stage checkpoints are still present - let headers_checkpoint = provider.get_stage_checkpoint(StageId::Headers).unwrap(); - assert!( - headers_checkpoint.is_some(), - "Headers checkpoint should still exist after reopening database" - ); - assert_eq!( - headers_checkpoint.unwrap().block_number, - 5, - "Headers checkpoint should still be at block 5" - ); - - // Verify we can read blocks - let block_5_hash = provider.block_hash(5).unwrap(); - assert!(block_5_hash.is_some(), "Block 5 should exist in database"); - assert_eq!(block_5_hash.unwrap(), test_blocks[4].hash(), "Block 5 hash should match"); - - // Check all stage checkpoints - debug!(target: "e2e::import", "All stage checkpoints after reopening:"); - for stage in StageId::ALL { - let checkpoint = provider.get_stage_checkpoint(stage).unwrap(); - debug!(target: "e2e::import", " Stage {stage:?}: {checkpoint:?}"); - } - } - } - - /// Helper to create test chain spec - fn create_test_chain_spec() -> Arc { - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!("testsuite/assets/genesis.json")).unwrap(), - ) - .london_activated() - .shanghai_activated() - .build(), - ) - } - - /// Helper to setup test blocks and write to RLP - async fn setup_test_blocks_and_rlp( + /// Helper to setup test blocks and write to RLP. + fn setup_test_blocks_and_rlp( chain_spec: &ChainSpec, block_count: u64, temp_dir: &Path, @@ -472,104 +275,79 @@ mod tests { (test_blocks, rlp_path) } + /// Reopens the database in `datadir` that was written by [`import_chain`]. + fn reopen_provider_factory( + datadir: &Path, + chain_spec: Arc, + runtime: Runtime, + ) -> ProviderFactory> { + let db = reth_db::init_db(datadir.join("db"), DatabaseArguments::default()).unwrap(); + ProviderFactory::new( + db, + chain_spec, + StaticFileProvider::read_only(datadir.join("static_files")).unwrap(), + RocksDBProvider::builder(datadir.join("rocksdb")) + .with_default_tables() + .build() + .unwrap(), + runtime, + ) + .expect("failed to create provider factory") + } + #[tokio::test] - async fn test_import_blocks_only() { - // Tests just the block import functionality without full node setup + async fn test_import_blocks_and_reopen() { + // Tests the block import without a node, and that the imported chain and stage + // checkpoints persist when reopening the database. reth_tracing::init_test_tracing(); - let chain_spec = create_test_chain_spec(); + let runtime = Runtime::test(); + let chain_spec = test_chain_spec(EthereumHardfork::Shanghai); let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); - let (test_blocks, rlp_path) = - setup_test_blocks_and_rlp(&chain_spec, 10, temp_dir.path()).await; + let (test_blocks, rlp_path) = setup_test_blocks_and_rlp(&chain_spec, 10, temp_dir.path()); - // Create a test database let datadir = temp_dir.path().join("datadir"); std::fs::create_dir_all(&datadir).unwrap(); - let db_path = datadir.join("db"); - let db_args = reth_node_core::args::DatabaseArgs::default().database_args(); - let db_env = reth_db::init_db(&db_path, db_args).unwrap(); - let db = Arc::new(reth_db::test_utils::TempDatabase::new(db_env, db_path)); - - // Create static files path - let static_files_path = datadir.join("static_files"); - - // Create rocksdb path - let rocksdb_dir_path = datadir.join("rocksdb"); - - // Create a provider factory - let provider_factory: ProviderFactory = ProviderFactory::new( - db.clone(), + let storage_settings = StorageSettings { storage_v2: StorageArgs::default().v2 }; + let result = import_chain( + &datadir, chain_spec.clone(), - reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(), - reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path) - .with_default_tables() - .build() - .unwrap(), - reth_tasks::Runtime::test(), - ) - .expect("failed to create provider factory"); - - // Initialize genesis - reth_db_common::init::init_genesis(&provider_factory).unwrap(); - - // Import the chain data - let import_config = ImportConfig::default(); - let config = Config::default(); - let evm_config = reth_node_ethereum::EthEvmConfig::new(chain_spec.clone()); - // Use NoopConsensus to skip gas limit validation for test imports - let consensus = reth_consensus::noop::NoopConsensus::arc(); - let runtime = reth_tasks::Runtime::test(); - - let result = import_blocks_from_file( + storage_settings, &rlp_path, - import_config, - provider_factory.clone(), - &config, - evm_config, - consensus, - runtime, + runtime.clone(), ) .await .unwrap(); - - debug!(target: "e2e::import", - "Import result: decoded {} blocks, imported {} blocks", - result.total_decoded_blocks, result.total_imported_blocks - ); - - // Verify the import was successful assert_eq!(result.total_decoded_blocks, 10); assert_eq!(result.total_imported_blocks, 10); assert_eq!(result.total_decoded_txns, 0); assert_eq!(result.total_imported_txns, 0); - // Verify we can read the imported blocks + let provider_factory = reopen_provider_factory(&datadir, chain_spec, runtime); let provider = provider_factory.database_provider_ro().unwrap(); - let latest_block = provider.last_block_number().unwrap(); - assert_eq!(latest_block, 10, "Should have imported up to block 10"); - - let block_10_hash = provider.block_hash(10).unwrap().expect("Block 10 should exist"); - assert_eq!(block_10_hash, test_blocks[9].hash(), "Block 10 hash should match"); + assert_eq!(provider.last_block_number().unwrap(), 10); + assert_eq!(provider.block_hash(10).unwrap(), Some(test_blocks[9].hash())); + let headers_checkpoint = provider.get_stage_checkpoint(StageId::Headers).unwrap(); + assert_eq!(headers_checkpoint.map(|checkpoint| checkpoint.block_number), Some(10)); } #[tokio::test] async fn test_import_with_node_integration() { - // Tests the full integration with node setup, forkchoice updates, and syncing + // Tests the full integration with node setup, forkchoice updates, and syncing. reth_tracing::init_test_tracing(); - let chain_spec = create_test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Shanghai); let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); - let (test_blocks, rlp_path) = - setup_test_blocks_and_rlp(&chain_spec, 10, temp_dir.path()).await; + let (test_blocks, rlp_path) = setup_test_blocks_and_rlp(&chain_spec, 10, temp_dir.path()); - // Create FCU data for the tip + // Create FCU data for the tip. let tip = test_blocks.last().expect("Should have generated blocks"); let fcu_path = temp_dir.path().join("test_fcu.json"); std::fs::write(&fcu_path, create_fcu_json(tip).to_string()) .expect("Failed to write FCU data"); for storage_v2 in [false, true] { - // Setup nodes with imported chain + // Setup nodes with imported chain. let result = setup_engine_with_chain_import( 1, chain_spec.clone(), @@ -577,12 +355,11 @@ mod tests { storage_v2, TreeConfig::default(), &rlp_path, - |_| PayloadAttributes::default(), ) .await .expect("Failed to setup nodes with chain import"); - // Load and apply forkchoice state + // Load and apply forkchoice state. let fcu_state = load_forkchoice_state(&fcu_path).expect("Failed to load forkchoice state"); @@ -598,15 +375,15 @@ mod tests { .expect("Failed to read storage settings"); assert_eq!(settings, Some(StorageSettings { storage_v2 })); - // Send forkchoice update to make the imported chain canonical + // Send forkchoice update to make the imported chain canonical. node.update_forkchoice(fcu_state.finalized_block_hash, fcu_state.head_block_hash) .await .expect("Failed to update forkchoice"); - // Wait for the node to sync to the head + // Wait for the node to sync to the head. node.sync_to(fcu_state.head_block_hash).await.expect("Failed to sync to head"); - // Verify the chain tip + // Verify the chain tip. let latest = node .inner .provider diff --git a/crates/e2e-test-utils/src/testsuite/mod.rs b/crates/e2e-test-utils/src/testsuite/mod.rs index 5bf829fd050..b7776ae425e 100644 --- a/crates/e2e-test-utils/src/testsuite/mod.rs +++ b/crates/e2e-test-utils/src/testsuite/mod.rs @@ -15,6 +15,7 @@ pub mod setup; use crate::testsuite::setup::Setup; use alloy_provider::{Provider, ProviderBuilder}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; +use reth_chainspec::ChainSpec; use reth_engine_primitives::ConsensusEngineHandle; use reth_provider::{BlockNumReader, ProviderResult}; use reth_rpc_builder::auth::AuthServerHandle; @@ -391,7 +392,7 @@ where /// Run the test scenario pub async fn run(mut self) -> Result<()> where - N: NodeBuilderHelper, + N: NodeBuilderHelper>, { let mut setup = self.setup.take(); diff --git a/crates/e2e-test-utils/src/testsuite/setup.rs b/crates/e2e-test-utils/src/testsuite/setup.rs index 28a8407ccb9..80b2330e074 100644 --- a/crates/e2e-test-utils/src/testsuite/setup.rs +++ b/crates/e2e-test-utils/src/testsuite/setup.rs @@ -1,14 +1,13 @@ //! Test setup utilities for configuring the initial state. -use crate::{testsuite::Environment, E2ETestSetupBuilder, NodeBuilderHelper}; +use crate::{testsuite::Environment, E2ETestSetupExt, NodeBuilderHelper}; use alloy_eips::BlockNumberOrTag; -use alloy_primitives::B256; -use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; +use alloy_rpc_types_engine::ForkchoiceState; use eyre::{eyre, Result}; use reth_chainspec::ChainSpec; use reth_ethereum_primitives::Block; use reth_network_p2p::sync::{NetworkSyncUpdater, SyncState}; -use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes, TreeConfig}; +use reth_node_api::{EngineTypes, PayloadTypes, TreeConfig}; use reth_node_core::{args::StorageArgs, primitives::RecoveredBlock}; use revm::state::EvmState; use std::{marker::PhantomData, path::Path, sync::Arc}; @@ -182,7 +181,7 @@ where /// Apply the setup to the environment pub async fn apply(&mut self, env: &mut Environment) -> Result<()> where - N: NodeBuilderHelper, + N: NodeBuilderHelper>, { // Note: this future is quite large so we box it Box::pin(self.apply_::(env)).await @@ -191,7 +190,7 @@ where /// Apply the setup to the environment async fn apply_(&mut self, env: &mut Environment) -> Result<()> where - N: NodeBuilderHelper, + N: NodeBuilderHelper>, { // If import_rlp_path is set, use apply_with_import instead if let Some(rlp_path) = self.import_rlp_path.take() { @@ -204,20 +203,16 @@ where self.shutdown_tx = Some(shutdown_tx); let is_dev = self.is_dev; - let node_count = self.network.node_count; let tree_config = self.tree_config.clone(); - let attributes_generator = Self::create_static_attributes_generator::(); - - let result = E2ETestSetupBuilder::::new( - node_count, + let result = N::test_setup( + self.network.node_count, Arc::::new((*chain_spec).clone().into()), - attributes_generator, ) .with_tree_config_modifier(move |base| { tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size()) }) - .with_node_config_modifier(move |config| config.set_dev(is_dev)) + .with_dev_mode(is_dev) .with_storage_v2(self.storage_v2) .with_connect_nodes(self.network.connect_nodes) .build() @@ -261,16 +256,6 @@ where let chain_spec = self.chain_spec.clone().ok_or_else(|| eyre!("Chain specification is required"))?; - let attributes_generator = move |timestamp| PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: alloy_primitives::Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: None, - ..Default::default() - }; - crate::setup_import::setup_engine_with_chain_import( self.network.node_count, chain_spec, @@ -278,31 +263,10 @@ where self.storage_v2, self.tree_config.clone(), rlp_path, - attributes_generator, ) .await } - /// Create a static attributes generator that doesn't capture any instance data - fn create_static_attributes_generator( - ) -> impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadAttributes + Copy + use - where - N: NodeBuilderHelper, - { - move |timestamp| { - PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: alloy_primitives::Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: None, - ..Default::default() - } - .into() - } - } - /// Common finalization logic for both apply methods async fn finalize_setup( &self, diff --git a/crates/e2e-test-utils/src/transaction.rs b/crates/e2e-test-utils/src/transaction.rs index 4d91da5877e..0125203374b 100644 --- a/crates/e2e-test-utils/src/transaction.rs +++ b/crates/e2e-test-utils/src/transaction.rs @@ -134,6 +134,12 @@ impl TransactionTestContext { .unwrap() } + /// Signs an arbitrary [`TransactionRequest`] using the provided wallet, returning the EIP-2718 + /// encoded bytes. + pub async fn sign_tx_bytes(wallet: PrivateKeySigner, tx: TransactionRequest) -> Bytes { + Self::sign_tx(wallet, tx).await.encoded_2718().into() + } + /// Creates a tx with blob sidecar and sign it, returning bytes pub async fn tx_with_blobs_bytes( chain_id: u64, diff --git a/crates/e2e-test-utils/src/wait.rs b/crates/e2e-test-utils/src/wait.rs new file mode 100644 index 00000000000..c0f51af4839 --- /dev/null +++ b/crates/e2e-test-utils/src/wait.rs @@ -0,0 +1,33 @@ +//! Helpers for waiting on conditions of test nodes. + +use eyre::eyre; +use std::{fmt::Display, future::Future, time::Duration}; + +/// Maximum time the wait helpers wait for a condition, e.g. for a node to sync to or commit a +/// block. +pub const WAIT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Interval at which [`poll_until`] polls. +pub const POLL_INTERVAL: Duration = Duration::from_millis(20); + +/// Calls `poll` every [`POLL_INTERVAL`] until it returns a value. +/// +/// Returns an error describing `what` was awaited if the condition is not met within +/// [`WAIT_TIMEOUT`], or the first error returned by `poll`. +pub async fn poll_until(what: impl Display, mut poll: F) -> eyre::Result +where + F: FnMut() -> Fut, + Fut: Future>>, +{ + let wait = async { + loop { + if let Some(value) = poll().await? { + return Ok(value) + } + tokio::time::sleep(POLL_INTERVAL).await; + } + }; + tokio::time::timeout(WAIT_TIMEOUT, wait) + .await + .map_err(|_| eyre!("timed out waiting for {what}"))? +} diff --git a/crates/e2e-test-utils/src/wallet.rs b/crates/e2e-test-utils/src/wallet.rs index 92af590e705..11976ceb3e7 100644 --- a/crates/e2e-test-utils/src/wallet.rs +++ b/crates/e2e-test-utils/src/wallet.rs @@ -1,6 +1,18 @@ +use crate::transaction::TransactionTestContext; +use alloy_eips::BlockId; +use alloy_network::Network; +use alloy_primitives::{Address, Bytes}; +use alloy_provider::Provider; +use alloy_rpc_types_eth::TransactionRequest; use alloy_signer::Signer; use alloy_signer_local::{coins_bip39::English, MnemonicBuilder, PrivateKeySigner}; +/// Mnemonic of the test accounts funded by the [`test_genesis`](crate::test_genesis). +pub const TEST_MNEMONIC: &str = "test test test test test test test test test test test junk"; + +/// Default derivation path prefix of the test accounts, followed by the account index. +const DEFAULT_DERIVATION_PATH: &str = "m/44'/60'/0'/0/"; + /// One of the accounts of the genesis allocations. #[derive(Debug)] pub struct Wallet { @@ -28,31 +40,136 @@ impl Wallet { } fn get_derivation_path(&self) -> &str { - self.derivation_path.as_deref().unwrap_or("m/44'/60'/0'/0/") + self.derivation_path.as_deref().unwrap_or(DEFAULT_DERIVATION_PATH) } /// Generates a list of wallets pub fn wallet_gen(&self) -> Vec { - let builder = MnemonicBuilder::::default().phrase(TEST_MNEMONIC); + (0..self.amount as u32).map(|idx| self.signer(idx)).collect() + } - // use the derivation path - let derivation_path = self.get_derivation_path(); + /// Returns the signer of the test account at `index`, with the chain id of the wallet. + pub fn signer(&self, index: u32) -> PrivateKeySigner { + derive_signer(self.get_derivation_path(), index).with_chain_id(Some(self.chain_id)) + } - let mut wallets = Vec::with_capacity(self.amount); - for idx in 0..self.amount { - let builder = - builder.clone().derivation_path(format!("{derivation_path}{idx}")).unwrap(); - let wallet = builder.build().unwrap().with_chain_id(Some(self.chain_id)); - wallets.push(wallet); - } - wallets + /// Returns the test account at `index`, with the chain id of the wallet and nonce 0. + pub fn account(&self, index: u32) -> TestAccount { + TestAccount::new(self.signer(index), self.chain_id) } } -const TEST_MNEMONIC: &str = "test test test test test test test test test test test junk"; - impl Default for Wallet { fn default() -> Self { Self::new(1) } } + +/// Returns the signer of the test account at `index`, derived from [`TEST_MNEMONIC`]. +pub fn test_signer(index: u32) -> PrivateKeySigner { + derive_signer(DEFAULT_DERIVATION_PATH, index) +} + +/// Derives the signer at `index` below `derivation_path` from [`TEST_MNEMONIC`]. +fn derive_signer(derivation_path: &str, index: u32) -> PrivateKeySigner { + MnemonicBuilder::::default() + .phrase(TEST_MNEMONIC) + .derivation_path(format!("{derivation_path}{index}")) + .expect("valid derivation path") + .build() + .expect("valid test mnemonic") +} + +/// A test account that tracks its nonce, e.g. one of the accounts funded by the +/// [`test_genesis`](crate::test_genesis). +/// +/// [`Self::sign_tx_bytes`] fills the nonce, chain id, gas limit and fees of transaction requests +/// that do not set them, so tests only need to set the fields they care about. +#[derive(Debug, Clone)] +pub struct TestAccount { + signer: PrivateKeySigner, + chain_id: u64, + nonce: u64, +} + +impl TestAccount { + /// Gas limit of transactions signed by [`Self::sign_tx_bytes`] that do not set one. + pub const DEFAULT_GAS_LIMIT: u64 = 1_000_000; + + /// Max fee per gas of transactions signed by [`Self::sign_tx_bytes`] that set no fees. + /// + /// High enough that transactions are accepted regardless of the base fee of test chains. + pub const DEFAULT_MAX_FEE_PER_GAS: u128 = 1_000_000_000_000; + + /// Max priority fee per gas of transactions signed by [`Self::sign_tx_bytes`] that set no + /// fees. + pub const DEFAULT_MAX_PRIORITY_FEE_PER_GAS: u128 = 1_000_000_000; + + /// Creates a new account for the given signer and chain id, starting at nonce 0. + pub const fn new(signer: PrivateKeySigner, chain_id: u64) -> Self { + Self { signer, chain_id, nonce: 0 } + } + + /// Returns the signer of the account. + pub const fn signer(&self) -> &PrivateKeySigner { + &self.signer + } + + /// Returns the address of the account. + pub const fn address(&self) -> Address { + self.signer.address() + } + + /// Returns the chain id transactions of the account are signed for. + pub const fn chain_id(&self) -> u64 { + self.chain_id + } + + /// Returns the nonce of the next transaction of the account. + pub const fn nonce(&self) -> u64 { + self.nonce + } + + /// Returns the nonce of the next transaction and increments it. + pub const fn next_nonce(&mut self) -> u64 { + let nonce = self.nonce; + self.nonce += 1; + nonce + } + + /// Sets the nonce to the pending transaction count of the account on the node behind + /// `provider`, and returns it. + pub async fn sync_nonce( + &mut self, + provider: &impl Provider, + ) -> eyre::Result { + self.nonce = + provider.get_transaction_count(self.address()).block_id(BlockId::pending()).await?; + Ok(self.nonce) + } + + /// Signs the transaction request, returning the EIP-2718 encoded bytes. + /// + /// Fills unset fields: the nonce with [`Self::next_nonce`], the chain id of the account, the + /// gas limit with [`Self::DEFAULT_GAS_LIMIT`] and, unless the request sets a legacy gas price, + /// the EIP-1559 fees with [`Self::DEFAULT_MAX_FEE_PER_GAS`] and + /// [`Self::DEFAULT_MAX_PRIORITY_FEE_PER_GAS`]. An explicit nonce does not advance the tracked + /// nonce. Contract creations must set `to`, e.g. with + /// [`TransactionBuilder::into_create`](alloy_network::TransactionBuilder::into_create). + /// + /// # Panics + /// + /// If the request can not be built into a signed transaction. + pub async fn sign_tx_bytes(&mut self, mut tx: TransactionRequest) -> Bytes { + if tx.nonce.is_none() { + tx.nonce = Some(self.next_nonce()); + } + tx.chain_id.get_or_insert(self.chain_id); + tx.gas.get_or_insert(Self::DEFAULT_GAS_LIMIT); + if tx.gas_price.is_none() { + tx.max_fee_per_gas.get_or_insert(Self::DEFAULT_MAX_FEE_PER_GAS); + tx.max_priority_fee_per_gas.get_or_insert(Self::DEFAULT_MAX_PRIORITY_FEE_PER_GAS); + } + TransactionTestContext::sign_tx_bytes(self.signer.clone(), tx).await + } +} diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index d182dc92cd0..2d1c625900a 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -3,8 +3,9 @@ use alloy_primitives::{Address, B256}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes, PayloadStatusEnum}; use eyre::Result; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ + test_chain_spec, test_rlp_utils::{generate_test_blocks, write_blocks_to_rlp}, testsuite::{ actions::{ @@ -15,12 +16,11 @@ use reth_e2e_test_utils::{ setup::{NetworkSetup, Setup}, Environment, TestBuilder, }, - E2ETestSetupBuilder, + E2ETestSetupExt, }; use reth_node_api::TreeConfig; use reth_node_ethereum::{EthEngineTypes, EthereumNode}; use reth_provider::{DatabaseProviderFactory, MetadataProvider, StorageSettings}; -use std::sync::Arc; use tempfile::TempDir; use tracing::debug; @@ -29,20 +29,7 @@ async fn test_apply_with_import() -> Result<()> { reth_tracing::init_test_tracing(); // Create test chain spec - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .london_activated() - .shanghai_activated() - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); // Generate test blocks let test_blocks = generate_test_blocks(&chain_spec, 10); @@ -124,18 +111,7 @@ async fn test_testsuite_assert_mine_block() -> Result<()> { reth_tracing::init_test_tracing(); let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .paris_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Paris)) .with_network(NetworkSetup::single_node()); let test = @@ -168,18 +144,7 @@ async fn test_testsuite_produce_blocks() -> Result<()> { reth_tracing::init_test_tracing(); let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::single_node()); let test = TestBuilder::new() @@ -198,18 +163,7 @@ async fn test_testsuite_create_fork() -> Result<()> { reth_tracing::init_test_tracing(); let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::single_node()); let test = TestBuilder::new() @@ -228,18 +182,7 @@ async fn test_testsuite_reorg_with_tagging() -> Result<()> { reth_tracing::init_test_tracing(); let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::single_node()); let test = TestBuilder::new() @@ -270,18 +213,7 @@ async fn test_testsuite_deep_reorg() -> Result<()> { reth_tracing::init_test_tracing(); let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::single_node()) .with_tree_config(TreeConfig::default().with_state_root_fallback(true)); @@ -322,18 +254,7 @@ async fn test_testsuite_multinode_block_production() -> Result<()> { reth_tracing::init_test_tracing(); let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::multi_node(2)) // Create 2 nodes .with_tree_config(TreeConfig::default().with_state_root_fallback(true)); @@ -364,31 +285,14 @@ async fn test_testsuite_multinode_block_production() -> Result<()> { async fn test_setup_builder_with_custom_tree_config() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - ); - - let (nodes, _wallet) = E2ETestSetupBuilder::::new(1, chain_spec, |_| { - PayloadAttributes::default() - }) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(5) - }) - .build() - .await?; - - assert_eq!(nodes.len(), 1); + let (node, _) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(5) + }) + .build_single() + .await?; - let genesis_hash = nodes[0].block_hash(0); + let genesis_hash = node.block_hash(0); assert_ne!(genesis_hash, B256::ZERO); Ok(()) @@ -398,31 +302,17 @@ async fn test_setup_builder_with_custom_tree_config() -> Result<()> { async fn test_setup_builder_with_storage_v2() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); for storage_v2 in [false, true] { // A node config modifier set afterwards must not reset the storage mode. - let (nodes, _wallet) = - E2ETestSetupBuilder::::new(1, chain_spec.clone(), |_| { - PayloadAttributes::default() - }) + let (node, _) = EthereumNode::test_setup(1, chain_spec.clone()) .with_storage_v2(storage_v2) .with_node_config_modifier(|config| config.set_dev(true)) - .build() + .build_single() .await?; - let provider = nodes[0].inner.provider.database_provider_ro()?; + let provider = node.inner.provider.database_provider_ro()?; assert_eq!(provider.storage_settings()?, Some(StorageSettings { storage_v2 })); } diff --git a/crates/e2e-test-utils/tests/rocksdb/main.rs b/crates/e2e-test-utils/tests/rocksdb/main.rs index 11283fd1c2f..eabc0e8e6cb 100644 --- a/crates/e2e-test-utils/tests/rocksdb/main.rs +++ b/crates/e2e-test-utils/tests/rocksdb/main.rs @@ -1,98 +1,45 @@ //! E2E tests for `RocksDB` provider functionality. use alloy_consensus::BlockHeader; -use alloy_eips::eip2718::Encodable2718; use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; -use alloy_rpc_types_engine::PayloadAttributes; use alloy_rpc_types_eth::{Transaction, TransactionInput, TransactionReceipt, TransactionRequest}; -use eyre::Result; +use eyre::{ensure, Result}; use jsonrpsee::core::client::ClientT; -use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_db::tables; -use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder}; +use reth_e2e_test_utils::{ + test_chain_spec, transaction::TransactionTestContext, wait::poll_until, wallet, E2ETestSetupExt, +}; use reth_node_ethereum::EthereumNode; use reth_provider::RocksDBProviderFactory; -use std::{sync::Arc, time::Duration}; - -const ROCKSDB_POLL_TIMEOUT: Duration = Duration::from_secs(60); -const ROCKSDB_POLL_INTERVAL: Duration = Duration::from_millis(50); +use std::time::Duration; /// Polls RPC until the given `tx_hash` is visible as pending (not yet mined). /// Prevents race conditions where `advance_block` is called before txs are in the pool. /// Returns the pending transaction. -async fn wait_for_pending_tx(client: &C, tx_hash: B256) -> Transaction { - let start = std::time::Instant::now(); - loop { - let tx: Option = client - .request("eth_getTransactionByHash", [tx_hash]) - .await - .expect("RPC request failed"); - if let Some(tx) = tx { - assert!( +async fn wait_for_pending_tx(client: &C, tx_hash: B256) -> Result { + poll_until(format_args!("transaction {tx_hash} in the pending pool"), || async move { + let tx: Option = client.request("eth_getTransactionByHash", [tx_hash]).await?; + if let Some(tx) = &tx { + ensure!( tx.block_number.is_none(), - "Expected pending tx but tx_hash={tx_hash:?} is already mined in block {:?}", + "expected pending transaction {tx_hash}, but it is mined in block {:?}", tx.block_number ); - return tx; } - assert!( - start.elapsed() < ROCKSDB_POLL_TIMEOUT, - "Timed out after {:?} waiting for tx_hash={tx_hash:?} to appear in pending pool", - start.elapsed() - ); - tokio::time::sleep(ROCKSDB_POLL_INTERVAL).await; - } + Ok(tx) + }) + .await } -/// Polls `RocksDB` until the given `tx_hash` appears in `TransactionHashNumbers`. -/// Returns the `tx_number` on success, or panics on timeout. -async fn poll_tx_in_rocksdb(provider: &P, tx_hash: B256) -> u64 { - let start = std::time::Instant::now(); - let mut interval = ROCKSDB_POLL_INTERVAL; - loop { +/// Polls `RocksDB` until the given `tx_hash` appears in `TransactionHashNumbers` and returns its +/// `tx_number`. +async fn poll_tx_in_rocksdb(provider: &P, tx_hash: B256) -> Result { + poll_until(format_args!("transaction {tx_hash} in RocksDB"), || async move { // Re-acquire handle each iteration to avoid stale snapshot reads - let rocksdb = provider.rocksdb_provider(); - let tx_number: Option = - rocksdb.get::(tx_hash).expect("RocksDB get failed"); - if let Some(n) = tx_number { - return n; - } - assert!( - start.elapsed() < ROCKSDB_POLL_TIMEOUT, - "Timed out after {:?} waiting for tx_hash={tx_hash:?} in RocksDB", - start.elapsed() - ); - tokio::time::sleep(interval).await; - // Simple backoff: 50ms -> 100ms -> 200ms (capped) - interval = std::cmp::min(interval * 2, Duration::from_millis(200)); - } -} - -/// Returns the test chain spec for `RocksDB` tests. -fn test_chain_spec() -> Arc { - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!("../../src/testsuite/assets/genesis.json")) - .expect("failed to parse genesis.json"), - ) - .cancun_activated() - .build(), - ) -} - -/// Returns test payload attributes for the given timestamp. -fn test_attributes_generator(timestamp: u64) -> PayloadAttributes { - PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: alloy_primitives::Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: None, - ..Default::default() - } + Ok(provider.rocksdb_provider().get::(tx_hash)?) + }) + .await } /// Smoke test: node boots with `RocksDB` routing enabled. @@ -100,13 +47,10 @@ fn test_attributes_generator(timestamp: u64) -> PayloadAttributes { async fn test_rocksdb_node_startup() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); - - let (nodes, _wallet) = - E2ETestSetupBuilder::::new(1, chain_spec, test_attributes_generator) - .with_storage_v2(true) - .build() - .await?; + let (nodes, _wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + .with_storage_v2(true) + .build() + .await?; assert_eq!(nodes.len(), 1); @@ -127,14 +71,11 @@ async fn test_rocksdb_node_startup() -> Result<()> { async fn test_rocksdb_block_mining() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); let (mut nodes, _wallet) = - E2ETestSetupBuilder::::new(1, chain_spec, test_attributes_generator) - .with_storage_v2(true) - .build() - .await?; + EthereumNode::test_setup(1, chain_spec).with_storage_v2(true).build().await?; assert_eq!(nodes.len(), 1); @@ -142,8 +83,7 @@ async fn test_rocksdb_block_mining() -> Result<()> { assert_ne!(genesis_hash, B256::ZERO); // Mine 3 blocks with transactions - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); let client = nodes[0].rpc_client().expect("RPC client should be available"); for i in 1..=3u64 { @@ -153,7 +93,7 @@ async fn test_rocksdb_block_mining() -> Result<()> { let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; // Wait for tx to enter pending pool before mining - wait_for_pending_tx(&client, tx_hash).await; + wait_for_pending_tx(&client, tx_hash).await?; let payload = nodes[0].advance_block().await?; let block = payload.block(); @@ -181,33 +121,28 @@ async fn test_rocksdb_block_mining() -> Result<()> { async fn test_rocksdb_transaction_queries() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .build() + .await?; assert_eq!(nodes.len(), 1); // Inject and mine a transaction - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); let client = nodes[0].rpc_client().expect("RPC client should be available"); let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await; let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; // Wait for tx to enter pending pool before mining - wait_for_pending_tx(&client, tx_hash).await; + wait_for_pending_tx(&client, tx_hash).await?; let payload = nodes[0].advance_block().await?; assert_eq!(payload.block().number(), 1); @@ -224,7 +159,7 @@ async fn test_rocksdb_transaction_queries() -> Result<()> { assert!(receipt.status()); // Direct RocksDB assertion - poll with timeout since persistence is async - let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await; + let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; assert_eq!(tx_number, 0, "First tx should have TxNumber 0"); // Verify missing hash returns None @@ -250,24 +185,19 @@ async fn test_rocksdb_transaction_queries() -> Result<()> { async fn test_rocksdb_multi_tx_same_block() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .build() + .await?; // Create 3 txs from the same wallet with sequential nonces - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); let client = nodes[0].rpc_client().expect("RPC client"); let mut tx_hashes = Vec::new(); @@ -281,7 +211,7 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> { // Wait for all txs to appear in pending pool before mining for tx_hash in &tx_hashes { - wait_for_pending_tx(&client, *tx_hash).await; + wait_for_pending_tx(&client, *tx_hash).await?; } // Mine one block containing all 3 txs @@ -304,7 +234,7 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> { // Poll RocksDB for all tx hashes and collect tx_numbers let mut tx_numbers = Vec::new(); for tx_hash in &tx_hashes { - let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await; + let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await?; tx_numbers.push(n); } @@ -320,23 +250,18 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> { async fn test_rocksdb_txs_across_blocks() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .build() + .await?; - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); let client = nodes[0].rpc_client().expect("RPC client"); // Block 1: 2 transactions @@ -354,8 +279,8 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { .await?; // Wait for both txs to appear in pending pool - wait_for_pending_tx(&client, tx_hash_0).await; - wait_for_pending_tx(&client, tx_hash_1).await; + wait_for_pending_tx(&client, tx_hash_0).await?; + wait_for_pending_tx(&client, tx_hash_1).await?; let payload1 = nodes[0].advance_block().await?; assert_eq!(payload1.block().number(), 1); @@ -368,7 +293,7 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { ) .await?; - wait_for_pending_tx(&client, tx_hash_2).await; + wait_for_pending_tx(&client, tx_hash_2).await?; let payload2 = nodes[0].advance_block().await?; assert_eq!(payload2.block().number(), 2); @@ -386,7 +311,7 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { let all_tx_hashes = [tx_hash_0, tx_hash_1, tx_hash_2]; let mut tx_numbers = Vec::new(); for tx_hash in &all_tx_hashes { - let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await; + let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await?; tx_numbers.push(n); } @@ -407,23 +332,18 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .build() + .await?; - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); // Inject tx but do NOT mine let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await; @@ -431,7 +351,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { // Verify tx is in pending pool via RPC let client = nodes[0].rpc_client().expect("RPC client"); - wait_for_pending_tx(&client, tx_hash).await; + wait_for_pending_tx(&client, tx_hash).await?; let pending_tx: Option = client.request("eth_getTransactionByHash", [tx_hash]).await?; @@ -452,7 +372,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { assert_eq!(payload.block().number(), 1); // Poll until tx appears in RocksDB - let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await; + let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; assert_eq!(tx_number, 0, "First tx should have tx_number 0"); // Verify tx is now mined via RPC @@ -473,67 +393,62 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { async fn test_rocksdb_reorg_unwind() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .build() + .await?; assert_eq!(nodes.len(), 1); // Use two separate wallets to avoid nonce conflicts during reorg - let wallets = wallet::Wallet::new(2).with_chain_id(chain_id).wallet_gen(); - let signer1 = wallets[0].clone(); - let signer2 = wallets[1].clone(); + let wallet = wallet::Wallet::new(2).with_chain_id(chain_id); + let (signer1, signer2) = (wallet.signer(0), wallet.signer(1)); let client = nodes[0].rpc_client().expect("RPC client"); // Mine block 1 with a transaction from signer1 let raw_tx1 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 0).await; let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?; - wait_for_pending_tx(&client, tx_hash1).await; + wait_for_pending_tx(&client, tx_hash1).await?; let payload1 = nodes[0].advance_block().await?; let block1_hash = payload1.block().hash(); assert_eq!(payload1.block().number(), 1); // Poll until tx1 appears in RocksDB (ensures persistence happened) - let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await; + let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await?; assert_eq!(tx_number1, 0, "First tx should have tx_number 0"); // Mine block 2 with transaction from signer1 (nonce 1) let raw_tx2 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 1).await; let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?; - wait_for_pending_tx(&client, tx_hash2).await; + wait_for_pending_tx(&client, tx_hash2).await?; let payload2 = nodes[0].advance_block().await?; assert_eq!(payload2.block().number(), 2); // Poll until tx2 appears in RocksDB - let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await; + let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await?; assert_eq!(tx_number2, 1, "Second tx should have tx_number 1"); // Mine block 3 with transaction from signer1 (nonce 2) let raw_tx3 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 2).await; let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?; - wait_for_pending_tx(&client, tx_hash3).await; + wait_for_pending_tx(&client, tx_hash3).await?; let payload3 = nodes[0].advance_block().await?; assert_eq!(payload3.block().number(), 3); // Poll until tx3 appears in RocksDB - let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await; + let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await?; assert_eq!(tx_number3, 2, "Third tx should have tx_number 2"); // Now create an alternate block 2 using signer2 (different wallet, avoids nonce conflict) @@ -541,7 +456,7 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> { let raw_alt_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 0).await; let alt_tx_hash = nodes[0].rpc.inject_tx(raw_alt_tx).await?; - wait_for_pending_tx(&client, alt_tx_hash).await; + wait_for_pending_tx(&client, alt_tx_hash).await?; // Build an alternate payload (this builds on top of the current head, i.e., block 3) // But we want to reorg back to block 1, so we'll use the payload and then FCU to it @@ -573,7 +488,7 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> { let raw_tx_final = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 1).await; let tx_hash_final = nodes[0].rpc.inject_tx(raw_tx_final).await?; - wait_for_pending_tx(&client, tx_hash_final).await; + wait_for_pending_tx(&client, tx_hash_final).await?; let final_payload = nodes[0].advance_block().await?; assert!(final_payload.block().number() > 3, "Should be able to mine block after reorg"); @@ -598,25 +513,20 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> { async fn test_rocksdb_historical_account_queries() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .build() + .await?; assert_eq!(nodes.len(), 1); - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); let sender: Address = signer.address(); let client = nodes[0].rpc_client().expect("RPC client"); @@ -630,11 +540,11 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { let raw_tx1 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await; let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?; - wait_for_pending_tx(&client, tx_hash1).await; + wait_for_pending_tx(&client, tx_hash1).await?; let payload1 = nodes[0].advance_block().await?; assert_eq!(payload1.block().number(), 1); - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await; + poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await?; // Record state after block 1 let balance_at_1: U256 = client.request("eth_getBalance", (sender, "0x1")).await?; @@ -646,11 +556,11 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { let raw_tx2 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await; let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?; - wait_for_pending_tx(&client, tx_hash2).await; + wait_for_pending_tx(&client, tx_hash2).await?; let payload2 = nodes[0].advance_block().await?; assert_eq!(payload2.block().number(), 2); - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await; + poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await?; let balance_at_2: U256 = client.request("eth_getBalance", (sender, "0x2")).await?; let nonce_at_2: U256 = client.request("eth_getTransactionCount", (sender, "0x2")).await?; @@ -661,11 +571,11 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { let raw_tx3 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await; let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?; - wait_for_pending_tx(&client, tx_hash3).await; + wait_for_pending_tx(&client, tx_hash3).await?; let payload3 = nodes[0].advance_block().await?; assert_eq!(payload3.block().number(), 3); - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await; + poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await?; let balance_at_3: U256 = client.request("eth_getBalance", (sender, "0x3")).await?; let nonce_at_3: U256 = client.request("eth_getTransactionCount", (sender, "0x3")).await?; @@ -683,7 +593,7 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce) .await; let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; - wait_for_pending_tx(&client, tx_hash).await; + wait_for_pending_tx(&client, tx_hash).await?; nodes[0].advance_block().await?; } // Allow the engine loop to process the persistence completions @@ -744,34 +654,29 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { async fn test_rocksdb_account_history_pruning() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); const PRUNE_DISTANCE: u64 = 5; const TOTAL_BLOCKS: u64 = 20; - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .with_node_config_modifier(|mut config| { - config.pruning.account_history_distance = Some(PRUNE_DISTANCE); - config.pruning.minimum_distance = Some(PRUNE_DISTANCE); - config.pruning.block_interval = Some(1); - config - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .with_node_config_modifier(|mut config| { + config.pruning.account_history_distance = Some(PRUNE_DISTANCE); + config.pruning.minimum_distance = Some(PRUNE_DISTANCE); + config.pruning.block_interval = Some(1); + config + }) + .build() + .await?; assert_eq!(nodes.len(), 1); - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); let sender: Address = signer.address(); let client = nodes[0].rpc_client().expect("RPC client"); @@ -779,26 +684,22 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { // completes independently. The race fires every cycle (the pruner reads stale // committed state that doesn't include save_blocks' pending batch), but processing // one block at a time makes the outcome deterministic. - let mut last_tx_hash = B256::ZERO; for nonce in 0..TOTAL_BLOCKS { let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce) .await; let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; - wait_for_pending_tx(&client, tx_hash).await; + wait_for_pending_tx(&client, tx_hash).await?; let payload = nodes[0].advance_block().await?; assert_eq!(payload.block().number(), nonce + 1); - last_tx_hash = tx_hash; // Let the persistence cycle (save_blocks → pruner → commit) complete before - // producing the next block, so each cycle processes exactly one block. - tokio::time::sleep(Duration::from_millis(300)).await; + // producing the next block, so each cycle processes exactly one block. The + // transaction becomes visible in RocksDB with the commit. + poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; } - // Wait for the last block to be fully persisted to RocksDB. - poll_tx_in_rocksdb(&nodes[0].inner.provider, last_tx_hash).await; - // Read the AccountsHistory shard for `sender` directly from RocksDB. // This is the data structure corrupted by the race. let rocksdb = nodes[0].inner.provider.rocksdb_provider(); @@ -843,34 +744,29 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { async fn test_rocksdb_storage_history_pruning() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let chain_id = chain_spec.chain().id(); const PRUNE_DISTANCE: u64 = 5; const TOTAL_BLOCKS: u64 = 20; - let (mut nodes, _) = E2ETestSetupBuilder::::new( - 1, - chain_spec.clone(), - test_attributes_generator, - ) - .with_storage_v2(true) - .with_tree_config_modifier(|config| { - config.with_persistence_threshold(0).with_memory_block_buffer_target(0) - }) - .with_node_config_modifier(|mut config| { - config.pruning.storage_history_distance = Some(PRUNE_DISTANCE); - config.pruning.minimum_distance = Some(PRUNE_DISTANCE); - config.pruning.block_interval = Some(1); - config - }) - .build() - .await?; + let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_storage_v2(true) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(0) + }) + .with_node_config_modifier(|mut config| { + config.pruning.storage_history_distance = Some(PRUNE_DISTANCE); + config.pruning.minimum_distance = Some(PRUNE_DISTANCE); + config.pruning.block_interval = Some(1); + config + }) + .build() + .await?; assert_eq!(nodes.len(), 1); - let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen(); - let signer = wallets[0].clone(); + let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); let client = nodes[0].rpc_client().expect("RPC client"); // Deploy a minimal contract that stores CALLDATA[0..32] into slot 0: @@ -922,17 +818,14 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { input: TransactionInput { input: None, data: Some(init_code) }, ..Default::default() }; - let signed_deploy = TransactionTestContext::sign_tx(signer.clone(), deploy_tx).await; - let deploy_bytes: Bytes = signed_deploy.encoded_2718().into(); + let deploy_bytes = TransactionTestContext::sign_tx_bytes(signer.clone(), deploy_tx).await; let deploy_hash = nodes[0].rpc.inject_tx(deploy_bytes).await?; - wait_for_pending_tx(&client, deploy_hash).await; + wait_for_pending_tx(&client, deploy_hash).await?; let payload1 = nodes[0].advance_block().await?; assert_eq!(payload1.block().number(), 1); - poll_tx_in_rocksdb(&nodes[0].inner.provider, deploy_hash).await; - // Let the persistence cycle complete before the next block (same cadence as the loop below) - tokio::time::sleep(Duration::from_millis(300)).await; + poll_tx_in_rocksdb(&nodes[0].inner.provider, deploy_hash).await?; // Get the deployed contract address from the receipt let receipt: Option = @@ -956,7 +849,6 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { // Mine TOTAL_BLOCKS - 1 more blocks (block 2..=TOTAL_BLOCKS), each calling the // contract to write a new value to slot 0. This creates a storage changeset entry // per block for (contract_address, slot 0). - let mut last_tx_hash = deploy_hash; for nonce in 1..TOTAL_BLOCKS { // calldata = abi encode the block number so each write is unique let block_num = nonce + 1; @@ -973,22 +865,17 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { input: TransactionInput { input: None, data: Some(Bytes::from(calldata.0)) }, ..Default::default() }; - let signed_call = TransactionTestContext::sign_tx(signer.clone(), call_tx).await; - let call_bytes: Bytes = signed_call.encoded_2718().into(); + let call_bytes = TransactionTestContext::sign_tx_bytes(signer.clone(), call_tx).await; let tx_hash = nodes[0].rpc.inject_tx(call_bytes).await?; - wait_for_pending_tx(&client, tx_hash).await; + wait_for_pending_tx(&client, tx_hash).await?; let payload = nodes[0].advance_block().await?; assert_eq!(payload.block().number(), block_num); - last_tx_hash = tx_hash; // Let the persistence cycle complete before the next block - tokio::time::sleep(Duration::from_millis(300)).await; + poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; } - // Wait for the last block to be fully persisted to RocksDB - poll_tx_in_rocksdb(&nodes[0].inner.provider, last_tx_hash).await; - // Read StoragesHistory shard for (contract_address, slot 0) directly from RocksDB let rocksdb = nodes[0].inner.provider.rocksdb_provider(); let shards = rocksdb.storage_history_shards(contract_address, storage_slot).unwrap(); diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index d121dfa2df7..83449813055 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -96,7 +96,6 @@ reth-e2e-test-utils.workspace = true assert_matches.workspace = true eyre.workspace = true -serde_json.workspace = true proptest.workspace = true rand.workspace = true rand_08.workspace = true diff --git a/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs b/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs index 2fd579d55f4..dc44dd28969 100644 --- a/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs +++ b/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs @@ -1,35 +1,26 @@ //! E2E tests for forkchoice updates to canonical ancestors around the finalized block. use eyre::Result; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::testsuite::{ - actions::{ - AssertChainTip, BlockReference, CaptureBlock, CreateFork, FinalizeBlock, MakeCanonical, - ProduceBlocks, SendForkchoiceUpdate, UpdateBlockInfo, +use reth_chainspec::EthereumHardfork; +use reth_e2e_test_utils::{ + test_chain_spec, + testsuite::{ + actions::{ + AssertChainTip, BlockReference, CaptureBlock, CreateFork, FinalizeBlock, MakeCanonical, + ProduceBlocks, SendForkchoiceUpdate, UpdateBlockInfo, + }, + setup::{NetworkSetup, Setup}, + TestBuilder, }, - setup::{NetworkSetup, Setup}, - TestBuilder, }; use reth_engine_tree::tree::TreeConfig; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_node_ethereum::EthereumNode; -use std::sync::Arc; /// Creates the standard setup for engine tree e2e tests. fn default_engine_tree_setup() -> Setup { Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::single_node()) .with_tree_config(TreeConfig::default().with_has_enough_parallelism(true)) } diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index 68d901f54a1..55b88a3d212 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -8,8 +8,9 @@ use alloy_rpc_types_engine::{ }; use eyre::Result; use futures::future::BoxFuture; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ + test_chain_spec, testsuite::{ actions::{ Action, AssertChainTip, BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, @@ -26,10 +27,7 @@ use reth_e2e_test_utils::{ use reth_engine_tree::tree::TreeConfig; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_node_ethereum::EthereumNode; -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use tokio::time::sleep; /// Waits for in-flight persistence by resubmitting the latest built payload to a node. @@ -261,18 +259,7 @@ impl Action for WaitForPersistedChain { /// Uses the node's default storage mode. fn default_engine_tree_setup() -> Setup { Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::single_node()) .with_tree_config(TreeConfig::default().with_has_enough_parallelism(true)) } @@ -516,18 +503,7 @@ async fn test_engine_tree_buffered_blocks_are_eventually_connected_e2e() -> Resu let test = TestBuilder::new() .with_setup( Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::multi_node_unconnected(2)) // Need 2 disconnected nodes .with_tree_config(TreeConfig::default().with_has_enough_parallelism(true)), ) @@ -609,18 +585,7 @@ async fn test_engine_tree_live_sync_transition_eventually_canonical_e2e() -> Res let test = TestBuilder::new() .with_setup( Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::multi_node(2)) // Two connected nodes .with_tree_config(TreeConfig::default().with_has_enough_parallelism(true)), ) @@ -670,18 +635,7 @@ async fn test_engine_tree_pipeline_sync_catches_up_masked_state_e2e() -> Result< let test = TestBuilder::new() .with_setup( Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::multi_node(2)) .with_storage_v2(true) .with_tree_config( @@ -821,18 +775,7 @@ async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> { /// on the chain lengths and not on how fast persistence keeps up with block production. fn disk_reorg_setup(storage_v2: bool) -> Setup { Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::multi_node_unconnected(2)) .with_storage_v2(storage_v2) .with_tree_config( diff --git a/crates/ethereum/node/tests/assets/genesis.json b/crates/ethereum/node/tests/assets/genesis.json deleted file mode 100644 index 76abaaaea5d..00000000000 --- a/crates/ethereum/node/tests/assets/genesis.json +++ /dev/null @@ -1 +0,0 @@ -{"config":{"chainId":1,"homesteadBlock":0,"daoForkSupport":true,"eip150Block":0,"eip155Block":0,"eip158Block":0,"byzantiumBlock":0,"constantinopleBlock":0,"petersburgBlock":0,"istanbulBlock":0,"muirGlacierBlock":0,"berlinBlock":0,"londonBlock":0,"arrowGlacierBlock":0,"grayGlacierBlock":0,"shanghaiTime":0,"cancunTime":0,"terminalTotalDifficulty":"0x0","terminalTotalDifficultyPassed":true},"nonce":"0x0","timestamp":"0x0","extraData":"0x00","gasLimit":"0x1c9c380","difficulty":"0x0","mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000","coinbase":"0x0000000000000000000000000000000000000000","alloc":{"0x14dc79964da2c08b23698b3d3cc7ca32193d9955":{"balance":"0xd3c21bcecceda1000000"},"0x15d34aaf54267db7d7c367839aaf71a00a2c6a65":{"balance":"0xd3c21bcecceda1000000"},"0x1cbd3b2770909d4e10f157cabc84c7264073c9ec":{"balance":"0xd3c21bcecceda1000000"},"0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f":{"balance":"0xd3c21bcecceda1000000"},"0x2546bcd3c84621e976d8185a91a922ae77ecec30":{"balance":"0xd3c21bcecceda1000000"},"0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc":{"balance":"0xd3c21bcecceda1000000"},"0x70997970c51812dc3a010c7d01b50e0d17dc79c8":{"balance":"0xd3c21bcecceda1000000"},"0x71be63f3384f5fb98995898a86b02fb2426c5788":{"balance":"0xd3c21bcecceda1000000"},"0x8626f6940e2eb28930efb4cef49b2d1f2c9c1199":{"balance":"0xd3c21bcecceda1000000"},"0x90f79bf6eb2c4f870365e785982e1f101e93b906":{"balance":"0xd3c21bcecceda1000000"},"0x976ea74026e726554db657fa54763abd0c3a0aa9":{"balance":"0xd3c21bcecceda1000000"},"0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc":{"balance":"0xd3c21bcecceda1000000"},"0x9c41de96b2088cdc640c6182dfcf5491dc574a57":{"balance":"0xd3c21bcecceda1000000"},"0xa0ee7a142d267c1f36714e4a8f75612f20a79720":{"balance":"0xd3c21bcecceda1000000"},"0xbcd4042de499d14e55001ccbb24a551f3b954096":{"balance":"0xd3c21bcecceda1000000"},"0xbda5747bfd65f08deb54cb465eb87d40e51b197e":{"balance":"0xd3c21bcecceda1000000"},"0xcd3b766ccdd6ae721141f452c550ca635964ce71":{"balance":"0xd3c21bcecceda1000000"},"0xdd2fd4581271e230360230f9337d5c0430bf44c0":{"balance":"0xd3c21bcecceda1000000"},"0xdf3e18d64bc6a983f673ab319ccae4f1a57c7097":{"balance":"0xd3c21bcecceda1000000"},"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266":{"balance":"0xd3c21bcecceda1000000"},"0xfabb0ac9d68b0b445fb7357272ff202c5651694a":{"balance":"0xd3c21bcecceda1000000"}},"number":"0x0"} diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index 830cc8befc8..f6dc7278d01 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -1,16 +1,12 @@ -use crate::utils::eth_payload_attributes; use alloy_eips::Decodable2718; -use alloy_genesis::Genesis; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ - node::NodeTestContext, transaction::TransactionTestContext, wallet::Wallet, + test_chain_spec, test_chain_spec_builder, transaction::TransactionTestContext, wallet::Wallet, + E2ETestSetupExt, }; use reth_ethereum_engine_primitives::BlobSidecars; use reth_ethereum_primitives::PooledTransactionVariant; -use reth_node_builder::{NodeBuilder, NodeHandle}; -use reth_node_core::{args::RpcServerArgs, node_config::NodeConfig}; use reth_node_ethereum::EthereumNode; -use reth_tasks::Runtime; use reth_transaction_pool::TransactionPool; use std::{ sync::Arc, @@ -20,28 +16,10 @@ use std::{ #[tokio::test] async fn can_handle_blobs() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let genesis_hash = chain_spec.genesis_hash(); - let node_config = NodeConfig::test() - .with_chain(chain_spec) - .with_unused_ports() - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .node(EthereumNode::default()) - .launch() - .await?; - - let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; + let (mut node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; let wallets = Wallet::new(2).wallet_gen(); let blob_wallet = wallets.first().unwrap(); @@ -91,27 +69,14 @@ async fn can_handle_blobs() -> eyre::Result<()> { #[tokio::test] async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Osaka); let genesis_hash = chain_spec.genesis_hash(); - let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_force_blob_sidecar_upcasting(), - ); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .node(EthereumNode::default()) - .launch() + let (mut node, _) = EthereumNode::test_setup(1, chain_spec) + .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) + .build_single() .await?; - let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; - let wallets = Wallet::new(2).wallet_gen(); let blob_wallet = wallets.first().unwrap(); @@ -144,36 +109,20 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { #[tokio::test] async fn blob_conversion_at_osaka() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); let current_timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); // Osaka activates in 2 slots let osaka_timestamp = current_timestamp + 24; - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .prague_activated() - .with_osaka_at(osaka_timestamp) - .build(), + test_chain_spec_builder().prague_activated().with_osaka_at(osaka_timestamp).build(), ); let genesis_hash = chain_spec.genesis_hash(); - let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_force_blob_sidecar_upcasting(), - ); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .node(EthereumNode::default()) - .launch() + let (mut node, _) = EthereumNode::test_setup(1, chain_spec) + .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) + .build_single() .await?; - let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; - let mut wallets = Wallet::new(3).wallet_gen(); let first = wallets.pop().unwrap(); let second = wallets.pop().unwrap(); diff --git a/crates/ethereum/node/tests/e2e/custom_genesis.rs b/crates/ethereum/node/tests/e2e/custom_genesis.rs index 63e366f00d4..dafb06c79d0 100644 --- a/crates/ethereum/node/tests/e2e/custom_genesis.rs +++ b/crates/ethereum/node/tests/e2e/custom_genesis.rs @@ -1,8 +1,7 @@ -use crate::utils::eth_payload_attributes; -use alloy_genesis::Genesis; use alloy_primitives::B256; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::{setup, transaction::TransactionTestContext}; +use reth_e2e_test_utils::{ + test_chain_spec_builder, test_genesis, transaction::TransactionTestContext, E2ETestSetupExt, +}; use reth_node_ethereum::EthereumNode; use reth_provider::{HeaderProvider, StageCheckpointReader}; use reth_stages_types::StageId; @@ -14,23 +13,13 @@ async fn can_run_eth_node_with_custom_genesis_number() -> eyre::Result<()> { reth_tracing::init_test_tracing(); // Create genesis with custom block number (e.g., 1000) - let mut genesis: Genesis = - serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); + let mut genesis = test_genesis(); genesis.number = Some(1000); genesis.parent_hash = Some(B256::random()); + let chain_spec = + Arc::new(test_chain_spec_builder().genesis(genesis).cancun_activated().build()); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); - - let (mut nodes, wallet) = - setup::(1, chain_spec, false, eth_payload_attributes).await?; - - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec).build_single().await?; // Verify stage checkpoints are initialized to genesis block number (1000) for stage in StageId::ALL { @@ -46,8 +35,7 @@ async fn can_run_eth_node_with_custom_genesis_number() -> eyre::Result<()> { // Advance the chain (block 1001) let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - let tx_hash = node.rpc.inject_tx(raw_tx).await?; - let payload = node.advance_block().await?; + let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; let block_hash = payload.block().hash(); let block_number = payload.block().number; @@ -68,23 +56,13 @@ async fn custom_genesis_block_query_boundaries() -> eyre::Result<()> { let genesis_number = 5000u64; - let mut genesis: Genesis = - serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); + let mut genesis = test_genesis(); genesis.number = Some(genesis_number); genesis.parent_hash = Some(B256::random()); + let chain_spec = + Arc::new(test_chain_spec_builder().genesis(genesis).cancun_activated().build()); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); - - let (mut nodes, _wallet) = - setup::(1, chain_spec, false, eth_payload_attributes).await?; - - let node = nodes.pop().unwrap(); + let (node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; // Query genesis block should succeed let genesis_header = node.inner.provider.header_by_number(genesis_number)?; @@ -98,3 +76,25 @@ async fn custom_genesis_block_query_boundaries() -> eyre::Result<()> { Ok(()) } + +/// Tests that payloads are built on top of a genesis that is newer than the default payload +/// timestamp of the test context. +#[tokio::test] +async fn can_advance_on_genesis_newer_than_payload_timestamp() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let genesis_timestamp = 2_000_000_000; + let mut genesis = test_genesis(); + genesis.timestamp = genesis_timestamp; + let chain_spec = + Arc::new(test_chain_spec_builder().genesis(genesis).cancun_activated().build()); + + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; + let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; + assert!(payload.block().timestamp > genesis_timestamp); + node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; + + Ok(()) +} diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index ba413cae5bd..ce8cde3cc2c 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -2,41 +2,34 @@ use alloy_genesis::Genesis; use alloy_primitives::{b256, hex, Address}; use futures::StreamExt; use reth_chainspec::ChainSpec; +use reth_e2e_test_utils::E2ETestSetupExt; use reth_node_api::{BlockBody, FullNodeComponents}; -use reth_node_builder::{rpc::RethRpcAddOns, FullNode, NodeBuilder, NodeConfig, NodeHandle}; -use reth_node_core::args::DevArgs; -use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; +use reth_node_builder::{rpc::RethRpcAddOns, FullNode}; +use reth_node_ethereum::EthereumNode; use reth_primitives_traits::transaction::TxHashRef; -use reth_provider::{ - providers::BlockchainProvider, BlockIdReader, BlockNumReader, CanonStateSubscriptions, -}; +use reth_provider::{BlockIdReader, BlockNumReader, CanonStateSubscriptions}; use reth_rpc_eth_api::{helpers::EthTransactions, EthApiServer}; -use reth_tasks::Runtime; use std::{num::NonZeroUsize, sync::Arc, time::Duration}; #[tokio::test] async fn can_run_dev_node() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - - let node_config = NodeConfig::test().with_chain(custom_chain()).with_dev(DevArgs { - dev: true, - finality_depth: NonZeroUsize::new(1).unwrap(), - ..Default::default() - }); - let NodeHandle { node, .. } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .with_types_and_provider::>() - .with_components(EthereumNode::components()) - .with_add_ons(EthereumAddOns::default()) - .launch_with_debug_capabilities() + + let (node, _) = EthereumNode::test_setup(1, custom_chain()) + .with_dev_mining(None) + .with_node_config_modifier(|mut config| { + config.dev.finality_depth = NonZeroUsize::new(1).unwrap(); + config + }) + .build_single() .await?; + let node = &node.inner; let canon_state = node.provider.canonical_in_memory_state(); let mut safe_block = canon_state.subscribe_safe_block(); let mut finalized_block = canon_state.subscribe_finalized_block(); - assert_chain_advances(&node).await; + assert_chain_advances(node).await; let chain_info = node.provider.chain_info()?; // Startup can leave an unread genesis notification, and the canonical head notification @@ -63,25 +56,19 @@ async fn can_run_dev_node() -> eyre::Result<()> { #[tokio::test] async fn can_run_dev_node_custom_attributes() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - let node_config = NodeConfig::test() - .with_chain(custom_chain()) - .with_dev(DevArgs { dev: true, ..Default::default() }); let fee_recipient = Address::random(); - let NodeHandle { node, .. } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .with_types_and_provider::>() - .with_components(EthereumNode::components()) - .with_add_ons(EthereumAddOns::default()) - .launch_with_debug_capabilities() - .map_debug_payload_attributes(move |mut attributes| { + let (node, _) = EthereumNode::test_setup(1, custom_chain()) + .with_dev_mining(None) + .map_dev_payload_attributes(move |mut attributes| { attributes.suggested_fee_recipient = fee_recipient; attributes }) + .build_single() .await?; + let node = &node.inner; - assert_chain_advances(&node).await; + assert_chain_advances(node).await; assert!( node.rpc_registry.eth_api().balance(fee_recipient, Default::default()).await.unwrap() > 0 @@ -102,6 +89,27 @@ async fn can_run_dev_node_custom_attributes() -> eyre::Result<()> { Ok(()) } +#[tokio::test] +async fn can_run_dev_node_with_block_time() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (node, _) = EthereumNode::test_setup(1, custom_chain()) + .with_dev_mining(Some(Duration::from_millis(100))) + .build_single() + .await?; + + // The local miner builds a block on every interval, even without pending transactions. + tokio::time::timeout(Duration::from_secs(10), async { + while node.inner.provider.best_block_number()? < 2 { + tokio::time::sleep(Duration::from_millis(50)).await; + } + eyre::Ok(()) + }) + .await??; + + Ok(()) +} + async fn assert_chain_advances(node: &FullNode) where N: FullNodeComponents, diff --git a/crates/ethereum/node/tests/e2e/eip6780.rs b/crates/ethereum/node/tests/e2e/eip6780.rs index 157d3e96f6b..a40de697b8b 100644 --- a/crates/ethereum/node/tests/e2e/eip6780.rs +++ b/crates/ethereum/node/tests/e2e/eip6780.rs @@ -25,18 +25,17 @@ //! - destroy/recreate cycles at the same `CREATE2` address across transactions and blocks, //! including the collision of a recreate attempt with a persisting destroyed contract. -use alloy_network::{EthereumWallet, TransactionBuilder}; +use alloy_network::TransactionBuilder; use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; -use alloy_provider::{Provider, ProviderBuilder}; +use alloy_provider::Provider; use alloy_rpc_types_eth::{TransactionReceipt, TransactionRequest}; use futures::StreamExt; -use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ - eth_payload_attributes_for_fork, setup_engine, + receipt::await_successful_receipts, trie::{assert_trie_consistency, wait_for_persisted_block}, - NodeHelperType, + E2ETestSetupExt, NodeHelperType, }; -use reth_node_api::TreeConfig; use reth_node_ethereum::EthereumNode; use reth_provider::Chain; use reth_revm::db::BundleAccount; @@ -69,20 +68,17 @@ async fn run_selfdestruct_suite(fork: EthereumHardfork) -> eyre::Result<()> { // low persistence thresholds so the scenario blocks reach the database and the persisted // trie representation can be verified at the end of the suite - let tree_config = TreeConfig::default() - .with_num_state_masking_blocks(0) - .with_persistence_threshold(2) - .with_memory_block_buffer_target(1); - let (mut nodes, wallet) = - setup_engine::(1, fork_spec(fork), false, tree_config, move |timestamp| { - eth_payload_attributes_for_fork(fork, timestamp) + let (node, wallet) = EthereumNode::test_setup_for(fork) + .with_tree_config_modifier(|config| { + config + .with_num_state_masking_blocks(0) + .with_persistence_threshold(2) + .with_memory_block_buffer_target(1) }) + .build_single() .await?; - let node = nodes.pop().unwrap(); let signer = wallet.inner.clone(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(signer.clone())) - .connect_http(node.rpc_url()); + let provider = node.rpc_provider_with_wallet(signer.clone()); let mut ctx = SuiteCtx { node, @@ -164,19 +160,7 @@ impl SuiteCtx

{ .ok_or_else(|| eyre::eyre!("canonical stream ended"))?; self.last_committed = Some(notification.committed()); - let mut receipts = Vec::with_capacity(expected); - for tx in pending { - receipts.push(tx.get_receipt().await?); - } - for (index, receipt) in receipts.iter().enumerate() { - assert!( - receipt.status(), - "transaction {index} in block {:?} reverted: gas used {}", - receipt.block_number, - receipt.gas_used, - ); - } - Ok(receipts) + await_successful_receipts(pending).await } /// Returns the account entry of the most recently committed block's bundle state. @@ -692,16 +676,3 @@ fn call_then_store_runtime(target: Address) -> Bytes { code.push(0x00); // STOP code.into() } - -fn fork_spec(fork: EthereumHardfork) -> Arc { - let builder = ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()); - let builder = match fork { - EthereumHardfork::Cancun => builder.cancun_activated(), - EthereumHardfork::Osaka => builder.osaka_activated(), - EthereumHardfork::Amsterdam => builder.amsterdam_activated(), - fork => unimplemented!("no activation configured for {fork}"), - }; - Arc::new(builder.build()) -} diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index 3f30a025a11..886b6d0ad55 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -1,9 +1,6 @@ -use crate::utils::{ - advance_with_random_transactions, eth_payload_attributes, eth_payload_attributes_amsterdam, -}; +use crate::utils::{advance_with_random_transactions, EngineSszRequestExt, EngineSszResponseExt}; use alloy_eips::eip7685::RequestsOrHash; -use alloy_genesis::Genesis; -use alloy_primitives::{Address, Bytes, B256}; +use alloy_primitives::{Bytes, B256}; use alloy_rpc_types_engine::{ ssz_engine_types::{ BlobsV1Request, BlobsV1Response, BlobsV2Response, BlobsV3Response, BlobsV4Request, @@ -15,14 +12,13 @@ use alloy_rpc_types_engine::{ ClientVersionV1, ForkchoiceState, PayloadAttributes, PayloadStatusEnum, }; use jsonrpsee_core::client::ClientT; -use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; +use reth_chainspec::{EthChainSpec, EthereumHardfork}; use reth_e2e_test_utils::{ - node::NodeTestContext, setup, setup_engine, transaction::TransactionTestContext, wallet::Wallet, + eth_payload_attributes, test_chain_spec, transaction::TransactionTestContext, wallet::Wallet, + E2ETestSetupExt, }; -use reth_node_api::TreeConfig; use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{ - args::RpcServerArgs, node_config::NodeConfig, version::{version_metadata, CLIENT_CODE}, }; @@ -31,13 +27,11 @@ use reth_node_ethereum::{ }; use reth_provider::{BlockNumReader, StateProviderFactory}; use reth_rpc_api::TestingBuildBlockRequestV1; -use reth_rpc_layer::secret_to_bearer_header; use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; use reth_tasks::Runtime; -use ssz::{Decode, Encode}; +use ssz::Encode; use std::sync::Arc; -const ENGINE_EXECUTION_VERSION_HEADER: &str = "Eth-Execution-Version"; const ENGINE_PRAGUE_FORK_HEADER: &str = "prague"; const ENGINE_PAYLOADS_ROUTE: &str = "/engine/v1/payloads"; const ENGINE_FORKCHOICE_ROUTE: &str = "/engine/v1/forkchoice"; @@ -49,34 +43,15 @@ const ENGINE_IDENTITY_ROUTE: &str = "/engine/v1/identity"; async fn can_run_eth_node() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let (mut nodes, wallet) = setup::( - 1, - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ), - false, - eth_payload_attributes, - ) - .await?; - - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; // make the node advance - let tx_hash = node.rpc.inject_tx(raw_tx).await?; - - // make the node advance - let payload = node.advance_block().await?; - - let block_hash = payload.block().hash(); - let block_number = payload.block().number; + let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; // assert the block has been committed to the blockchain - node.assert_new_block(tx_hash, block_hash, block_number).await?; + node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; Ok(()) } @@ -85,45 +60,20 @@ async fn can_run_eth_node() -> eyre::Result<()> { #[cfg(unix)] async fn can_run_eth_node_with_auth_engine_api_over_ipc() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - - // Chain spec with test allocs - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); - - // Node setup - let node_config = NodeConfig::test() - .with_chain(chain_spec) - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http().with_auth_ipc()); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) - .testing_node(runtime) - .node(EthereumNode::default()) - .launch() + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + .with_rpc_modifier(|rpc| rpc.with_auth_ipc()) + .build_single() .await?; - let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; - // Configure wallet from test mnemonic and create dummy transfer tx - let wallet = Wallet::default(); + // Create dummy transfer tx let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; // make the node advance - let tx_hash = node.rpc.inject_tx(raw_tx).await?; - - // make the node advance - let payload = node.advance_block().await?; - - let block_hash = payload.block().hash(); - let block_number = payload.block().number; + let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; // assert the block has been committed to the blockchain - node.assert_new_block(tx_hash, block_hash, block_number).await?; + node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; Ok(()) } @@ -132,27 +82,8 @@ async fn can_run_eth_node_with_auth_engine_api_over_ipc() -> eyre::Result<()> { #[cfg(unix)] async fn test_failed_run_eth_node_with_no_auth_engine_api_over_ipc_opts() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - - // Chain spec with test allocs - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); - - // Node setup - let node_config = NodeConfig::test().with_chain(chain_spec); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) - .testing_node(runtime) - .node(EthereumNode::default()) - .launch() - .await?; - let node = NodeTestContext::new(node, eth_payload_attributes).await?; + let (node, _) = EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; // Ensure that the engine api client is not available let client = node.inner.engine_ipc_client().await; @@ -165,25 +96,11 @@ async fn test_failed_run_eth_node_with_no_auth_engine_api_over_ipc_opts() -> eyr async fn test_engine_graceful_shutdown() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let (mut nodes, wallet) = setup::( - 1, - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ), - false, - eth_payload_attributes, - ) - .await?; - - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - let tx_hash = node.rpc.inject_tx(raw_tx).await?; - let payload = node.advance_block().await?; + let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; // Get block number before shutdown @@ -216,45 +133,24 @@ async fn test_engine_graceful_shutdown() -> eyre::Result<()> { #[tokio::test] async fn test_testing_build_block_v1_osaka() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Osaka); let genesis_hash = chain_spec.genesis_hash(); - - let node_config = - NodeConfig::test().with_chain(chain_spec.clone()).with_unused_ports().with_rpc( - RpcServerArgs::default().with_unused_ports().with_http().with_http_api( - RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing]), - ), - ); - - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) - .testing_node(runtime) - .node(EthereumNode::default()) - .launch() + let (node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_rpc_modifier(|rpc| { + rpc.with_http_api(RpcModuleSelection::from([ + RethRpcModule::Eth, + RethRpcModule::Testing, + ])) + }) + .build_single() .await?; - let node = NodeTestContext::new(node, eth_payload_attributes).await?; - - let wallet = Wallet::default(); let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - let payload_attributes = PayloadAttributes { - timestamp: chain_spec.genesis().timestamp + 1, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: None, - ..Default::default() - }; - let request = TestingBuildBlockRequestV1 { parent_block_hash: genesis_hash, - payload_attributes, + payload_attributes: eth_payload_attributes(&chain_spec, chain_spec.genesis().timestamp + 1), transactions: vec![raw_tx], extra_data: None, }; @@ -287,51 +183,29 @@ async fn test_testing_build_block_v1_osaka() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); - - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .prague_activated() - .build(), - ); + + let chain_spec = test_chain_spec(EthereumHardfork::Prague); let genesis_hash = chain_spec.genesis_hash(); - let node_config = - NodeConfig::test().with_chain(chain_spec.clone()).with_unused_ports().with_rpc( - RpcServerArgs::default().with_unused_ports().with_http().with_http_api( - RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing]), - ), - ); - - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) - .testing_node(runtime) - .with_types::() - .with_components(EthereumNode::components()) - .with_add_ons(EthereumAddOns::default()) - .launch() + let (node, _) = EthereumNode::test_setup(1, chain_spec.clone()) + .with_rpc_modifier(|rpc| { + rpc.with_http_api(RpcModuleSelection::from([ + RethRpcModule::Eth, + RethRpcModule::Testing, + ])) + }) + .build_single() .await?; - let node = NodeTestContext::new(node, eth_payload_attributes).await?; - let wallets = Wallet::new(2).wallet_gen(); let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallets[0].clone()).await; - let payload_attributes = PayloadAttributes { - timestamp: chain_spec.genesis().timestamp + 1, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: None, - ..Default::default() - }; - let envelope = node .testing_build_block_v1(TestingBuildBlockRequestV1 { parent_block_hash: genesis_hash, - payload_attributes, + payload_attributes: eth_payload_attributes( + &chain_spec, + chain_spec.genesis().timestamp + 1, + ), transactions: vec![raw_tx], extra_data: None, }) @@ -340,9 +214,8 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let payload = envelope.execution_payload; let block_hash = payload.payload_inner.payload_inner.block_hash; let client = reqwest::Client::new(); - let auth_server = node.auth_server_handle(); - let auth_url = auth_server.http_url(); - let auth_header = secret_to_bearer_header(auth_server.jwt_secret()); + let auth = node.auth_server_handle(); + let auth_url = auth.http_url(); for route in [ENGINE_CAPABILITIES_ROUTE, ENGINE_PAYLOADS_ROUTE] { let response = client.get(format!("{auth_url}{route}")).send().await?; @@ -351,7 +224,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let capabilities_response = client .get(format!("{auth_url}{ENGINE_CAPABILITIES_ROUTE}")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .jwt(&auth) .header(reqwest::header::ACCEPT, "application/json") .send() .await?; @@ -384,7 +257,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let identity_response = client .get(format!("{auth_url}{ENGINE_IDENTITY_ROUTE}")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .jwt(&auth) .header(reqwest::header::ACCEPT, "application/json") .send() .await?; @@ -408,40 +281,36 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { }] ); - let new_payload_response = client + let status: SszPayloadStatus = client .post(format!("{auth_url}{ENGINE_PAYLOADS_ROUTE}")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(ENGINE_EXECUTION_VERSION_HEADER, ENGINE_PRAGUE_FORK_HEADER) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .jwt(&auth) + .fork(ENGINE_PRAGUE_FORK_HEADER) .header(reqwest::header::ACCEPT, "application/octet-stream") - .body((payload, B256::ZERO, envelope.execution_requests.take()).as_ssz_bytes()) + .ssz(&(payload, B256::ZERO, envelope.execution_requests.take())) .send() + .await? + .ssz() .await?; - assert_eq!(new_payload_response.status(), reqwest::StatusCode::OK); - - let status = SszPayloadStatus::from_ssz_bytes(&new_payload_response.bytes().await?).unwrap(); assert_eq!(status.status, PayloadStatusKind::Valid); - let fcu_response = client + let fcu: SszForkchoiceUpdateResponse = client .post(format!("{auth_url}{ENGINE_FORKCHOICE_ROUTE}")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(ENGINE_EXECUTION_VERSION_HEADER, ENGINE_PRAGUE_FORK_HEADER) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .jwt(&auth) + .fork(ENGINE_PRAGUE_FORK_HEADER) .header(reqwest::header::ACCEPT, "application/octet-stream") - .body( - ( - ForkchoiceState { - head_block_hash: block_hash, - safe_block_hash: genesis_hash, - finalized_block_hash: genesis_hash, - }, - Vec::::new(), - ) - .as_ssz_bytes(), - ) + .ssz(&( + ForkchoiceState { + head_block_hash: block_hash, + safe_block_hash: genesis_hash, + finalized_block_hash: genesis_hash, + }, + Vec::::new(), + )) .send() + .await? + .ssz() .await?; - assert_eq!(fcu_response.status(), reqwest::StatusCode::OK); + assert_eq!(fcu.payload_status.status, PayloadStatusKind::Valid); let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, wallets[1].clone()).await?; let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?; @@ -450,49 +319,43 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let mut requested_hashes = versioned_hashes.clone(); requested_hashes.push(B256::ZERO); - let blobs_response = client + let blobs: BlobsV1Response = client .post(format!("{auth_url}{ENGINE_V1_BLOBS_ROUTE}")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .jwt(&auth) .header(reqwest::header::ACCEPT, "application/octet-stream") - .body(BlobsV1Request { versioned_hashes: requested_hashes }.as_ssz_bytes()) + .ssz(&BlobsV1Request { versioned_hashes: requested_hashes }) .send() + .await? + .ssz() .await?; - assert_eq!(blobs_response.status(), reqwest::StatusCode::OK); - - let blobs = BlobsV1Response::from_ssz_bytes(&blobs_response.bytes().await?).unwrap(); assert_eq!(blobs.entries.len(), versioned_hashes.len() + 1); assert!(blobs.entries[..versioned_hashes.len()].iter().all(|entry| entry.available)); assert!(!blobs.entries.last().unwrap().available); - let fcu = SszForkchoiceUpdateResponse::from_ssz_bytes(&fcu_response.bytes().await?).unwrap(); - assert_eq!(fcu.payload_status.status, PayloadStatusKind::Valid); - node.wait_block(1, block_hash, false).await?; for (fork, available) in [("prague", true), ("cancun", false)] { - let response = client + let bodies: BodiesResponsePrague = client .post(format!("{auth_url}/engine/v1/bodies/hash")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(ENGINE_EXECUTION_VERSION_HEADER, fork) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(BodiesByHashRequest { block_hashes: vec![block_hash, B256::ZERO] }.as_ssz_bytes()) + .jwt(&auth) + .fork(fork) + .ssz(&BodiesByHashRequest { block_hashes: vec![block_hash, B256::ZERO] }) .send() + .await? + .ssz() .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let bodies = BodiesResponsePrague::from_ssz_bytes(&response.bytes().await?).unwrap(); assert_eq!(bodies.entries.len(), 2); assert_eq!(bodies.entries[0].available, available); assert!(!bodies.entries[1].available); } - let response = client + let bodies: BodiesResponsePrague = client .get(format!("{auth_url}/engine/v1/bodies?from=1&count=32")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(ENGINE_EXECUTION_VERSION_HEADER, "prague") + .jwt(&auth) + .fork("prague") .send() + .await? + .ssz() .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let bodies = BodiesResponsePrague::from_ssz_bytes(&response.bytes().await?).unwrap(); assert_eq!(bodies.entries.len(), 1); assert!(bodies.entries[0].available); @@ -501,55 +364,40 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_proxy_blob_revisions() -> eyre::Result<()> { - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .osaka_activated() - .build(), - ); - let (mut nodes, _) = - setup::(1, chain_spec, false, eth_payload_attributes).await?; - let mut node = nodes.pop().unwrap(); + let (mut node, _) = + EthereumNode::test_setup_for(EthereumHardfork::Osaka).build_single().await?; node.advance_block().await?; let client = reqwest::Client::new(); - let auth_server = node.auth_server_handle(); - let auth_url = auth_server.http_url(); - let auth_header = secret_to_bearer_header(auth_server.jwt_secret()); + let auth = node.auth_server_handle(); + let auth_url = auth.http_url(); // Missing blobs retain one unavailable entry, except V2's all-or-nothing response. for version in 2..=4 { - let body = if version == 4 { - BlobsV4Request { + let request = client.post(format!("{auth_url}/engine/v1/blobs/v{version}")).jwt(&auth); + let request = if version == 4 { + request.ssz(&BlobsV4Request { versioned_hashes: vec![B256::ZERO], indices_bitarray: alloy_primitives::B128::ZERO, - } - .as_ssz_bytes() + }) } else { - BlobsV1Request { versioned_hashes: vec![B256::ZERO] }.as_ssz_bytes() + request.ssz(&BlobsV1Request { versioned_hashes: vec![B256::ZERO] }) }; - let response = client - .post(format!("{auth_url}/engine/v1/blobs/v{version}")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(body) - .send() - .await?; + let response = request.send().await?; if version == 2 { assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT); assert!(response.bytes().await?.is_empty()); continue } - assert_eq!(response.status(), reqwest::StatusCode::OK); - let bytes = response.bytes().await?; let availability = match version { - 3 => BlobsV3Response::from_ssz_bytes(&bytes) - .unwrap() + 3 => response + .ssz::() + .await? .entries .into_iter() .map(|entry| entry.available) .collect::>(), - 4 => BlobsV4Response::from_ssz_bytes(&bytes) - .unwrap() + 4 => response + .ssz::() + .await? .entries .into_iter() .map(|entry| entry.available) @@ -559,15 +407,15 @@ async fn test_engine_ssz_proxy_blob_revisions() -> eyre::Result<()> { assert_eq!(availability, [false]); } // A container with no requested hashes is a successful empty V2 response. - let response = client + let blobs: BlobsV2Response = client .post(format!("{auth_url}/engine/v1/blobs/v2")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .body(BlobsV1Request { versioned_hashes: vec![] }.as_ssz_bytes()) + .jwt(&auth) + .ssz(&BlobsV1Request { versioned_hashes: vec![] }) .send() + .await? + .ssz() .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - assert!(BlobsV2Response::from_ssz_bytes(&response.bytes().await?).unwrap().entries.is_empty()); + assert!(blobs.entries.is_empty()); for (body, status) in [ (vec![0; 32], reqwest::StatusCode::BAD_REQUEST), @@ -578,13 +426,13 @@ async fn test_engine_ssz_proxy_blob_revisions() -> eyre::Result<()> { ] { let response = client .post(format!("{auth_url}/engine/v1/blobs/v1")) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) + .jwt(&auth) .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") .body(body) .send() .await?; assert_eq!(response.status(), status); - assert_eq!(response.headers()[reqwest::header::CONTENT_TYPE], "application/problem+json"); + response.problem_type().await?; } Ok(()) @@ -592,17 +440,9 @@ async fn test_engine_ssz_proxy_blob_revisions() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_proxy_returns_canonical_witness() -> eyre::Result<()> { - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .amsterdam_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Amsterdam); let genesis_hash = chain_spec.genesis_hash(); - let (mut nodes, _) = - setup::(1, chain_spec, false, eth_payload_attributes_amsterdam).await?; - let mut node = nodes.pop().unwrap(); + let (mut node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; let payload = node.new_payload().await?; let envelope = payload.try_into_v6()?; let request = ExecutionPayloadEnvelopeAmsterdam { @@ -611,35 +451,17 @@ async fn test_engine_ssz_proxy_returns_canonical_witness() -> eyre::Result<()> { execution_requests: envelope.execution_requests, }; let client = reqwest::Client::new(); - let auth_server = node.auth_server_handle(); - let auth_url = auth_server.http_url(); - let auth_header = secret_to_bearer_header(auth_server.jwt_secret()); - let url = format!("{auth_url}/engine/v1/payloads/witness"); - let response = client - .post(&url) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") - .body(request.as_ssz_bytes()) - .send() - .await?; + let auth = node.auth_server_handle(); + let url = format!("{}/engine/v1/payloads/witness", auth.http_url()); + let response = client.post(&url).fork("amsterdam").ssz(&request).send().await?; assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED); for fork in ["prague", "amsterdam"] { - let response = client - .post(&url) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, fork) - .body(request.as_ssz_bytes()) - .send() - .await?; + let response = client.post(&url).jwt(&auth).fork(fork).ssz(&request).send().await?; if fork == "prague" { assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); continue } - let status = response.status(); - let bytes = response.bytes().await?; - assert_eq!(status, reqwest::StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); - let response = PayloadStatusWithWitness::from_ssz_bytes(&bytes).unwrap(); + let response = response.ssz::().await?; assert_eq!(response.payload_status.status, PayloadStatusKind::Valid); let witness = response.witness.as_ref().expect("valid payload includes a witness"); assert!(!witness.state.is_empty()); @@ -650,33 +472,15 @@ async fn test_engine_ssz_proxy_returns_canonical_witness() -> eyre::Result<()> { } let mut invalid = request; invalid.payload.payload_inner.payload_inner.payload_inner.block_hash = B256::ZERO; - let response = client - .post(&url) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") - .body(invalid.as_ssz_bytes()) - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let response = PayloadStatusWithWitness::from_ssz_bytes(&response.bytes().await?).unwrap(); + let response: PayloadStatusWithWitness = + client.post(&url).jwt(&auth).fork("amsterdam").ssz(&invalid).send().await?.ssz().await?; assert_eq!(response.payload_status.status, PayloadStatusKind::Invalid); assert!(response.witness.is_none()); invalid.payload.payload_inner.payload_inner.payload_inner.transactions = vec![alloy_primitives::Bytes::from_static(&[2])]; - let response = client - .post(&url) - .header(reqwest::header::AUTHORIZATION, auth_header.to_str()?) - .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") - .body(invalid.as_ssz_bytes()) - .send() - .await?; + let response = client.post(&url).jwt(&auth).fork("amsterdam").ssz(&invalid).send().await?; assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY); - assert_eq!( - response.json::().await?["type"], - "/engine-api/errors/invalid-body" - ); + assert_eq!(response.problem_type().await?, "/engine-api/errors/invalid-body"); Ok(()) } @@ -693,27 +497,14 @@ async fn test_engine_ssz_proxy_returns_canonical_witness() -> eyre::Result<()> { async fn test_share_sparse_trie_with_payload_builder() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let tree_config = TreeConfig::default() - .with_share_execution_cache_with_payload_builder(true) - .with_share_sparse_trie_with_payload_builder(true); - - let (mut nodes, _wallet) = setup_engine::( - 1, - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .prague_activated() - .build(), - ), - false, - tree_config, - eth_payload_attributes, - ) - .await?; - - let mut node = nodes.pop().unwrap(); + let (mut node, _) = EthereumNode::test_setup_for(EthereumHardfork::Prague) + .with_tree_config_modifier(|config| { + config + .with_share_execution_cache_with_payload_builder(true) + .with_share_sparse_trie_with_payload_builder(true) + }) + .build_single() + .await?; let mut rng = rand::rng(); let num_blocks = 5; @@ -740,25 +531,10 @@ async fn test_sparse_trie_reuse_across_blocks() -> eyre::Result<()> { reth_tracing::init_test_tracing(); // Use the state-root task with pruning enabled. - let tree_config = TreeConfig::default().with_sparse_trie_prune_depth(2); - - let (mut nodes, _wallet) = setup_engine::( - 1, - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .prague_activated() - .build(), - ), - false, - tree_config, - eth_payload_attributes, - ) - .await?; - - let mut node = nodes.pop().unwrap(); + let (mut node, _) = EthereumNode::test_setup_for(EthereumHardfork::Prague) + .with_tree_config_modifier(|config| config.with_sparse_trie_prune_depth(2)) + .build_single() + .await?; // Use a seeded RNG for reproducibility let mut rng = rand::rng(); @@ -778,19 +554,10 @@ async fn test_sparse_trie_reuse_across_blocks() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_request_validation() -> eyre::Result<()> { - let chain = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .prague_activated() - .build(), - ); - let (mut nodes, _) = - setup::(1, chain.clone(), false, eth_payload_attributes).await?; - let node = nodes.pop().unwrap(); + let chain = test_chain_spec(EthereumHardfork::Prague); + let (node, _) = EthereumNode::test_setup(1, chain.clone()).build_single().await?; let auth = node.auth_server_handle(); let url = auth.http_url(); - let jwt = secret_to_bearer_header(auth.jwt_secret()); let client = reqwest::Client::new(); let state = ForkchoiceState { head_block_hash: chain.genesis_hash(), @@ -810,38 +577,29 @@ async fn test_engine_ssz_request_validation() -> eyre::Result<()> { }; let response = client .post(format!("{url}{ENGINE_FORKCHOICE_ROUTE}")) - .header("Authorization", jwt.to_str()?) - .header("Content-Type", "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, fork) - .body( - ForkchoiceUpdateCancun { - forkchoice_state: state, - payload_attributes: Optional::some(attrs), - } - .as_ssz_bytes(), - ) + .jwt(&auth) + .fork(fork) + .ssz(&ForkchoiceUpdateCancun { + forkchoice_state: state, + payload_attributes: Optional::some(attrs), + }) .send() .await?; if let Some(error) = expected_error { assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); - assert_eq!( - response.json::().await?["type"], - format!("/engine-api/errors/{error}") - ); + assert_eq!(response.problem_type().await?, format!("/engine-api/errors/{error}")); } else { - assert_eq!(response.status(), reqwest::StatusCode::OK); - let fcu = - SszForkchoiceUpdateResponse::from_ssz_bytes(&response.bytes().await?).unwrap(); + let fcu = response.ssz::().await?; assert_eq!(fcu.payload_status.status, PayloadStatusKind::Valid); let id = fcu.payload_id.into_option().unwrap(); - let response = client + let built: BuiltPayloadPrague = client .get(format!("{url}{ENGINE_PAYLOADS_ROUTE}/{id}")) - .header("Authorization", jwt.to_str()?) - .header(ENGINE_EXECUTION_VERSION_HEADER, fork) + .jwt(&auth) + .fork(fork) .send() + .await? + .ssz() .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let built = BuiltPayloadPrague::from_ssz_bytes(&response.bytes().await?).unwrap(); assert_eq!(built.payload.payload_inner.withdrawals.len(), 16); } } @@ -849,16 +607,12 @@ async fn test_engine_ssz_request_validation() -> eyre::Result<()> { // The timestamp restriction only applies to builds, not historical head updates. let response = client .post(format!("{url}{ENGINE_FORKCHOICE_ROUTE}")) - .header("Authorization", jwt.to_str()?) - .header("Content-Type", "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, "cancun") - .body( - ForkchoiceUpdateCancun { - forkchoice_state: state, - payload_attributes: Optional::none(), - } - .as_ssz_bytes(), - ) + .jwt(&auth) + .fork("cancun") + .ssz(&ForkchoiceUpdateCancun { + forkchoice_state: state, + payload_attributes: Optional::none(), + }) .send() .await?; assert_eq!(response.status(), reqwest::StatusCode::OK); @@ -887,25 +641,17 @@ async fn test_engine_ssz_request_validation() -> eyre::Result<()> { payload.payload_inner.payload_inner.transactions = transactions; let response = client .post(format!("{url}{ENGINE_PAYLOADS_ROUTE}")) - .header("Authorization", jwt.to_str()?) - .header("Content-Type", "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, "prague") - .body( - ExecutionPayloadEnvelopePrague { - payload, - parent_beacon_block_root: B256::ZERO, - execution_requests: Default::default(), - } - .as_ssz_bytes(), - ) + .jwt(&auth) + .fork("prague") + .ssz(&ExecutionPayloadEnvelopePrague { + payload, + parent_beacon_block_root: B256::ZERO, + execution_requests: Default::default(), + }) .send() .await?; assert_eq!(response.status(), status); - assert_eq!(response.headers()[reqwest::header::CONTENT_TYPE], "application/problem+json"); - assert_eq!( - response.json::().await?["type"], - format!("/engine-api/errors/{error}") - ); + assert_eq!(response.problem_type().await?, format!("/engine-api/errors/{error}")); } Ok(()) } @@ -913,7 +659,6 @@ async fn test_engine_ssz_request_validation() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_custom_engine_and_middleware() -> eyre::Result<()> { use std::sync::atomic::{AtomicUsize, Ordering}; - let runtime = Runtime::test(); let requests = Arc::new(AtomicUsize::new(0)); let observed = requests.clone(); let middleware = @@ -923,16 +668,10 @@ async fn test_engine_ssz_custom_engine_and_middleware() -> eyre::Result<()> { } request }); - let chain = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .prague_activated() - .build(), - ); + let chain = test_chain_spec(EthereumHardfork::Prague); let NodeHandle { node, .. } = NodeBuilder::new(NodeConfig::test().with_chain(chain).with_unused_ports()) - .testing_node(runtime) + .testing_node(Runtime::test()) .with_types::() .with_components(EthereumNode::components()) .with_add_ons( @@ -943,19 +682,11 @@ async fn test_engine_ssz_custom_engine_and_middleware() -> eyre::Result<()> { .launch() .await?; let auth = &node.add_ons_handle.rpc_server_handles.auth; - let jwt = secret_to_bearer_header(auth.jwt_secret()); let client = reqwest::Client::new(); for route in [ENGINE_CAPABILITIES_ROUTE, ENGINE_IDENTITY_ROUTE] { - let response = client - .get(format!("{}{route}", auth.http_url())) - .header("Authorization", jwt.to_str()?) - .send() - .await?; + let response = client.get(format!("{}{route}", auth.http_url())).jwt(auth).send().await?; assert_eq!(response.status(), reqwest::StatusCode::NOT_FOUND); - assert_eq!( - response.json::().await?["type"], - "/engine-api/errors/method-not-found" - ); + assert_eq!(response.problem_type().await?, "/engine-api/errors/method-not-found"); } assert_eq!(requests.load(Ordering::Relaxed), 2); Ok(()) @@ -963,15 +694,8 @@ async fn test_engine_ssz_custom_engine_and_middleware() -> eyre::Result<()> { #[tokio::test] async fn test_engine_ssz_witness_omitted_without_provider_parent_state() -> eyre::Result<()> { - let chain = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .amsterdam_activated() - .build(), - ); let (mut nodes, _) = - setup::(2, chain, false, eth_payload_attributes_amsterdam).await?; + EthereumNode::test_setup_for(EthereumHardfork::Amsterdam).with_num_nodes(2).build().await?; let target = nodes.pop().unwrap(); let mut source = nodes.pop().unwrap(); let first = source.advance_block().await?; @@ -986,17 +710,15 @@ async fn test_engine_ssz_witness_omitted_without_provider_parent_state() -> eyre execution_requests: child.execution_requests, }; let auth = target.auth_server_handle(); - let jwt = secret_to_bearer_header(auth.jwt_secret()); - let response = reqwest::Client::new() + let response: PayloadStatusWithWitness = reqwest::Client::new() .post(format!("{}/engine/v1/payloads/witness", auth.http_url())) - .header("Authorization", jwt.to_str()?) - .header("Content-Type", "application/octet-stream") - .header(ENGINE_EXECUTION_VERSION_HEADER, "amsterdam") - .body(request.as_ssz_bytes()) + .jwt(&auth) + .fork("amsterdam") + .ssz(&request) .send() + .await? + .ssz() .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - let response = PayloadStatusWithWitness::from_ssz_bytes(&response.bytes().await?).unwrap(); assert_eq!(response.payload_status.status, PayloadStatusKind::Valid); assert!(response.witness.is_none()); Ok(()) diff --git a/crates/ethereum/node/tests/e2e/forkchoice.rs b/crates/ethereum/node/tests/e2e/forkchoice.rs index 6e0e5806922..3dc23e654b8 100644 --- a/crates/ethereum/node/tests/e2e/forkchoice.rs +++ b/crates/ethereum/node/tests/e2e/forkchoice.rs @@ -1,46 +1,41 @@ //! Tests for canonical chain and atomic forkchoice state updates via the Engine API. -use crate::utils::eth_payload_attributes; use alloy_eips::BlockNumberOrTag; use alloy_primitives::B256; -use alloy_provider::{Provider, ProviderBuilder}; +use alloy_provider::Provider; use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; use jsonrpsee_core::client::Error; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::{trie::wait_for_persisted_block, E2ETestSetupBuilder}; +use reth_chainspec::EthereumHardfork; +use reth_e2e_test_utils::{ + eth_payload_attributes, test_chain_spec, trie::wait_for_persisted_block, E2ETestSetupExt, +}; use reth_node_ethereum::{EthEngineTypes, EthereumNode}; use reth_provider::{DatabaseProviderFactory, HeaderProvider}; use reth_rpc_api::{EngineApiClient, TestingBuildBlockRequestV1}; use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; -use std::{sync::Arc, time::Duration}; +use std::time::Duration; #[tokio::test] async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - let (mut nodes, _) = - E2ETestSetupBuilder::::new(2, chain_spec, eth_payload_attributes) - .with_connect_nodes(false) - .with_node_config_modifier(|mut config| { - config.rpc.http_api = - Some(RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing])); - config - }) - .build() - .await?; + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); + let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()) + .with_connect_nodes(false) + .with_rpc_modifier(|rpc| { + rpc.with_http_api(RpcModuleSelection::from([ + RethRpcModule::Eth, + RethRpcModule::Testing, + ])) + }) + .build() + .await?; let node = nodes.pop().unwrap(); let producer = nodes.pop().unwrap(); let genesis = node.block_hash(0); let engine = node.auth_server_handle().http_client(); let producer_engine = producer.auth_server_handle().http_client(); - let rpc = ProviderBuilder::new().connect_http(node.rpc_url()); + let rpc = node.rpc_provider(); // Build a1 <- a2 <- a3 and b1 <- b2, both rooted at genesis. Only chain A is canonical. let mut chains = [vec![genesis], vec![genesis]]; @@ -49,7 +44,10 @@ async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { let envelope = producer .testing_build_block_v1(TestingBuildBlockRequestV1 { parent_block_hash: *chains[branch].last().unwrap(), - payload_attributes: eth_payload_attributes(number + branch as u64 * 10), + payload_attributes: eth_payload_attributes( + &chain_spec, + number + branch as u64 * 10, + ), transactions: vec![], extra_data: None, }) @@ -147,7 +145,7 @@ async fn invalid_forkchoice_preserves_canonical_state() -> eyre::Result<()> { let err = EngineApiClient::::fork_choice_updated_v3( &engine, ForkchoiceState::same_hash(b[2]), - Some(eth_payload_attributes(1)), + Some(eth_payload_attributes(&chain_spec, 1)), ) .await .unwrap_err(); @@ -177,35 +175,29 @@ async fn fcu_restores_reorged_out_persisted_head_above_shorter_branch() -> eyre: async fn assert_fcu_restores_reorged_out_persisted_head(sibling_len: u64) -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - let (mut nodes, _) = - E2ETestSetupBuilder::::new(2, chain_spec, eth_payload_attributes) - .with_connect_nodes(false) - .with_tree_config_modifier(|config| { - config - .with_num_state_masking_blocks(0) - .with_persistence_threshold(0) - .with_memory_block_buffer_target(0) - }) - .with_node_config_modifier(|mut config| { - config.rpc.http_api = - Some(RpcModuleSelection::from([RethRpcModule::Eth, RethRpcModule::Testing])); - config - }) - .build() - .await?; + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); + let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()) + .with_connect_nodes(false) + .with_tree_config_modifier(|config| { + config + .with_num_state_masking_blocks(0) + .with_persistence_threshold(0) + .with_memory_block_buffer_target(0) + }) + .with_rpc_modifier(|rpc| { + rpc.with_http_api(RpcModuleSelection::from([ + RethRpcModule::Eth, + RethRpcModule::Testing, + ])) + }) + .build() + .await?; let node = nodes.pop().unwrap(); let producer = nodes.pop().unwrap(); let genesis = node.block_hash(0); let engine = node.auth_server_handle().http_client(); let producer_engine = producer.auth_server_handle().http_client(); - let rpc = ProviderBuilder::new().connect_http(node.rpc_url()); + let rpc = node.rpc_provider(); // Persist A through block 3, then import a sibling branch without canonicalizing it. let mut chains = [vec![genesis], vec![genesis]]; @@ -214,7 +206,10 @@ async fn assert_fcu_restores_reorged_out_persisted_head(sibling_len: u64) -> eyr let envelope = producer .testing_build_block_v1(TestingBuildBlockRequestV1 { parent_block_hash: *chains[branch].last().unwrap(), - payload_attributes: eth_payload_attributes(number + branch as u64 * 10), + payload_attributes: eth_payload_attributes( + &chain_spec, + number + branch as u64 * 10, + ), transactions: vec![], extra_data: None, }) diff --git a/crates/ethereum/node/tests/e2e/invalid_payload.rs b/crates/ethereum/node/tests/e2e/invalid_payload.rs index bb5eec4e627..a8b3c354d05 100644 --- a/crates/ethereum/node/tests/e2e/invalid_payload.rs +++ b/crates/ethereum/node/tests/e2e/invalid_payload.rs @@ -3,22 +3,18 @@ //! This module tests the scenario where a node receives invalid payloads (e.g., with modified //! state roots) before receiving valid ones, ensuring the node can recover and continue. -use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; use alloy_consensus::proofs::calculate_transaction_root; use alloy_eips::{eip2718::Decodable2718, eip7685::RequestsOrHash}; use alloy_primitives::{bytes, keccak256, Bytes, B256}; use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV3, PayloadStatusEnum}; use rand::{rngs::StdRng, Rng, SeedableRng}; -use reth_chainspec::{ChainSpecBuilder, EthereumHardfork, MAINNET}; -use reth_e2e_test_utils::{ - eth_payload_attributes_for_fork, setup_engine, transaction::TransactionTestContext, -}; +use reth_chainspec::EthereumHardfork; +use reth_e2e_test_utils::{test_chain_spec, transaction::TransactionTestContext, E2ETestSetupExt}; use reth_ethereum_primitives::TransactionSigned; use reth_node_ethereum::EthereumNode; use reth_primitives_traits::SignedTransaction; use reth_rpc_api::EngineApiClient; -use std::sync::Arc; /// Tests that a node can handle receiving an invalid payload (with wrong state root) /// followed by the correct payload, and continue operating normally. @@ -35,22 +31,9 @@ async fn can_handle_invalid_payload_then_valid() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let (mut nodes, wallet) = setup_engine::( - 2, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; + let (mut nodes, wallet) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; let mut producer = nodes.pop().unwrap(); let receiver = nodes.pop().unwrap(); @@ -152,22 +135,9 @@ async fn can_handle_multiple_invalid_payloads() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let (mut nodes, wallet) = setup_engine::( - 2, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; + let (mut nodes, wallet) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; let mut producer = nodes.pop().unwrap(); let receiver = nodes.pop().unwrap(); @@ -253,22 +223,9 @@ async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let (mut nodes, wallet) = setup_engine::( - 2, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; + let (mut nodes, wallet) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; let mut producer = nodes.pop().unwrap(); let receiver = nodes.pop().unwrap(); @@ -368,19 +325,8 @@ async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> { async fn unrecoverable_signature_is_invalid_payload() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .paris_activated() - .build(), - ); - let (mut nodes, wallet) = - setup_engine::(1, chain_spec, false, Default::default(), |timestamp| { - eth_payload_attributes_for_fork(EthereumHardfork::Paris, timestamp) - }) - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Paris).build_single().await?; let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; node.rpc.inject_tx(raw_tx).await?; @@ -432,23 +378,8 @@ async fn unrecoverable_signature_is_invalid_payload() -> eyre::Result<()> { async fn undecodable_bal_is_invalid_payload() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .amsterdam_activated() - .build(), - ); - - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes_amsterdam, - ) - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Amsterdam).build_single().await?; // Build a valid Amsterdam payload without making it canonical. let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index 8d139639ed9..b8ffdf95ac1 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -1,18 +1,15 @@ -use crate::utils::{ - advance_with_random_transactions, eth_payload_attributes, eth_payload_attributes_amsterdam, -}; +use crate::utils::advance_with_random_transactions; use alloy_consensus::{SignableTransaction, TxEip1559, TxEnvelope}; use alloy_eips::{BlockNumberOrTag, Encodable2718}; use alloy_network::TxSignerSync; use alloy_primitives::B256; -use alloy_provider::{Provider, ProviderBuilder}; +use alloy_provider::Provider; use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; use futures::{future::JoinAll, StreamExt}; use rand::{rngs::StdRng, seq::IndexedRandom, Rng, SeedableRng}; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ - setup, setup_engine, setup_engine_with_connection, transaction::TransactionTestContext, - wallet::Wallet, + test_chain_spec, transaction::TransactionTestContext, wallet::Wallet, E2ETestSetupExt, }; use reth_engine_primitives::ConsensusEngineEvent; use reth_ethereum_primitives::EthPrimitives; @@ -31,13 +28,7 @@ async fn can_launch_with_net_if_and_shared_discovery_port() -> eyre::Result<()> reth_tracing::init_test_tracing(); let runtime = Runtime::test(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let discovery_port = unused_udp_port(); let mut network = NetworkArgs::default().with_unused_p2p_port(); @@ -103,35 +94,19 @@ fn unused_udp_port() -> u16 { async fn can_sync() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let (mut nodes, wallet) = setup::( - 2, - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ), - false, - eth_payload_attributes, - ) - .await?; + let (mut nodes, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).with_num_nodes(2).build().await?; let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; let mut second_node = nodes.pop().unwrap(); let mut first_node = nodes.pop().unwrap(); - // Make the first node advance - let tx_hash = first_node.rpc.inject_tx(raw_tx).await?; - // make the node advance - let payload = first_node.advance_block().await?; - + let (tx_hash, payload) = first_node.inject_and_advance(raw_tx).await?; let block_hash = payload.block().hash(); - let block_number = payload.block().number; // assert the block has been committed to the blockchain - first_node.assert_new_block(tx_hash, block_hash, block_number).await?; + first_node.assert_new_block(tx_hash, block_hash, payload.block().number).await?; // only send forkchoice update to second node second_node.update_forkchoice(block_hash, block_hash).await?; @@ -146,23 +121,9 @@ async fn can_sync() -> eyre::Result<()> { async fn rejects_downloaded_block_with_invalid_bal_hash() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .amsterdam_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Amsterdam); - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes_amsterdam, - ) - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()).build_single().await?; // Build a valid Amsterdam block without submitting it to the engine, then change only its BAL // commitment so all other execution-derived header fields remain valid. @@ -241,30 +202,16 @@ async fn e2e_test_send_transactions() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .prague_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Prague); - let (mut nodes, _) = setup_engine::( - 2, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; + let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + let provider = node.rpc_provider(); advance_with_random_transactions(&mut node, 100, &mut rng, true).await?; let second_node = nodes.pop().unwrap(); - let second_provider = ProviderBuilder::new().connect_http(second_node.rpc_url()); + let second_provider = second_node.rpc_provider(); assert_eq!(second_provider.get_block_number().await?, 0); @@ -283,28 +230,14 @@ async fn test_long_reorg() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .prague_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Prague); - let (mut nodes, _) = setup_engine::( - 2, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; + let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; let mut first_node = nodes.pop().unwrap(); let mut second_node = nodes.pop().unwrap(); - let first_provider = ProviderBuilder::new().connect_http(first_node.rpc_url()); + let first_provider = first_node.rpc_provider(); // Advance first node 100 blocks. advance_with_random_transactions(&mut first_node, 100, &mut rng, false).await?; @@ -341,29 +274,15 @@ async fn test_pipeline_sync_target_head_becomes_finalized() -> eyre::Result<()> let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .prague_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Prague); - let (mut nodes, _) = setup_engine::( - 2, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; + let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; let mut first_node = nodes.pop().unwrap(); let second_node = nodes.pop().unwrap(); - let first_provider = ProviderBuilder::new().connect_http(first_node.rpc_url()); - let second_provider = ProviderBuilder::new().connect_http(second_node.rpc_url()); + let first_provider = first_node.rpc_provider(); + let second_provider = second_node.rpc_provider(); advance_with_random_transactions(&mut first_node, TARGET_BLOCK as usize, &mut rng, false) .await?; @@ -403,28 +322,14 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .prague_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Prague); - let (mut nodes, _) = setup_engine::( - 2, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; + let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; let mut first_node = nodes.pop().unwrap(); let mut second_node = nodes.pop().unwrap(); - let first_provider = ProviderBuilder::new().connect_http(first_node.rpc_url()); + let first_provider = first_node.rpc_provider(); // Advance first node 100 blocks and finalize the chain. advance_with_random_transactions(&mut first_node, 100, &mut rng, true).await?; @@ -448,14 +353,7 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { async fn test_tx_propagation() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .prague_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Prague); // Setup wallet let chain_id = chain_spec.chain().into(); @@ -476,15 +374,8 @@ async fn test_tx_propagation() -> eyre::Result<()> { }; // Setup 10 nodes - let (mut nodes, _) = setup_engine_with_connection::( - 10, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - false, - ) - .await?; + let (mut nodes, _) = + EthereumNode::test_setup(10, chain_spec.clone()).with_connect_nodes(false).build().await?; // Connect all nodes to the first one let (first, rest) = nodes.split_at_mut(1); @@ -494,8 +385,7 @@ async fn test_tx_propagation() -> eyre::Result<()> { // Advance all nodes for 1 block so that they don't consider themselves unsynced let tx = build_tx(); - nodes[0].rpc.inject_tx(tx.encoded_2718().into()).await?; - let payload = nodes[0].advance_block().await?; + let (_, payload) = nodes[0].inject_and_advance(tx.encoded_2718().into()).await?; nodes[1..] .iter_mut() .map(|node| async { diff --git a/crates/ethereum/node/tests/e2e/pool.rs b/crates/ethereum/node/tests/e2e/pool.rs index 82cde6a9998..79579be78a5 100644 --- a/crates/ethereum/node/tests/e2e/pool.rs +++ b/crates/ethereum/node/tests/e2e/pool.rs @@ -1,29 +1,23 @@ -use crate::utils::eth_payload_attributes; use alloy_consensus::{EthereumTxEnvelope, TxEip4844}; use alloy_eips::{eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M, Encodable2718}; -use alloy_genesis::Genesis; use alloy_primitives::{Address, TxKind, B256, U256}; -use alloy_provider::{Provider, ProviderBuilder}; +use alloy_provider::Provider; use alloy_rpc_types_eth::TransactionRequest; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ - node::NodeTestContext, transaction::TransactionTestContext, wallet::Wallet, E2ETestSetupBuilder, -}; -use reth_node_builder::{NodeBuilder, NodeHandle}; -use reth_node_core::{ - args::{RpcServerArgs, TxPoolArgs}, - node_config::NodeConfig, + test_chain_spec, transaction::TransactionTestContext, wait::poll_until, wallet::Wallet, + E2ETestSetupExt, }; +use reth_node_core::args::TxPoolArgs; use reth_node_ethereum::EthereumNode; use reth_primitives_traits::Recovered; use reth_provider::CanonStateSubscriptions; -use reth_tasks::Runtime; use reth_transaction_pool::{ blobstore::InMemoryBlobStore, test_utils::OkValidator, BlockInfo, CoinbaseTipOrdering, EthPooledTransaction, Pool, PoolTransaction, TransactionOrigin, TransactionPool, TransactionPoolExt, }; -use std::{sync::Arc, time::Duration}; +use std::time::Duration; #[tokio::test] async fn rpc_enforces_minimum_priority_fee() -> eyre::Result<()> { @@ -31,26 +25,16 @@ async fn rpc_enforces_minimum_priority_fee() -> eyre::Result<()> { const MINIMUM_PRIORITY_FEE: u128 = 1; - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json"))?; - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); - let (mut nodes, wallet) = - E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .with_node_config_modifier(|config| { - config.with_txpool(TxPoolArgs { - minimum_priority_fee: Some(MINIMUM_PRIORITY_FEE), - ..Default::default() - }) + let (node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + .with_node_config_modifier(|config| { + config.with_txpool(TxPoolArgs { + minimum_priority_fee: Some(MINIMUM_PRIORITY_FEE), + ..Default::default() }) - .build() - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + }) + .build_single() + .await?; + let provider = node.rpc_provider(); let transaction = |max_priority_fee_per_gas| TransactionRequest { nonce: Some(0), @@ -63,8 +47,9 @@ async fn rpc_enforces_minimum_priority_fee() -> eyre::Result<()> { ..Default::default() }; - let below_minimum = TransactionTestContext::sign_tx(wallet.inner.clone(), transaction(0)).await; - let err = provider.send_raw_transaction(&below_minimum.encoded_2718()).await.unwrap_err(); + let below_minimum = + TransactionTestContext::sign_tx_bytes(wallet.inner.clone(), transaction(0)).await; + let err = provider.send_raw_transaction(&below_minimum).await.unwrap_err(); assert!( err.to_string().contains("transaction priority fee below minimum required priority fee 1"), "{err}" @@ -72,8 +57,9 @@ async fn rpc_enforces_minimum_priority_fee() -> eyre::Result<()> { assert!(node.inner.pool.is_empty()); let at_minimum = - TransactionTestContext::sign_tx(wallet.inner, transaction(MINIMUM_PRIORITY_FEE)).await; - let pending = provider.send_raw_transaction(&at_minimum.encoded_2718()).await?; + TransactionTestContext::sign_tx_bytes(wallet.inner, transaction(MINIMUM_PRIORITY_FEE)) + .await; + let pending = provider.send_raw_transaction(&at_minimum).await?; assert_eq!(node.inner.pool.len(), 1); assert!(node.inner.pool.contains(pending.tx_hash())); @@ -84,7 +70,6 @@ async fn rpc_enforces_minimum_priority_fee() -> eyre::Result<()> { #[tokio::test] async fn maintain_txpool_stale_eviction() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); let txpool = Pool::new( OkValidator::default(), @@ -95,27 +80,9 @@ async fn maintain_txpool_stale_eviction() -> eyre::Result<()> { // Directly generate a node to simulate various traits such as `StateProviderFactory` required // by the pool maintenance task - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); - let node_config = NodeConfig::test() - .with_chain(chain_spec) - .with_unused_ports() - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .node(EthereumNode::default()) - .launch() - .await?; - - let node = NodeTestContext::new(node, eth_payload_attributes).await?; - - let wallet = Wallet::default(); + let (node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + let runtime = node.inner.task_executor.clone(); let config = reth_transaction_pool::maintain::MaintainPoolConfig { max_tx_lifetime: Duration::from_secs(1), @@ -157,7 +124,6 @@ async fn maintain_txpool_stale_eviction() -> eyre::Result<()> { #[tokio::test] async fn maintain_txpool_reorg() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); let txpool = Pool::new( OkValidator::default(), @@ -168,26 +134,10 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> { // Directly generate a node to simulate various traits such as `StateProviderFactory` required // by the pool maintenance task - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let genesis_hash = chain_spec.genesis_hash(); - let node_config = NodeConfig::test() - .with_chain(chain_spec) - .with_unused_ports() - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .node(EthereumNode::default()) - .launch() - .await?; - - let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; + let (mut node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + let runtime = node.inner.task_executor.clone(); let wallets = Wallet::new(2).wallet_gen(); let w1 = wallets.first().unwrap(); @@ -258,28 +208,24 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> { node.update_forkchoice(genesis_hash, block_hash1).await?; - loop { - // wait for pool to process `CanonStateNotification::Commit` event correctly, and finally - // tx1 will be removed and tx2 is still in the pool - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - if txpool.get(&tx_hash1).is_none() && txpool.get(&tx_hash2).is_some() { - break; - } - } + // wait for pool to process `CanonStateNotification::Commit` event correctly, and finally tx1 + // will be removed and tx2 is still in the pool + poll_until("pool to process the commit", || async { + Ok((txpool.get(&tx_hash1).is_none() && txpool.get(&tx_hash2).is_some()).then_some(())) + }) + .await?; // submit payload2 let block_hash2 = node.submit_payload(payload2).await?; node.update_forkchoice(genesis_hash, block_hash2).await?; - loop { - // wait for pool to process `CanonStateNotification::Reorg` event properly, and finally tx1 - // will be added back to the pool and tx2 will be removed. - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - if txpool.get(&tx_hash1).is_some() && txpool.get(&tx_hash2).is_none() { - break; - } - } + // wait for pool to process `CanonStateNotification::Reorg` event properly, and finally tx1 + // will be added back to the pool and tx2 will be removed. + poll_until("pool to process the reorg", || async { + Ok((txpool.get(&tx_hash1).is_some() && txpool.get(&tx_hash2).is_none()).then_some(())) + }) + .await?; Ok(()) } @@ -289,7 +235,6 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> { #[tokio::test] async fn maintain_txpool_commit() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let runtime = Runtime::test(); let txpool = Pool::new( OkValidator::default(), @@ -300,27 +245,9 @@ async fn maintain_txpool_commit() -> eyre::Result<()> { // Directly generate a node to simulate various traits such as `StateProviderFactory` required // by the pool maintenance task - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(genesis) - .cancun_activated() - .build(), - ); - let node_config = NodeConfig::test() - .with_chain(chain_spec) - .with_unused_ports() - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(runtime.clone()) - .node(EthereumNode::default()) - .launch() - .await?; - - let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; - - let wallet = Wallet::default(); + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + let runtime = node.inner.task_executor.clone(); runtime.spawn_critical_task( "txpool maintenance task", @@ -358,14 +285,10 @@ async fn maintain_txpool_commit() -> eyre::Result<()> { let _ = node.rpc.inject_tx(envelop.encoded_2718().into()).await.unwrap(); let _ = node.advance_block().await.unwrap(); - loop { - // wait for pool to process `CanonStateNotification::Commit` event correctly, and finally - // the pool will be cleared - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - if txpool.is_empty() { - break; - } - } + // wait for pool to process `CanonStateNotification::Commit` event correctly, and finally the + // pool will be cleared + poll_until("pool to process the commit", || async { Ok(txpool.is_empty().then_some(())) }) + .await?; Ok(()) } diff --git a/crates/ethereum/node/tests/e2e/prestate.rs b/crates/ethereum/node/tests/e2e/prestate.rs index d1c35063ecb..5183a891a68 100644 --- a/crates/ethereum/node/tests/e2e/prestate.rs +++ b/crates/ethereum/node/tests/e2e/prestate.rs @@ -8,11 +8,8 @@ use alloy_rpc_types_trace::geth::{ }; use eyre::{eyre, Result}; use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_node_builder::{NodeBuilder, NodeHandle}; -use reth_node_core::{args::RpcServerArgs, node_config::NodeConfig}; +use reth_e2e_test_utils::E2ETestSetupExt; use reth_node_ethereum::EthereumNode; -use reth_rpc_server_types::RpcModuleSelection; -use reth_tasks::Runtime; use serde::Deserialize; use std::sync::Arc; @@ -29,8 +26,6 @@ async fn debug_trace_call_matches_geth_prestate_snapshot() -> Result<()> { let mut genesis: Genesis = MAINNET.genesis().clone(); genesis.coinbase = address!("0x95222290dd7278aa3ddd389cc1e1d165cc4bafe5"); - let runtime = Runtime::test(); - let expected_frame = expected_snapshot_frame()?; let prestate_mode = match &expected_frame { PreStateFrame::Default(mode) => mode.clone(), @@ -54,20 +49,8 @@ async fn debug_trace_call_matches_geth_prestate_snapshot() -> Result<()> { .build(), ); - let node_config = NodeConfig::test().with_chain(chain_spec).with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::all_modules().into()), - ); - - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) - .testing_node(runtime) - .node(EthereumNode::default()) - .launch() - .await?; - - let provider = node.rpc_server_handle().eth_http_provider().unwrap(); + let (node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + let provider = node.rpc_provider(); // let tx = r#"{ diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index 38380eb0108..976d6c6e331 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -1,13 +1,9 @@ -use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; use alloy_eips::{ eip2718::Encodable2718, eip7910::EthConfig, eip7928::BlockAccessList, BlockNumberOrTag, }; -use alloy_genesis::Genesis; use alloy_primitives::{keccak256, Address, Bytes, B256, U256}; use alloy_provider::{ - ext::DebugApi, - network::{EthereumWallet, TransactionBuilder}, - Provider, ProviderBuilder, SendableTx, + ext::DebugApi, network::TransactionBuilder, Provider, ProviderBuilder, SendableTx, }; use alloy_rpc_types_beacon::relay::{ BidTrace, BuilderBlockValidationRequestV3, BuilderBlockValidationRequestV4, @@ -28,10 +24,10 @@ use alloy_rpc_types_trace::geth::{ }; use jsonrpsee::core::client::{ClientT, Subscription, SubscriptionClientT}; use rand::{rngs::StdRng, Rng, SeedableRng}; -use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; +use reth_chainspec::{ChainSpecBuilder, EthChainSpec, EthereumHardfork}; use reth_e2e_test_utils::{ - setup_engine, transaction::TransactionTestContext, wallet::Wallet, E2ETestSetupBuilder, - NodeHelperType, + receipt::PendingTransactionExt, test_chain_spec, test_chain_spec_builder, + transaction::TransactionTestContext, wallet::Wallet, E2ETestSetupExt, NodeHelperType, }; use reth_network::{types::NatResolver, PeersInfo}; use reth_node_builder::{NodeBuilder, NodeHandle}; @@ -71,11 +67,7 @@ async fn test_rpc_shares_sender_recovery_cache_with_execution() -> eyre::Result< use reth_transaction_pool::test_utils::TransactionGenerator; let chain_spec = Arc::new(ChainSpecBuilder::mainnet().cancun_activated().build()); - let (mut nodes, _) = - E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .build() - .await?; - let node = nodes.pop().unwrap(); + let (node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; let cache = node.inner.evm_config.sender_recovery_cache.as_ref().unwrap(); let client = node.rpc_client().unwrap(); let transaction = @@ -97,7 +89,7 @@ async fn inject_blob_transaction( node: &NodeHelperType, wallet: &Wallet, ) -> eyre::Result<()> { - let blob_wallet = wallet.wallet_gen().swap_remove(0); + let blob_wallet = wallet.signer(0); let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, blob_wallet).await?; node.rpc.inject_tx(blob_tx).await?; Ok(()) @@ -107,23 +99,7 @@ async fn inject_blob_transaction( async fn test_block_access_list_lookup_semantics() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - - let (mut nodes, _) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let node = nodes.pop().unwrap(); + let (node, _) = EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; let client = node.rpc_client().unwrap(); let pending: Option = @@ -147,26 +123,15 @@ async fn test_block_access_list_lookup_semantics() -> eyre::Result<()> { #[tokio::test(flavor = "multi_thread")] async fn test_bal_prewarming_for_transaction_replay() -> eyre::Result<()> { for (cache_computed, prewarm) in [(false, false), (true, false), (false, true)] { - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .cancun_activated() - .build(), - ); - let (mut nodes, wallet) = - E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .with_node_config_modifier(move |mut config| { - config.rpc.rpc_state_cache.cache_computed_bals = cache_computed; - config.rpc.rpc_state_cache.prewarm_bals = prewarm.then_some(0); - config - }) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + .with_node_config_modifier(move |mut config| { + config.rpc.rpc_state_cache.cache_computed_bals = cache_computed; + config.rpc.rpc_state_cache.prewarm_bals = prewarm.then_some(0); + config + }) + .build_single() + .await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let first = provider .send_transaction(TransactionRequest::default().to(Address::ZERO).value(U256::from(1))) .await?; @@ -236,26 +201,10 @@ async fn test_fee_history() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let fee_history = provider.get_fee_history(10, 0_u64.into(), &[]).await?; @@ -270,8 +219,7 @@ async fn test_fee_history() -> eyre::Result<()> { // Spend some gas let builder = GasWaster::deploy_builder(&provider, U256::from(500)).send().await?; node.advance_block().await?; - let receipt = builder.get_receipt().await?; - assert!(receipt.status()); + let receipt = builder.successful_receipt().await?; let block = provider.get_block_by_number(1.into()).await?.unwrap(); assert_eq!(block.header.gas_used, receipt.gas_used,); @@ -321,31 +269,11 @@ async fn test_fee_history() -> eyre::Result<()> { async fn test_debug_trace_chain_subscription() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - let (mut nodes, wallet) = - E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .with_node_config_modifier(|config| { - config.with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All) - .with_ws() - .with_ws_api(RpcModuleSelection::All), - ) - }) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + .with_rpc_modifier(|rpc| rpc.with_ws().with_ws_api(RpcModuleSelection::All)) + .build_single() + .await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); // Geth suppresses empty intermediate blocks but always emits the end block. node.advance_block().await?; @@ -404,32 +332,15 @@ async fn test_debug_trace_eip8037_gas() -> eyre::Result<()> { // block and lets the same node exercise both sides of the fork. const AMSTERDAM_TIMESTAMP: u64 = 1_710_338_137; let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .osaka_activated() - .with_amsterdam_at(AMSTERDAM_TIMESTAMP) - .build(), + test_chain_spec_builder().osaka_activated().with_amsterdam_at(AMSTERDAM_TIMESTAMP).build(), ); - let payload_attributes = |timestamp| { - if timestamp >= AMSTERDAM_TIMESTAMP { - eth_payload_attributes_amsterdam(timestamp) - } else { - eth_payload_attributes(timestamp) - } - }; - let (mut nodes, wallet) = - setup_engine::(1, chain_spec, false, Default::default(), payload_attributes) - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let pre_amsterdam = GasWaster::deploy_builder(&provider, U256::from(1)).send().await?; let pre_amsterdam_hash = *pre_amsterdam.tx_hash(); node.advance_block().await?; - assert!(pre_amsterdam.get_receipt().await?.status()); + pre_amsterdam.successful_receipt().await?; let GethTrace::Default(frame) = provider .debug_trace_transaction(pre_amsterdam_hash, GethDebugTracingOptions::default()) @@ -461,7 +372,7 @@ async fn test_debug_trace_eip8037_gas() -> eyre::Result<()> { let amsterdam = GasWaster::deploy_builder(&provider, U256::from(1)).send().await?; let amsterdam_hash = *amsterdam.tx_hash(); node.advance_block().await?; - assert!(amsterdam.get_receipt().await?.status()); + amsterdam.successful_receipt().await?; let GethTrace::StateGasTracer(state_gas) = provider .debug_trace_transaction(amsterdam_hash, GethDebugTracingOptions::state_gas_tracer()) @@ -505,26 +416,10 @@ async fn test_debug_trace_eip8037_gas() -> eyre::Result<()> { async fn test_flashbots_validate_v3() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); node.advance(100, |_| { let provider = provider.clone(); @@ -587,25 +482,13 @@ async fn test_flashbots_validate_v3() -> eyre::Result<()> { async fn test_flashbots_validate_uses_shared_sender_recovery_cache() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - - let (mut nodes, wallet) = - E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .build() - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; let cache = node.inner.evm_config.sender_recovery_cache.clone().expect("cache is enabled"); - let signer = wallet.wallet_gen().swap_remove(0); + let signer = wallet.signer(0); let sender = signer.address(); - let provider = - ProviderBuilder::new().wallet(EthereumWallet::new(signer)).connect_http(node.rpc_url()); + let provider = node.rpc_provider_with_wallet(signer); let tx_hash = *provider .send_transaction(TransactionRequest::default().to(Address::ZERO)) @@ -649,26 +532,10 @@ async fn test_flashbots_validate_uses_shared_sender_recovery_cache() -> eyre::Re async fn test_flashbots_validate_v4() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .prague_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Prague); - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); node.advance(100, |_| { let provider = provider.clone(); @@ -733,31 +600,11 @@ async fn test_flashbots_validate_v4() -> eyre::Result<()> { async fn test_flashbots_validate_v5() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .osaka_activated() - .build(), - ); - - let (mut nodes, wallet) = - E2ETestSetupBuilder::::new(1, chain_spec, eth_payload_attributes) - .with_node_config_modifier(|config| { - config.with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All) - .with_force_blob_sidecar_upcasting(), - ) - }) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Osaka) + .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) + .build_single() + .await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); inject_blob_transaction(&node, &wallet).await?; let payload = node.new_payload().await?; @@ -817,34 +664,11 @@ async fn test_flashbots_validate_v5() -> eyre::Result<()> { async fn test_flashbots_validate_v6() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .amsterdam_activated() - .build(), - ); - - let (mut nodes, wallet) = E2ETestSetupBuilder::::new( - 1, - chain_spec, - eth_payload_attributes_amsterdam, - ) - .with_node_config_modifier(|config| { - config.with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All) - .with_force_blob_sidecar_upcasting(), - ) - }) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Amsterdam) + .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) + .build_single() + .await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); inject_blob_transaction(&node, &wallet).await?; let payload = node.new_payload().await?; @@ -947,28 +771,12 @@ async fn test_flashbots_validate_v6() -> eyre::Result<()> { async fn test_estimate_gas_basic_transfers_post_amsterdam() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .amsterdam_activated() - .build(), - ); + let (node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Amsterdam).build_single().await?; + let provider = node.rpc_provider(); - let (mut nodes, _) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes_amsterdam, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new().connect_http(node.rpc_url()); - - let mut signers = Wallet::new(2).wallet_gen(); - let from = signers.remove(0).address(); - let existing_recipient = signers.remove(0).address(); + let from = wallet.signer(0).address(); + let existing_recipient = wallet.signer(1).address(); let self_send_gas = provider .estimate_gas( @@ -1059,27 +867,15 @@ async fn test_eth_config() -> eyre::Result<()> { let osaka_timestamp = timestamp + 10000000; let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + test_chain_spec_builder() .cancun_activated() .with_prague_at(prague_timestamp) .with_osaka_at(osaka_timestamp) .build(), ); - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let _ = provider.send_transaction(TransactionRequest::default().to(Address::ZERO)).await?; node.advance_block().await?; @@ -1151,9 +947,7 @@ async fn test_admin_external_ip() -> eyre::Result<()> { let runtime = Runtime::test(); // Chain spec with test allocs - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = - Arc::new(ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).build()); + let chain_spec = Arc::new(test_chain_spec_builder().build()); let external_ip = "10.64.128.71".parse().unwrap(); // Node setup @@ -1186,9 +980,7 @@ async fn test_admin_node_info_uses_discv5_port_when_discv4_is_disabled() -> eyre let runtime = Runtime::test(); - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = - Arc::new(ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).build()); + let chain_spec = Arc::new(test_chain_spec_builder().build()); let mut network = NetworkArgs::default().with_unused_ports(); network.bootnodes = Some(Vec::new()); @@ -1231,9 +1023,7 @@ async fn test_admin_node_info_discv5_enr_uses_nat_extip_when_discv4_is_disabled( let runtime = Runtime::test(); - let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); - let chain_spec = - Arc::new(ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).build()); + let chain_spec = Arc::new(test_chain_spec_builder().build()); let mut network = NetworkArgs::default().with_unused_ports(); network.bootnodes = Some(Vec::new()); diff --git a/crates/ethereum/node/tests/e2e/selfdestruct.rs b/crates/ethereum/node/tests/e2e/selfdestruct.rs index 626f9d3b5e0..8efcbaa9c22 100644 --- a/crates/ethereum/node/tests/e2e/selfdestruct.rs +++ b/crates/ethereum/node/tests/e2e/selfdestruct.rs @@ -7,40 +7,32 @@ //! We disable prewarming to ensure deterministic cache behavior and verify the execution //! output state contains the expected account status after SELFDESTRUCT. -use crate::utils::{eth_payload_attributes, eth_payload_attributes_shanghai}; -use alloy_network::{EthereumWallet, TransactionBuilder}; +use alloy_network::TransactionBuilder; use alloy_primitives::{bytes, Address, Bytes, TxKind, U256}; -use alloy_provider::{Provider, ProviderBuilder}; +use alloy_provider::Provider; use alloy_rpc_types_eth::TransactionRequest; use futures::StreamExt; -use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::setup_engine; -use reth_node_api::TreeConfig; +use reth_chainspec::EthereumHardfork; +use reth_e2e_test_utils::{ + receipt::PendingTransactionExt, wallet::Wallet, E2ETestSetupExt, NodeHelperType, +}; use reth_node_ethereum::EthereumNode; use reth_revm::db::BundleAccount; -use std::sync::Arc; const MAX_FEE_PER_GAS: u128 = 20_000_000_000; const MAX_PRIORITY_FEE_PER_GAS: u128 = 1_000_000_000; -fn cancun_spec() -> Arc { - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ) -} - -fn shanghai_spec() -> Arc { - Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .shanghai_activated() - .build(), - ) +/// Launches a node for the given hardfork with prewarming disabled, so the execution output state +/// is deterministic. +async fn setup_node( + fork: EthereumHardfork, +) -> eyre::Result<(NodeHelperType, Wallet)> { + EthereumNode::test_setup_for(fork) + .with_tree_config_modifier(|config| { + config.without_prewarming(true).without_state_cache(false) + }) + .build_single() + .await } fn deploy_tx(from: Address, nonce: u64, init_code: Bytes) -> TransactionRequest { @@ -141,23 +133,16 @@ fn selfdestruct_contract_init_code() -> Bytes { async fn test_selfdestruct_post_dencun() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false); - let (mut nodes, wallet) = - setup_engine::(1, cancun_spec(), false, tree_config, eth_payload_attributes) - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = setup_node(EthereumHardfork::Cancun).await?; let signer = wallet.inner.clone(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(signer.clone())) - .connect_http(node.rpc_url()); + let provider = node.rpc_provider_with_wallet(signer.clone()); // Deploy contract that stores 0x42 at slot 0 and selfdestructs on any call let pending = provider .send_transaction(deploy_tx(signer.address(), 0, selfdestruct_contract_init_code())) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "Contract deployment should succeed"); + let receipt = pending.successful_receipt().await?; let contract_address = receipt.contract_address.expect("Should have contract address"); @@ -167,8 +152,7 @@ async fn test_selfdestruct_post_dencun() -> eyre::Result<()> { // Trigger SELFDESTRUCT by calling the contract let pending = provider.send_transaction(call_tx(signer.address(), contract_address, 1)).await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "Selfdestruct tx should succeed"); + pending.successful_receipt().await?; // Get the canonical notification for the selfdestruct block let notification = node.canonical_stream.next().await.unwrap(); @@ -195,8 +179,7 @@ async fn test_selfdestruct_post_dencun() -> eyre::Result<()> { // balance) let pending = provider.send_transaction(call_tx(signer.address(), contract_address, 2)).await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "Second call to contract should succeed"); + pending.successful_receipt().await?; // Consume the canonical notification let notification = node.canonical_stream.next().await.unwrap(); @@ -235,23 +218,16 @@ async fn test_selfdestruct_post_dencun() -> eyre::Result<()> { async fn test_selfdestruct_same_tx_post_dencun() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false); - let (mut nodes, wallet) = - setup_engine::(1, cancun_spec(), false, tree_config, eth_payload_attributes) - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = setup_node(EthereumHardfork::Cancun).await?; let signer = wallet.inner.clone(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(signer.clone())) - .connect_http(node.rpc_url()); + let provider = node.rpc_provider_with_wallet(signer.clone()); // Deploy contract that selfdestructs during its constructor let pending = provider .send_transaction(deploy_tx(signer.address(), 0, selfdestruct_in_constructor_init_code())) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "Contract deployment with selfdestruct should succeed"); + pending.successful_receipt().await?; // Calculate the contract address (CREATE uses sender + nonce) let contract_address = signer.address().create(0); @@ -280,8 +256,7 @@ async fn test_selfdestruct_same_tx_post_dencun() -> eyre::Result<()> { .send_transaction(transfer_tx(signer.address(), contract_address, 1, U256::from(1000))) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "ETH transfer to destroyed address should succeed"); + pending.successful_receipt().await?; // Consume the canonical notification let _ = node.canonical_stream.next().await; @@ -310,28 +285,16 @@ async fn test_selfdestruct_same_tx_post_dencun() -> eyre::Result<()> { async fn test_selfdestruct_pre_dencun() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false); - let (mut nodes, wallet) = setup_engine::( - 1, - shanghai_spec(), - false, - tree_config, - eth_payload_attributes_shanghai, - ) - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = setup_node(EthereumHardfork::Shanghai).await?; let signer = wallet.inner.clone(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(signer.clone())) - .connect_http(node.rpc_url()); + let provider = node.rpc_provider_with_wallet(signer.clone()); // Deploy contract that stores 0x42 at slot 0 and selfdestructs on any call let pending = provider .send_transaction(deploy_tx(signer.address(), 0, selfdestruct_contract_init_code())) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "Contract deployment should succeed"); + let receipt = pending.successful_receipt().await?; let contract_address = receipt.contract_address.expect("Should have contract address"); @@ -341,8 +304,7 @@ async fn test_selfdestruct_pre_dencun() -> eyre::Result<()> { // Trigger SELFDESTRUCT by calling the contract let pending = provider.send_transaction(call_tx(signer.address(), contract_address, 1)).await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "Selfdestruct tx should succeed"); + pending.successful_receipt().await?; // Get the canonical notification for the selfdestruct block let notification = node.canonical_stream.next().await.unwrap(); @@ -370,8 +332,7 @@ async fn test_selfdestruct_pre_dencun() -> eyre::Result<()> { .send_transaction(transfer_tx(signer.address(), contract_address, 2, U256::from(1000))) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "ETH transfer to destroyed contract address should succeed"); + pending.successful_receipt().await?; // Consume the canonical notification let notification = node.canonical_stream.next().await.unwrap(); @@ -420,15 +381,9 @@ async fn test_selfdestruct_pre_dencun() -> eyre::Result<()> { async fn test_selfdestruct_same_tx_preexisting_account_post_dencun() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false); - let (mut nodes, wallet) = - setup_engine::(1, cancun_spec(), false, tree_config, eth_payload_attributes) - .await?; - let mut node = nodes.pop().unwrap(); + let (mut node, wallet) = setup_node(EthereumHardfork::Cancun).await?; let signer = wallet.inner.clone(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(signer.clone())) - .connect_http(node.rpc_url()); + let provider = node.rpc_provider_with_wallet(signer.clone()); // Calculate where the contract will be deployed (CREATE uses sender + nonce) // We'll use nonce 1 for deployment, so first send ETH with nonce 0 @@ -444,8 +399,7 @@ async fn test_selfdestruct_same_tx_preexisting_account_post_dencun() -> eyre::Re )) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "ETH transfer should succeed"); + pending.successful_receipt().await?; // Consume the canonical notification let _ = node.canonical_stream.next().await; @@ -459,8 +413,7 @@ async fn test_selfdestruct_same_tx_preexisting_account_post_dencun() -> eyre::Re .send_transaction(deploy_tx(signer.address(), 1, selfdestruct_in_constructor_init_code())) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "Contract deployment with selfdestruct should succeed"); + let receipt = pending.successful_receipt().await?; // Verify deployment went to the expected address assert_eq!( @@ -509,8 +462,7 @@ async fn test_selfdestruct_same_tx_preexisting_account_post_dencun() -> eyre::Re )) .await?; node.advance_block().await?; - let receipt = pending.get_receipt().await?; - assert!(receipt.status(), "ETH transfer should succeed"); + pending.successful_receipt().await?; // Consume notification let _ = node.canonical_stream.next().await; diff --git a/crates/ethereum/node/tests/e2e/simulate.rs b/crates/ethereum/node/tests/e2e/simulate.rs index 49367310c77..16d9ef2e656 100644 --- a/crates/ethereum/node/tests/e2e/simulate.rs +++ b/crates/ethereum/node/tests/e2e/simulate.rs @@ -1,37 +1,21 @@ -use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; use alloy_primitives::{bytes, Address, U256}; -use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder}; -use alloy_rpc_types_engine::PayloadAttributes; +use alloy_provider::Provider; use alloy_rpc_types_eth::{ simulate::{SimBlock, SimulatePayload, SimulatedBlock}, state::{AccountOverride, StateOverride, StateOverridesBuilder}, BlockOverrides, TransactionRequest, TransactionTrait as _, }; -use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::setup_engine; +use reth_chainspec::EthereumHardfork; +use reth_e2e_test_utils::{test_chain_spec, test_chain_spec_builder, E2ETestSetupExt}; use reth_node_ethereum::EthereumNode; use std::sync::Arc; #[tokio::test] async fn test_simulate_v1_transfer_logs_across_amsterdam() -> eyre::Result<()> { - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .osaka_activated() - .with_amsterdam_at(24) - .build(), - ); - let (mut nodes, _) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes_amsterdam, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + let chain_spec = + Arc::new(test_chain_spec_builder().osaka_activated().with_amsterdam_at(24).build()); + let (node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + let provider = node.rpc_provider(); let from = Address::with_last_byte(0x41); let to = Address::with_last_byte(0x42); let reverting = Address::with_last_byte(0x43); @@ -92,24 +76,10 @@ async fn test_simulate_v1_transfer_logs_across_amsterdam() -> eyre::Result<()> { #[tokio::test] async fn test_simulate_v1_block_access_list_hash_across_amsterdam() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json"))?) - .osaka_activated() - .with_amsterdam_at(24) - .build(), - ); - let (mut nodes, _) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes_amsterdam, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + let chain_spec = + Arc::new(test_chain_spec_builder().osaka_activated().with_amsterdam_at(24).build()); + let (node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + let provider = node.rpc_provider(); let from = Address::with_last_byte(0x41); let contract = Address::with_last_byte(0x42); let state_overrides = StateOverride::from_iter([ @@ -153,26 +123,9 @@ async fn test_simulate_v1_block_access_list_hash_across_amsterdam() -> eyre::Res async fn test_simulate_v1_explicit_gas_uses_remaining_block_gas() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let tx = TransactionRequest::default().to(Address::ZERO).gas_limit(3_000_000); let sim_block = SimBlock::default().call(tx.clone()).call(tx); @@ -192,26 +145,9 @@ async fn test_simulate_v1_explicit_gas_uses_remaining_block_gas() -> eyre::Resul async fn test_simulate_v1_no_fields_call_defaults_to_remaining_block_gas() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let first_tx_gas_limit = 21_000; let sim_block = SimBlock::default() @@ -237,19 +173,11 @@ async fn test_simulate_v1_no_fields_call_defaults_to_remaining_block_gas() -> ey Ok(()) } -async fn assert_validation_uses_remaining_gas( - chain_spec: Arc, - payload_attributes: fn(u64) -> PayloadAttributes, -) -> eyre::Result<()> { +async fn assert_validation_uses_remaining_gas(fork: EthereumHardfork) -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let (mut nodes, wallet) = - setup_engine::(1, chain_spec, false, Default::default(), payload_attributes) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (node, wallet) = EthereumNode::test_setup_for(fork).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let from: Address = "0xc000000000000000000000000000000000000000".parse()?; let state_overrides = @@ -288,53 +216,22 @@ async fn assert_validation_uses_remaining_gas( #[tokio::test] async fn test_simulate_v1_validation_omitted_gas_uses_remaining_gas_osaka() -> eyre::Result<()> { - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .osaka_activated() - .build(), - ); - assert_validation_uses_remaining_gas(chain_spec, eth_payload_attributes).await + assert_validation_uses_remaining_gas(EthereumHardfork::Osaka).await } #[tokio::test] async fn test_simulate_v1_validation_omitted_gas_uses_remaining_gas_amsterdam() -> eyre::Result<()> { - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .amsterdam_activated() - .build(), - ); - assert_validation_uses_remaining_gas(chain_spec, eth_payload_attributes_amsterdam).await + assert_validation_uses_remaining_gas(EthereumHardfork::Amsterdam).await } #[tokio::test] async fn test_simulate_v1_blockhash_reads_prior_simulated_block() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let contract = Address::with_last_byte(0x42); let mut state_overrides = StateOverride::default(); @@ -370,26 +267,9 @@ async fn test_simulate_v1_blockhash_reads_prior_simulated_block() -> eyre::Resul async fn test_simulate_v1_explicit_gas_over_remaining_block_gas_errors() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let sim_block = SimBlock::default() .with_block_overrides(BlockOverrides { gas_limit: Some(50_000), ..Default::default() }) @@ -418,26 +298,10 @@ async fn test_simulate_v1_explicit_gas_over_remaining_block_gas_errors() -> eyre async fn test_simulate_v1_with_max_fee_per_blob_gas_only() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec.clone(), - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let mut node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let _ = provider.send_transaction(TransactionRequest::default().to(Address::ZERO)).await?; node.advance_block().await?; @@ -483,26 +347,9 @@ async fn test_simulate_v1_with_max_fee_per_blob_gas_only() -> eyre::Result<()> { async fn test_simulate_v1_too_many_blocks_error() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - ); - - let (mut nodes, wallet) = setup_engine::( - 1, - chain_spec, - false, - Default::default(), - eth_payload_attributes, - ) - .await?; - let node = nodes.pop().unwrap(); - let provider = ProviderBuilder::new() - .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) - .connect_http(node.rpc_url()); + let (node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let payload: SimulatePayload = (0..257).fold(SimulatePayload::default(), |payload, _| payload.extend(SimBlock::default())); diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 1c92e5dc5cc..4bacb42e3d9 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -4,45 +4,28 @@ use alloy_provider::{ network::{ Ethereum, EthereumWallet, NetworkWallet, TransactionBuilder, TransactionBuilder7702, }, - Provider, ProviderBuilder, SendableTx, + Provider, SendableTx, }; -use alloy_rpc_types_engine::PayloadAttributes; use alloy_rpc_types_eth::TransactionRequest; use alloy_signer::SignerSync; +use eyre::{ensure, eyre}; use rand::{seq::IndexedRandom, Rng}; -use reth_chainspec::EthereumHardfork; -use reth_e2e_test_utils::{eth_payload_attributes_for_fork, wallet::Wallet, NodeHelperType, TmpDB}; +use reqwest::{header, RequestBuilder, Response, StatusCode}; +use reth_e2e_test_utils::{wallet::Wallet, NodeHelperType}; use reth_ethereum_primitives::TxType; -use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; -use reth_provider::FullProvider; - -/// Helper function to create a new eth payload attributes -pub(crate) fn eth_payload_attributes(timestamp: u64) -> PayloadAttributes { - eth_payload_attributes_for_fork(EthereumHardfork::Cancun, timestamp) -} - -/// Helper function to create pre-Cancun (Shanghai) payload attributes. -pub(crate) fn eth_payload_attributes_shanghai(timestamp: u64) -> PayloadAttributes { - eth_payload_attributes_for_fork(EthereumHardfork::Shanghai, timestamp) -} - -/// Helper function to create Amsterdam payload attributes. -pub(crate) fn eth_payload_attributes_amsterdam(timestamp: u64) -> PayloadAttributes { - eth_payload_attributes_for_fork(EthereumHardfork::Amsterdam, timestamp) -} +use reth_rpc_builder::auth::AuthServerHandle; +use reth_rpc_layer::secret_to_bearer_header; +use ssz::{Decode, Encode}; /// Advances node by producing blocks with random transactions. -pub(crate) async fn advance_with_random_transactions( - node: &mut NodeHelperType, +pub(crate) async fn advance_with_random_transactions( + node: &mut NodeHelperType, num_blocks: usize, rng: &mut impl Rng, finalize: bool, -) -> eyre::Result<()> -where - Provider: FullProvider>, -{ - let provider = ProviderBuilder::new().connect_http(node.rpc_url()); +) -> eyre::Result<()> { + let provider = node.rpc_provider(); let signers = Wallet::new(1).with_chain_id(provider.get_chain_id().await?).wallet_gen(); // simple contract which writes to storage on any call @@ -142,3 +125,60 @@ where Ok(()) } + +/// Header selecting the fork of fork-scoped SSZ engine API endpoints. +pub(crate) const ENGINE_EXECUTION_VERSION_HEADER: &str = "Eth-Execution-Version"; + +/// Extension trait for requests against the SSZ engine API. +pub(crate) trait EngineSszRequestExt { + /// Authenticates the request with the JWT secret of the auth server. + fn jwt(self, auth: &AuthServerHandle) -> Self; + + /// Selects the fork of a fork-scoped endpoint. + fn fork(self, fork: &str) -> Self; + + /// Sets the SSZ encoded body. + fn ssz(self, body: &impl Encode) -> Self; +} + +impl EngineSszRequestExt for RequestBuilder { + fn jwt(self, auth: &AuthServerHandle) -> Self { + self.header(header::AUTHORIZATION, secret_to_bearer_header(auth.jwt_secret())) + } + + fn fork(self, fork: &str) -> Self { + self.header(ENGINE_EXECUTION_VERSION_HEADER, fork) + } + + fn ssz(self, body: &impl Encode) -> Self { + self.header(header::CONTENT_TYPE, "application/octet-stream").body(body.as_ssz_bytes()) + } +} + +/// Extension trait for responses of the SSZ engine API. +pub(crate) trait EngineSszResponseExt { + /// Decodes the SSZ body of a `200 OK` response. + async fn ssz(self) -> eyre::Result; + + /// Returns the `type` of a problem details error response. + async fn problem_type(self) -> eyre::Result; +} + +impl EngineSszResponseExt for Response { + async fn ssz(self) -> eyre::Result { + let status = self.status(); + let bytes = self.bytes().await?; + ensure!(status == StatusCode::OK, "{status}: {}", String::from_utf8_lossy(&bytes)); + T::from_ssz_bytes(&bytes).map_err(|err| eyre!("failed to decode SSZ response: {err:?}")) + } + + async fn problem_type(self) -> eyre::Result { + let content_type = self.headers().get(header::CONTENT_TYPE).cloned(); + ensure!( + content_type.as_ref().is_some_and(|value| value == "application/problem+json"), + "expected a problem details response, got {content_type:?}" + ); + let problem = self.json::().await?; + problem["type"].as_str().map(str::to_owned).ok_or_else(|| eyre!("missing type: {problem}")) + } +} diff --git a/crates/ethereum/node/tests/it/testing.rs b/crates/ethereum/node/tests/it/testing.rs index 6ddc532ab6c..d736afcdd35 100644 --- a/crates/ethereum/node/tests/it/testing.rs +++ b/crates/ethereum/node/tests/it/testing.rs @@ -1,11 +1,12 @@ //! E2E tests for the testing RPC namespace. -use alloy_primitives::{Address, Bytes, B256}; +use alloy_primitives::{Bytes, B256}; use alloy_rpc_types_engine::ExecutionPayloadEnvelopeV4; use alloy_rpc_types_eth::BlockNumberOrTag; use jsonrpsee_core::client::ClientT; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::EthereumHardfork; use reth_db::test_utils::create_test_rw_db; +use reth_e2e_test_utils::{eth_payload_attributes, test_chain_spec}; use reth_ethereum_engine_primitives::EthPayloadAttributes; use reth_node_builder::{NodeBuilder, NodeConfig}; use reth_node_core::{ @@ -17,7 +18,7 @@ use reth_rpc_api::TestingBuildBlockRequestV1; use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; use reth_tasks::Runtime; use serde_json::Value; -use std::{str::FromStr, sync::Arc}; +use std::str::FromStr; use tempfile::tempdir; use tokio::sync::oneshot; @@ -54,15 +55,7 @@ async fn testing_rpc_build_block_works() -> eyre::Result<()> { let chain = ctx.config().chain.clone(); let parent_block_hash = chain.genesis_hash(); - let payload_attributes = EthPayloadAttributes { - timestamp: chain.genesis().timestamp + 1, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: None, - parent_beacon_block_root: None, - slot_number: None, - ..Default::default() - }; + let payload_attributes = eth_payload_attributes(&chain, chain.genesis().timestamp + 1); let request = TestingBuildBlockRequestV1 { parent_block_hash, @@ -108,13 +101,7 @@ async fn testing_rpc_commit_block_works() -> eyre::Result<()> { rocksdb_path: Some(tempdir.path().join("rocksdb")), pprof_dumps_path: Some(tempdir.path().join("pprof")), }; - let chain_spec = Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) - .amsterdam_activated() - .build(), - ); + let chain_spec = test_chain_spec(EthereumHardfork::Amsterdam); let config = NodeConfig::test() .with_chain(chain_spec) .with_datadir_args(datadir_args) @@ -136,16 +123,9 @@ async fn testing_rpc_commit_block_works() -> eyre::Result<()> { let chain = ctx.config().chain.clone(); let timestamp = chain.genesis().timestamp + 1; let target_gas_limit = chain.genesis().gas_limit - 100; - #[allow(clippy::needless_update)] let payload_attributes = EthPayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - slot_number: Some(timestamp), target_gas_limit: Some(target_gas_limit), - ..Default::default() + ..eth_payload_attributes(&chain, timestamp) }; tokio::spawn(async move { diff --git a/examples/exex-test/Cargo.toml b/examples/exex-test/Cargo.toml index 44d52db4564..c46e77c6a87 100644 --- a/examples/exex-test/Cargo.toml +++ b/examples/exex-test/Cargo.toml @@ -16,6 +16,5 @@ tokio = { workspace = true, features = ["full"] } # other dependencies eyre.workspace = true -serde_json.workspace = true [lints] diff --git a/examples/exex-test/src/main.rs b/examples/exex-test/src/main.rs index 59d36d36adc..5038befb115 100644 --- a/examples/exex-test/src/main.rs +++ b/examples/exex-test/src/main.rs @@ -1,11 +1,14 @@ use futures_util::StreamExt; -use reth_e2e_test_utils::testsuite::{ - actions::ProduceBlocks, - setup::{NetworkSetup, Setup}, - TestBuilder, +use reth_e2e_test_utils::{ + test_chain_spec, + testsuite::{ + actions::ProduceBlocks, + setup::{NetworkSetup, Setup}, + TestBuilder, + }, }; use reth_ethereum::{ - chainspec::{ChainSpecBuilder, MAINNET}, + chainspec::EthereumHardfork, exex::{ExExContext, ExExEvent}, node::{ api::{FullNodeComponents, NodeTypes}, @@ -100,18 +103,7 @@ async fn run_exex_test() -> eyre::Result<()> { // Set up the test environment let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis( - serde_json::from_str(include_str!( - "../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" - )) - .unwrap(), - ) - .cancun_activated() - .build(), - )) + .with_chain_spec(test_chain_spec(EthereumHardfork::Cancun)) .with_network(NetworkSetup::single_node()); println!("Test environment set up"); From 1b4558f563c3fff8576a3e6eed6f1fcb674101fe Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 17:41:20 +0000 Subject: [PATCH 096/133] refactor: stop relying on account Copy (#27449) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- .../tree/state_root_strategy/sparse_trie.rs | 22 ++++--- crates/net/downloaders/src/snap/mod.rs | 7 +- crates/net/eth-wire-types/src/snap.rs | 20 +++--- .../src/segments/user/account_history.rs | 19 +++--- .../src/segments/user/storage_history.rs | 18 +++--- crates/revm/src/test_utils.rs | 3 +- crates/rpc/rpc/src/reth.rs | 2 +- crates/snap-sync/src/account/store.rs | 5 +- crates/snap-sync/src/catch_up/apply.rs | 2 +- .../stages/stages/src/stages/execution/mod.rs | 13 ++-- crates/stages/stages/src/stages/merkle.rs | 3 +- .../stages/stages/src/test_utils/test_db.rs | 3 +- crates/storage/db-common/src/init.rs | 3 +- crates/storage/db-models/src/accounts.rs | 2 +- .../storage/db/src/implementation/mdbx/mod.rs | 3 +- .../src/providers/blockchain_provider.rs | 51 ++++++++------- .../provider/src/providers/consistent.rs | 35 ++++++---- .../src/providers/database/provider.rs | 10 +-- .../storage/storage-overlay/src/provider.rs | 6 +- crates/trie/common/src/hashed_state.rs | 11 ++-- crates/trie/common/src/proofs.rs | 3 +- crates/trie/db/src/state.rs | 8 +-- crates/trie/db/tests/post_state.rs | 22 +++---- crates/trie/db/tests/trie.rs | 64 +++++++++---------- crates/trie/parallel/src/state_root_task.rs | 4 +- crates/trie/parallel/src/value_encoder.rs | 5 +- crates/trie/sparse/src/state.rs | 5 +- .../trie/trie/src/hashed_cursor/post_state.rs | 12 ++-- crates/trie/trie/src/node_iter.rs | 43 ++++++------- crates/trie/trie/src/witness.rs | 2 + testing/testing-utils/src/generators.rs | 5 +- 31 files changed, 231 insertions(+), 180 deletions(-) diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 16f0d8bf257..bcf4ea14121 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -655,6 +655,7 @@ where target = "engine::tree::payload_processor::sparse_trie", skip_all )] + #[allow(clippy::clone_on_copy)] fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) { for (&address, storage) in &hashed_state_update.storages { if !storage.storage.is_empty() { @@ -680,7 +681,7 @@ where self.pending_account_updates.entry(address).or_insert(None); } - for (&address, &account) in &hashed_state_update.accounts { + for (&address, account) in &hashed_state_update.accounts { // Track account as touched. // // This might overwrite an existing update, which is fine, because storage root from it @@ -689,7 +690,7 @@ where // Track account in `pending_account_updates` so that once storage root is computed, // it will be updated in the accounts trie. - self.pending_account_updates.insert(address, Some(account)); + self.pending_account_updates.insert(address, Some(account.clone())); } self.final_hashed_state.extend(hashed_state_update); @@ -1611,7 +1612,8 @@ fn encode_account_leaf_value( storage_root: B256, account_rlp_buf: &mut Vec, ) -> Vec { - if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH { + if account.as_ref().is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH + { return Vec::new(); } @@ -1812,9 +1814,9 @@ mod tests { panic!("expected HashedState message"); }; - let account = received.accounts.get(&address).unwrap().unwrap(); - assert_eq!(account.balance, expected_state.accounts[&address].unwrap().balance); - assert_eq!(account.nonce, expected_state.accounts[&address].unwrap().nonce); + let account = received.accounts.get(&address).unwrap().as_ref().unwrap(); + assert_eq!(account.balance, expected_state.accounts[&address].as_ref().unwrap().balance); + assert_eq!(account.nonce, expected_state.accounts[&address].as_ref().unwrap().nonce); let storage = received.storages.get(&address).unwrap(); assert_eq!(*storage.storage.get(&slot).unwrap(), value); @@ -1978,6 +1980,7 @@ mod tests { } #[test] + #[allow(clippy::clone_on_copy)] fn in_flight_storage_updates_keep_latest_values_and_deletions() { let runtime = Runtime::test(); let trie = SparseStateTrie::default() @@ -2002,7 +2005,7 @@ mod tests { }; let mut state = HashedPostState::default(); - state.accounts.insert(address, Some(account)); + state.accounts.insert(address, Some(account.clone())); state.storages.entry(address).or_default().storage.extend([ (removed_slot, U256::from(8)), (changed_slot, U256::from(9)), @@ -2239,6 +2242,7 @@ mod tests { } #[test] + #[allow(clippy::clone_on_copy)] fn run_waits_for_storage_tries_hashed_off_thread() { let runtime = Runtime::test(); let default_trie = RevealableSparseTrie::::revealed_empty(); @@ -2270,7 +2274,7 @@ mod tests { let mut state = HashedPostState::default(); for (address, account, storage) in &accounts { - state.accounts.insert(*address, Some(*account)); + state.accounts.insert(*address, Some(account.clone())); state.storages.entry(*address).or_default().storage.extend(storage.iter().copied()); } updates_tx.send(StateRootMessage::HashedStateUpdate(state)).unwrap(); @@ -2281,7 +2285,7 @@ mod tests { let expected = accounts.iter().map(|(address, account, storage)| { let storage_root = reth_trie_common::root::storage_root_unsorted(storage.iter().copied()); - (*address, account.into_trie_account(storage_root)) + (*address, account.clone().into_trie_account(storage_root)) }); assert_eq!(outcome.state_root, reth_trie_common::root::state_root_unsorted(expected)); assert_eq!(task.storage_in_flight, 0); diff --git a/crates/net/downloaders/src/snap/mod.rs b/crates/net/downloaders/src/snap/mod.rs index 9deac39b784..b8c32e65db6 100644 --- a/crates/net/downloaders/src/snap/mod.rs +++ b/crates/net/downloaders/src/snap/mod.rs @@ -333,6 +333,7 @@ fn verify_proof( } #[cfg(test)] +#[allow(clippy::clone_on_copy)] mod tests { use super::{request::MAX_RETRIES, test_utils::TestSnapClient, *}; use alloy_primitives::{Bytes, U256}; @@ -587,7 +588,7 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, origin: B256::ZERO, - accounts: vec![accounts[0]], + accounts: vec![accounts[0].clone()], has_more: false, next: Some(key(3)), }) @@ -732,7 +733,7 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, origin: B256::ZERO, - accounts: vec![accounts[0]], + accounts: vec![accounts[0].clone()], has_more: false, next: Some(key(9)), }) @@ -760,7 +761,7 @@ mod tests { AccountRangeOutcome::Verified(VerifiedAccountRange { state_root: root_hash, origin: B256::ZERO, - accounts: vec![accounts[0]], + accounts: vec![accounts[0].clone()], has_more: true, next: Some(key(3)), }) diff --git a/crates/net/eth-wire-types/src/snap.rs b/crates/net/eth-wire-types/src/snap.rs index 6e9641254df..3a2c40f0920 100644 --- a/crates/net/eth-wire-types/src/snap.rs +++ b/crates/net/eth-wire-types/src/snap.rs @@ -123,12 +123,14 @@ impl AccountData { /// Returns the account the trie leaf commits to. /// /// Default storage roots and code hashes are restored when decoding the slim body. - pub const fn trie_account(&self) -> TrieAccount { - self.body.0 + #[allow(clippy::clone_on_copy)] + pub fn trie_account(&self) -> TrieAccount { + self.body.0.clone() } /// Consumes the wire value and returns its hashed key with the decoded trie account. - pub const fn into_trie_entry(self) -> (B256, TrieAccount) { + #[allow(clippy::missing_const_for_fn)] + pub fn into_trie_entry(self) -> (B256, TrieAccount) { (self.hash, self.body.0) } } @@ -512,7 +514,7 @@ impl SnapProtocolMessage { } /// A trie account encoded with default storage and code hashes replaced by empty byte strings. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct SlimAccountBody(TrieAccount); impl SlimAccountBody { @@ -535,8 +537,9 @@ impl SlimAccountBody { } impl From<&TrieAccount> for SlimAccountBody { + #[allow(clippy::clone_on_copy)] fn from(account: &TrieAccount) -> Self { - Self(*account) + Self(account.clone()) } } @@ -977,7 +980,7 @@ mod tests { let hash = B256::repeat_byte(1); let encoded = AccountData::from_trie_account(hash, &account); - let body = alloy_rlp::encode(encoded.body); + let body = alloy_rlp::encode(&encoded.body); assert_eq!(body, alloy_primitives::hex!("c4072a8080")); assert_eq!(alloy_rlp::decode_exact::(&body).unwrap(), encoded.body); assert_eq!(encoded.trie_account(), account); @@ -985,12 +988,13 @@ mod tests { } #[test] + #[allow(clippy::clone_on_copy)] fn slim_body_keeps_non_default_storage_and_code() { let account = trie_account(B256::repeat_byte(2), B256::repeat_byte(3)); let encoded = AccountData::from_trie_account(B256::repeat_byte(1), &account); - let body = alloy_rlp::encode(encoded.body); - assert_eq!(body, alloy_rlp::encode(account)); + let body = alloy_rlp::encode(&encoded.body); + assert_eq!(body, alloy_rlp::encode(account.clone())); assert_eq!(alloy_rlp::decode_exact::(&body).unwrap(), encoded.body); assert_eq!(encoded.trie_account(), account); } diff --git a/crates/prune/prune/src/segments/user/account_history.rs b/crates/prune/prune/src/segments/user/account_history.rs index e1c24267951..440617f8ac7 100644 --- a/crates/prune/prune/src/segments/user/account_history.rs +++ b/crates/prune/prune/src/segments/user/account_history.rs @@ -629,6 +629,7 @@ mod tests { /// Tests that when a limiter stops mid-block (with multiple changes for the same block), /// the checkpoint is set to `block_number - 1` to avoid dangling index entries. #[test] + #[allow(clippy::clone_on_copy)] fn prune_partial_progress_mid_block() { use alloy_primitives::{Address, U256}; use reth_primitives_traits::Account; @@ -656,17 +657,17 @@ mod tests { // Build changesets: blocks 0-4 have 1 change each, block 5 has 4 changes, block 6 has 1 let changesets: Vec = vec![ - vec![(addr1, account, vec![])], // block 0 - vec![(addr1, account, vec![])], // block 1 - vec![(addr1, account, vec![])], // block 2 - vec![(addr1, account, vec![])], // block 3 - vec![(addr1, account, vec![])], // block 4 + vec![(addr1, account.clone(), vec![])], // block 0 + vec![(addr1, account.clone(), vec![])], // block 1 + vec![(addr1, account.clone(), vec![])], // block 2 + vec![(addr1, account.clone(), vec![])], // block 3 + vec![(addr1, account.clone(), vec![])], // block 4 // block 5: 4 different account changes (sorted by address for consistency) vec![ - (addr1, account, vec![]), - (addr2, account, vec![]), - (addr3, account, vec![]), - (addr4, account, vec![]), + (addr1, account.clone(), vec![]), + (addr2, account.clone(), vec![]), + (addr3, account.clone(), vec![]), + (addr4, account.clone(), vec![]), ], vec![(addr5, account, vec![])], // block 6 ]; diff --git a/crates/prune/prune/src/segments/user/storage_history.rs b/crates/prune/prune/src/segments/user/storage_history.rs index b2062d3d819..79eb40b4ef6 100644 --- a/crates/prune/prune/src/segments/user/storage_history.rs +++ b/crates/prune/prune/src/segments/user/storage_history.rs @@ -539,6 +539,7 @@ mod tests { /// Tests that when a limiter stops mid-block (with multiple storage changes for the same /// block), the checkpoint is set to `block_number - 1` to avoid dangling index entries. #[test] + #[allow(clippy::clone_on_copy)] fn prune_partial_progress_mid_block() { use alloy_primitives::{Address, U256}; use reth_primitives_traits::Account; @@ -570,16 +571,16 @@ mod tests { // Build changesets: blocks 0-4 have 1 storage change each, block 5 has 4 changes, block 6 // has 1. Entries within each account must be sorted by key. let changesets: Vec = vec![ - vec![(addr1, account, vec![storage_entry(1)])], // block 0 - vec![(addr1, account, vec![storage_entry(1)])], // block 1 - vec![(addr1, account, vec![storage_entry(1)])], // block 2 - vec![(addr1, account, vec![storage_entry(1)])], // block 3 - vec![(addr1, account, vec![storage_entry(1)])], // block 4 + vec![(addr1, account.clone(), vec![storage_entry(1)])], // block 0 + vec![(addr1, account.clone(), vec![storage_entry(1)])], // block 1 + vec![(addr1, account.clone(), vec![storage_entry(1)])], // block 2 + vec![(addr1, account.clone(), vec![storage_entry(1)])], // block 3 + vec![(addr1, account.clone(), vec![storage_entry(1)])], // block 4 // block 5: 4 different storage changes (2 addresses, each with 2 storage slots) // Sorted by address, then by storage key within each address vec![ - (addr1, account, vec![storage_entry(1), storage_entry(2)]), - (addr2, account, vec![storage_entry(1), storage_entry(2)]), + (addr1, account.clone(), vec![storage_entry(1), storage_entry(2)]), + (addr2, account.clone(), vec![storage_entry(1), storage_entry(2)]), ], vec![(addr1, account, vec![storage_entry(3)])], // block 6 ]; @@ -797,6 +798,7 @@ mod tests { /// walk deletes no changesets, so a checkpoint rewound below such a block would make every /// later run reread it and never advance. #[test] + #[allow(clippy::clone_on_copy)] fn dense_block_advances_rocksdb_checkpoint() { use alloy_primitives::U256; use reth_db_api::models::storage_sharded_key::StorageShardedKey; @@ -822,7 +824,7 @@ mod tests { .map(|_| { vec![( address, - account, + account.clone(), keys.iter() .map(|key| StorageEntry { key: *key, value: U256::from(1) }) .collect(), diff --git a/crates/revm/src/test_utils.rs b/crates/revm/src/test_utils.rs index b1f0ecb631a..5c9fcedb414 100644 --- a/crates/revm/src/test_utils.rs +++ b/crates/revm/src/test_utils.rs @@ -47,8 +47,9 @@ impl StateProviderTest { } impl AccountReader for StateProviderTest { + #[allow(clippy::clone_on_copy)] fn basic_account(&self, address: &Address) -> ProviderResult> { - Ok(self.accounts.get(address).map(|(_, acc)| *acc)) + Ok(self.accounts.get(address).map(|(_, acc)| acc.clone())) } } diff --git a/crates/rpc/rpc/src/reth.rs b/crates/rpc/rpc/src/reth.rs index 121a25f9b59..3a728f26a6c 100644 --- a/crates/rpc/rpc/src/reth.rs +++ b/crates/rpc/rpc/src/reth.rs @@ -95,7 +95,7 @@ where AddressMap::default(), |mut hash_map, account_before| -> RethResult<_> { let current_balance = state.account_balance(&account_before.address)?; - let prev_balance = account_before.info.map(|info| info.balance); + let prev_balance = account_before.info.as_ref().map(|info| info.balance); if current_balance != prev_balance { hash_map.insert(account_before.address, current_balance.unwrap_or_default()); } diff --git a/crates/snap-sync/src/account/store.rs b/crates/snap-sync/src/account/store.rs index d4a8ed971f7..b0029233cc3 100644 --- a/crates/snap-sync/src/account/store.rs +++ b/crates/snap-sync/src/account/store.rs @@ -231,6 +231,7 @@ struct RangeDependencies { } impl RangeDependencies { + #[allow(clippy::clone_on_copy)] fn new( accounts: &[(B256, TrieAccount)], storages: B256Map, @@ -238,7 +239,9 @@ impl RangeDependencies { ) -> Self { let state = HashedPostState::default() .with_accounts( - accounts.iter().map(|(hash, account)| (*hash, Some(Account::from(*account)))), + accounts + .iter() + .map(|(hash, account)| (*hash, Some(Account::from(account.clone())))), ) .with_storages(storages) .into_sorted(); diff --git a/crates/snap-sync/src/catch_up/apply.rs b/crates/snap-sync/src/catch_up/apply.rs index f694118458a..efaa524e50d 100644 --- a/crates/snap-sync/src/catch_up/apply.rs +++ b/crates/snap-sync/src/catch_up/apply.rs @@ -52,7 +52,7 @@ impl BalStateUpdate { } /// What the downloaded state holds for an account a list changes. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum DownloadedAccount { /// The account's range is not downloaded yet. Unknown, diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index 482738e3ca2..9051cc731dd 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -1148,6 +1148,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::clone_on_copy)] async fn sanity_execute_unwind() { let factory = create_test_provider_factory(); let provider = factory.provider_rw().unwrap(); @@ -1186,8 +1187,8 @@ mod tests { let acc2 = address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"); let acc2_info = Account { balance, ..Default::default() }; - db_tx.put::(acc1, acc1_info).unwrap(); - db_tx.put::(acc2, acc2_info).unwrap(); + db_tx.put::(acc1, acc1_info.clone()).unwrap(); + db_tx.put::(acc2, acc2_info.clone()).unwrap(); db_tx.put::(code_hash, Bytecode::new_raw(code.to_vec().into())).unwrap(); provider.commit().unwrap(); @@ -1327,6 +1328,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::clone_on_copy)] async fn test_selfdestruct() { let test_db = TestStageDB::default(); let provider = test_db.factory.database_provider_rw().unwrap(); @@ -1367,10 +1369,13 @@ mod tests { // set account let provider = test_db.factory.provider_rw().unwrap(); - provider.tx_ref().put::(caller_address, caller_info).unwrap(); provider .tx_ref() - .put::(destroyed_address, destroyed_info) + .put::(caller_address, caller_info.clone()) + .unwrap(); + provider + .tx_ref() + .put::(destroyed_address, destroyed_info.clone()) .unwrap(); provider .tx_ref() diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index 5dd05a247d2..a6eca099b26 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -672,6 +672,7 @@ mod tests { impl ExecuteStageTestRunner for MerkleTestRunner { type Seed = Vec>; + #[allow(clippy::clone_on_copy)] fn seed_execution(&mut self, input: ExecInput) -> Result { let stage_progress = input.checkpoint().block_number; let start = stage_progress + 1; @@ -698,7 +699,7 @@ mod tests { .collect::>(); self.db.insert_accounts_and_storages( - accounts.iter().map(|(addr, acc)| (*addr, (*acc, std::iter::empty()))), + accounts.iter().map(|(addr, acc)| (*addr, (acc.clone(), std::iter::empty()))), )?; let (header, body) = random_block( diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index 4f7f2186d58..85f8d7583e4 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -404,6 +404,7 @@ impl TestStageDB { } /// Insert collection of ([Address], [Account]) into corresponding tables. + #[allow(clippy::clone_on_copy)] pub fn insert_accounts_and_storages(&self, accounts: I) -> ProviderResult<()> where I: IntoIterator, @@ -414,7 +415,7 @@ impl TestStageDB { let hashed_address = keccak256(address); // Insert into account tables. - tx.put::(address, account)?; + tx.put::(address, account.clone())?; tx.put::(hashed_address, account)?; // Insert into storage tables. diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index 32924a67a1e..39b6958960d 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -973,6 +973,7 @@ where /// `StorageChangeSets` receive data in sorted order within each account). For `HashedAccounts` /// and `HashedStorages`, insertion order is unsorted (keccak scrambles address order), so we /// use `put`/`upsert` which do a full B-tree lookup. +#[allow(clippy::clone_on_copy)] fn write_account_to_db( tx: &TX, address: &Address, @@ -1002,7 +1003,7 @@ fn write_account_to_db( let hashed_address = keccak256(address); // plain state — sorted by address (ETL order), use append - tx.put::(*address, account)?; + tx.put::(*address, account.clone())?; // hashed state — unsorted (keccak scrambles order), must use put tx.put::(hashed_address, account)?; diff --git a/crates/storage/db-models/src/accounts.rs b/crates/storage/db-models/src/accounts.rs index 4f846d1927c..f3e9d54bdeb 100644 --- a/crates/storage/db-models/src/accounts.rs +++ b/crates/storage/db-models/src/accounts.rs @@ -35,7 +35,7 @@ impl reth_codecs::Compact for AccountBeforeTx { // for now put full bytes and later compress it. buf.put_slice(self.address.as_slice()); - let acc_len = if let Some(account) = self.info { account.to_compact(buf) } else { 0 }; + let acc_len = if let Some(account) = &self.info { account.to_compact(buf) } else { 0 }; acc_len + 20 } diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 20a3d92a9ce..bd5024ff8e4 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -1412,6 +1412,7 @@ mod tests { #[test] #[allow(clippy::needless_update)] + #[allow(clippy::clone_on_copy)] fn db_closure_put_get() { let tempdir = TempDir::new().expect(ERROR_TEMPDIR); let path = tempdir.path(); @@ -1430,7 +1431,7 @@ mod tests { // PUT let result = env.update(|tx| { - tx.put::(key, value).expect(ERROR_PUT); + tx.put::(key, value.clone()).expect(ERROR_PUT); 200 }); assert_eq!(result.expect(ERROR_RETURN_VALUE), 200); diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 23077ae21db..78e2bdfcd38 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1052,6 +1052,7 @@ impl StateReader for BlockchainProvider { } #[cfg(test)] +#[allow(clippy::clone_on_copy)] mod tests { use super::SNAPSHOT_STATE_RETENTION; use crate::{ @@ -1137,8 +1138,8 @@ mod tests { ( HashedPostState::default() .with_accounts([ - (hashed_address, Some(account)), - (keccak256(other_address), Some(account)), + (hashed_address, Some(account.clone())), + (keccak256(other_address), Some(account.clone())), ]) .with_storages([( hashed_address, @@ -1179,8 +1180,8 @@ mod tests { let mut updates = TrieUpdates::default(); updates.storage_tries.insert(hashed_address, storage_updates); let state_root = reth_trie::test_utils::state_root([ - (address, (account, storage.clone())), - (other_address, (account, BTreeMap::new())), + (address, (account.clone(), storage.clone())), + (other_address, (account.clone(), BTreeMap::new())), ]); storage_roots.push(reth_trie::test_utils::storage_root(storage.clone())); let block = Block { @@ -2111,9 +2112,9 @@ mod tests { let (in_memory_changesets, in_memory_state) = random_changeset_range( &mut rng, &in_memory_blocks, - database_state - .iter() - .map(|(address, (account, storage))| (*address, (*account, storage.clone()))), + database_state.iter().map(|(address, (account, storage))| { + (*address, (account.clone(), storage.clone())) + }), 0..0, 0..0, ); @@ -2133,7 +2134,7 @@ mod tests { }), database_changesets.iter().map(|block_changesets| { block_changesets.iter().map(|(address, account, _)| { - (*address, Some(Some((*account).into())), []) + (*address, Some(Some((account.clone()).into())), []) }) }), Vec::new(), @@ -2164,7 +2165,7 @@ mod tests { (address, None, Some(account.into()), Default::default()) }), [in_memory_changesets.iter().map(|(address, account, _)| { - (*address, Some(Some((*account).into())), Vec::new()) + (*address, Some(Some((account.clone()).into())), Vec::new()) })], [], ), @@ -3070,14 +3071,16 @@ mod tests { let accounts: Vec<_> = (0..5u64).map(random_account).collect(); provider_rw.insert_account_for_hashing( - accounts.iter().map(|(address, account)| (*address, Some(*account))), + accounts.iter().map(|(address, account)| (*address, Some(account.clone()))), )?; provider_rw.commit()?; let provider = BlockchainProvider::new(factory)?; - let mut expected: Vec<_> = - accounts.iter().map(|(address, account)| (keccak256(address), *account)).collect(); + let mut expected: Vec<_> = accounts + .iter() + .map(|(address, account)| (keccak256(address), account.clone())) + .collect(); expected.sort_by_key(|(hash, _)| *hash); let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); @@ -3101,7 +3104,7 @@ mod tests { let accounts: Vec<_> = (0..5u64).map(random_account).collect(); provider_rw.insert_account_for_hashing( - accounts.iter().map(|(address, account)| (*address, Some(*account))), + accounts.iter().map(|(address, account)| (*address, Some(account.clone()))), )?; provider_rw.commit()?; @@ -3272,7 +3275,7 @@ mod tests { let value = U256::from(42); let mut hashed_state = HashedPostState::default(); - hashed_state.accounts.insert(hashed_address, Some(account)); + hashed_state.accounts.insert(hashed_address, Some(account.clone())); hashed_state .storages .insert(hashed_address, HashedStorage::from_iter([(keccak256(slot_key), value)])); @@ -3297,7 +3300,7 @@ mod tests { blob_gas_used: 0, }, state: BundleState::new( - [(address, None, Some(account.into()), storage)], + [(address, None, Some(account.clone().into()), storage)], [[(address, Some(None), [])]], [], ), @@ -3329,7 +3332,7 @@ mod tests { provider.latest()?, provider.state_by_block_hash(block_hash)?, ] { - assert_eq!(state.basic_account(&address)?, Some(account)); + assert_eq!(state.basic_account(&address)?, Some(account.clone())); assert_eq!(state.storage(address, slot_key)?, Some(value)); } @@ -3345,7 +3348,7 @@ mod tests { let (address, account) = random_account(1); let hashed_address = keccak256(address); let mut hashed_state = HashedPostState::default(); - hashed_state.accounts.insert(hashed_address, Some(account)); + hashed_state.accounts.insert(hashed_address, Some(account.clone())); // A root only the in-memory block carries, so a match proves the in-memory path (not // persisted history, which has no block with this root) resolved it. @@ -3398,7 +3401,7 @@ mod tests { let (target_address, target_account) = random_account(1); let target_hashed = keccak256(target_address); let mut target_state = HashedPostState::default(); - target_state.accounts.insert(target_hashed, Some(target_account)); + target_state.accounts.insert(target_hashed, Some(target_account.clone())); let unique_root = B256::repeat_byte(0x77); let mut block = random_block( @@ -3440,7 +3443,7 @@ mod tests { .try_recover() .expect("failed to seal block with senders"); let mut noise_state = HashedPostState::default(); - noise_state.accounts.insert(keccak256(noise_address), Some(noise_account)); + noise_state.accounts.insert(keccak256(noise_address), Some(noise_account.clone())); let provider_rw = provider.database.provider_rw()?; provider_rw.append_blocks_with_state( vec![noise_block], @@ -3484,7 +3487,7 @@ mod tests { let factory = test_provider_factory_with_genesis()?; let provider_rw = factory.provider_rw()?; - provider_rw.insert_account_for_hashing([(address, Some(account_a))])?; + provider_rw.insert_account_for_hashing([(address, Some(account_a.clone()))])?; provider_rw.insert_storage_for_hashing([( address, [StorageEntry { key: slot_key, value: value_a }], @@ -3510,14 +3513,14 @@ mod tests { provider_rw.commit()?; // State B: a later block changes both the account and its storage slot. - let account_b = Account { nonce: 2, balance: U256::from(2), ..account_a }; + let account_b = Account { nonce: 2, balance: U256::from(2), ..account_a.clone() }; let value_b = U256::from(2); let mut storage = HashMap::default(); storage.insert(slot, (value_a, value_b)); let mut state_b = HashedPostState::default(); - state_b.accounts.insert(hashed_address, Some(account_b)); + state_b.accounts.insert(hashed_address, Some(account_b.clone())); state_b.storages.insert(hashed_address, HashedStorage::from_iter([(hashed_slot, value_b)])); let state_b_root = factory.latest()?.state_root(state_b.clone())?; @@ -3536,8 +3539,8 @@ mod tests { vec![later_block], &ExecutionOutcome { bundle: BundleState::new( - [(address, Some(account_a.into()), Some(account_b.into()), storage)], - [[(address, Some(Some(account_a.into())), [(slot, value_a)])]], + [(address, Some(account_a.clone().into()), Some(account_b.into()), storage)], + [[(address, Some(Some(account_a.clone().into())), [(slot, value_a)])]], [], ), first_block: 2, diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index adce953c744..8dd750ef921 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -1412,6 +1412,7 @@ impl ChangeSetReader for ConsistentProvider { } #[cfg(test)] +#[allow(clippy::clone_on_copy)] mod tests { use crate::{ providers::blockchain_provider::BlockchainProvider, @@ -1719,9 +1720,9 @@ mod tests { let (in_memory_changesets, in_memory_state) = random_changeset_range( &mut rng, &in_memory_blocks, - database_state - .iter() - .map(|(address, (account, storage))| (*address, (*account, storage.clone()))), + database_state.iter().map(|(address, (account, storage))| { + (*address, (account.clone(), storage.clone())) + }), 0..0, 0..0, ); @@ -1741,7 +1742,7 @@ mod tests { }), database_changesets.iter().map(|block_changesets| { block_changesets.iter().map(|(address, account, _)| { - (*address, Some(Some((*account).into())), []) + (*address, Some(Some((account.clone()).into())), []) }) }), Vec::new(), @@ -1772,7 +1773,7 @@ mod tests { (address, None, Some(account.into()), Default::default()) }), [in_memory_changesets.iter().map(|(address, account, _)| { - (*address, Some(Some((*account).into())), Vec::new()) + (*address, Some(Some((account.clone()).into())), Vec::new()) })], [], ), @@ -1849,14 +1850,18 @@ mod tests { .collect(), &ExecutionOutcome { bundle: BundleState::new( - [(address, None, Some(account.into()), { + [(address, None, Some(account.clone().into()), { let mut s = HashMap::default(); s.insert(slot, (U256::ZERO, U256::from(100))); s })], [ Vec::new(), - vec![(address, Some(Some(account.into())), vec![(slot, U256::ZERO)])], + vec![( + address, + Some(Some(account.clone().into())), + vec![(slot, U256::ZERO)], + )], ], [], ), @@ -1921,12 +1926,12 @@ mod tests { .collect(), &ExecutionOutcome { bundle: BundleState::new( - [(address, None, Some(account.into()), { + [(address, None, Some(account.clone().into()), { let mut s = HashMap::default(); s.insert(slot, (U256::ZERO, U256::from(100))); s })], - [[(address, Some(Some(account.into())), vec![(slot, U256::ZERO)])]], + [[(address, Some(Some(account.clone().into())), vec![(slot, U256::ZERO)])]], [], ), first_block: 0, @@ -1948,7 +1953,7 @@ mod tests { )), execution_output: Arc::new(BlockExecutionOutput { state: BundleState::new( - [(address, None, Some(account.into()), { + [(address, None, Some(account.clone().into()), { let mut s = HashMap::default(); s.insert(slot, (U256::from(100), U256::from(200))); s @@ -2020,13 +2025,17 @@ mod tests { .collect(), &ExecutionOutcome { bundle: BundleState::new( - [(address, None, Some(account.into()), { + [(address, None, Some(account.clone().into()), { let mut s = HashMap::default(); s.insert(slot, (U256::ZERO, U256::from(100))); s })], vec![ - vec![(address, Some(Some(account.into())), vec![(slot, U256::ZERO)])], + vec![( + address, + Some(Some(account.clone().into())), + vec![(slot, U256::ZERO)], + )], vec![], ], [], @@ -2050,7 +2059,7 @@ mod tests { )), execution_output: Arc::new(BlockExecutionOutput { state: BundleState::new( - [(address, None, Some(account.into()), { + [(address, None, Some(account.clone().into()), { let mut s = HashMap::default(); s.insert(slot, (U256::from(100), U256::from(200))); s diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 7edd4facdbb..4aee893a1bd 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1258,6 +1258,7 @@ impl DatabaseProvider { /// Populate a [`BundleStateInit`] and [`RevertsInit`] using cursors over the /// [`tables::PlainAccountState`] and [`tables::PlainStorageState`] tables, based on the given /// storage and account changesets. + #[allow(clippy::clone_on_copy)] pub(crate) fn populate_bundle_state( &self, account_changeset: Vec<(u64, AccountBeforeTx)>, @@ -1283,11 +1284,11 @@ impl DatabaseProvider { match state.entry(address) { hash_map::Entry::Vacant(entry) => { let new_info = get_account(address)?; - entry.insert((old_info, new_info, HashMap::default())); + entry.insert((old_info.clone(), new_info, HashMap::default())); } hash_map::Entry::Occupied(mut entry) => { // overwrite old account state. - entry.get_mut().0 = old_info; + entry.get_mut().0 = old_info.clone(); } } // insert old info into reverts. @@ -1301,7 +1302,7 @@ impl DatabaseProvider { let account_state = match state.entry(address) { hash_map::Entry::Vacant(entry) => { let present_info = get_account(address)?; - entry.insert((present_info, present_info, HashMap::default())) + entry.insert((present_info.clone(), present_info, HashMap::default())) } hash_map::Entry::Occupied(entry) => entry.into_mut(), }; @@ -3247,6 +3248,7 @@ impl StorageTrieWriter for DatabaseP } impl HashingWriter for DatabaseProvider { + #[allow(clippy::clone_on_copy)] fn unwind_account_hashing<'a>( &self, changesets: impl Iterator, @@ -3256,7 +3258,7 @@ impl HashingWriter for DatabaseProvi // changes are applied in the correct order. let hashed_accounts = changesets .into_iter() - .map(|(_, e)| (keccak256(e.address), e.info)) + .map(|(_, e)| (keccak256(e.address), e.info.clone())) .collect::>() .into_iter() .rev() diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index cf2614c56a7..0999d125b45 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -1300,6 +1300,7 @@ mod tests { } #[test] + #[allow(clippy::clone_on_copy)] fn historical_execution_reads_use_history_indexes() { let (factory, blocks) = setup_frontiers(1, 3); let address = Address::with_last_byte(1); @@ -1317,7 +1318,10 @@ mod tests { .unwrap(); provider_rw .tx_ref() - .put::(2, AccountBeforeTx { address, info: Some(account) }) + .put::( + 2, + AccountBeforeTx { address, info: Some(account.clone()) }, + ) .unwrap(); provider_rw .tx_ref() diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index a154be9c279..6a8e3a87f32 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -193,8 +193,9 @@ impl HashedPostState { self.extend_inner(Cow::Borrowed(other)); } + #[allow(clippy::clone_on_copy)] fn extend_inner(&mut self, other: Cow<'_, Self>) { - self.accounts.extend(other.accounts.iter().map(|(&k, &v)| (k, v))); + self.accounts.extend(other.accounts.iter().map(|(&k, v)| (k, v.clone()))); self.storages.reserve(other.storages.len()); match other { @@ -226,13 +227,14 @@ impl HashedPostState { /// Extend this hashed post state with sorted data, converting directly into the unsorted /// `HashMap` representation. This is more efficient than first converting to `HashedPostState` /// and then extending, as it avoids creating intermediate `HashMap` allocations. + #[allow(clippy::clone_on_copy)] pub fn extend_from_sorted(&mut self, sorted: &HashedPostStateSorted) { // Reserve capacity for accounts self.accounts.reserve(sorted.accounts.len()); // Insert accounts (Some = updated, None = destroyed) for (address, account) in &sorted.accounts { - self.accounts.insert(*address, *account); + self.accounts.insert(*address, account.clone()); } // Reserve capacity for storages @@ -269,8 +271,9 @@ impl HashedPostState { /// Creates a sorted copy without consuming self. /// More efficient than `.clone().into_sorted()` as it avoids cloning `HashMap` metadata. + #[allow(clippy::clone_on_copy)] pub fn clone_into_sorted(&self) -> HashedPostStateSorted { - let mut accounts: Vec<_> = self.accounts.iter().map(|(&k, &v)| (k, v)).collect(); + let mut accounts: Vec<_> = self.accounts.iter().map(|(&k, v)| (k, v.clone())).collect(); accounts.sort_unstable_by_key(|(address, _)| *address); let storages = self @@ -1192,7 +1195,7 @@ mod tests { assert_eq!(state1.accounts[0].0, B256::from([1; 32])); assert_eq!(state1.accounts[1].0, B256::from([2; 32])); assert_eq!(state1.accounts[2].0, B256::from([3; 32])); - assert_eq!(state1.accounts[2].1.unwrap().nonce, 1); // Should have state2's value + assert_eq!(state1.accounts[2].1.as_ref().unwrap().nonce, 1); // Should have state2's value assert_eq!(state1.accounts[3].0, B256::from([4; 32])); assert_eq!(state1.accounts[4].0, B256::from([5; 32])); assert_eq!(state1.accounts[4].1, None); diff --git a/crates/trie/common/src/proofs.rs b/crates/trie/common/src/proofs.rs index 95e7ff7c123..b90ee5b0123 100644 --- a/crates/trie/common/src/proofs.rs +++ b/crates/trie/common/src/proofs.rs @@ -968,6 +968,7 @@ impl AccountProof { } /// Verify the storage proofs and account proof against the provided state root. + #[allow(clippy::clone_on_copy)] pub fn verify(&self, root: B256) -> Result<(), ProofVerificationError> { // Verify storage proofs. for storage_proof in &self.storage_proofs { @@ -979,7 +980,7 @@ impl AccountProof { None } else { Some(alloy_rlp::encode( - self.info.unwrap_or_default().into_trie_account(self.storage_root), + self.info.clone().unwrap_or_default().into_trie_account(self.storage_root), )) }; let nibbles = Nibbles::unpack(keccak256(self.address)); diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index 2c985cca0d8..61ffb4848a6 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -498,7 +498,7 @@ mod tests { assert_eq!(sorted.accounts.len(), 2); let hashed_addr1 = keccak256(address1); let account1 = sorted.accounts.iter().find(|(addr, _)| *addr == hashed_addr1).unwrap(); - assert_eq!(account1.1.unwrap().nonce, 1); + assert_eq!(account1.1.as_ref().unwrap().nonce, 1); // Ordering guarantees - accounts sorted by hashed address assert!(sorted.accounts.windows(2).all(|w| w[0].0 <= w[1].0)); @@ -621,7 +621,7 @@ mod tests { let hashed_addr2 = keccak256(address2); let account1 = sorted.accounts.iter().find(|(addr, _)| *addr == hashed_addr1).unwrap(); - assert_eq!(account1.1.unwrap().nonce, 1); + assert_eq!(account1.1.as_ref().unwrap().nonce, 1); let account2 = sorted.accounts.iter().find(|(addr, _)| *addr == hashed_addr2).unwrap(); assert!(account2.1.is_none()); @@ -714,11 +714,11 @@ mod tests { let account1 = sorted.accounts.iter().find(|(addr, _)| *addr == expected_hashed_addr1).unwrap(); - assert_eq!(account1.1.unwrap().nonce, 10); + assert_eq!(account1.1.as_ref().unwrap().nonce, 10); let account2 = sorted.accounts.iter().find(|(addr, _)| *addr == expected_hashed_addr2).unwrap(); - assert_eq!(account2.1.unwrap().nonce, 20); + assert_eq!(account2.1.as_ref().unwrap().nonce, 20); assert!(sorted.accounts.windows(2).all(|w| w[0].0 <= w[1].0)); diff --git a/crates/trie/db/tests/post_state.rs b/crates/trie/db/tests/post_state.rs index b071487fbed..c7689bdbfdf 100644 --- a/crates/trie/db/tests/post_state.rs +++ b/crates/trie/db/tests/post_state.rs @@ -1,4 +1,4 @@ -#![allow(missing_docs)] +#![allow(missing_docs, clippy::clone_on_copy)] use alloy_primitives::{B256, U256}; use proptest::prelude::*; @@ -59,7 +59,7 @@ fn post_state_only_accounts() { let mut hashed_post_state = HashedPostState::default(); for (hashed_address, account) in &accounts { - hashed_post_state.accounts.insert(*hashed_address, Some(*account)); + hashed_post_state.accounts.insert(*hashed_address, Some(account.clone())); } let db = create_test_rw_db(); @@ -78,7 +78,7 @@ fn db_only_accounts() { let db = create_test_rw_db(); db.update(|tx| { for (key, account) in &accounts { - tx.put::(*key, *account).unwrap(); + tx.put::(*key, account.clone()).unwrap(); } }) .unwrap(); @@ -101,14 +101,14 @@ fn account_cursor_correct_order() { let db = create_test_rw_db(); db.update(|tx| { for (key, account) in accounts.iter().filter(|x| x.0[31].is_multiple_of(2)) { - tx.put::(*key, *account).unwrap(); + tx.put::(*key, account.clone()).unwrap(); } }) .unwrap(); let mut hashed_post_state = HashedPostState::default(); for (hashed_address, account) in accounts.iter().filter(|x| !x.0[31].is_multiple_of(2)) { - hashed_post_state.accounts.insert(*hashed_address, Some(*account)); + hashed_post_state.accounts.insert(*hashed_address, Some(account.clone())); } let sorted = hashed_post_state.into_sorted(); @@ -128,7 +128,7 @@ fn removed_accounts_are_discarded() { let db = create_test_rw_db(); db.update(|tx| { for (key, account) in accounts.iter().filter(|x| x.0[31].is_multiple_of(2)) { - tx.put::(*key, *account).unwrap(); + tx.put::(*key, account.clone()).unwrap(); } }) .unwrap(); @@ -137,7 +137,7 @@ fn removed_accounts_are_discarded() { for (hashed_address, account) in accounts.iter().filter(|x| !x.0[31].is_multiple_of(2)) { hashed_post_state.accounts.insert( *hashed_address, - if removed_keys.contains(hashed_address) { None } else { Some(*account) }, + if removed_keys.contains(hashed_address) { None } else { Some(account.clone()) }, ); } @@ -165,7 +165,7 @@ fn post_state_accounts_take_precedence() { let mut hashed_post_state = HashedPostState::default(); for (hashed_address, account) in &accounts { - hashed_post_state.accounts.insert(*hashed_address, Some(*account)); + hashed_post_state.accounts.insert(*hashed_address, Some(account.clone())); } let sorted = hashed_post_state.into_sorted(); @@ -180,21 +180,21 @@ fn fuzz_hashed_account_cursor() { let db = create_test_rw_db(); db.update(|tx| { for (key, account) in &db_accounts { - tx.put::(*key, *account).unwrap(); + tx.put::(*key, account.clone()).unwrap(); } }) .unwrap(); let mut hashed_post_state = HashedPostState::default(); for (hashed_address, account) in &post_state_accounts { - hashed_post_state.accounts.insert(*hashed_address, *account); + hashed_post_state.accounts.insert(*hashed_address, account.clone()); } let mut expected = db_accounts; // overwrite or remove accounts from the expected result for (key, account) in &post_state_accounts { if let Some(account) = account { - expected.insert(*key, *account); + expected.insert(*key, account.clone()); } else { expected.remove(key); } diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index bc670160298..126b46e00f6 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -1,4 +1,4 @@ -#![allow(missing_docs)] +#![allow(missing_docs, clippy::clone_on_copy)] use alloy_consensus::EMPTY_ROOT_HASH; use alloy_primitives::{ @@ -410,7 +410,7 @@ fn arbitrary_state_root_with_progress() { let tx = factory.provider_rw().unwrap(); for (address, (account, storage)) in &state { - insert_account(tx.tx_ref(), *address, *account, storage) + insert_account(tx.tx_ref(), *address, account.clone(), storage) } tx.commit().unwrap(); let tx = factory.provider_rw().unwrap(); @@ -450,7 +450,7 @@ fn test_state_root_with_state(state: State) { let tx = factory.provider_rw().unwrap(); for (address, (account, storage)) in &state { - insert_account(tx.tx_ref(), *address, *account, storage) + insert_account(tx.tx_ref(), *address, account.clone(), storage) } tx.commit().unwrap(); let expected = state_root(state); @@ -462,8 +462,8 @@ fn test_state_root_with_state(state: State) { }); } -fn encode_account(account: Account, storage_root: Option) -> Vec { - let account = account.into_trie_account(storage_root.unwrap_or(EMPTY_ROOT_HASH)); +fn encode_account(account: &Account, storage_root: Option) -> Vec { + let account = account.clone().into_trie_account(storage_root.unwrap_or(EMPTY_ROOT_HASH)); let mut account_rlp = Vec::with_capacity(account.length()); account.encode(&mut account_rlp); account_rlp @@ -531,7 +531,7 @@ fn account_and_storage_trie() { let key1 = b256!("0xb000000000000000000000000000000000000000000000000000000000000000"); let account1 = Account { balance: U256::from(3).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key1, &account1).unwrap(); - hash_builder.add_leaf(Nibbles::unpack(key1), &encode_account(account1, None)); + hash_builder.add_leaf(Nibbles::unpack(key1), &encode_account(&account1, None)); // Some address whose hash starts with 0xB040 let address2 = address!("0x7db3e81b72d2695e19764583f6d219dbee0f35ca"); @@ -540,7 +540,7 @@ fn account_and_storage_trie() { assert_eq!(key2[1], 0x40); let account2 = Account { balance: ether, ..Default::default() }; hashed_account_cursor.upsert(key2, &account2).unwrap(); - hash_builder.add_leaf(Nibbles::unpack(key2), &encode_account(account2, None)); + hash_builder.add_leaf(Nibbles::unpack(key2), &encode_account(&account2, None)); // Some address whose hash starts with 0xB041 let address3 = address!("0x16b07afd1c635f77172e842a000ead9a2a222459"); @@ -568,32 +568,32 @@ fn account_and_storage_trie() { DbStorageRoot::<_, A>::from_tx(tx.tx_ref(), address3).root().unwrap() }); hash_builder - .add_leaf(Nibbles::unpack(key3), &encode_account(account3, Some(account3_storage_root))); + .add_leaf(Nibbles::unpack(key3), &encode_account(&account3, Some(account3_storage_root))); let key4a = b256!("0xB1A0000000000000000000000000000000000000000000000000000000000000"); let account4a = Account { balance: U256::from(4).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key4a, &account4a).unwrap(); - hash_builder.add_leaf(Nibbles::unpack(key4a), &encode_account(account4a, None)); + hash_builder.add_leaf(Nibbles::unpack(key4a), &encode_account(&account4a, None)); let key5 = b256!("0xB310000000000000000000000000000000000000000000000000000000000000"); let account5 = Account { balance: U256::from(8).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key5, &account5).unwrap(); - hash_builder.add_leaf(Nibbles::unpack(key5), &encode_account(account5, None)); + hash_builder.add_leaf(Nibbles::unpack(key5), &encode_account(&account5, None)); let key6 = b256!("0xB340000000000000000000000000000000000000000000000000000000000000"); let account6 = Account { balance: U256::from(1).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key6, &account6).unwrap(); - hash_builder.add_leaf(Nibbles::unpack(key6), &encode_account(account6, None)); + hash_builder.add_leaf(Nibbles::unpack(key6), &encode_account(&account6, None)); // Populate account & storage trie DB tables let expected_root = b256!("0x72861041bc90cd2f93777956f058a545412b56de79af5eb6b8075fe2eabbe015"); let computed_expected_root: B256 = triehash::trie_root::([ - (key1, encode_account(account1, None)), - (key2, encode_account(account2, None)), - (key3, encode_account(account3, Some(account3_storage_root))), - (key4a, encode_account(account4a, None)), - (key5, encode_account(account5, None)), - (key6, encode_account(account6, None)), + (key1, encode_account(&account1, None)), + (key2, encode_account(&account2, None)), + (key3, encode_account(&account3, Some(account3_storage_root))), + (key4a, encode_account(&account4a, None)), + (key5, encode_account(&account5, None)), + (key6, encode_account(&account6, None)), ]); // Check computed trie root to ensure correctness assert_eq!(computed_expected_root, expected_root); @@ -688,13 +688,13 @@ fn account_and_storage_trie() { account_prefix_set.insert(Nibbles::unpack(account.0)); let computed_expected_root: B256 = triehash::trie_root::([ - (key1, encode_account(account1, None)), - // DELETED: (key2, encode_account(account2, None)), - (key3, encode_account(account3, Some(account3_storage_root))), - (key4a, encode_account(account4a, None)), - (key4b, encode_account(account4b, None)), - (key5, encode_account(account5, None)), - (key6, encode_account(account6, None)), + (key1, encode_account(&account1, None)), + // DELETED: (key2, encode_account(&account2, None)), + (key3, encode_account(&account3, Some(account3_storage_root))), + (key4a, encode_account(&account4a, None)), + (key4b, encode_account(&account4b, None)), + (key5, encode_account(&account5, None)), + (key6, encode_account(&account6, None)), ]); let (root, trie_updates) = reth_trie_db::with_adapter!(tx, |A| { @@ -745,13 +745,13 @@ fn account_and_storage_trie() { account_prefix_set.insert(Nibbles::unpack(account3.0)); let computed_expected_root: B256 = triehash::trie_root::([ - (key1, encode_account(account1, None)), - // DELETED: (key2, encode_account(account2, None)), - // DELETED: (key3, encode_account(account3, Some(account3_storage_root))), - (key4a, encode_account(account4a, None)), - (key4b, encode_account(account4b, None)), - (key5, encode_account(account5, None)), - (key6, encode_account(account6, None)), + (key1, encode_account(&account1, None)), + // DELETED: (key2, encode_account(&account2, None)), + // DELETED: (key3, encode_account(&account3, Some(account3_storage_root))), + (key4a, encode_account(&account4a, None)), + (key4b, encode_account(&account4b, None)), + (key5, encode_account(&account5, None)), + (key6, encode_account(&account6, None)), ]); let (root, trie_updates) = reth_trie_db::with_adapter!(tx, |A| { @@ -926,7 +926,7 @@ fn extension_node_trie( bytecode_hash: Some(B256::random()), ..Default::default() }; - let val = encode_account(a, None); + let val = encode_account(&a, None); let mut hashed_accounts = tx.tx_ref().cursor_write::().unwrap(); let mut hb = HashBuilder::default(); diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index 382124aa63a..937130c6dc5 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -694,8 +694,8 @@ mod tests { let from_hook = evm_state_to_hashed_post_state(evm_state); assert_eq!( - from_hook.accounts.get(&keccak256(address)).copied().flatten(), - from_bundle.accounts.get(&keccak256(address)).copied().flatten(), + from_hook.accounts.get(&keccak256(address)).and_then(Option::as_ref), + from_bundle.accounts.get(&keccak256(address)).and_then(Option::as_ref), "state-hook and bundle producers disagree about a created-empty account" ); } diff --git a/crates/trie/parallel/src/value_encoder.rs b/crates/trie/parallel/src/value_encoder.rs index 139ba0e03b9..6f2b4745ca0 100644 --- a/crates/trie/parallel/src/value_encoder.rs +++ b/crates/trie/parallel/src/value_encoder.rs @@ -117,6 +117,7 @@ where TC: TrieStorageCursor, HC: HashedStorageCursor, { + #[allow(clippy::clone_on_copy)] fn encode(mut self, buf: &mut Vec) -> Result<(), StateProofError> { let (account, root) = match &mut self { Self::Dispatched { @@ -128,7 +129,7 @@ where storage_calculator, } => { let hashed_address = *hashed_address; - let account = *account; + let account = account.clone(); // Take the receiver so Drop won't try to receive on it again let proof_result_rx = proof_result_rx .take() @@ -169,7 +170,7 @@ where } Self::Sync { storage_calculator, hashed_address, account } => { let hashed_address = *hashed_address; - let account = *account; + let account = account.clone(); let mut calculator = storage_calculator.borrow_mut(); let root_node = calculator.storage_root_node(hashed_address)?; let storage_root = calculator diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index 5648f3cd2c9..a43139571b4 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -1014,6 +1014,7 @@ mod tests { } #[test] + #[allow(clippy::clone_on_copy)] fn take_trie_updates() { reth_tracing::init_test_tracing(); @@ -1053,7 +1054,7 @@ mod tests { let address_1 = b256!("0x1000000000000000000000000000000000000000000000000000000000000000"); let address_path_1 = Nibbles::unpack(address_1); let account_1 = Account::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(); - let mut trie_account_1 = account_1.into_trie_account(storage_root); + let mut trie_account_1 = account_1.clone().into_trie_account(storage_root); let address_2 = b256!("0x1100000000000000000000000000000000000000000000000000000000000000"); let address_path_2 = Nibbles::unpack(address_2); let account_2 = Account::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(); @@ -1061,7 +1062,7 @@ mod tests { let mut hash_builder = HashBuilder::default() .with_proof_retainer(ProofRetainer::from_iter([address_path_1, address_path_2])); - hash_builder.add_leaf(address_path_1, &alloy_rlp::encode(trie_account_1)); + hash_builder.add_leaf(address_path_1, &alloy_rlp::encode(trie_account_1.clone())); hash_builder.add_leaf(address_path_2, &alloy_rlp::encode(trie_account_2)); let root = hash_builder.root(); diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index a5e6e593347..ee460545e53 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -57,11 +57,11 @@ where /// This design allows us to use `U256::ZERO`, rather than an Option, to indicate deletion for /// storage (which maps cleanly to how changesets are stored in the DB) while not requiring two /// different cursor implementations. -pub trait HashedPostStateCursorValue: Copy { +pub trait HashedPostStateCursorValue: Clone { /// The non-zero type returned by `into_option`. /// For `Option`, this is `Account`. /// For `U256`, this is `U256`. - type NonZero: Copy + std::fmt::Debug; + type NonZero: Clone + std::fmt::Debug; /// Returns `Some(&NonZero)` if the value is present, `None` if deleted. fn into_option(self) -> Option; @@ -248,7 +248,7 @@ where fn choose_next_entry(&mut self) -> Result, DatabaseError> { loop { let post_state_current = - self.post_state_cursor.current().copied().map(|(k, v)| (k, v.into_option())); + self.post_state_cursor.current().cloned().map(|(k, v)| (k, v.into_option())); let db_entry = self.db_cursor_state.entry(); match (post_state_current, db_entry) { @@ -277,7 +277,7 @@ where // - mem_key > db_key // - overlay is exhausted // Return the db_entry. If DB is also exhausted then this returns None. - _ => return Ok(db_entry.copied()), + _ => return Ok(db_entry.cloned()), } } } @@ -300,7 +300,7 @@ where /// [`HashedCursor::seek`] or [`HashedCursor::next`] are called. fn seek(&mut self, key: B256) -> Result, DatabaseError> { let post_state_entry = - self.post_state_cursor.seek(&key).copied().map(|(k, v)| (k, v.into_option())); + self.post_state_cursor.seek(&key).cloned().map(|(k, v)| (k, v.into_option())); if let Some((mem_key, Some(value))) = post_state_entry && mem_key == key @@ -511,7 +511,7 @@ mod tests { ) -> Vec<(B256, V::NonZero)> where V: HashedPostStateCursorValue, - V::NonZero: Copy, + V::NonZero: Clone, { db_nodes .into_iter() diff --git a/crates/trie/trie/src/node_iter.rs b/crates/trie/trie/src/node_iter.rs index 77eef810225..c47917adca1 100644 --- a/crates/trie/trie/src/node_iter.rs +++ b/crates/trie/trie/src/node_iter.rs @@ -80,7 +80,7 @@ pub struct TrieNodeIter { impl TrieNodeIter where - H::Value: Copy, + H::Value: Clone, K: AsRef, { /// Creates a new [`TrieNodeIter`] for the state trie. @@ -123,14 +123,13 @@ where /// /// If `metrics` feature is enabled, it also updates the metrics. fn seek_hashed_entry(&mut self, key: B256) -> Result, DatabaseError> { - if let Some((last_key, last_value)) = self.last_next_result && - last_key == key + if let Some((last_key, _)) = self.last_next_result.as_ref() && + *last_key == key { trace!(target: "trie::node_iter", seek_key = ?key, "reusing result from last next() call instead of seeking"); - self.last_next_result = None; // Consume the cached value - - let result = Some((last_key, last_value)); - self.last_seeked_hashed_entry = Some(SeekedHashedEntry { seeked_key: key, result }); + let result = self.last_next_result.take(); + self.last_seeked_hashed_entry = + Some(SeekedHashedEntry { seeked_key: key, result: result.clone() }); return Ok(result); } @@ -139,7 +138,7 @@ where .last_seeked_hashed_entry .as_ref() .filter(|entry| entry.seeked_key == key) - .map(|entry| entry.result) + .map(|entry| entry.result.clone()) { #[cfg(feature = "metrics")] self.metrics.inc_leaf_nodes_same_seeked(); @@ -148,7 +147,8 @@ where trace!(target: "trie::node_iter", ?key, "performing hashed cursor seek"); let result = self.hashed_cursor.seek(key)?; - self.last_seeked_hashed_entry = Some(SeekedHashedEntry { seeked_key: key, result }); + self.last_seeked_hashed_entry = + Some(SeekedHashedEntry { seeked_key: key, result: result.clone() }); #[cfg(feature = "metrics")] { @@ -163,7 +163,7 @@ where fn next_hashed_entry(&mut self) -> Result, DatabaseError> { let next = self.hashed_cursor.next()?; - self.last_next_result = next; + self.last_next_result = next.clone(); #[cfg(feature = "metrics")] { @@ -177,7 +177,7 @@ impl TrieNodeIter where C: TrieCursor, H: HashedCursor, - H::Value: Copy, + H::Value: Clone, K: AsRef, { /// Return the next trie node to be added to the hash builder. @@ -407,14 +407,13 @@ mod tests { let account_3 = b256!("0x0000000000000000000000000000000000000000000000000000000000000100"); let account_4 = b256!("0x0000000000000000000000000000000000000000000000000000000000000101"); let account_5 = b256!("0x0000000000000000000000000000000000000000000000000000000000000110"); - let empty_account = Account::default(); let hash_builder_branch_nodes = get_hash_builder_branch_nodes(vec![ - (Nibbles::unpack(account_1), empty_account), - (Nibbles::unpack(account_2), empty_account), - (Nibbles::unpack(account_3), empty_account), - (Nibbles::unpack(account_4), empty_account), - (Nibbles::unpack(account_5), empty_account), + (Nibbles::unpack(account_1), Account::default()), + (Nibbles::unpack(account_2), Account::default()), + (Nibbles::unpack(account_3), Account::default()), + (Nibbles::unpack(account_4), Account::default()), + (Nibbles::unpack(account_5), Account::default()), ]); let branch_node_1_rlp = RlpNode::from_rlp(&alloy_rlp::encode(BranchNode::new( @@ -479,11 +478,11 @@ mod tests { let hashed_cursor_factory = MockHashedCursorFactory::new( BTreeMap::from([ - (account_1, empty_account), - (account_2, empty_account), - (account_3, empty_account), - (account_4, empty_account), - (account_5, empty_account), + (account_1, Account::default()), + (account_2, Account::default()), + (account_3, Account::default()), + (account_4, Account::default()), + (account_5, Account::default()), ]), B256Map::default(), ); diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index 1a78c512767..851d3eb434a 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -105,6 +105,7 @@ where /// # Arguments /// /// `state` - state transition containing both modified and touched accounts and storage slots. + #[allow(clippy::clone_on_copy)] pub fn compute(mut self, state: HashedPostState) -> Result, TrieWitnessError> { let is_state_empty = state.is_empty(); if is_state_empty && !self.always_include_root_node { @@ -236,6 +237,7 @@ where .accounts .get(&hashed_address) .ok_or(TrieWitnessError::MissingAccount(hashed_address))? + .clone() .unwrap_or_default(); let storage_root = diff --git a/testing/testing-utils/src/generators.rs b/testing/testing-utils/src/generators.rs index 3e601f1d101..89cc4c943a6 100644 --- a/testing/testing-utils/src/generators.rs +++ b/testing/testing-utils/src/generators.rs @@ -300,6 +300,7 @@ type AccountState = (Account, Vec); /// /// Returns a Vec of account and storage changes for each block, /// along with the final state of all accounts and storages. +#[allow(clippy::clone_on_copy)] pub fn random_changeset_range<'a, R: Rng, IBlk, IAcc>( rng: &mut R, blocks: IBlk, @@ -331,7 +332,7 @@ where // extract from sending account let (prev_from, _) = state.get_mut(&from).unwrap(); - changeset.push((from, *prev_from, Vec::new())); + changeset.push((from, prev_from.clone(), Vec::new())); transfer = max(min(transfer, prev_from.balance), U256::from(1)); prev_from.balance = prev_from.balance.wrapping_sub(transfer); @@ -356,7 +357,7 @@ where .collect(); old_entries.sort_by_key(|entry| entry.key); - changeset.push((to, *prev_to, old_entries)); + changeset.push((to, prev_to.clone(), old_entries)); changeset.sort_by_key(|(address, _, _)| *address); From df7b7fdf55f9774610bb0209d981fd6550ac1b0d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 25 Sep 2026 20:08:22 +0200 Subject: [PATCH 097/133] refactor(e2e): clean up test utils (#27452) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- Cargo.lock | 1 - crates/e2e-test-utils/Cargo.toml | 1 - crates/e2e-test-utils/src/node.rs | 57 ++-- crates/e2e-test-utils/src/setup_import.rs | 9 +- crates/e2e-test-utils/src/testsuite/setup.rs | 13 +- crates/e2e-test-utils/src/wait.rs | 5 +- crates/e2e-test-utils/src/wallet.rs | 25 +- crates/e2e-test-utils/tests/rocksdb/main.rs | 257 ++++++++---------- crates/ethereum/node/tests/e2e/blobs.rs | 32 +-- .../ethereum/node/tests/e2e/custom_genesis.rs | 12 +- crates/ethereum/node/tests/e2e/dev.rs | 11 +- crates/ethereum/node/tests/e2e/eth.rs | 32 +-- .../node/tests/e2e/invalid_payload.rs | 17 +- crates/ethereum/node/tests/e2e/p2p.rs | 60 ++-- crates/ethereum/node/tests/e2e/pool.rs | 13 +- crates/ethereum/node/tests/e2e/simulate.rs | 7 +- crates/ethereum/node/tests/e2e/utils.rs | 2 +- 17 files changed, 224 insertions(+), 330 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c7e38c492c..763acfdb286 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8475,7 +8475,6 @@ dependencies = [ "reth-engine-primitives", "reth-ethereum-primitives", "reth-network-api", - "reth-network-p2p", "reth-network-peers", "reth-node-api", "reth-node-builder", diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index 4fdd0e9b7d4..fcf902070fa 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -15,7 +15,6 @@ reth-chainspec.workspace = true reth-tracing.workspace = true reth-db = { workspace = true, features = ["test-utils"] } reth-network-api.workspace = true -reth-network-p2p.workspace = true reth-rpc-server-types.workspace = true reth-rpc-builder.workspace = true reth-rpc-eth-api.workspace = true diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index b26bb343641..7eb43c73b7d 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -116,21 +116,17 @@ where { let mut chain = Vec::with_capacity(length as usize); for i in 0..length { - let raw_tx = tx_generator(i).await; - let tx_hash = self.rpc.inject_tx(raw_tx).await?; - let payload = self.advance_block().await?; - let block_hash = payload.block().hash(); - let block_number = payload.block().number(); - self.assert_new_block(tx_hash, block_hash, block_number).await?; + let (_, payload) = self.inject_and_advance(tx_generator(i).await).await?; chain.push(payload); } Ok(chain) } - /// Injects the raw transaction into the pool and advances the node one block, returning the - /// transaction hash and the built payload. + /// Injects the raw transaction into the pool, advances the node one block and waits until the + /// block is committed, returning the transaction hash and the built payload. /// - /// Returns an error if the transaction is not included in the block. + /// Returns an error if the transaction is not included in the block, see + /// [`Self::assert_new_block`]. pub async fn inject_and_advance( &mut self, raw_tx: Bytes, @@ -142,12 +138,7 @@ where { let tx_hash = self.rpc.inject_tx(raw_tx).await?; let payload = self.advance_block().await?; - let block = payload.block(); - ensure!( - block.body().transactions().iter().any(|tx| *tx.tx_hash() == tx_hash), - "transaction {tx_hash} was not included in block {}", - block.number() - ); + self.assert_new_block(tx_hash, payload.block().hash(), payload.block().number()).await?; Ok((tx_hash, payload)) } @@ -228,7 +219,7 @@ where wait_finish_checkpoint: bool, ) -> eyre::Result<()> { let provider = &self.inner.provider; - poll_until(format_args!("block {number}"), move || async move { + poll_until(format!("block {number}"), move || async move { if wait_finish_checkpoint && provider .get_stage_checkpoint(StageId::Finish)? @@ -258,7 +249,7 @@ where /// Returns an error if the node does not unwind within [`WAIT_TIMEOUT`]. pub async fn wait_unwind(&self, number: BlockNumber) -> eyre::Result<()> { let provider = &self.inner.provider; - poll_until(format_args!("unwind to block {number}"), move || async move { + poll_until(format!("unwind to block {number}"), move || async move { let checkpoint = provider.get_stage_checkpoint(StageId::Headers)?; Ok(checkpoint.is_some_and(|checkpoint| checkpoint.block_number == number).then_some(())) }) @@ -291,8 +282,8 @@ where block_hash: B256, block_number: BlockNumber, ) -> eyre::Result<()> { - // get head block from notifications stream and verify the tx has been pushed to the pool is - // actually present in the canonical block + // Get the head block from the notification stream and verify the tx is included in the + // canonical block. let head = tokio::time::timeout(WAIT_TIMEOUT, self.canonical_stream.next()) .await .map_err(|_| eyre!("timed out waiting for block {block_number}"))? @@ -303,10 +294,10 @@ where head.tip().number() ); - // wait for the block to commit and make sure the block hash we submitted via FCU engine - // api is the new latest block + // Wait for the block to commit and make sure the block hash we submitted via FCU engine + // api is the new latest block. let provider = &self.inner.provider; - poll_until(format_args!("block {block_number} to be committed"), move || async move { + poll_until(format!("block {block_number} to be committed"), move || async move { let Some(latest) = provider.block_by_number_or_tag(BlockNumberOrTag::Latest)? else { return Ok(None) }; @@ -526,3 +517,25 @@ where eyre::Ok(res) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::NodeHelperType; + use reth_node_ethereum::EthereumNode; + + fn assert_send(_: T) {} + + /// Tests of downstream nodes await these helpers in spawned tasks, so their futures must be + /// `Send`. + #[expect(dead_code)] + fn test_helper_futures_are_send(node: &mut NodeHelperType) { + assert_send(node.advance_block()); + assert_send(node.inject_and_advance(Bytes::new())); + assert_send(node.wait_block(0, B256::ZERO, false)); + assert_send(node.wait_unwind(0)); + assert_send(node.wait_for_pool(|_| true)); + assert_send(node.assert_new_block(B256::ZERO, B256::ZERO, 0)); + assert_send(node.sync_to(B256::ZERO)); + } +} diff --git a/crates/e2e-test-utils/src/setup_import.rs b/crates/e2e-test-utils/src/setup_import.rs index 60c4d4aa5c3..b1ddda23f09 100644 --- a/crates/e2e-test-utils/src/setup_import.rs +++ b/crates/e2e-test-utils/src/setup_import.rs @@ -71,6 +71,10 @@ pub async fn setup_engine_with_chain_import( rlp_path: &Path, ) -> eyre::Result { let runtime = Runtime::test(); + let attributes_generator = { + let chain_spec = chain_spec.clone(); + Arc::new(move |timestamp| eth_payload_attributes(&chain_spec, timestamp)) + }; // Create nodes with imported data. let mut nodes = Vec::with_capacity(num_nodes); @@ -103,15 +107,12 @@ pub async fn setup_engine_with_chain_import( // Now launch the node with the pre-populated datadir. debug!(target: "e2e::import", "Launching node with datadir: {:?}", datadir); - let chain_spec_for_attributes = chain_spec.clone(); let node = launch_test_node::(LaunchArgs { node_config, runtime: runtime.clone(), tree_config: tree_config.clone(), datadir, - attributes_generator: Arc::new(move |timestamp| { - eth_payload_attributes(&chain_spec_for_attributes, timestamp) - }), + attributes_generator: attributes_generator.clone(), dev_payload_attributes: None, }) .instrument(span) diff --git a/crates/e2e-test-utils/src/testsuite/setup.rs b/crates/e2e-test-utils/src/testsuite/setup.rs index 80b2330e074..3e10ea822e0 100644 --- a/crates/e2e-test-utils/src/testsuite/setup.rs +++ b/crates/e2e-test-utils/src/testsuite/setup.rs @@ -6,7 +6,6 @@ use alloy_rpc_types_engine::ForkchoiceState; use eyre::{eyre, Result}; use reth_chainspec::ChainSpec; use reth_ethereum_primitives::Block; -use reth_network_p2p::sync::{NetworkSyncUpdater, SyncState}; use reth_node_api::{EngineTypes, PayloadTypes, TreeConfig}; use reth_node_core::{args::StorageArgs, primitives::RecoveredBlock}; use revm::state::EvmState; @@ -202,7 +201,6 @@ where let (shutdown_tx, mut shutdown_rx) = mpsc::channel(1); self.shutdown_tx = Some(shutdown_tx); - let is_dev = self.is_dev; let tree_config = self.tree_config.clone(); let result = N::test_setup( @@ -212,7 +210,7 @@ where .with_tree_config_modifier(move |base| { tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size()) }) - .with_dev_mode(is_dev) + .with_dev_mode(self.is_dev) .with_storage_v2(self.storage_v2) .with_connect_nodes(self.network.connect_nodes) .build() @@ -320,15 +318,6 @@ where self.network.node_count, initial_block_info.number, initial_block_info.hash ); - // In test environments, explicitly set sync state to Idle after initialization - // This ensures that eth_syncing returns false as expected by tests - if let Some(import_result) = &self.import_result_holder { - for (idx, node_ctx) in import_result.nodes.iter().enumerate() { - debug!("Setting sync state to Idle for node {}", idx); - node_ctx.inner.network.update_sync_state(SyncState::Idle); - } - } - Ok(()) } diff --git a/crates/e2e-test-utils/src/wait.rs b/crates/e2e-test-utils/src/wait.rs index c0f51af4839..aa669ebe8e8 100644 --- a/crates/e2e-test-utils/src/wait.rs +++ b/crates/e2e-test-utils/src/wait.rs @@ -14,7 +14,10 @@ pub const POLL_INTERVAL: Duration = Duration::from_millis(20); /// /// Returns an error describing `what` was awaited if the condition is not met within /// [`WAIT_TIMEOUT`], or the first error returned by `poll`. -pub async fn poll_until(what: impl Display, mut poll: F) -> eyre::Result +/// +/// `what` is held across awaits, so it must be `Send` for the returned future to be `Send`. This +/// rules out `format_args!`, use `format!` instead. +pub async fn poll_until(what: impl Display + Send, mut poll: F) -> eyre::Result where F: FnMut() -> Fut, Fut: Future>>, diff --git a/crates/e2e-test-utils/src/wallet.rs b/crates/e2e-test-utils/src/wallet.rs index 11976ceb3e7..ed93166b758 100644 --- a/crates/e2e-test-utils/src/wallet.rs +++ b/crates/e2e-test-utils/src/wallet.rs @@ -10,9 +10,6 @@ use alloy_signer_local::{coins_bip39::English, MnemonicBuilder, PrivateKeySigner /// Mnemonic of the test accounts funded by the [`test_genesis`](crate::test_genesis). pub const TEST_MNEMONIC: &str = "test test test test test test test test test test test junk"; -/// Default derivation path prefix of the test accounts, followed by the account index. -const DEFAULT_DERIVATION_PATH: &str = "m/44'/60'/0'/0/"; - /// One of the accounts of the genesis allocations. #[derive(Debug)] pub struct Wallet { @@ -23,14 +20,12 @@ pub struct Wallet { /// The chain id pub chain_id: u64, amount: usize, - derivation_path: Option, } impl Wallet { /// Creates a new account from one of the secret/pubkeys of the genesis allocations (test.json) pub fn new(amount: usize) -> Self { - let inner = MnemonicBuilder::::default().phrase(TEST_MNEMONIC).build().unwrap(); - Self { inner, chain_id: 1, amount, derivation_path: None, inner_nonce: 0 } + Self { inner: test_signer(0), chain_id: 1, amount, inner_nonce: 0 } } /// Sets chain id @@ -39,10 +34,6 @@ impl Wallet { self } - fn get_derivation_path(&self) -> &str { - self.derivation_path.as_deref().unwrap_or(DEFAULT_DERIVATION_PATH) - } - /// Generates a list of wallets pub fn wallet_gen(&self) -> Vec { (0..self.amount as u32).map(|idx| self.signer(idx)).collect() @@ -50,7 +41,7 @@ impl Wallet { /// Returns the signer of the test account at `index`, with the chain id of the wallet. pub fn signer(&self, index: u32) -> PrivateKeySigner { - derive_signer(self.get_derivation_path(), index).with_chain_id(Some(self.chain_id)) + test_signer(index).with_chain_id(Some(self.chain_id)) } /// Returns the test account at `index`, with the chain id of the wallet and nonce 0. @@ -65,17 +56,13 @@ impl Default for Wallet { } } -/// Returns the signer of the test account at `index`, derived from [`TEST_MNEMONIC`]. +/// Returns the signer of the test account at `index`, derived from [`TEST_MNEMONIC`] with the +/// default Ethereum derivation path `m/44'/60'/0'/0/{index}`. pub fn test_signer(index: u32) -> PrivateKeySigner { - derive_signer(DEFAULT_DERIVATION_PATH, index) -} - -/// Derives the signer at `index` below `derivation_path` from [`TEST_MNEMONIC`]. -fn derive_signer(derivation_path: &str, index: u32) -> PrivateKeySigner { MnemonicBuilder::::default() .phrase(TEST_MNEMONIC) - .derivation_path(format!("{derivation_path}{index}")) - .expect("valid derivation path") + .index(index) + .expect("valid derivation index") .build() .expect("valid test mnemonic") } diff --git a/crates/e2e-test-utils/tests/rocksdb/main.rs b/crates/e2e-test-utils/tests/rocksdb/main.rs index eabc0e8e6cb..15d5d00b131 100644 --- a/crates/e2e-test-utils/tests/rocksdb/main.rs +++ b/crates/e2e-test-utils/tests/rocksdb/main.rs @@ -7,9 +7,7 @@ use eyre::{ensure, Result}; use jsonrpsee::core::client::ClientT; use reth_chainspec::EthereumHardfork; use reth_db::tables; -use reth_e2e_test_utils::{ - test_chain_spec, transaction::TransactionTestContext, wait::poll_until, wallet, E2ETestSetupExt, -}; +use reth_e2e_test_utils::{transaction::TransactionTestContext, wait::poll_until, E2ETestSetupExt}; use reth_node_ethereum::EthereumNode; use reth_provider::RocksDBProviderFactory; use std::time::Duration; @@ -18,7 +16,7 @@ use std::time::Duration; /// Prevents race conditions where `advance_block` is called before txs are in the pool. /// Returns the pending transaction. async fn wait_for_pending_tx(client: &C, tx_hash: B256) -> Result { - poll_until(format_args!("transaction {tx_hash} in the pending pool"), || async move { + poll_until(format!("transaction {tx_hash} in the pending pool"), || async move { let tx: Option = client.request("eth_getTransactionByHash", [tx_hash]).await?; if let Some(tx) = &tx { ensure!( @@ -35,7 +33,7 @@ async fn wait_for_pending_tx(client: &C, tx_hash: B256) -> Result(provider: &P, tx_hash: B256) -> Result { - poll_until(format_args!("transaction {tx_hash} in RocksDB"), || async move { + poll_until(format!("transaction {tx_hash} in RocksDB"), || async move { // Re-acquire handle each iteration to avoid stale snapshot reads Ok(provider.rocksdb_provider().get::(tx_hash)?) }) @@ -47,20 +45,18 @@ async fn poll_tx_in_rocksdb(provider: &P, tx_hash: B2 async fn test_rocksdb_node_startup() -> Result<()> { reth_tracing::init_test_tracing(); - let (nodes, _wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + let (node, _) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) - .build() + .build_single() .await?; - assert_eq!(nodes.len(), 1); - // Verify RocksDB provider is functional (can query without error) - let rocksdb = nodes[0].inner.provider.rocksdb_provider(); + let rocksdb = node.inner.provider.rocksdb_provider(); let missing_hash = B256::from([0xab; 32]); let result: Option = rocksdb.get::(missing_hash)?; assert!(result.is_none(), "Missing hash should return None"); - let genesis_hash = nodes[0].block_hash(0); + let genesis_hash = node.block_hash(0); assert_ne!(genesis_hash, B256::ZERO); Ok(()) @@ -71,31 +67,29 @@ async fn test_rocksdb_node_startup() -> Result<()> { async fn test_rocksdb_block_mining() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - - let (mut nodes, _wallet) = - EthereumNode::test_setup(1, chain_spec).with_storage_v2(true).build().await?; - - assert_eq!(nodes.len(), 1); + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) + .with_storage_v2(true) + .build_single() + .await?; + let chain_id = wallet.chain_id; - let genesis_hash = nodes[0].block_hash(0); + let genesis_hash = node.block_hash(0); assert_ne!(genesis_hash, B256::ZERO); // Mine 3 blocks with transactions - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); - let client = nodes[0].rpc_client().expect("RPC client should be available"); + let signer = wallet.signer(0); + let client = node.rpc_client().expect("RPC client should be available"); for i in 1..=3u64 { let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), i - 1) .await; - let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; // Wait for tx to enter pending pool before mining wait_for_pending_tx(&client, tx_hash).await?; - let payload = nodes[0].advance_block().await?; + let payload = node.advance_block().await?; let block = payload.block(); assert_eq!(block.number(), i); assert_ne!(block.hash(), B256::ZERO); @@ -109,7 +103,7 @@ async fn test_rocksdb_block_mining() -> Result<()> { // Verify all blocks are stored for i in 0..=3 { - let block_hash = nodes[0].block_hash(i); + let block_hash = node.block_hash(i); assert_ne!(block_hash, B256::ZERO); } @@ -121,30 +115,26 @@ async fn test_rocksdb_block_mining() -> Result<()> { async fn test_rocksdb_transaction_queries() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) - .build() + .build_single() .await?; - - assert_eq!(nodes.len(), 1); + let chain_id = wallet.chain_id; // Inject and mine a transaction - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); - let client = nodes[0].rpc_client().expect("RPC client should be available"); + let signer = wallet.signer(0); + let client = node.rpc_client().expect("RPC client should be available"); let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await; - let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; // Wait for tx to enter pending pool before mining wait_for_pending_tx(&client, tx_hash).await?; - let payload = nodes[0].advance_block().await?; + let payload = node.advance_block().await?; assert_eq!(payload.block().number(), 1); // Query each transaction by hash @@ -159,12 +149,12 @@ async fn test_rocksdb_transaction_queries() -> Result<()> { assert!(receipt.status()); // Direct RocksDB assertion - poll with timeout since persistence is async - let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; + let tx_number = poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; assert_eq!(tx_number, 0, "First tx should have TxNumber 0"); // Verify missing hash returns None let missing_hash = B256::from([0xde; 32]); - let rocksdb = nodes[0].inner.provider.rocksdb_provider(); + let rocksdb = node.inner.provider.rocksdb_provider(); let missing_tx_number: Option = rocksdb.get::(missing_hash)?; assert!(missing_tx_number.is_none()); @@ -185,27 +175,25 @@ async fn test_rocksdb_transaction_queries() -> Result<()> { async fn test_rocksdb_multi_tx_same_block() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) - .build() + .build_single() .await?; + let chain_id = wallet.chain_id; // Create 3 txs from the same wallet with sequential nonces - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); - let client = nodes[0].rpc_client().expect("RPC client"); + let signer = wallet.signer(0); + let client = node.rpc_client().expect("RPC client"); let mut tx_hashes = Vec::new(); for nonce in 0..3 { let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce) .await; - let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; tx_hashes.push(tx_hash); } @@ -215,7 +203,7 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> { } // Mine one block containing all 3 txs - let payload = nodes[0].advance_block().await?; + let payload = node.advance_block().await?; assert_eq!(payload.block().number(), 1); // Verify block contains all 3 txs @@ -234,7 +222,7 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> { // Poll RocksDB for all tx hashes and collect tx_numbers let mut tx_numbers = Vec::new(); for tx_hash in &tx_hashes { - let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await?; + let n = poll_tx_in_rocksdb(&node.inner.provider, *tx_hash).await?; tx_numbers.push(n); } @@ -250,28 +238,26 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> { async fn test_rocksdb_txs_across_blocks() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) - .build() + .build_single() .await?; + let chain_id = wallet.chain_id; - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); - let client = nodes[0].rpc_client().expect("RPC client"); + let signer = wallet.signer(0); + let client = node.rpc_client().expect("RPC client"); // Block 1: 2 transactions - let tx_hash_0 = nodes[0] + let tx_hash_0 = node .rpc .inject_tx( TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await, ) .await?; - let tx_hash_1 = nodes[0] + let tx_hash_1 = node .rpc .inject_tx( TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await, @@ -282,11 +268,11 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { wait_for_pending_tx(&client, tx_hash_0).await?; wait_for_pending_tx(&client, tx_hash_1).await?; - let payload1 = nodes[0].advance_block().await?; + let payload1 = node.advance_block().await?; assert_eq!(payload1.block().number(), 1); // Block 2: 1 transaction - let tx_hash_2 = nodes[0] + let tx_hash_2 = node .rpc .inject_tx( TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await, @@ -295,7 +281,7 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { wait_for_pending_tx(&client, tx_hash_2).await?; - let payload2 = nodes[0].advance_block().await?; + let payload2 = node.advance_block().await?; assert_eq!(payload2.block().number(), 2); // Verify block contents via RPC @@ -311,7 +297,7 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { let all_tx_hashes = [tx_hash_0, tx_hash_1, tx_hash_2]; let mut tx_numbers = Vec::new(); for tx_hash in &all_tx_hashes { - let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await?; + let n = poll_tx_in_rocksdb(&node.inner.provider, *tx_hash).await?; tx_numbers.push(n); } @@ -332,25 +318,23 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> { async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) - .build() + .build_single() .await?; + let chain_id = wallet.chain_id; - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); + let signer = wallet.signer(0); // Inject tx but do NOT mine let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await; - let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; // Verify tx is in pending pool via RPC - let client = nodes[0].rpc_client().expect("RPC client"); + let client = node.rpc_client().expect("RPC client"); wait_for_pending_tx(&client, tx_hash).await?; let pending_tx: Option = @@ -359,7 +343,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { assert!(pending_tx.unwrap().block_number.is_none(), "Pending tx should have no block_number"); // Assert tx is NOT in RocksDB before mining (single check - tx is confirmed pending) - let rocksdb = nodes[0].inner.provider.rocksdb_provider(); + let rocksdb = node.inner.provider.rocksdb_provider(); let tx_number: Option = rocksdb.get::(tx_hash)?; assert!( tx_number.is_none(), @@ -368,11 +352,11 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { ); // Now mine the block - let payload = nodes[0].advance_block().await?; + let payload = node.advance_block().await?; assert_eq!(payload.block().number(), 1); // Poll until tx appears in RocksDB - let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; + let tx_number = poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; assert_eq!(tx_number, 0, "First tx should have tx_number 0"); // Verify tx is now mined via RPC @@ -393,80 +377,75 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> { async fn test_rocksdb_reorg_unwind() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) - .build() + .build_single() .await?; + let chain_id = wallet.chain_id; - assert_eq!(nodes.len(), 1); - - // Use two separate wallets to avoid nonce conflicts during reorg - let wallet = wallet::Wallet::new(2).with_chain_id(chain_id); + // Use two separate accounts to avoid nonce conflicts during reorg let (signer1, signer2) = (wallet.signer(0), wallet.signer(1)); - let client = nodes[0].rpc_client().expect("RPC client"); + let client = node.rpc_client().expect("RPC client"); // Mine block 1 with a transaction from signer1 let raw_tx1 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 0).await; - let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?; + let tx_hash1 = node.rpc.inject_tx(raw_tx1).await?; wait_for_pending_tx(&client, tx_hash1).await?; - let payload1 = nodes[0].advance_block().await?; + let payload1 = node.advance_block().await?; let block1_hash = payload1.block().hash(); assert_eq!(payload1.block().number(), 1); // Poll until tx1 appears in RocksDB (ensures persistence happened) - let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await?; + let tx_number1 = poll_tx_in_rocksdb(&node.inner.provider, tx_hash1).await?; assert_eq!(tx_number1, 0, "First tx should have tx_number 0"); // Mine block 2 with transaction from signer1 (nonce 1) let raw_tx2 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 1).await; - let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?; + let tx_hash2 = node.rpc.inject_tx(raw_tx2).await?; wait_for_pending_tx(&client, tx_hash2).await?; - let payload2 = nodes[0].advance_block().await?; + let payload2 = node.advance_block().await?; assert_eq!(payload2.block().number(), 2); // Poll until tx2 appears in RocksDB - let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await?; + let tx_number2 = poll_tx_in_rocksdb(&node.inner.provider, tx_hash2).await?; assert_eq!(tx_number2, 1, "Second tx should have tx_number 1"); // Mine block 3 with transaction from signer1 (nonce 2) let raw_tx3 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 2).await; - let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?; + let tx_hash3 = node.rpc.inject_tx(raw_tx3).await?; wait_for_pending_tx(&client, tx_hash3).await?; - let payload3 = nodes[0].advance_block().await?; + let payload3 = node.advance_block().await?; assert_eq!(payload3.block().number(), 3); // Poll until tx3 appears in RocksDB - let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await?; + let tx_number3 = poll_tx_in_rocksdb(&node.inner.provider, tx_hash3).await?; assert_eq!(tx_number3, 2, "Third tx should have tx_number 2"); // Now create an alternate block 2 using signer2 (different wallet, avoids nonce conflict) // Inject a tx from signer2 (nonce 0) before building the alternate block let raw_alt_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 0).await; - let alt_tx_hash = nodes[0].rpc.inject_tx(raw_alt_tx).await?; + let alt_tx_hash = node.rpc.inject_tx(raw_alt_tx).await?; wait_for_pending_tx(&client, alt_tx_hash).await?; // Build an alternate payload (this builds on top of the current head, i.e., block 3) // But we want to reorg back to block 1, so we'll use the payload and then FCU to it - let alt_payload = nodes[0].new_payload().await?; - let alt_block_hash = nodes[0].submit_payload(alt_payload.clone()).await?; + let alt_payload = node.new_payload().await?; + let alt_block_hash = node.submit_payload(alt_payload.clone()).await?; // Trigger reorg: make the alternate chain canonical by sending FCU pointing to block 1's hash // as finalized, which should trigger an unwind of blocks 2 and 3 // The alt block becomes the new head - nodes[0].update_forkchoice(block1_hash, alt_block_hash).await?; + node.update_forkchoice(block1_hash, alt_block_hash).await?; // Give time for the reorg to complete tokio::time::sleep(Duration::from_millis(500)).await; @@ -487,10 +466,10 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> { // Mine another block to verify the chain can continue let raw_tx_final = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 1).await; - let tx_hash_final = nodes[0].rpc.inject_tx(raw_tx_final).await?; + let tx_hash_final = node.rpc.inject_tx(raw_tx_final).await?; wait_for_pending_tx(&client, tx_hash_final).await?; - let final_payload = nodes[0].advance_block().await?; + let final_payload = node.advance_block().await?; assert!(final_payload.block().number() > 3, "Should be able to mine block after reorg"); // Verify tx_final is included @@ -513,22 +492,18 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> { async fn test_rocksdb_historical_account_queries() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) }) - .build() + .build_single() .await?; + let chain_id = wallet.chain_id; - assert_eq!(nodes.len(), 1); - - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); + let signer = wallet.signer(0); let sender: Address = signer.address(); - let client = nodes[0].rpc_client().expect("RPC client"); + let client = node.rpc_client().expect("RPC client"); // Query the sender's balance and nonce at genesis (block 0) let genesis_balance: U256 = client.request("eth_getBalance", (sender, "0x0")).await?; @@ -539,12 +514,12 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { // Mine block 1 with a transfer (nonce 0) let raw_tx1 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await; - let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?; + let tx_hash1 = node.rpc.inject_tx(raw_tx1).await?; wait_for_pending_tx(&client, tx_hash1).await?; - let payload1 = nodes[0].advance_block().await?; + let payload1 = node.advance_block().await?; assert_eq!(payload1.block().number(), 1); - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await?; + poll_tx_in_rocksdb(&node.inner.provider, tx_hash1).await?; // Record state after block 1 let balance_at_1: U256 = client.request("eth_getBalance", (sender, "0x1")).await?; @@ -555,12 +530,12 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { // Mine block 2 with another transfer (nonce 1) let raw_tx2 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await; - let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?; + let tx_hash2 = node.rpc.inject_tx(raw_tx2).await?; wait_for_pending_tx(&client, tx_hash2).await?; - let payload2 = nodes[0].advance_block().await?; + let payload2 = node.advance_block().await?; assert_eq!(payload2.block().number(), 2); - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await?; + poll_tx_in_rocksdb(&node.inner.provider, tx_hash2).await?; let balance_at_2: U256 = client.request("eth_getBalance", (sender, "0x2")).await?; let nonce_at_2: U256 = client.request("eth_getTransactionCount", (sender, "0x2")).await?; @@ -570,12 +545,12 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { // Mine block 3 with a third transfer (nonce 2) let raw_tx3 = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await; - let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?; + let tx_hash3 = node.rpc.inject_tx(raw_tx3).await?; wait_for_pending_tx(&client, tx_hash3).await?; - let payload3 = nodes[0].advance_block().await?; + let payload3 = node.advance_block().await?; assert_eq!(payload3.block().number(), 3); - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await?; + poll_tx_in_rocksdb(&node.inner.provider, tx_hash3).await?; let balance_at_3: U256 = client.request("eth_getBalance", (sender, "0x3")).await?; let nonce_at_3: U256 = client.request("eth_getTransactionCount", (sender, "0x3")).await?; @@ -592,9 +567,9 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce) .await; - let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; wait_for_pending_tx(&client, tx_hash).await?; - nodes[0].advance_block().await?; + node.advance_block().await?; } // Allow the engine loop to process the persistence completions tokio::time::sleep(Duration::from_millis(500)).await; @@ -654,13 +629,10 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> { async fn test_rocksdb_account_history_pruning() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - const PRUNE_DISTANCE: u64 = 5; const TOTAL_BLOCKS: u64 = 20; - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) @@ -671,14 +643,13 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { config.pruning.block_interval = Some(1); config }) - .build() + .build_single() .await?; + let chain_id = wallet.chain_id; - assert_eq!(nodes.len(), 1); - - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); + let signer = wallet.signer(0); let sender: Address = signer.address(); - let client = nodes[0].rpc_client().expect("RPC client"); + let client = node.rpc_client().expect("RPC client"); // Mine blocks one at a time with a delay so each save_blocks + pruner cycle // completes independently. The race fires every cycle (the pruner reads stale @@ -688,21 +659,21 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce) .await; - let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; wait_for_pending_tx(&client, tx_hash).await?; - let payload = nodes[0].advance_block().await?; + let payload = node.advance_block().await?; assert_eq!(payload.block().number(), nonce + 1); // Let the persistence cycle (save_blocks → pruner → commit) complete before // producing the next block, so each cycle processes exactly one block. The // transaction becomes visible in RocksDB with the commit. - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; + poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; } // Read the AccountsHistory shard for `sender` directly from RocksDB. // This is the data structure corrupted by the race. - let rocksdb = nodes[0].inner.provider.rocksdb_provider(); + let rocksdb = node.inner.provider.rocksdb_provider(); let shards = rocksdb.account_history_shards(sender).unwrap(); let all_entries: Vec = shards.iter().flat_map(|(_, list)| list.iter()).collect(); @@ -744,13 +715,10 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { async fn test_rocksdb_storage_history_pruning() -> Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - let chain_id = chain_spec.chain().id(); - const PRUNE_DISTANCE: u64 = 5; const TOTAL_BLOCKS: u64 = 20; - let (mut nodes, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (mut node, wallet) = EthereumNode::test_setup_for(EthereumHardfork::Cancun) .with_storage_v2(true) .with_tree_config_modifier(|config| { config.with_persistence_threshold(0).with_memory_block_buffer_target(0) @@ -761,13 +729,12 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { config.pruning.block_interval = Some(1); config }) - .build() + .build_single() .await?; + let chain_id = wallet.chain_id; - assert_eq!(nodes.len(), 1); - - let signer = wallet::Wallet::new(1).with_chain_id(chain_id).signer(0); - let client = nodes[0].rpc_client().expect("RPC client"); + let signer = wallet.signer(0); + let client = node.rpc_client().expect("RPC client"); // Deploy a minimal contract that stores CALLDATA[0..32] into slot 0: // PUSH0 ; [0] @@ -819,13 +786,13 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { ..Default::default() }; let deploy_bytes = TransactionTestContext::sign_tx_bytes(signer.clone(), deploy_tx).await; - let deploy_hash = nodes[0].rpc.inject_tx(deploy_bytes).await?; + let deploy_hash = node.rpc.inject_tx(deploy_bytes).await?; wait_for_pending_tx(&client, deploy_hash).await?; - let payload1 = nodes[0].advance_block().await?; + let payload1 = node.advance_block().await?; assert_eq!(payload1.block().number(), 1); // Let the persistence cycle complete before the next block (same cadence as the loop below) - poll_tx_in_rocksdb(&nodes[0].inner.provider, deploy_hash).await?; + poll_tx_in_rocksdb(&node.inner.provider, deploy_hash).await?; // Get the deployed contract address from the receipt let receipt: Option = @@ -866,18 +833,18 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { ..Default::default() }; let call_bytes = TransactionTestContext::sign_tx_bytes(signer.clone(), call_tx).await; - let tx_hash = nodes[0].rpc.inject_tx(call_bytes).await?; + let tx_hash = node.rpc.inject_tx(call_bytes).await?; wait_for_pending_tx(&client, tx_hash).await?; - let payload = nodes[0].advance_block().await?; + let payload = node.advance_block().await?; assert_eq!(payload.block().number(), block_num); // Let the persistence cycle complete before the next block - poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await?; + poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; } // Read StoragesHistory shard for (contract_address, slot 0) directly from RocksDB - let rocksdb = nodes[0].inner.provider.rocksdb_provider(); + let rocksdb = node.inner.provider.rocksdb_provider(); let shards = rocksdb.storage_history_shards(contract_address, storage_slot).unwrap(); let all_entries: Vec = shards.iter().flat_map(|(_, list)| list.iter()).collect(); diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index f6dc7278d01..feef4cc18a9 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -1,8 +1,7 @@ use alloy_eips::Decodable2718; use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ - test_chain_spec, test_chain_spec_builder, transaction::TransactionTestContext, wallet::Wallet, - E2ETestSetupExt, + test_chain_spec, test_chain_spec_builder, transaction::TransactionTestContext, E2ETestSetupExt, }; use reth_ethereum_engine_primitives::BlobSidecars; use reth_ethereum_primitives::PooledTransactionVariant; @@ -19,14 +18,10 @@ async fn can_handle_blobs() -> eyre::Result<()> { let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let genesis_hash = chain_spec.genesis_hash(); - let (mut node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; - - let wallets = Wallet::new(2).wallet_gen(); - let blob_wallet = wallets.first().unwrap(); - let second_wallet = wallets.last().unwrap(); + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec).build_single().await?; // inject normal tx - let raw_tx = TransactionTestContext::transfer_tx_bytes(1, second_wallet.clone()).await; + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.signer(1)).await; let tx_hash = node.rpc.inject_tx(raw_tx).await?; // build payload with normal tx let payload = node.new_payload().await?; @@ -35,7 +30,7 @@ async fn can_handle_blobs() -> eyre::Result<()> { node.inner.pool.remove_transactions(vec![tx_hash]); // build blob tx - let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, blob_wallet.clone()).await?; + let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(0)).await?; // inject blob tx to the pool let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?; @@ -72,16 +67,13 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { let chain_spec = test_chain_spec(EthereumHardfork::Osaka); let genesis_hash = chain_spec.genesis_hash(); - let (mut node, _) = EthereumNode::test_setup(1, chain_spec) + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec) .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) .build_single() .await?; - let wallets = Wallet::new(2).wallet_gen(); - let blob_wallet = wallets.first().unwrap(); - // build blob tx - let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, blob_wallet.clone()).await?; + let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(0)).await?; let tx = PooledTransactionVariant::decode_2718_exact(&blob_tx).unwrap(); assert!(tx.as_eip4844().unwrap().tx().sidecar.is_eip4844()); @@ -118,24 +110,20 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { test_chain_spec_builder().prague_activated().with_osaka_at(osaka_timestamp).build(), ); let genesis_hash = chain_spec.genesis_hash(); - let (mut node, _) = EthereumNode::test_setup(1, chain_spec) + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec) .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) .build_single() .await?; - let mut wallets = Wallet::new(3).wallet_gen(); - let first = wallets.pop().unwrap(); - let second = wallets.pop().unwrap(); - // build a dummy payload at `current_timestamp` - let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallets.pop().unwrap()).await; + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.signer(0)).await; node.rpc.inject_tx(raw_tx).await?; node.payload.timestamp = current_timestamp - 1; node.advance_block().await?; // build blob txs - let first_blob = TransactionTestContext::tx_with_blobs_bytes(1, first.clone()).await?; - let second_blob = TransactionTestContext::tx_with_blobs_bytes(1, second.clone()).await?; + let first_blob = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(1)).await?; + let second_blob = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(2)).await?; // assert both txs have legacy sidecars assert!(PooledTransactionVariant::decode_2718_exact(&first_blob) diff --git a/crates/ethereum/node/tests/e2e/custom_genesis.rs b/crates/ethereum/node/tests/e2e/custom_genesis.rs index dafb06c79d0..0b11d6a1e14 100644 --- a/crates/ethereum/node/tests/e2e/custom_genesis.rs +++ b/crates/ethereum/node/tests/e2e/custom_genesis.rs @@ -33,19 +33,14 @@ async fn can_run_eth_node_with_custom_genesis_number() -> eyre::Result<()> { ); } - // Advance the chain (block 1001) + // Advance the chain (block 1001) and assert the block has been committed let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; - - let block_hash = payload.block().hash(); + let (_, payload) = node.inject_and_advance(raw_tx).await?; let block_number = payload.block().number; // Verify we're at block 1001 (genesis + 1) assert_eq!(block_number, 1001, "Block number should be 1001 after advancing from genesis 1000"); - // Assert the block has been committed - node.assert_new_block(tx_hash, block_hash, block_number).await?; - Ok(()) } @@ -92,9 +87,8 @@ async fn can_advance_on_genesis_newer_than_payload_timestamp() -> eyre::Result<( let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec).build_single().await?; let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; + let (_, payload) = node.inject_and_advance(raw_tx).await?; assert!(payload.block().timestamp > genesis_timestamp); - node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; Ok(()) } diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index ce8cde3cc2c..ebe920d88cc 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -2,7 +2,7 @@ use alloy_genesis::Genesis; use alloy_primitives::{b256, hex, Address}; use futures::StreamExt; use reth_chainspec::ChainSpec; -use reth_e2e_test_utils::E2ETestSetupExt; +use reth_e2e_test_utils::{wait::poll_until, E2ETestSetupExt}; use reth_node_api::{BlockBody, FullNodeComponents}; use reth_node_builder::{rpc::RethRpcAddOns, FullNode}; use reth_node_ethereum::EthereumNode; @@ -99,13 +99,10 @@ async fn can_run_dev_node_with_block_time() -> eyre::Result<()> { .await?; // The local miner builds a block on every interval, even without pending transactions. - tokio::time::timeout(Duration::from_secs(10), async { - while node.inner.provider.best_block_number()? < 2 { - tokio::time::sleep(Duration::from_millis(50)).await; - } - eyre::Ok(()) + poll_until("two dev blocks", || async { + Ok((node.inner.provider.best_block_number()? >= 2).then_some(())) }) - .await??; + .await?; Ok(()) } diff --git a/crates/ethereum/node/tests/e2e/eth.rs b/crates/ethereum/node/tests/e2e/eth.rs index 886b6d0ad55..b7f2e263816 100644 --- a/crates/ethereum/node/tests/e2e/eth.rs +++ b/crates/ethereum/node/tests/e2e/eth.rs @@ -14,8 +14,7 @@ use alloy_rpc_types_engine::{ use jsonrpsee_core::client::ClientT; use reth_chainspec::{EthChainSpec, EthereumHardfork}; use reth_e2e_test_utils::{ - eth_payload_attributes, test_chain_spec, transaction::TransactionTestContext, wallet::Wallet, - E2ETestSetupExt, + eth_payload_attributes, test_chain_spec, transaction::TransactionTestContext, E2ETestSetupExt, }; use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{ @@ -32,7 +31,6 @@ use reth_tasks::Runtime; use ssz::Encode; use std::sync::Arc; -const ENGINE_PRAGUE_FORK_HEADER: &str = "prague"; const ENGINE_PAYLOADS_ROUTE: &str = "/engine/v1/payloads"; const ENGINE_FORKCHOICE_ROUTE: &str = "/engine/v1/forkchoice"; const ENGINE_V1_BLOBS_ROUTE: &str = "/engine/v1/blobs/v1"; @@ -47,11 +45,8 @@ async fn can_run_eth_node() -> eyre::Result<()> { EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - // make the node advance - let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; - - // assert the block has been committed to the blockchain - node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; + // make the node advance and assert the block has been committed to the blockchain + node.inject_and_advance(raw_tx).await?; Ok(()) } @@ -69,11 +64,8 @@ async fn can_run_eth_node_with_auth_engine_api_over_ipc() -> eyre::Result<()> { // Create dummy transfer tx let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - // make the node advance - let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; - - // assert the block has been committed to the blockchain - node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; + // make the node advance and assert the block has been committed to the blockchain + node.inject_and_advance(raw_tx).await?; Ok(()) } @@ -100,8 +92,7 @@ async fn test_engine_graceful_shutdown() -> eyre::Result<()> { EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await; - let (tx_hash, payload) = node.inject_and_advance(raw_tx).await?; - node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?; + node.inject_and_advance(raw_tx).await?; // Get block number before shutdown let block_before = node.inner.provider.best_block_number()?; @@ -186,7 +177,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let chain_spec = test_chain_spec(EthereumHardfork::Prague); let genesis_hash = chain_spec.genesis_hash(); - let (node, _) = EthereumNode::test_setup(1, chain_spec.clone()) + let (node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()) .with_rpc_modifier(|rpc| { rpc.with_http_api(RpcModuleSelection::from([ RethRpcModule::Eth, @@ -196,8 +187,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { .build_single() .await?; - let wallets = Wallet::new(2).wallet_gen(); - let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallets[0].clone()).await; + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.signer(0)).await; let envelope = node .testing_build_block_v1(TestingBuildBlockRequestV1 { @@ -284,7 +274,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let status: SszPayloadStatus = client .post(format!("{auth_url}{ENGINE_PAYLOADS_ROUTE}")) .jwt(&auth) - .fork(ENGINE_PRAGUE_FORK_HEADER) + .fork("prague") .header(reqwest::header::ACCEPT, "application/octet-stream") .ssz(&(payload, B256::ZERO, envelope.execution_requests.take())) .send() @@ -296,7 +286,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { let fcu: SszForkchoiceUpdateResponse = client .post(format!("{auth_url}{ENGINE_FORKCHOICE_ROUTE}")) .jwt(&auth) - .fork(ENGINE_PRAGUE_FORK_HEADER) + .fork("prague") .header(reqwest::header::ACCEPT, "application/octet-stream") .ssz(&( ForkchoiceState { @@ -312,7 +302,7 @@ async fn test_engine_ssz_proxy_can_mine_block() -> eyre::Result<()> { .await?; assert_eq!(fcu.payload_status.status, PayloadStatusKind::Valid); - let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, wallets[1].clone()).await?; + let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(1)).await?; let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?; let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; let versioned_hashes = TransactionTestContext::validate_sidecar(envelope); diff --git a/crates/ethereum/node/tests/e2e/invalid_payload.rs b/crates/ethereum/node/tests/e2e/invalid_payload.rs index a8b3c354d05..bb52ef19e0b 100644 --- a/crates/ethereum/node/tests/e2e/invalid_payload.rs +++ b/crates/ethereum/node/tests/e2e/invalid_payload.rs @@ -9,7 +9,7 @@ use alloy_primitives::{bytes, keccak256, Bytes, B256}; use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV3, PayloadStatusEnum}; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_chainspec::EthereumHardfork; -use reth_e2e_test_utils::{test_chain_spec, transaction::TransactionTestContext, E2ETestSetupExt}; +use reth_e2e_test_utils::{transaction::TransactionTestContext, E2ETestSetupExt}; use reth_ethereum_primitives::TransactionSigned; use reth_node_ethereum::EthereumNode; use reth_primitives_traits::SignedTransaction; @@ -31,9 +31,8 @@ async fn can_handle_invalid_payload_then_valid() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - - let (mut nodes, wallet) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; + let (mut nodes, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).with_num_nodes(2).build().await?; let mut producer = nodes.pop().unwrap(); let receiver = nodes.pop().unwrap(); @@ -135,9 +134,8 @@ async fn can_handle_multiple_invalid_payloads() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - - let (mut nodes, wallet) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; + let (mut nodes, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).with_num_nodes(2).build().await?; let mut producer = nodes.pop().unwrap(); let receiver = nodes.pop().unwrap(); @@ -223,9 +221,8 @@ async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - - let (mut nodes, wallet) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; + let (mut nodes, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).with_num_nodes(2).build().await?; let mut producer = nodes.pop().unwrap(); let receiver = nodes.pop().unwrap(); diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index b8ffdf95ac1..cb076baf906 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -5,7 +5,7 @@ use alloy_network::TxSignerSync; use alloy_primitives::B256; use alloy_provider::Provider; use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; -use futures::{future::JoinAll, StreamExt}; +use futures::StreamExt; use rand::{rngs::StdRng, seq::IndexedRandom, Rng, SeedableRng}; use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ @@ -19,8 +19,8 @@ use reth_node_core::{args::NetworkArgs, node_config::NodeConfig}; use reth_node_ethereum::EthereumNode; use reth_primitives_traits::SealedBlock; use reth_provider::test_utils::MockEthProvider; -use reth_rpc_api::EthApiServer; use reth_tasks::Runtime; +use reth_transaction_pool::TransactionPool; use std::{net::UdpSocket, sync::Arc, time::Duration}; #[tokio::test] @@ -105,9 +105,6 @@ async fn can_sync() -> eyre::Result<()> { let (tx_hash, payload) = first_node.inject_and_advance(raw_tx).await?; let block_hash = payload.block().hash(); - // assert the block has been committed to the blockchain - first_node.assert_new_block(tx_hash, block_hash, payload.block().number).await?; - // only send forkchoice update to second node second_node.update_forkchoice(block_hash, block_hash).await?; @@ -202,9 +199,8 @@ async fn e2e_test_send_transactions() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = test_chain_spec(EthereumHardfork::Prague); - - let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; + let (mut nodes, _) = + EthereumNode::test_setup_for(EthereumHardfork::Prague).with_num_nodes(2).build().await?; let mut node = nodes.pop().unwrap(); let provider = node.rpc_provider(); @@ -230,9 +226,8 @@ async fn test_long_reorg() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = test_chain_spec(EthereumHardfork::Prague); - - let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; + let (mut nodes, _) = + EthereumNode::test_setup_for(EthereumHardfork::Prague).with_num_nodes(2).build().await?; let mut first_node = nodes.pop().unwrap(); let mut second_node = nodes.pop().unwrap(); @@ -274,9 +269,8 @@ async fn test_pipeline_sync_target_head_becomes_finalized() -> eyre::Result<()> let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = test_chain_spec(EthereumHardfork::Prague); - - let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; + let (mut nodes, _) = + EthereumNode::test_setup_for(EthereumHardfork::Prague).with_num_nodes(2).build().await?; let mut first_node = nodes.pop().unwrap(); let second_node = nodes.pop().unwrap(); @@ -322,9 +316,8 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { let mut rng = StdRng::from_seed(seed); println!("Seed: {seed:?}"); - let chain_spec = test_chain_spec(EthereumHardfork::Prague); - - let (mut nodes, _) = EthereumNode::test_setup(2, chain_spec.clone()).build().await?; + let (mut nodes, _) = + EthereumNode::test_setup_for(EthereumHardfork::Prague).with_num_nodes(2).build().await?; let mut first_node = nodes.pop().unwrap(); let mut second_node = nodes.pop().unwrap(); @@ -375,7 +368,7 @@ async fn test_tx_propagation() -> eyre::Result<()> { // Setup 10 nodes let (mut nodes, _) = - EthereumNode::test_setup(10, chain_spec.clone()).with_connect_nodes(false).build().await?; + EthereumNode::test_setup(10, chain_spec).with_connect_nodes(false).build().await?; // Connect all nodes to the first one let (first, rest) = nodes.split_at_mut(1); @@ -383,31 +376,14 @@ async fn test_tx_propagation() -> eyre::Result<()> { node.connect(&mut first[0]).await; } - // Advance all nodes for 1 block so that they don't consider themselves unsynced - let tx = build_tx(); - let (_, payload) = nodes[0].inject_and_advance(tx.encoded_2718().into()).await?; - nodes[1..] - .iter_mut() - .map(|node| async { - node.submit_payload(payload.clone()).await.unwrap(); - node.sync_to(payload.block().hash()).await.unwrap(); - }) - .collect::>() - .await; - // Build and send transaction to first node let tx = build_tx(); let tx_hash = *tx.tx_hash(); let _ = nodes[0].rpc.inject_tx(tx.encoded_2718().into()).await?; - tokio::time::sleep(Duration::from_millis(100)).await; - - // Assert that all nodes have the transaction - for (i, node) in nodes.iter().enumerate() { - assert!( - node.rpc.inner.eth_api().transaction_by_hash(tx_hash).await?.is_some(), - "Node {i} should have the transaction" - ); + // Wait until all nodes have the transaction + for node in &nodes { + node.wait_for_pool(|pool| pool.contains(&tx_hash)).await?; } // Build and send one more transaction to a random node @@ -415,11 +391,9 @@ async fn test_tx_propagation() -> eyre::Result<()> { let tx_hash = *tx.tx_hash(); let _ = nodes.choose(&mut rand::rng()).unwrap().rpc.inject_tx(tx.encoded_2718().into()).await?; - tokio::time::sleep(Duration::from_millis(100)).await; - - // Assert that all nodes have the transaction - for node in nodes { - assert!(node.rpc.inner.eth_api().transaction_by_hash(tx_hash).await?.is_some()); + // Wait until all nodes have the transaction + for node in &nodes { + node.wait_for_pool(|pool| pool.contains(&tx_hash)).await?; } Ok(()) diff --git a/crates/ethereum/node/tests/e2e/pool.rs b/crates/ethereum/node/tests/e2e/pool.rs index 79579be78a5..9b68ff9ff80 100644 --- a/crates/ethereum/node/tests/e2e/pool.rs +++ b/crates/ethereum/node/tests/e2e/pool.rs @@ -5,8 +5,7 @@ use alloy_provider::Provider; use alloy_rpc_types_eth::TransactionRequest; use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ - test_chain_spec, transaction::TransactionTestContext, wait::poll_until, wallet::Wallet, - E2ETestSetupExt, + test_chain_spec, transaction::TransactionTestContext, wait::poll_until, E2ETestSetupExt, }; use reth_node_core::args::TxPoolArgs; use reth_node_ethereum::EthereumNode; @@ -136,12 +135,10 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> { // by the pool maintenance task let chain_spec = test_chain_spec(EthereumHardfork::Cancun); let genesis_hash = chain_spec.genesis_hash(); - let (mut node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec).build_single().await?; let runtime = node.inner.task_executor.clone(); - let wallets = Wallet::new(2).wallet_gen(); - let w1 = wallets.first().unwrap(); - let w2 = wallets.last().unwrap(); + let (w1, w2) = (wallet.signer(0), wallet.signer(1)); runtime.spawn_critical_task( "txpool maintenance task", @@ -209,7 +206,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> { node.update_forkchoice(genesis_hash, block_hash1).await?; // wait for pool to process `CanonStateNotification::Commit` event correctly, and finally tx1 - // will be removed and tx2 is still in the pool + // will be removed and tx2 is still in the pool. poll_until("pool to process the commit", || async { Ok((txpool.get(&tx_hash1).is_none() && txpool.get(&tx_hash2).is_some()).then_some(())) }) @@ -286,7 +283,7 @@ async fn maintain_txpool_commit() -> eyre::Result<()> { let _ = node.advance_block().await.unwrap(); // wait for pool to process `CanonStateNotification::Commit` event correctly, and finally the - // pool will be cleared + // pool will be cleared. poll_until("pool to process the commit", || async { Ok(txpool.is_empty().then_some(())) }) .await?; diff --git a/crates/ethereum/node/tests/e2e/simulate.rs b/crates/ethereum/node/tests/e2e/simulate.rs index 16d9ef2e656..3beed879114 100644 --- a/crates/ethereum/node/tests/e2e/simulate.rs +++ b/crates/ethereum/node/tests/e2e/simulate.rs @@ -6,7 +6,7 @@ use alloy_rpc_types_eth::{ BlockOverrides, TransactionRequest, TransactionTrait as _, }; use reth_chainspec::EthereumHardfork; -use reth_e2e_test_utils::{test_chain_spec, test_chain_spec_builder, E2ETestSetupExt}; +use reth_e2e_test_utils::{test_chain_spec_builder, E2ETestSetupExt}; use reth_node_ethereum::EthereumNode; use std::sync::Arc; @@ -298,9 +298,8 @@ async fn test_simulate_v1_explicit_gas_over_remaining_block_gas_errors() -> eyre async fn test_simulate_v1_with_max_fee_per_blob_gas_only() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let chain_spec = test_chain_spec(EthereumHardfork::Cancun); - - let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec.clone()).build_single().await?; + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; let provider = node.rpc_provider_with_wallet(wallet.signer(0)); let _ = provider.send_transaction(TransactionRequest::default().to(Address::ZERO)).await?; diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 4bacb42e3d9..e7bf3e9c543 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -127,7 +127,7 @@ pub(crate) async fn advance_with_random_transactions( } /// Header selecting the fork of fork-scoped SSZ engine API endpoints. -pub(crate) const ENGINE_EXECUTION_VERSION_HEADER: &str = "Eth-Execution-Version"; +const ENGINE_EXECUTION_VERSION_HEADER: &str = "Eth-Execution-Version"; /// Extension trait for requests against the SSZ engine API. pub(crate) trait EngineSszRequestExt { From d85d9deaef3b63161a339e2049f1917574eb1d2a Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Sat, 26 Sep 2026 04:57:59 +0000 Subject: [PATCH 098/133] fix(rpc): end syncing subscription on close (#27455) Co-authored-by: Amp --- crates/rpc/rpc-builder/tests/it/ws.rs | 140 +++++++++++++++++++++++--- crates/rpc/rpc/src/eth/pubsub.rs | 13 ++- 2 files changed, 137 insertions(+), 16 deletions(-) diff --git a/crates/rpc/rpc-builder/tests/it/ws.rs b/crates/rpc/rpc-builder/tests/it/ws.rs index 9c2685214f7..1eb8dde8cd1 100644 --- a/crates/rpc/rpc-builder/tests/it/ws.rs +++ b/crates/rpc/rpc-builder/tests/it/ws.rs @@ -1,20 +1,68 @@ #![allow(unreachable_pub)] //! `WebSocket` subscription tests for `eth_subscribe` / `eth_unsubscribe` -use crate::utils::{launch_ws, test_rpc_builder}; -use jsonrpsee::core::client::{Subscription, SubscriptionClientT}; -use reth_rpc_server_types::RpcModuleSelection; +use crate::utils::{launch_ws, test_address, test_rpc_builder}; +use jsonrpsee::{ + core::client::{Subscription, SubscriptionClientT}, + server::ServerConfigBuilder, +}; +use reth_consensus::noop::NoopConsensus; +use reth_evm_ethereum::EthEvmConfig; +use reth_network_api::noop::NoopNetwork; +use reth_primitives_traits::SealedHeader; +use reth_provider::{ + providers::BlockchainProvider, + test_utils::{create_test_provider_factory, NoopProvider}, +}; +use reth_rpc_server_types::{RethRpcModule, RpcModuleSelection}; +use reth_tasks::Runtime; use reth_tokio_util::EventSender; +use reth_transaction_pool::{ + test_utils::{TestPool, TestPoolBuilder}, + PoolTransaction, TransactionOrigin, TransactionPool, +}; use serde_json::Value; use std::time::Duration; +use tokio::time::Instant; -use reth_rpc_builder::{RpcServerConfig, TransportRpcModuleConfig}; +use reth_rpc_builder::{RpcModuleBuilder, RpcServerConfig, TransportRpcModuleConfig}; /// Helper to launch a WS server with the Eth module. async fn launch_ws_eth() -> reth_rpc_builder::RpcServerHandle { launch_ws(vec![reth_rpc_server_types::RethRpcModule::Eth]).await } +/// Launches a WS server with the Eth module, backed by a provider whose canonical state +/// notification sender stays alive. +/// +/// `NoopProvider` drops the sender immediately, which ends canonical state streams right away. +/// With a live sender, subscription tasks block on the stream the same way they do on a running +/// node. +async fn launch_ws_eth_with_canon_state( + max_subscriptions_per_connection: u32, +) -> reth_rpc_builder::RpcServerHandle { + let provider = + BlockchainProvider::with_latest(create_test_provider_factory(), SealedHeader::default()) + .unwrap(); + let pool: TestPool = TestPoolBuilder::default().into(); + let builder = RpcModuleBuilder::default() + .with_provider(provider) + .with_pool(pool) + .with_network(NoopNetwork::default()) + .with_executor(Runtime::test()) + .with_evm_config(EthEvmConfig::mainnet()) + .with_consensus(NoopConsensus::default()); + let eth_api = builder.bootstrap_eth_api(); + let server = builder.build( + TransportRpcModuleConfig::set_ws(vec![RethRpcModule::Eth]), + eth_api, + EventSender::new(1), + ); + let config = ServerConfigBuilder::default() + .max_subscriptions_per_connection(max_subscriptions_per_connection); + RpcServerConfig::ws(config).with_ws_address(test_address()).start(&server).await.unwrap() +} + #[tokio::test(flavor = "multi_thread")] async fn test_eth_subscribe_all_supported_kinds_accept() { reth_tracing::init_test_tracing(); @@ -148,17 +196,6 @@ async fn test_eth_subscribe_not_available_over_http() { #[tokio::test(flavor = "multi_thread")] async fn test_eth_subscribe_pending_transactions_receives_tx() { - use reth_consensus::noop::NoopConsensus; - use reth_evm_ethereum::EthEvmConfig; - use reth_network_api::noop::NoopNetwork; - use reth_provider::test_utils::NoopProvider; - use reth_rpc_builder::RpcModuleBuilder; - use reth_tasks::Runtime; - use reth_transaction_pool::{ - test_utils::{TestPool, TestPoolBuilder}, - PoolTransaction, TransactionOrigin, TransactionPool, - }; - reth_tracing::init_test_tracing(); let pool: TestPool = TestPoolBuilder::default().into(); @@ -213,3 +250,76 @@ async fn test_eth_subscribe_pending_transactions_receives_tx() { sub.unsubscribe().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth_subscribe_syncing_releases_permit_on_unsubscribe() { + reth_tracing::init_test_tracing(); + + // A single permit means each subscription must release it before the next one is accepted. + let handle = launch_ws_eth_with_canon_state(1).await; + let client = handle.ws_client().await.unwrap(); + + for i in 0..3 { + // The previous subscription task releases its permit asynchronously after unsubscribe. + let deadline = Instant::now() + Duration::from_secs(5); + let mut sub: Subscription = loop { + match client + .subscribe("eth_subscribe", jsonrpsee::rpc_params!["syncing"], "eth_unsubscribe") + .await + { + Ok(sub) => break sub, + Err(err) if Instant::now() >= deadline => { + panic!("syncing subscription #{i} rejected after unsubscribe: {err}") + } + Err(_) => tokio::time::sleep(Duration::from_millis(50)).await, + } + }; + + let initial = tokio::time::timeout(Duration::from_secs(5), sub.next()) + .await + .expect("timed out waiting for initial syncing status") + .expect("subscription closed before initial status") + .unwrap(); + assert_eq!(initial, Value::Bool(false)); + + sub.unsubscribe().await.unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth_subscribe_syncing_task_exits_on_disconnect() { + reth_tracing::init_test_tracing(); + + const SUBSCRIPTIONS: usize = 50; + + let metrics = tokio::runtime::Handle::current().metrics(); + let handle = launch_ws_eth_with_canon_state(SUBSCRIPTIONS as u32).await; + + // Open and close one connection first so tasks spawned lazily by the server are not counted. + drop(handle.ws_client().await.unwrap()); + tokio::time::sleep(Duration::from_millis(500)).await; + let baseline = metrics.num_alive_tasks(); + + let client = handle.ws_client().await.unwrap(); + let mut subs = Vec::with_capacity(SUBSCRIPTIONS); + for _ in 0..SUBSCRIPTIONS { + let mut sub: Subscription = client + .subscribe("eth_subscribe", jsonrpsee::rpc_params!["syncing"], "eth_unsubscribe") + .await + .unwrap(); + assert_eq!(sub.next().await.unwrap().unwrap(), Value::Bool(false)); + subs.push(sub); + } + assert!(metrics.num_alive_tasks() >= baseline + SUBSCRIPTIONS); + + // Close the connection before dropping the subscriptions so no `eth_unsubscribe` is sent. + drop(client); + drop(subs); + + let deadline = Instant::now() + Duration::from_secs(5); + while metrics.num_alive_tasks() > baseline && Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + let alive = metrics.num_alive_tasks(); + assert!(alive <= baseline, "subscription tasks outlived the connection: {baseline} -> {alive}"); +} diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 3f510ce60a2..3ca43e1c3dd 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -163,7 +163,18 @@ where return Ok(()) } - while canon_state.next().await.is_some() { + loop { + // Sends only happen when the sync status changes, so a failed send cannot be + // relied on to detect a closed subscription. + tokio::select! { + _ = accepted_sink.closed() => break, + maybe_event = canon_state.next() => { + if maybe_event.is_none() { + break + } + } + } + let current_syncing = self.inner.eth_api.network().is_syncing(); // Only send a new response if the sync status has changed if current_syncing != initial_sync_status { From a68f9e19fe423eeb99f267c02ee94229648fb3ec Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 26 Sep 2026 07:53:57 +0200 Subject: [PATCH 099/133] fix(ipc): enforce max subscriptions per connection (#27459) --- crates/rpc/ipc/src/server/mod.rs | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index 718561db871..c3837e91251 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -395,9 +395,7 @@ where trace!("{:?}", request); let cfg = RpcServiceCfg { - bounded_subscriptions: BoundedSubscriptions::new( - self.inner.server_cfg.max_subscriptions_per_connection, - ), + bounded_subscriptions: self.inner.bounded_subscriptions.clone(), id_provider: self.inner.id_provider.clone(), sink: self.inner.method_sink.clone(), }; @@ -780,7 +778,7 @@ mod tests { params::BatchRequestBuilder, }, rpc_params, - types::Request, + types::{error::TOO_MANY_SUBSCRIPTIONS_CODE, Request}, PendingSubscriptionSink, RpcModule, SubscriptionMessage, }; use reth_tracing::init_test_tracing; @@ -1016,6 +1014,38 @@ mod tests { assert_eq!(items.len(), 16); } + #[tokio::test(flavor = "multi_thread")] + async fn test_max_subscriptions_per_connection() { + let endpoint = &dummy_name(); + let server = Builder::default().max_subscriptions_per_connection(1).build(endpoint.clone()); + let mut module = RpcModule::new(()); + module + .register_subscription( + "subscribe_hello", + "s_hello", + "unsubscribe_hello", + |_, pending, _, _| async move { + let Ok(sink) = pending.accept().await else { return }; + sink.closed().await; + }, + ) + .unwrap(); + let handle = server.start(module).await.unwrap(); + tokio::spawn(handle.stopped()); + + let client = IpcClientBuilder::default().build(endpoint).await.unwrap(); + let _sub: Subscription = + client.subscribe("subscribe_hello", rpc_params![], "unsubscribe_hello").await.unwrap(); + let err = client + .subscribe::("subscribe_hello", rpc_params![], "unsubscribe_hello") + .await + .unwrap_err(); + assert!( + matches!(&err, Error::Call(err) if err.code() == TOO_MANY_SUBSCRIPTIONS_CODE), + "{err}" + ); + } + #[tokio::test] async fn test_rpc_middleware() { #[derive(Clone)] From f57be2b11c39b5544ce12a2c100f27c0314a456d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 26 Sep 2026 08:05:10 +0200 Subject: [PATCH 100/133] fix(examples): stop subscription tasks when the client goes away (#27461) --- examples/exex-subscription/src/main.rs | 9 ++++++++- examples/node-custom-rpc/src/main.rs | 20 +++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/examples/exex-subscription/src/main.rs b/examples/exex-subscription/src/main.rs index fd9595c5d9c..8d584b76b2e 100644 --- a/examples/exex-subscription/src/main.rs +++ b/examples/exex-subscription/src/main.rs @@ -78,7 +78,14 @@ impl StorageWatcherApiServer for StorageWatcherRpc { let Ok(mut rx) = resp_rx.await else { return }; - while let Some(diff) = rx.recv().await { + loop { + // diffs only arrive when the address changes, so a failed send alone would not + // notice that the client went away + let diff = tokio::select! { + _ = sink.closed() => break, + diff = rx.recv() => diff, + }; + let Some(diff) = diff else { break }; let msg = SubscriptionMessage::from( serde_json::value::to_raw_value(&diff).expect("serialize"), ); diff --git a/examples/node-custom-rpc/src/main.rs b/examples/node-custom-rpc/src/main.rs index 4cdb5473e37..fa9eeedc5af 100644 --- a/examples/node-custom-rpc/src/main.rs +++ b/examples/node-custom-rpc/src/main.rs @@ -128,12 +128,18 @@ where }; loop { - sleep(Duration::from_secs(delay)).await; + // stop once the client unsubscribes or disconnects + tokio::select! { + _ = sink.closed() => break, + _ = sleep(Duration::from_secs(delay)) => {} + } let msg = SubscriptionMessage::from( serde_json::value::to_raw_value(&pool.pool_size().total).expect("serialize"), ); - let _ = sink.send(msg).await; + if sink.send(msg).await.is_err() { + break; + } } }); @@ -183,14 +189,18 @@ mod tests { // Send pool size repeatedly, with a 10-second delay loop { - sleep(Duration::from_millis(delay)).await; + tokio::select! { + _ = sink.closed() => break, + _ = sleep(Duration::from_millis(delay)) => {} + } let message = SubscriptionMessage::from( serde_json::value::to_raw_value(&pool.pool_size().total) .expect("serialize usize"), ); - // Just ignore errors if a client has dropped - let _ = sink.send(message).await; + if sink.send(message).await.is_err() { + break; + } } }); Ok(()) From c3d46f3d09c14d50f47b14768807c0d0a491e8fe Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 26 Sep 2026 08:51:39 +0200 Subject: [PATCH 101/133] fix(ipc): abort in-flight calls when the connection closes (#27458) --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/rpc/ipc/Cargo.toml | 2 +- crates/rpc/ipc/src/server/ipc.rs | 20 +++++-- crates/rpc/ipc/src/server/mod.rs | 90 ++++++++++++++++++++++++++++++-- 5 files changed, 104 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 763acfdb286..a92818b5202 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12889,6 +12889,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", + "futures-util", "libc", "pin-project-lite", "slab", diff --git a/Cargo.toml b/Cargo.toml index ce3344448cd..2232794a18f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -538,7 +538,7 @@ metrics-util = { default-features = false, version = "0.20.3" } tokio = { version = "1.52.3", default-features = false } tokio-stream = "0.1.11" tokio-tungstenite = "0.29.0" -tokio-util = { version = "0.7.4", features = ["codec"] } +tokio-util = { version = "0.7.11", features = ["codec"] } # async async-compression = { version = "0.4", default-features = false } diff --git a/crates/rpc/ipc/Cargo.toml b/crates/rpc/ipc/Cargo.toml index 58639f2c44e..41fff176146 100644 --- a/crates/rpc/ipc/Cargo.toml +++ b/crates/rpc/ipc/Cargo.toml @@ -15,7 +15,7 @@ workspace = true # async/net futures.workspace = true tokio = { workspace = true, features = ["net", "time", "rt-multi-thread"] } -tokio-util = { workspace = true, features = ["codec"] } +tokio-util = { workspace = true, features = ["codec", "rt"] } tokio-stream.workspace = true pin-project.workspace = true tower.workspace = true diff --git a/crates/rpc/ipc/src/server/ipc.rs b/crates/rpc/ipc/src/server/ipc.rs index f3d5f4383f1..26135966d63 100644 --- a/crates/rpc/ipc/src/server/ipc.rs +++ b/crates/rpc/ipc/src/server/ipc.rs @@ -1,6 +1,6 @@ //! IPC request handling adapted from [`jsonrpsee`] http request handling -use futures::{stream::FuturesOrdered, StreamExt}; +use futures::{stream::FuturesOrdered, FutureExt, StreamExt}; use jsonrpsee::{ batch_response_error, core::{server::helpers::prepare_error, JsonRawValue}, @@ -11,7 +11,7 @@ use jsonrpsee::{ }, BatchResponseBuilder, MethodResponse, }; -use std::sync::Arc; +use std::{future::Future, panic::AssertUnwindSafe, sync::Arc}; use tokio::sync::OwnedSemaphorePermit; use tokio_util::either::Either; use tracing::instrument; @@ -45,7 +45,7 @@ where .into_iter() .filter_map(|v| { if let Ok(req) = serde_json::from_str::>(v.get()) { - Some(Either::Right(rpc_service.call(req))) + Some(Either::Right(catch_call_panic(req.id(), rpc_service.call(req)))) } else if let Ok(_notif) = serde_json::from_str::>(v.get()) { // notifications should not be answered. got_notif = true; @@ -109,7 +109,19 @@ pub(crate) async fn execute_call_with_tracing<'a, S>( where S: RpcServiceT + Send, { - rpc_service.call(req).await + catch_call_panic(req.id(), rpc_service.call(req)).await +} + +/// Answers a panicking call with an internal error, otherwise the panic would only surface as a +/// failed call task and the client would never receive a response for `id`. +async fn catch_call_panic( + id: Id<'_>, + call: impl Future, +) -> MethodResponse { + AssertUnwindSafe(call) + .catch_unwind() + .await + .unwrap_or_else(|_| MethodResponse::error(id, ErrorObject::from(ErrorCode::InternalError))) } pub(crate) async fn call_with_service( diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index c3837e91251..4f392432f81 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -39,6 +39,7 @@ use crate::{ }; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use tokio_util::task::AbortOnDropHandle; use tower::layer::{util::Stack, LayerFn}; mod connection; @@ -415,7 +416,10 @@ where // example tracing calls are relatively CPU expensive on serde::serialize alone, moving this // work to a separate task takes the pressure off the connection so all concurrent responses // are also serialized concurrently and the connection can focus on read+write - let f = tokio::task::spawn(async move { + // + // The connection drops its pending calls when it closes, so the call must not outlive the + // returned future, otherwise it keeps running without anyone waiting for the response. + let f = AbortOnDropHandle::new(tokio::task::spawn(async move { ipc::call_with_service( request, rpc_service, @@ -424,9 +428,17 @@ where conn, ) .await - }); - - Box::pin(async move { f.await.map_err(|err| err.into()) }) + })); + + Box::pin(async move { + // Call panics are answered by the call itself. Anything left here has no request id to + // respond to, and the connection writes errors verbatim, which would corrupt the + // stream. + Ok(f.await.unwrap_or_else(|err| { + warn!(%err, "IPC call task failed"); + None + })) + }) } } @@ -778,7 +790,7 @@ mod tests { params::BatchRequestBuilder, }, rpc_params, - types::{error::TOO_MANY_SUBSCRIPTIONS_CODE, Request}, + types::{error::TOO_MANY_SUBSCRIPTIONS_CODE, ErrorCode, Request}, PendingSubscriptionSink, RpcModule, SubscriptionMessage, }; use reth_tracing::init_test_tracing; @@ -924,6 +936,74 @@ mod tests { assert!(response4.is_ok()); } + #[tokio::test(flavor = "multi_thread")] + async fn test_pending_call_aborted_on_disconnect() { + init_test_tracing(); + + let endpoint = &dummy_name(); + let server = Builder::default().build(endpoint.clone()); + let (started_tx, started_rx) = oneshot::channel::<()>(); + let (dropped_tx, dropped_rx) = oneshot::channel::<()>(); + let mut module = RpcModule::new(std::sync::Mutex::new(Some((started_tx, dropped_tx)))); + module + .register_async_method("hang", |_, ctx, _| async move { + // `dropped_tx` is dropped together with the call + let (started_tx, _dropped_tx) = ctx.lock().unwrap().take().unwrap(); + let _ = started_tx.send(()); + std::future::pending::<()>().await; + "unreachable" + }) + .unwrap(); + let handle = server.start(module).await.unwrap(); + tokio::spawn(handle.stopped()); + + let client = IpcClientBuilder::default().build(endpoint).await.unwrap(); + tokio::select! { + _ = client.request::("hang", rpc_params![]) => panic!("call completed"), + _ = started_rx => {} + } + drop(client); + + let dropped = tokio::time::timeout(std::time::Duration::from_secs(5), dropped_rx).await; + assert!(dropped.is_ok(), "call kept running after the connection closed"); + } + + #[tokio::test] + async fn test_panicking_call_returns_internal_error() { + init_test_tracing(); + + let endpoint = &dummy_name(); + let server = Builder::default().build(endpoint.clone()); + let mut module = RpcModule::new(()); + module + .register_async_method("maybe_panic", |params, _, _| async move { + assert!(!params.one::().unwrap(), "requested panic"); + "ok" + }) + .unwrap(); + let handle = server.start(module).await.unwrap(); + tokio::spawn(handle.stopped()); + + let client = IpcClientBuilder::default().build(endpoint).await.unwrap(); + let err = client.request::("maybe_panic", rpc_params![true]).await.unwrap_err(); + assert!( + matches!(&err, Error::Call(err) if err.code() == ErrorCode::InternalError.code()), + "{err:?}" + ); + + let mut batch_request_builder = BatchRequestBuilder::new(); + let _ = batch_request_builder.insert("maybe_panic", rpc_params![true]); + let _ = batch_request_builder.insert("maybe_panic", rpc_params![false]); + let responses = client + .batch_request::(batch_request_builder) + .await + .unwrap() + .into_iter() + .collect::>(); + assert!(matches!(&responses[0], Err(err) if err.code() == ErrorCode::InternalError.code())); + assert_eq!(responses[1].as_deref(), Ok("ok")); + } + #[tokio::test] async fn test_rpc_request() { init_test_tracing(); From 621f663edf4be141150ece866325b18d993b50f7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 26 Sep 2026 09:34:18 +0200 Subject: [PATCH 102/133] refactor(tasks): move CancelOnDrop from reth-revm to reth-tasks (#27466) --- crates/payload/basic/src/lib.rs | 4 ++-- crates/revm/src/lib.rs | 4 +--- crates/rpc/rpc-eth-api/src/helpers/call.rs | 2 +- .../src/cancelled.rs => tasks/src/cancel.rs} | 24 +++++++++++-------- crates/tasks/src/lib.rs | 2 ++ 5 files changed, 20 insertions(+), 16 deletions(-) rename crates/{revm/src/cancelled.rs => tasks/src/cancel.rs} (89%) diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index fb67012927f..ddb602d4685 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -22,9 +22,9 @@ use reth_payload_builder::{ use reth_payload_builder_primitives::PayloadBuilderError; use reth_payload_primitives::{BuiltPayload, PayloadAttributes, PayloadKind}; use reth_primitives_traits::{HeaderTy, NodePrimitives, SealedHeader}; -use reth_revm::{cached::CachedReads, cancelled::CancelOnDrop}; +use reth_revm::cached::CachedReads; use reth_storage_api::{BlockReaderIdExt, StateProviderFactory}; -use reth_tasks::Runtime; +use reth_tasks::{CancelOnDrop, Runtime}; use reth_trie_parallel::state_root_task::PayloadStateRootHandle; use std::{ fmt, diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index acf7548304b..c666fe02ee0 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -9,15 +9,13 @@ #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(any(test, feature = "test-utils", feature = "witness"))] extern crate alloc; /// Cache database that reads from an underlying [`DatabaseRef`]. /// Database adapters for payload building. pub mod cached; -/// A marker that can be used to cancel execution. -pub mod cancelled; - /// Contains glue code for integrating reth database into revm's [Database]. pub mod database; diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index e323aaec0bb..0f40b949275 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -27,7 +27,6 @@ use reth_evm::{ use reth_node_api::BlockBody; use reth_primitives_traits::Recovered; use reth_revm::{ - cancelled::CancelOnDrop, database::StateProviderDatabase, db::{ bal::{BalState, EvmDatabaseError}, @@ -42,6 +41,7 @@ use reth_rpc_eth_types::{ EthApiError, StateCacheDb, }; use reth_storage_api::{BlockIdReader, ProviderTx, StateProvider}; +use reth_tasks::CancelOnDrop; use revm::{ context::Block, context_interface::{result::ResultAndState, Cfg, Transaction}, diff --git a/crates/revm/src/cancelled.rs b/crates/tasks/src/cancel.rs similarity index 89% rename from crates/revm/src/cancelled.rs rename to crates/tasks/src/cancel.rs index 591cd3d376d..0d09172958b 100644 --- a/crates/revm/src/cancelled.rs +++ b/crates/tasks/src/cancel.rs @@ -1,18 +1,23 @@ -use alloc::sync::Arc; -use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +//! Cooperative cancellation of spawned work. -/// Payload building is still in progress. +use std::sync::{ + atomic::{AtomicBool, AtomicU8, Ordering}, + Arc, +}; + +/// The work is still in progress. const RUNNING: u8 = 0; -/// Payload building should stop accepting transactions and seal the accumulated work. +/// The work should wrap up and keep what it has produced so far. const FINALIZATION_REQUESTED: u8 = 1; -/// Payload building should stop and discard the accumulated work. +/// The work should stop and discard what it has produced so far. const CANCELLED: u8 = 2; /// Cancels execution on drop and supports cooperative finalization. /// /// If dropped, it will set the `cancelled` flag to true. /// -/// This is most useful when a payload job needs to be cancelled. +/// This is most useful when a spawned job should stop once its owner goes away, e.g. a payload +/// job or a blocking RPC call whose caller disconnected. #[derive(Default, Clone, Debug)] pub struct CancelOnDrop(Arc); @@ -56,8 +61,7 @@ impl Drop for CancelOnDrop { /// If dropped, it will NOT set the `cancelled` flag to true. /// If `cancel` is called, the `cancelled` flag will be set to true. /// -/// This is useful in prewarming, when an external signal is received to cancel many prewarming -/// tasks. +/// This is useful when an external signal should cancel many tasks at once. #[derive(Default, Clone, Debug)] pub struct ManualCancel(Arc); @@ -66,12 +70,12 @@ pub struct ManualCancel(Arc); impl ManualCancel { /// Returns true if the job was cancelled. pub fn is_cancelled(&self) -> bool { - self.0.load(core::sync::atomic::Ordering::Relaxed) + self.0.load(Ordering::Relaxed) } /// Drops the [`ManualCancel`], setting the cancelled flag to true. pub fn cancel(self) { - self.0.store(true, core::sync::atomic::Ordering::Relaxed); + self.0.store(true, Ordering::Relaxed); } } diff --git a/crates/tasks/src/lib.rs b/crates/tasks/src/lib.rs index 9b2bff7b46b..e95ccf06323 100644 --- a/crates/tasks/src/lib.rs +++ b/crates/tasks/src/lib.rs @@ -30,6 +30,7 @@ use tokio::{ }; use tracing::debug; +pub mod cancel; pub mod lazy; pub mod metrics; pub mod runtime; @@ -48,6 +49,7 @@ pub mod for_each_ordered; #[cfg(feature = "rayon")] pub use for_each_ordered::ForEachOrdered; +pub use cancel::{CancelOnDrop, ManualCancel}; pub use lazy::LazyHandle; #[cfg(feature = "rayon")] pub use runtime::RayonConfig; From 619a1e565f7b94f42dfc7454db6dbd2216301344 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 26 Sep 2026 11:11:22 +0200 Subject: [PATCH 103/133] perf(rpc): stop block re-execution and builder validation of dropped requests (#27463) --- crates/rpc/rpc/src/reth.rs | 25 ++++++++---- crates/rpc/rpc/src/validation.rs | 70 ++++++++++++++------------------ 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/crates/rpc/rpc/src/reth.rs b/crates/rpc/rpc/src/reth.rs index 3a728f26a6c..4f4b006c3ee 100644 --- a/crates/rpc/rpc/src/reth.rs +++ b/crates/rpc/rpc/src/reth.rs @@ -20,7 +20,7 @@ use reth_storage_api::{ BlockReader, BlockReaderIdExt, ChangeSetReader, StateProvider, StateProviderFactory, TransactionVariant, }; -use reth_tasks::{pool::BlockingTaskGuard, Runtime}; +use reth_tasks::{pool::BlockingTaskGuard, CancelOnDrop, Runtime}; use serde::Serialize; use tokio::sync::oneshot; @@ -139,17 +139,23 @@ where .acquire_owned() .await .map_err(|_| EthApiError::InternalEthError)?; - self.on_blocking_task(async move |this| { - let _permit = permit; - this.try_block_execution_outcome(block_id, block_count) - }) - .await + let guard = CancelOnDrop::default(); + let cancel = guard.clone(); + let outcome = self + .on_blocking_task(async move |this| { + let _permit = permit; + this.try_block_execution_outcome(block_id, block_count, &cancel) + }) + .await; + drop(guard); + outcome } fn try_block_execution_outcome( &self, block_id: BlockId, block_count: u64, + cancel: &CancelOnDrop, ) -> EthResult>> { let Some(start_block) = self.provider().block_number_for_id(block_id)? else { return Ok(None) @@ -178,11 +184,16 @@ where blocks.push(block); } - let outcome = self.evm_config().executor(db).execute_batch(&blocks).map_err( + // stop between blocks once the request is dropped + let blocks = blocks.iter().take_while(|_| !cancel.is_cancelled()); + let outcome = self.evm_config().executor(db).execute_batch(blocks).map_err( |e: reth_evm::execute::BlockExecutionError| { EthApiError::Internal(reth_errors::RethError::Other(e.into())) }, )?; + if cancel.is_cancelled() { + return Err(EthApiError::InternalEthError) + } Ok(Some(outcome)) } diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index 8ea1e0ba920..2cd5cbaf854 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -121,6 +121,28 @@ where *cache = (head, cached_state) } } + + /// Runs the validation on a blocking task. + /// + /// The validation is skipped if the request is dropped before it starts and stopped if the + /// request is dropped while the validation waits for the cached reads. + async fn spawn_validation(&self, validation: F) -> RpcResult<()> + where + F: Future> + Send + 'static, + { + let (mut tx, rx) = oneshot::channel(); + + self.task_spawner.spawn_blocking_task(async move { + let result = tokio::select! { + biased; + _ = tx.closed() => return, + result = validation => result.map_err(ErrorObject::from), + }; + let _ = tx.send(result); + }); + + rx.await.map_err(|_| internal_rpc_err("Internal blocking task error"))? + } } impl ValidationApi @@ -593,16 +615,8 @@ where request: BuilderBlockValidationRequestV3, ) -> RpcResult<()> { let this = self.clone(); - let (tx, rx) = oneshot::channel(); - - self.task_spawner.spawn_blocking_task(async move { - let result = Self::validate_builder_submission_v3(&this, request) - .await - .map_err(ErrorObject::from); - let _ = tx.send(result); - }); - - rx.await.map_err(|_| internal_rpc_err("Internal blocking task error"))? + let validation = async move { Self::validate_builder_submission_v3(&this, request).await }; + self.spawn_validation(validation).await } /// Validates a block submitted to the relay @@ -611,16 +625,8 @@ where request: BuilderBlockValidationRequestV4, ) -> RpcResult<()> { let this = self.clone(); - let (tx, rx) = oneshot::channel(); - - self.task_spawner.spawn_blocking_task(async move { - let result = Self::validate_builder_submission_v4(&this, request) - .await - .map_err(ErrorObject::from); - let _ = tx.send(result); - }); - - rx.await.map_err(|_| internal_rpc_err("Internal blocking task error"))? + let validation = async move { Self::validate_builder_submission_v4(&this, request).await }; + self.spawn_validation(validation).await } /// Validates a block submitted to the relay @@ -629,16 +635,8 @@ where request: BuilderBlockValidationRequestV5, ) -> RpcResult<()> { let this = self.clone(); - let (tx, rx) = oneshot::channel(); - - self.task_spawner.spawn_blocking_task(async move { - let result = Self::validate_builder_submission_v5(&this, request) - .await - .map_err(ErrorObject::from); - let _ = tx.send(result); - }); - - rx.await.map_err(|_| internal_rpc_err("Internal blocking task error"))? + let validation = async move { Self::validate_builder_submission_v5(&this, request).await }; + self.spawn_validation(validation).await } /// Validates a block submitted to the relay @@ -647,16 +645,8 @@ where request: BuilderBlockValidationRequestV6, ) -> RpcResult<()> { let this = self.clone(); - let (tx, rx) = oneshot::channel(); - - self.task_spawner.spawn_blocking_task(async move { - let result = Self::validate_builder_submission_v6(&this, request) - .await - .map_err(ErrorObject::from); - let _ = tx.send(result); - }); - - rx.await.map_err(|_| internal_rpc_err("Internal blocking task error"))? + let validation = async move { Self::validate_builder_submission_v6(&this, request).await }; + self.spawn_validation(validation).await } } From 7e31e83e9f6952a6eda0073eab03cca53a99d60d Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Sat, 26 Sep 2026 13:50:35 +0200 Subject: [PATCH 104/133] fix(storage): revert BAL store activation (#27470) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/cli/commands/src/common.rs | 16 +++-- crates/engine/tree/src/persistence.rs | 37 ----------- crates/node/builder/src/launch/common.rs | 19 +++--- crates/storage/provider/src/bal/rocksdb.rs | 14 ++++- .../provider/src/providers/rocksdb/metrics.rs | 2 - .../src/providers/rocksdb/provider.rs | 63 ++++++++++++------- 6 files changed, 66 insertions(+), 85 deletions(-) diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 77cd552633f..2748ba5ec9c 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -27,7 +27,8 @@ use reth_provider::{ BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider, StaticFileProviderBuilder, }, - BalStoreHandle, ProviderFactory, RocksDBBalStore, StaticFileProviderFactory, StorageSettings, + BalConfig, BalStoreHandle, InMemoryBalStore, ProviderFactory, StaticFileProviderFactory, + StorageSettings, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; @@ -195,14 +196,11 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let bal_store = self - .db - .balstore_cache_size - .map(|distance| { - RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) - }) - .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); - let bal_store = BalStoreHandle::new(bal_store); + let balstore_cache_size = + self.db.balstore_cache_size.unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); + let bal_store = BalStoreHandle::new(InMemoryBalStore::new( + BalConfig::with_in_memory_retention_distance(balstore_cache_size), + )); let factory = ProviderFactory::>::new( db, self.chain.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index f466dad914e..3c6261945c1 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -562,43 +562,6 @@ mod tests { assert_eq!(result.last_state_trie_block, result.last_block); } - #[test] - fn test_save_blocks_flushes_bal_store() { - use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; - - reth_tracing::init_test_tracing(); - let provider = create_test_provider_factory(); - let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); - let provider = provider.with_bal_store(bal_store); - init_genesis(&provider).unwrap(); - - let mut test_block_builder = TestBlockBuilder::eth(); - let executed = test_block_builder.get_executed_block_with_number(1, B256::random()); - let num_hash = executed.recovered_block().num_hash(); - let raw_bal = Bytes::from_static(&[0xc0]); - - provider.bal_store().insert(num_hash, RawBal::new(raw_bal.clone())).unwrap(); - - let (_finished_exex_height_tx, finished_exex_height_rx) = - tokio::sync::watch::channel(FinishedExExHeight::NoExExs); - let pruner = - Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); - let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); - let handle = PersistenceHandle::::spawn_service( - provider.clone(), - pruner, - sync_metrics_tx, - ); - let (tx, rx) = crossbeam_channel::bounded(1); - - handle.save_blocks(full_save_input(vec![executed]), tx).unwrap(); - - let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); - assert_eq!(result.last_block, num_hash); - let persisted_store = RocksDBBalStore::new(provider.rocksdb_provider()); - assert_eq!(persisted_store.get_by_hash(num_hash.hash).unwrap(), Some(raw_bal)); - } - #[test] fn test_save_blocks_multiple_blocks() { reth_tracing::init_test_tracing(); diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 1a7146c96c5..6f54bfca5a3 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -71,10 +71,10 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, - MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, - RocksDBBalStore, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, - StaticFileProviderFactory, StorageSettingsCache, + BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, + DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, + ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, + StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, }; use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; @@ -522,15 +522,14 @@ where .build()? }; - let bal_store = self + let balstore_cache_size = self .node_config() .db .balstore_cache_size - .map(|distance| { - RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) - }) - .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); - let bal_store = BalStoreHandle::new(bal_store); + .unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); + let bal_store = BalStoreHandle::new(InMemoryBalStore::new( + BalConfig::with_in_memory_retention_distance(balstore_cache_size), + )); let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index 8944ec0a014..34494ba3a34 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -333,12 +333,20 @@ struct RocksDBBalEntry { #[cfg(test)] mod tests { use super::*; - use crate::providers::RocksDBBuilder; + use crate::providers::{RocksDBBuilder, RocksDBProvider}; use alloy_primitives::B256; + fn test_rocksdb(dir: &tempfile::TempDir) -> RocksDBProvider { + RocksDBBuilder::new(dir.path()) + .with_table::() + .with_table::() + .build() + .unwrap() + } + fn test_store() -> (tempfile::TempDir, RocksDBBalStore) { let dir = tempfile::tempdir().unwrap(); - let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + let rocksdb = test_rocksdb(&dir); (dir, RocksDBBalStore::new(rocksdb)) } @@ -427,7 +435,7 @@ mod tests { #[test] fn configured_buffer_retention_distance_is_used() { let dir = tempfile::tempdir().unwrap(); - let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + let rocksdb = test_rocksdb(&dir); let store = RocksDBBalStore::with_buffer_retention_distance(rocksdb, 64); let old = NumHash::new(1, B256::with_last_byte(1)); let tip = NumHash::new(34, B256::with_last_byte(2)); diff --git a/crates/storage/provider/src/providers/rocksdb/metrics.rs b/crates/storage/provider/src/providers/rocksdb/metrics.rs index c22edf65176..3971ed978ba 100644 --- a/crates/storage/provider/src/providers/rocksdb/metrics.rs +++ b/crates/storage/provider/src/providers/rocksdb/metrics.rs @@ -8,8 +8,6 @@ use strum::{EnumIter, IntoEnumIterator}; pub(super) const ROCKSDB_TABLES: &[&str] = &[ Tables::TransactionHashNumbers.name(), - Tables::BlockAccessLists.name(), - Tables::BlockAccessListBlockNumbers.name(), Tables::StoragesHistory.name(), Tables::AccountsHistory.name(), ]; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 7a401053195..09dfba0e16a 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -333,14 +333,10 @@ impl RocksDBBuilder { /// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping /// - [`tables::AccountsHistory`] - Account history index /// - [`tables::StoragesHistory`] - Storage history index - /// - [`tables::BlockAccessLists`] - Block access list payloads - /// - [`tables::BlockAccessListBlockNumbers`] - Block access list hash index pub fn with_default_tables(self) -> Self { self.with_table::() .with_table::() .with_table::() - .with_table::() - .with_table::() } /// Enables metrics. @@ -3170,25 +3166,22 @@ mod tests { provider.put::(key.clone(), &value).unwrap(); assert!(provider.get::(key).unwrap().is_some()); - let bal_key = - reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); - let bal_value = - reth_db_api::models::StoredBlockAccessList::new(Bytes::from_static(&[0xc0])); - provider.put::(bal_key, &bal_value).unwrap(); - assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); - provider - .put::(bal_key.hash(), &bal_key.number()) - .unwrap(); - assert_eq!( - provider.get::(bal_key.hash()).unwrap(), - Some(bal_key.number()) - ); + drop(provider); + + let column_families = DB::list_cf(&Options::default(), temp_dir.path()).unwrap(); + assert!(!column_families.iter().any(|name| name == tables::BlockAccessLists::NAME)); + assert!(!column_families + .iter() + .any(|name| name == tables::BlockAccessListBlockNumbers::NAME)); } #[test] fn block_access_lists_store_large_payloads_in_blob_files() { let temp_dir = TempDir::new().unwrap(); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_table::() + .build() + .unwrap(); let bal_key = reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ @@ -3244,7 +3237,12 @@ mod tests { 1 ])); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_default_tables() + .with_table::() + .with_table::() + .build() + .unwrap(); provider.put::(bal_key, &bal_value).unwrap(); provider .put::(bal_key.hash(), &bal_key.number()) @@ -3252,17 +3250,34 @@ mod tests { provider.flush(&[tables::BlockAccessLists::NAME]).unwrap(); drop(provider); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); + assert_eq!( + provider.get::(bal_key.hash()).unwrap(), + Some(bal_key.number()) + ); + } + + #[test] + fn test_read_only_opens_legacy_database_without_bal_tables() { + let temp_dir = TempDir::new().unwrap(); + let tx_hash = B256::with_last_byte(1); let provider = RocksDBBuilder::new(temp_dir.path()) .with_table::() .with_table::() .with_table::() .build() .unwrap(); - assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); - assert_eq!( - provider.get::(bal_key.hash()).unwrap(), - Some(bal_key.number()) - ); + provider.put::(tx_hash, &42).unwrap(); + drop(provider); + + // Replay must open the old schema without requiring a writable upgrade first. + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_default_tables() + .with_read_only(true) + .build() + .unwrap(); + assert_eq!(provider.get::(tx_hash).unwrap(), Some(42)); } #[test] From 863f7055d848afd7d7b1ca49e71744130e19b8d0 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:34:00 +0000 Subject: [PATCH 105/133] fix(storage): open legacy BAL databases read-only (#27471) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: Emma Jamieson-Hoare <21029500+emmajam@users.noreply.github.com> --- crates/storage/provider/src/bal/rocksdb.rs | 77 +++++++++++++++++++ .../src/providers/rocksdb/provider.rs | 18 +++++ 2 files changed, 95 insertions(+) diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index 34494ba3a34..be87fd6f0b8 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -105,6 +105,14 @@ impl RocksDBBalStore { return Ok(Some(bal)) } + // Read-only legacy databases predate both BAL tables. + if self.rocksdb.is_read_only() && + !self.rocksdb.has_table::() && + !self.rocksdb.has_table::() + { + return Ok(None) + } + let Some(block_number) = self.rocksdb.get::(block_hash)? else { @@ -573,4 +581,73 @@ mod tests { vec![first, second] ); } + + #[test] + fn read_only_legacy_database_has_no_persisted_bals() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("db"); + let hash = B256::with_last_byte(1); + let rocksdb = RocksDBBuilder::new(&path) + .with_table::() + .with_table::() + .with_table::() + .build() + .unwrap(); + rocksdb.put::(hash, &42).unwrap(); + drop(rocksdb); + + let rocksdb = RocksDBBuilder::new(&path) + .with_default_tables() + .with_table::() + .with_table::() + .with_read_only(true) + .build() + .unwrap(); + assert_eq!(rocksdb.get::(hash).unwrap(), Some(42)); + let store = RocksDBBalStore::new(rocksdb); + assert_eq!(store.get_by_hash(hash).unwrap(), None); + + let raw = Bytes::from_static(&[0xc0]); + store.insert(NumHash::new(42, hash), RawBal::from(raw.clone())).unwrap(); + assert_eq!(store.get_by_hash(hash).unwrap(), Some(raw)); + drop(store); + + // The secondary open must not add tables to the primary database. + let rocksdb = RocksDBBuilder::new(&path) + .with_table::() + .with_table::() + .with_table::() + .build() + .unwrap(); + assert!(!rocksdb.has_table::()); + assert!(!rocksdb.has_table::()); + } + + #[test] + fn read_only_database_reads_persisted_bals() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("db"); + let rocksdb = RocksDBBuilder::new(&path) + .with_default_tables() + .with_table::() + .with_table::() + .build() + .unwrap(); + let store = RocksDBBalStore::new(rocksdb); + let block = NumHash::new(42, B256::with_last_byte(1)); + let raw = Bytes::from_static(&[0xc0]); + store.insert(block, RawBal::from(raw.clone())).unwrap(); + store.flush(&[block]).unwrap(); + drop(store); + + let rocksdb = RocksDBBuilder::new(&path) + .with_default_tables() + .with_table::() + .with_table::() + .with_read_only(true) + .build() + .unwrap(); + let store = RocksDBBalStore::new(rocksdb); + assert_eq!(store.get_by_hash(block.hash).unwrap(), Some(raw)); + } } diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 09dfba0e16a..846361af89a 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -415,6 +415,16 @@ impl RocksDBBuilder { code: -1, })) })?; + if self.read_only { + // Legacy databases have no BAL tables; secondary opens cannot create them. + // Keep existing BAL tables; only omit those absent from disk. + cf_descriptors.retain(|cf| { + !matches!( + cf.name(), + tables::BlockAccessLists::NAME | tables::BlockAccessListBlockNumbers::NAME + ) || existing_column_families.iter().any(|name| name == cf.name()) + }); + } let unknown_column_families: Vec = existing_column_families .into_iter() .filter(|name| { @@ -914,6 +924,14 @@ impl RocksDBProvider { self.0.cf_handle::() } + /// Returns whether this provider opened the given table. + pub(crate) fn has_table(&self) -> bool { + match self.0.as_ref() { + RocksDBProviderInner::ReadWrite { db, .. } => db.cf_handle(T::NAME).is_some(), + RocksDBProviderInner::Secondary { db, .. } => db.cf_handle(T::NAME).is_some(), + } + } + /// Executes a function and records metrics with the given operation and table name. fn execute_with_operation_metric( &self, From 5723a3fed5521fad105b44f86876580242f31be9 Mon Sep 17 00:00:00 2001 From: valvecitydev <271573222+valvecitydev@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:08:06 +0000 Subject: [PATCH 106/133] fix(rpc): log gas-cap clamping at debug without the request (#27477) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 0f40b949275..ce47b596fd0 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -49,7 +49,7 @@ use revm::{ }; use revm_inspectors::{access_list::AccessListInspector, transfer::TransferInspector}; use std::collections::BTreeMap; -use tracing::{trace, warn}; +use tracing::{debug, trace}; /// Result type for `eth_simulateV1` RPC method. pub type SimulatedBlocksResult = Result>>, E>; @@ -858,7 +858,9 @@ pub trait Call: if let Some(requested_gas) = request.as_ref().gas_limit() { let global_gas_cap = self.call_gas_limit(); if global_gas_cap != 0 && global_gas_cap < requested_gas { - warn!(target: "rpc::eth::call", ?request, ?global_gas_cap, "Capping gas limit to global gas cap"); + // Callers routinely send gas limits above the cap, so this is not worth a warning. + // Log only the limits: the request carries caller calldata and addresses. + debug!(target: "rpc::eth::call", requested_gas, global_gas_cap, "Capping gas limit to global gas cap"); request.as_mut().set_gas_limit(global_gas_cap); } } else { From 617bfc2143aa6896ae8ae0dd470987f02a347a0b Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Mon, 28 Sep 2026 05:54:37 +0000 Subject: [PATCH 107/133] chore: update gh-actions to ee2960a6 (#27481) Co-authored-by: grandizzy <38490174+grandizzy@users.noreply.github.com> --- .github/sts/cargo-update-pr.sts.yaml | 2 +- .github/workflows/bench-benchmarkoor.yml | 2 +- .github/workflows/bench-scheduled.yml | 2 +- .github/workflows/bench.yml | 2 +- .github/workflows/book.yml | 2 +- .github/workflows/check-alloy.yml | 2 +- .github/workflows/compact.yml | 2 +- .github/workflows/cyclops-audit.yml | 2 +- .github/workflows/dependencies.yml | 2 +- .github/workflows/docker-test.yml | 4 +-- .github/workflows/docker.yml | 4 +-- .github/workflows/e2e.yml | 8 ++--- .github/workflows/hive.yml | 6 ++-- .github/workflows/integration.yml | 8 ++--- .github/workflows/kurtosis.yml | 2 +- .github/workflows/lint-actions.yml | 2 +- .github/workflows/lint.yml | 40 +++++++++++----------- .github/workflows/release-dist.yml | 2 +- .github/workflows/release-reproducible.yml | 2 +- .github/workflows/release.yml | 4 +-- .github/workflows/reproducible-build.yml | 2 +- .github/workflows/scan-github-actions.yml | 4 +-- .github/workflows/stage.yml | 2 +- .github/workflows/sync-era.yml | 2 +- .github/workflows/sync.yml | 2 +- .github/workflows/unit.yml | 10 +++--- 26 files changed, 61 insertions(+), 61 deletions(-) diff --git a/.github/sts/cargo-update-pr.sts.yaml b/.github/sts/cargo-update-pr.sts.yaml index c61d5a403d9..75ec78bd80f 100644 --- a/.github/sts/cargo-update-pr.sts.yaml +++ b/.github/sts/cargo-update-pr.sts.yaml @@ -7,7 +7,7 @@ claim_pattern: repository_id: '537233603' event_name: '(schedule|workflow_dispatch)' workflow_ref: 'paradigmxyz/reth/\.github/workflows/dependencies\.yml@refs/heads/main' - job_workflow_ref: 'tempoxyz/gh-actions/\.github/workflows/cargo-update-pr\.yml@117919c943b804057be733b0c4034c5542e58959' + job_workflow_ref: 'tempoxyz/gh-actions/\.github/workflows/cargo-update-pr\.yml@ee2960a6cc25d99bda45614135bfe1197d879bac' permissions: contents: write diff --git a/.github/workflows/bench-benchmarkoor.yml b/.github/workflows/bench-benchmarkoor.yml index 77554b2ea02..e2b4c03eee6 100644 --- a/.github/workflows/bench-benchmarkoor.yml +++ b/.github/workflows/bench-benchmarkoor.yml @@ -177,7 +177,7 @@ jobs: BENCHMARKOOR_CACHE: ${{ github.workspace }}/bench-work/benchmarkoor-cache steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - name: Clean up previous bench-work run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 2ade0528b95..2cb2e34498b 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -328,7 +328,7 @@ jobs: BENCH_VICTORIAMETRICS_URL: ${{ secrets.BENCH_VICTORIAMETRICS_URL }} steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - name: Clean up previous bench-work run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index dcc760fbc21..afddfab67ac 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1171,7 +1171,7 @@ jobs: BENCH_VICTORIAMETRICS_URL: ${{ needs.bench-ack.outputs.metrics == 'true' && secrets.BENCH_VICTORIAMETRICS_URL || '' }} steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - name: Clean up previous bench-work run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 7f838a987fd..ffdb189146f 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -24,7 +24,7 @@ jobs: NODE_EXTRA_CA_CERTS: /etc/ssl/certs/ca-certificates.crt steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/check-alloy.yml b/.github/workflows/check-alloy.yml index cea21a21b4a..3f15cf387d1 100644 --- a/.github/workflows/check-alloy.yml +++ b/.github/workflows/check-alloy.yml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 60 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 0e0b498693a..edfa4252704 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -30,7 +30,7 @@ jobs: - cargo run --bin reth --features "dev" steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - name: Checkout PR uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/cyclops-audit.yml b/.github/workflows/cyclops-audit.yml index 29fcbf6b884..d468bfaba3d 100644 --- a/.github/workflows/cyclops-audit.yml +++ b/.github/workflows/cyclops-audit.yml @@ -60,7 +60,7 @@ jobs: pull-requests: write steps: - name: Handle audit command - uses: tempoxyz/gh-actions/actions/pr-audit-comment@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/pr-audit-comment@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: command-regex: '^(?:@decofe\s+)?(?:cyclops\s+(?:private\s+)?audit|derek\s+audit)\b' permission-check-mode: association diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 5d4542d1f2e..fab87788aa1 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -17,4 +17,4 @@ jobs: permissions: contents: read id-token: write - uses: tempoxyz/gh-actions/.github/workflows/cargo-update-pr.yml@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/.github/workflows/cargo-update-pr.yml@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index c009c9e0f0c..246f9cc5c46 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -27,7 +27,7 @@ jobs: contents: read steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -54,7 +54,7 @@ jobs: echo "is_fork=false" >> "$GITHUB_OUTPUT" fi - - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 if: steps.fork.outputs.is_fork == 'false' with: toolchain: "1.95.0" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0a3cb4a646c..1e40ad717b4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -42,12 +42,12 @@ jobs: id-token: write steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + - uses: tempoxyz/gh-actions/vendor/dtolnay/rust-toolchain@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: toolchain: "1.95.0" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6ec638c6b45..74739c415e3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,7 +31,7 @@ jobs: timeout-minutes: 90 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -42,7 +42,7 @@ jobs: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Install nextest - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: nextest@0.9.140 checksum: true @@ -75,7 +75,7 @@ jobs: timeout-minutes: 60 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -86,7 +86,7 @@ jobs: toolchain: stable - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Install nextest - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: nextest@0.9.140 checksum: true diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 02057d515a8..428ab9d65ed 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -44,7 +44,7 @@ jobs: name: Prepare Hive - ${{ matrix.variant == 'amsterdam' && 'Amsterdam' || 'Osaka' }} steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -218,7 +218,7 @@ jobs: issues: write steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -407,7 +407,7 @@ jobs: issues: write steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bae574ccfaa..412d6db76c5 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 60 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -50,7 +50,7 @@ jobs: - name: Install Geth run: .github/scripts/install_geth.sh - name: Install nextest - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: nextest@0.9.140 checksum: true @@ -90,7 +90,7 @@ jobs: contents: read steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -100,7 +100,7 @@ jobs: with: toolchain: stable - name: Install nextest - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: nextest@0.9.140 checksum: true diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index a39951b9ad0..f2a0f4c84fe 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -45,7 +45,7 @@ jobs: - build-reth steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml index f40d236c33d..e9950d740c9 100644 --- a/.github/workflows/lint-actions.yml +++ b/.github/workflows/lint-actions.yml @@ -18,7 +18,7 @@ jobs: contents: read steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 280bc12d1a0..6cd6ab01ae0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -33,7 +33,7 @@ jobs: features: "asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -61,7 +61,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -87,7 +87,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -97,7 +97,7 @@ jobs: toolchain: stable target: wasm32-wasip1 - name: Install cargo-hack - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: cargo-hack@0.6.45 checksum: true @@ -120,7 +120,7 @@ jobs: timeout-minutes: 60 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -130,7 +130,7 @@ jobs: toolchain: stable target: riscv32imac-unknown-none-elf - name: Install cargo-hack - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: cargo-hack@0.6.45 checksum: true @@ -156,7 +156,7 @@ jobs: timeout-minutes: 60 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -166,7 +166,7 @@ jobs: with: toolchain: stable - name: Install cargo-hack - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: cargo-hack@0.6.45 checksum: true @@ -186,7 +186,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -212,7 +212,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -240,7 +240,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -262,7 +262,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -276,7 +276,7 @@ jobs: with: cache-on-failure: true - name: Install cargo-udeps - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: cargo-udeps@0.1.61 checksum: true @@ -292,7 +292,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -320,7 +320,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -334,7 +334,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -352,7 +352,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -367,7 +367,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -387,7 +387,7 @@ jobs: timeout-minutes: 20 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -408,7 +408,7 @@ jobs: permissions: contents: read id-token: write - uses: tempoxyz/gh-actions/.github/workflows/rust-deny.yml@b35994a23aaf6acf4d5e2f91d18235a6e2f90e95 # 2026-09-23T08-07-21Z-b35994a2 + uses: tempoxyz/gh-actions/.github/workflows/rust-deny.yml@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: rust-toolchain: nightly diff --git a/.github/workflows/release-dist.yml b/.github/workflows/release-dist.yml index 03e25385e70..59bffb49b81 100644 --- a/.github/workflows/release-dist.yml +++ b/.github/workflows/release-dist.yml @@ -17,7 +17,7 @@ jobs: id-token: write steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - name: Update Homebrew formula uses: dawidd6/action-homebrew-bump-formula@b12d2bc99c46444125e606142f28200ec71a1214 # v10 with: diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 158dbb2aeb6..66b0b485b23 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -55,7 +55,7 @@ jobs: contents: write steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 745c5451134..859178ba70c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: if: ${{ github.event.inputs.dry_run != 'true' }} steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -117,7 +117,7 @@ jobs: binary: reth steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index 1778ade901d..950a0dc54a2 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -24,7 +24,7 @@ jobs: machine: machine-2 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/scan-github-actions.yml b/.github/workflows/scan-github-actions.yml index 6f251fad0c4..4bbf84d159c 100644 --- a/.github/workflows/scan-github-actions.yml +++ b/.github/workflows/scan-github-actions.yml @@ -10,10 +10,10 @@ permissions: {} jobs: scan: - uses: tempoxyz/gh-actions/.github/workflows/scan-github-actions.yml@b35994a23aaf6acf4d5e2f91d18235a6e2f90e95 # 2026-09-23T08-07-21Z-b35994a2 + uses: tempoxyz/gh-actions/.github/workflows/scan-github-actions.yml@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 permissions: actions: read contents: read id-token: write with: - secure-runner-fail-on-violation: false + ensure-secure-runner: false diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 89ac37ff13e..0b8af9b96e1 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -35,7 +35,7 @@ jobs: timeout-minutes: 60 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index a573990403f..2bd45132f01 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -40,7 +40,7 @@ jobs: unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a" steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 9782ce33043..a6748b5372e 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -40,7 +40,7 @@ jobs: unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a" steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index de697a5395e..cf325c776c9 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -38,7 +38,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -54,7 +54,7 @@ jobs: cache-bin: false cache-on-failure: true - name: Install nextest - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: nextest@0.9.140 checksum: true @@ -80,7 +80,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -108,7 +108,7 @@ jobs: with: toolchain: stable - name: Install nextest - uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: tool: nextest@0.9.140 checksum: true @@ -130,7 +130,7 @@ jobs: timeout-minutes: 30 steps: - name: Secure runner - uses: tempoxyz/gh-actions/actions/secure-runner@117919c943b804057be733b0c4034c5542e58959 # 2026-09-23T05-09-41Z-117919c9 + uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false From 912356df194fbd70e5d9eba83b368ccb1534c5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:28:14 +0200 Subject: [PATCH 108/133] docs(snap-sync): extend crate docs (#27483) --- crates/snap-sync/README.md | 66 +++++++++++++++++++++++++++++++++++++ crates/snap-sync/src/lib.rs | 32 +----------------- 2 files changed, 67 insertions(+), 31 deletions(-) create mode 100644 crates/snap-sync/README.md diff --git a/crates/snap-sync/README.md b/crates/snap-sync/README.md new file mode 100644 index 00000000000..374be8529aa --- /dev/null +++ b/crates/snap-sync/README.md @@ -0,0 +1,66 @@ +# reth-snap-sync + +snap/2 state synchronization, as specified by [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189). + +Instead of executing every block from genesis, a node downloads the state of a recent block from +peers and verifies it against that block's state root. snap/2 keeps that state current with +[EIP-7928 block access lists](https://eips.ethereum.org/EIPS/eip-7928) (BALs), which record every +field each block changes, replacing snap/1's trie-node healing. + +This crate owns the synchronization logic and its progress. Requests and proof checks come from +`reth-downloaders`, and running the sync inside a node is left to node integration. + +## How a sync runs + +`SnapBootstrap` drives one sync from start to hand-off: + +1. **Pick a pivot.** A recent finalized block is preferred, since it cannot be reorged; otherwise + the pivot sits 64 blocks behind the head by default, the EIP's example distance. With no eligible + block the sync waits. +2. **Download the state at the pivot.** Accounts are fetched in key-order ranges, proved against the + pivot's state root. A range commits only once its contracts' storage and code are persisted, + whether supplied with it or stored beforehand. +3. **Advance the pivot.** Peers only keep recent state, so once the pivot lags more than 96 blocks + by default the sync re-anchors to a newer block. BALs of the blocks in between carry the state + already downloaded forward, applied strictly in block order, and the remaining ranges download at + the new root. +4. **Hand off.** Once every account is downloaded and the BALs reach the pivot, the state goes to + the merkle stage, which rebuilds the trie. It is accepted only when the root matches the pivot's + header. + +```rust +use reth_snap_sync::SnapPivotPolicy; + +let policy = SnapPivotPolicy::default(); +// Without a finalized block, anchor at the EIP's example distance. +assert_eq!(policy.pivot_block(1_000, None), Some(936)); +// A recent finalized block is anchored to directly. +assert_eq!(policy.pivot_block(1_000, Some(950)), Some(950)); +// Stalled finality falls back to the example distance. +assert_eq!(policy.pivot_block(1_000, Some(500)), Some(936)); +// A chain shorter than the head distance has no pivot yet. +assert_eq!(policy.pivot_block(4, None), None); +``` + +## Design choices + +- **State is written in place.** Downloads land directly in the hashed state tables, so there is no + staging copy to move afterwards. An attempt record owns that state: every write carries the + attempt and pivot it was fetched for, and responses to an older attempt or pivot are refused. +- **Progress commits with its data.** Each range, storage chunk or BAL commits in the same + transaction as the progress it advances, so a stopped sync resumes from what the database holds. +- **Nothing counts before its dependencies.** An account range commits only with storage matching + its accounts' roots and code matching their hashes. Storage too large for one response is kept + ahead of its range, tied to the attempt and pivot. +- **BALs never skip a block.** A block whose BAL no peer serves holds back every later one, and a + BAL only applies on top of its parent. +- **The trie is rebuilt once, at the end.** The existing merkle stage builds it from the downloaded + state, instead of maintaining one during the download. +- **Starting over is the fallback.** An attempt restarts when catch-up falls behind the BALs peers + still serve. + +Reorg recovery is not implemented yet: a reorg that orphans the pivot is detected and restarts the +attempt, instead of repairing the state from the orphaned blocks' BALs as EIP-8189 describes. + +snap/1 synchronization is not covered: this design keeps the state current with BALs, which only +snap/2 serves, rather than with snap/1's trie-node healing. diff --git a/crates/snap-sync/src/lib.rs b/crates/snap-sync/src/lib.rs index 352c2f97ec8..4a8cdbfb2a6 100644 --- a/crates/snap-sync/src/lib.rs +++ b/crates/snap-sync/src/lib.rs @@ -1,34 +1,4 @@ -//! snap/2 state synchronization for [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189). -//! -//! Coordinates a state bootstrap that starts from a recent pivot block, downloads accounts, storage -//! and bytecode authenticated against that pivot's state root, and advances the pivot with -//! [EIP-7928 block access lists](https://eips.ethereum.org/EIPS/eip-7928) as the chain moves past -//! it. -//! -//! This crate owns download progress only. Authenticated downloads come from -//! `reth-downloaders`, and verified state is handed back to node integration once its trie root -//! matches the target header. [`SnapBootstrap`] runs these steps as one resumable sync. -//! -//! Downloaded state goes into the hashed state tables, owned by an attempt record that commits -//! with it, along with how far the account key space has been downloaded. An account range only -//! commits with storage matching its accounts' roots and code matching their hashes, so committed -//! progress never depends on work still pending. Storage too large for one response is persisted -//! ahead of its range, under progress tied to the attempt, pivot and range. -//! -//! ``` -//! use reth_snap_sync::SnapPivotPolicy; -//! -//! let policy = SnapPivotPolicy::default(); -//! // Without a finalized block, anchor at the EIP's example distance. -//! assert_eq!(policy.pivot_block(1_000, None), Some(936)); -//! // A recent finalized block is anchored to directly. -//! assert_eq!(policy.pivot_block(1_000, Some(950)), Some(950)); -//! // Stalled finality falls back to the example distance. -//! assert_eq!(policy.pivot_block(1_000, Some(500)), Some(936)); -//! // A chain shorter than the head distance has no pivot yet. -//! assert_eq!(policy.pivot_block(4, None), None); -//! ``` - +#![doc = include_str!("../README.md")] #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", From f5060248edb3f39b8798ad719c9ba0ff5099447e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 28 Sep 2026 12:29:59 +0200 Subject: [PATCH 109/133] revert: "fix(storage): revert BAL store activation" (#27472) --- crates/cli/commands/src/common.rs | 16 ++--- crates/engine/tree/src/persistence.rs | 37 +++++++++++ crates/node/builder/src/launch/common.rs | 19 +++--- crates/storage/provider/src/bal/rocksdb.rs | 14 +---- .../provider/src/providers/rocksdb/metrics.rs | 2 + .../src/providers/rocksdb/provider.rs | 63 +++++++------------ 6 files changed, 85 insertions(+), 66 deletions(-) diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 2748ba5ec9c..77cd552633f 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -27,8 +27,7 @@ use reth_provider::{ BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider, StaticFileProviderBuilder, }, - BalConfig, BalStoreHandle, InMemoryBalStore, ProviderFactory, StaticFileProviderFactory, - StorageSettings, + BalStoreHandle, ProviderFactory, RocksDBBalStore, StaticFileProviderFactory, StorageSettings, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; @@ -196,11 +195,14 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let balstore_cache_size = - self.db.balstore_cache_size.unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + let bal_store = self + .db + .balstore_cache_size + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::>::new( db, self.chain.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 3c6261945c1..f466dad914e 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -562,6 +562,43 @@ mod tests { assert_eq!(result.last_state_trie_block, result.last_block); } + #[test] + fn test_save_blocks_flushes_bal_store() { + use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; + + reth_tracing::init_test_tracing(); + let provider = create_test_provider_factory(); + let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); + let provider = provider.with_bal_store(bal_store); + init_genesis(&provider).unwrap(); + + let mut test_block_builder = TestBlockBuilder::eth(); + let executed = test_block_builder.get_executed_block_with_number(1, B256::random()); + let num_hash = executed.recovered_block().num_hash(); + let raw_bal = Bytes::from_static(&[0xc0]); + + provider.bal_store().insert(num_hash, RawBal::new(raw_bal.clone())).unwrap(); + + let (_finished_exex_height_tx, finished_exex_height_rx) = + tokio::sync::watch::channel(FinishedExExHeight::NoExExs); + let pruner = + Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); + let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); + let handle = PersistenceHandle::::spawn_service( + provider.clone(), + pruner, + sync_metrics_tx, + ); + let (tx, rx) = crossbeam_channel::bounded(1); + + handle.save_blocks(full_save_input(vec![executed]), tx).unwrap(); + + let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); + assert_eq!(result.last_block, num_hash); + let persisted_store = RocksDBBalStore::new(provider.rocksdb_provider()); + assert_eq!(persisted_store.get_by_hash(num_hash.hash).unwrap(), Some(raw_bal)); + } + #[test] fn test_save_blocks_multiple_blocks() { reth_tracing::init_test_tracing(); diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 6f54bfca5a3..1a7146c96c5 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -71,10 +71,10 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, - DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, - ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, - StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, + BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, + MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, + RocksDBBalStore, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, + StaticFileProviderFactory, StorageSettingsCache, }; use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; @@ -522,14 +522,15 @@ where .build()? }; - let balstore_cache_size = self + let bal_store = self .node_config() .db .balstore_cache_size - .unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index be87fd6f0b8..6940693ccd6 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -341,20 +341,12 @@ struct RocksDBBalEntry { #[cfg(test)] mod tests { use super::*; - use crate::providers::{RocksDBBuilder, RocksDBProvider}; + use crate::providers::RocksDBBuilder; use alloy_primitives::B256; - fn test_rocksdb(dir: &tempfile::TempDir) -> RocksDBProvider { - RocksDBBuilder::new(dir.path()) - .with_table::() - .with_table::() - .build() - .unwrap() - } - fn test_store() -> (tempfile::TempDir, RocksDBBalStore) { let dir = tempfile::tempdir().unwrap(); - let rocksdb = test_rocksdb(&dir); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); (dir, RocksDBBalStore::new(rocksdb)) } @@ -443,7 +435,7 @@ mod tests { #[test] fn configured_buffer_retention_distance_is_used() { let dir = tempfile::tempdir().unwrap(); - let rocksdb = test_rocksdb(&dir); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); let store = RocksDBBalStore::with_buffer_retention_distance(rocksdb, 64); let old = NumHash::new(1, B256::with_last_byte(1)); let tip = NumHash::new(34, B256::with_last_byte(2)); diff --git a/crates/storage/provider/src/providers/rocksdb/metrics.rs b/crates/storage/provider/src/providers/rocksdb/metrics.rs index 3971ed978ba..c22edf65176 100644 --- a/crates/storage/provider/src/providers/rocksdb/metrics.rs +++ b/crates/storage/provider/src/providers/rocksdb/metrics.rs @@ -8,6 +8,8 @@ use strum::{EnumIter, IntoEnumIterator}; pub(super) const ROCKSDB_TABLES: &[&str] = &[ Tables::TransactionHashNumbers.name(), + Tables::BlockAccessLists.name(), + Tables::BlockAccessListBlockNumbers.name(), Tables::StoragesHistory.name(), Tables::AccountsHistory.name(), ]; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 846361af89a..b681c382c64 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -333,10 +333,14 @@ impl RocksDBBuilder { /// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping /// - [`tables::AccountsHistory`] - Account history index /// - [`tables::StoragesHistory`] - Storage history index + /// - [`tables::BlockAccessLists`] - Block access list payloads + /// - [`tables::BlockAccessListBlockNumbers`] - Block access list hash index pub fn with_default_tables(self) -> Self { self.with_table::() .with_table::() .with_table::() + .with_table::() + .with_table::() } /// Enables metrics. @@ -3184,22 +3188,25 @@ mod tests { provider.put::(key.clone(), &value).unwrap(); assert!(provider.get::(key).unwrap().is_some()); - drop(provider); - - let column_families = DB::list_cf(&Options::default(), temp_dir.path()).unwrap(); - assert!(!column_families.iter().any(|name| name == tables::BlockAccessLists::NAME)); - assert!(!column_families - .iter() - .any(|name| name == tables::BlockAccessListBlockNumbers::NAME)); + let bal_key = + reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); + let bal_value = + reth_db_api::models::StoredBlockAccessList::new(Bytes::from_static(&[0xc0])); + provider.put::(bal_key, &bal_value).unwrap(); + assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); + provider + .put::(bal_key.hash(), &bal_key.number()) + .unwrap(); + assert_eq!( + provider.get::(bal_key.hash()).unwrap(), + Some(bal_key.number()) + ); } #[test] fn block_access_lists_store_large_payloads_in_blob_files() { let temp_dir = TempDir::new().unwrap(); - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_table::() - .build() - .unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); let bal_key = reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ @@ -3255,12 +3262,7 @@ mod tests { 1 ])); - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_default_tables() - .with_table::() - .with_table::() - .build() - .unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); provider.put::(bal_key, &bal_value).unwrap(); provider .put::(bal_key.hash(), &bal_key.number()) @@ -3268,34 +3270,17 @@ mod tests { provider.flush(&[tables::BlockAccessLists::NAME]).unwrap(); drop(provider); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); - assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); - assert_eq!( - provider.get::(bal_key.hash()).unwrap(), - Some(bal_key.number()) - ); - } - - #[test] - fn test_read_only_opens_legacy_database_without_bal_tables() { - let temp_dir = TempDir::new().unwrap(); - let tx_hash = B256::with_last_byte(1); let provider = RocksDBBuilder::new(temp_dir.path()) .with_table::() .with_table::() .with_table::() .build() .unwrap(); - provider.put::(tx_hash, &42).unwrap(); - drop(provider); - - // Replay must open the old schema without requiring a writable upgrade first. - let provider = RocksDBBuilder::new(temp_dir.path()) - .with_default_tables() - .with_read_only(true) - .build() - .unwrap(); - assert_eq!(provider.get::(tx_hash).unwrap(), Some(42)); + assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); + assert_eq!( + provider.get::(bal_key.hash()).unwrap(), + Some(bal_key.number()) + ); } #[test] From 3d592ece6de8c4559987416a544fc215fd6d6921 Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Mon, 28 Sep 2026 11:49:29 +0100 Subject: [PATCH 110/133] chore: release 2.7.0 --- Cargo.lock | 230 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 117 insertions(+), 117 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a92818b5202..7640e82587a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3354,7 +3354,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.5.2" +version = "2.7.0" dependencies = [ "clap", "ef-tests", @@ -3362,7 +3362,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3814,7 +3814,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "eyre", @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "clap", @@ -7806,7 +7806,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7860,7 +7860,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7886,7 +7886,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7933,7 +7933,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7962,7 +7962,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7982,7 +7982,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-genesis", "clap", @@ -7994,7 +7994,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8085,7 +8085,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.5.2" +version = "2.7.0" dependencies = [ "reth-tasks", "tokio", @@ -8094,7 +8094,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8148,7 +8148,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "eyre", @@ -8166,7 +8166,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8179,7 +8179,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8193,7 +8193,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8217,7 +8217,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8252,7 +8252,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8281,7 +8281,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8312,7 +8312,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8328,7 +8328,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8355,7 +8355,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8380,7 +8380,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8408,7 +8408,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8448,7 +8448,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8506,7 +8506,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.5.2" +version = "2.7.0" dependencies = [ "aes", "alloy-primitives", @@ -8532,7 +8532,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8554,7 +8554,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8581,7 +8581,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8650,7 +8650,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8678,7 +8678,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8703,7 +8703,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "bytes", @@ -8721,7 +8721,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8752,7 +8752,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.5.2" +version = "2.7.0" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8762,7 +8762,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8800,7 +8800,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8828,7 +8828,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8868,7 +8868,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.5.2" +version = "2.7.0" dependencies = [ "clap", "eyre", @@ -8891,7 +8891,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8907,7 +8907,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8923,7 +8923,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.9", @@ -8936,7 +8936,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8965,7 +8965,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8986,7 +8986,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "rayon", @@ -8996,7 +8996,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9022,7 +9022,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9047,7 +9047,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "fixed-cache", @@ -9065,7 +9065,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -9077,7 +9077,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9099,7 +9099,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9144,7 +9144,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "eyre", @@ -9175,7 +9175,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9192,7 +9192,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.5.2" +version = "2.7.0" dependencies = [ "serde", "serde_json", @@ -9201,7 +9201,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9232,7 +9232,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.5.2" +version = "2.7.0" dependencies = [ "bytes", "futures", @@ -9254,7 +9254,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.5.2" +version = "2.7.0" dependencies = [ "bitflags 2.13.2", "byteorder", @@ -9271,7 +9271,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.5.2" +version = "2.7.0" dependencies = [ "bindgen", "cc", @@ -9279,7 +9279,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.5.2" +version = "2.7.0" dependencies = [ "futures", "libc", @@ -9292,7 +9292,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "ipnet", @@ -9300,7 +9300,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.5.2" +version = "2.7.0" dependencies = [ "futures-util", "if-addrs", @@ -9314,7 +9314,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9385,7 +9385,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9409,7 +9409,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9433,7 +9433,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9450,7 +9450,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9463,7 +9463,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.5.2" +version = "2.7.0" dependencies = [ "anyhow", "bincode", @@ -9482,7 +9482,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9505,7 +9505,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9576,7 +9576,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9632,7 +9632,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9703,7 +9703,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9726,7 +9726,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9749,7 +9749,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.5.2" +version = "2.7.0" dependencies = [ "bytes", "eyre", @@ -9779,7 +9779,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9790,7 +9790,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9813,7 +9813,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.5.2" +version = "2.7.0" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9824,7 +9824,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9848,7 +9848,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9857,7 +9857,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9900,7 +9900,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9953,7 +9953,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9984,7 +9984,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10004,7 +10004,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10020,7 +10020,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10102,7 +10102,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-genesis", @@ -10132,7 +10132,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-network", @@ -10190,7 +10190,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10210,7 +10210,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10230,7 +10230,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10266,7 +10266,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10312,7 +10312,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10363,7 +10363,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-rpc-types-engine", "brotli", @@ -10383,7 +10383,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -10412,7 +10412,7 @@ dependencies = [ [[package]] name = "reth-snap-sync" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10449,7 +10449,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10512,7 +10512,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10546,7 +10546,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10562,7 +10562,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -10585,7 +10585,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "clap", @@ -10603,7 +10603,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10629,7 +10629,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10645,7 +10645,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10675,7 +10675,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10705,7 +10705,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.5.2" +version = "2.7.0" dependencies = [ "crossbeam-utils", "dashmap", @@ -10725,7 +10725,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10741,7 +10741,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.5.2" +version = "2.7.0" dependencies = [ "tokio", "tokio-stream", @@ -10750,7 +10750,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.5.2" +version = "2.7.0" dependencies = [ "clap", "eyre", @@ -10769,7 +10769,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.5.2" +version = "2.7.0" dependencies = [ "base64 0.22.1", "clap", @@ -10787,7 +10787,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10838,7 +10838,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10870,7 +10870,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10903,7 +10903,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10930,7 +10930,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10957,7 +10957,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.5.2" +version = "2.7.0" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 2232794a18f..83ad952b808 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.5.2" +version = "2.7.0" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 6aa891ac973..5e138998783 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.5.2', + text: 'v2.7.0', items: [ { text: 'Releases', From 6292e03778f425a091cfc9d7ccee48bf4a59d341 Mon Sep 17 00:00:00 2001 From: Akosh Farkash Date: Wed, 4 Nov 2026 17:53:58 +0000 Subject: [PATCH 111/133] ci: run fork validation through CircleCI act Squash the fork CI support from 04e73b7c2bdc8dcf3c1e8961c8d04b861bc15a8e and push-event support from b0d194cad81ddf9dcb954ecee95b22cc2d68b70b into one replayable maintenance commit. Preserve release action pins and advisory policy when resolving the replay. Select the book build job without entering Pages deployment. Pin the Rust toolchains the workflows install: stable to 1.97.0, nightly to nightly-2026-07-14. A maintenance branch outlives its release by weeks, and floating toolchains change under it. When replaying onto a new release, resolve each workflow hunk to upstream's side and re-apply only these pins, the llvm install in lint.yml's wasm job and the gcc-multilib `--yes`. Run cargo deny natively instead of through act (folded in from c279f31455e8b714c1123c19accb1a68bd566f7c): lint.yml's `deny` job calls a reusable tempoxyz workflow that starts with tempoxyz's secure-runner. That action requires a GitHub OIDC token unless the event is a fork pull request, and act never receives one, so the job failed on every push and same-repo PR. Run the same `cargo deny --all-features check all` in a CircleCI job with a digest-pinned Rust image and a checksum-pinned cargo-deny release, and drop `deny` from the act lint matrix. Build the compact reader in a separate target directory so act cannot reuse libraries compiled from the comparison base after restoring older source mtimes (folded in from 785b05757). Give the valid_blocks ef-tests group the ten-minute budget already used by grouped general-state fixtures instead of the single-test sixty-second timeout, which terminated it on every CircleCI attempt (folded in from d9bd9875f). Compare Compact vectors on pushes to an op-v branch against upstream tag v, derived from the branch name, instead of a hard-coded release commit (folded in from 447813d53, #21). Pin the docs build to the nightly the book job installs: the script reads RUST_NIGHTLY instead of resolving `+nightly` to the newest channel, which fails to compile boa_engine since 2026-11-01 (rust-lang/rust#165263). Adapted from upstream 121b0f73dc5528e0ee78c375793df79af542c725. Ignore RUSTSEC-2026-0391, -0398 and -0403, compression advisories pinned by reth-zstd-compressors 0.7 whose patched releases need reth-core 0.8 and Rust 1.96, and RUSTSEC-2026-0387, crypto-bigint 0.5 with no patched release compatible with elliptic-curve 0.13. All four still apply on this base (folded in from 22ab9316c, #30). Update the lockfile to patched releases for the advisories cargo deny reports on this base, re-resolved against its lockfile rather than replayed: rustls 0.23.46, libz-sys 1.1.31, rustls-webpki 0.103.16, tokio 1.53.3, webpki-root-certs 1.0.10, webpki-roots 1.0.10 and zstd-sys 2.1.2. No manifest changes. Co-authored-by: Sebastian Stammler Co-authored-by: Josh Klopfenstein Co-authored-by: einar-oplabs Co-authored-by: falcorocks <14293929+falcorocks@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Claude Fable 5.1 --- .circleci/config.yml | 246 ++++++++++++++++++++++++++ .circleci/prepare-act-event.sh | 102 +++++++++++ .config/nextest.toml | 6 + .github/scripts/check_rv32imac.sh | 2 +- .github/scripts/check_wasm.sh | 2 +- .github/workflows/book.yml | 4 +- .github/workflows/compact.yml | 34 +++- .github/workflows/e2e.yml | 4 +- .github/workflows/integration.yml | 4 +- .github/workflows/lint.yml | 25 +-- .github/workflows/stage.yml | 2 +- .github/workflows/unit.yml | 6 +- Cargo.lock | 56 +++--- deny.toml | 7 + docs/vocs/scripts/build-cargo-docs.sh | 2 +- 15 files changed, 446 insertions(+), 56 deletions(-) create mode 100644 .circleci/config.yml create mode 100755 .circleci/prepare-act-event.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..970fc6f12b5 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,246 @@ +version: 2.1 + +# Runs this fork's validation workflows through pinned act because GitHub Actions is disabled. +# Pull requests and branch pushes run the PR checks; merge queues add stage.yml. Event metadata is +# generated once per CircleCI revision and shared by all shards. +# +# The read-only GitHub token is exposed to PR code, so its CircleCI context must remain +# restricted to trusted contributors. Keep CircleCI auto-cancellation enabled. +# +# The act binary, checksum, and runner image are pinned. Compiler caches are no-ops because +# act's local cache is not reusable across these machines; Pages plumbing is skipped after the +# validation build. Lint and unit jobs are distributed; other workflows run up to four jobs. +# +# Excluded: pr-title.yml, label-pr.yml, and the privileged cyclops-audit.yml integration. + +jobs: + # Replaces lint.yml's `deny` job. That job calls a reusable workflow from tempoxyz, whose deny job + # starts with tempoxyz's secure-runner. secure-runner needs a GitHub OIDC token that act never + # receives, so the check itself runs natively here. Its policy comes from deny.toml in this repo. + # + # When re-applying this CI patch onto a new reth release, follow lint.yml's `deny:` job through the + # reusable workflows it calls until you reach the step that runs cargo deny. Compare its cargo-deny + # version, `check` command and flags with this job (on v2.7.0 the step is a vendored + # EmbarkStudios/cargo-deny-action, whose Dockerfile pins the version). When the version moves, + # update DENY_VERSION and DENY_SHA256 here. The Rust image stays on the MSRV on purpose, while + # upstream installs a floating nightly. + cargo-deny: + docker: + - image: rust:1.95-bookworm@sha256:6258907abe69656e41cd992e0b705cdcfabcbbe3db374f92ed2d47121282d4a1 + resource_class: medium + steps: + - checkout + - run: + name: Install pinned cargo-deny + command: | + set -euo pipefail + readonly DENY_VERSION="0.20.2" + readonly DENY_SHA256="9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f" + readonly DENY_ARCHIVE="cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl.tar.gz" + curl --fail --silent --show-error --location \ + "https://github.com/EmbarkStudios/cargo-deny/releases/download/${DENY_VERSION}/${DENY_ARCHIVE}" \ + --output "/tmp/${DENY_ARCHIVE}" + echo "${DENY_SHA256} /tmp/${DENY_ARCHIVE}" | sha256sum --check + tar -xzf "/tmp/${DENY_ARCHIVE}" -C /usr/local/bin --strip-components=1 \ + "cargo-deny-${DENY_VERSION}-x86_64-unknown-linux-musl/cargo-deny" + test "$(cargo-deny --version)" = "cargo-deny ${DENY_VERSION}" + - run: + name: cargo deny check + command: cargo deny --log-level warn --all-features check all + + prepare-act-event: + machine: + image: ubuntu-2404:2025.09.1 + resource_class: medium + steps: + - checkout + - run: + name: Snapshot GitHub event + command: | + mkdir -p /tmp/act-event + if ! .circleci/prepare-act-event.sh; then + echo "GitHub event preparation failed; act shards will report the failure" | + tee /tmp/act-event/failed + fi + - persist_to_workspace: + root: /tmp/act-event + paths: + - . + + act: + parameters: + wf: + type: string + job: + type: string + default: "" + concurrent_jobs: + type: integer + default: 4 + machine: + image: ubuntu-2404:2025.09.1 + resource_class: xlarge + steps: + - checkout + - attach_workspace: + at: /tmp/act-event + - run: + name: Check event snapshot + command: | + if [ -f /tmp/act-event/failed ]; then + cat /tmp/act-event/failed >&2 + exit 1 + fi + - run: + name: Install pinned act + command: | + set -euo pipefail + readonly ACT_VERSION="0.2.89-action-cache-fix.1" + readonly ACT_RELEASE="v0.2.89-action-cache-fix.4" + readonly ACT_BINARY="act_Linux_x86_64" + readonly ACT_SHA256="28bfcf1502f59a834b14f2fb23c39e5b253c2234d875592b5aa0a175f9674dce" + curl --fail --silent --show-error --location \ + "https://github.com/ethereum-optimism/act/releases/download/${ACT_RELEASE}/${ACT_BINARY}" \ + --output "/tmp/${ACT_BINARY}" + echo "${ACT_SHA256} /tmp/${ACT_BINARY}" | sha256sum --check + sudo install -m 0755 "/tmp/${ACT_BINARY}" /usr/local/bin/act + test "$(act --version)" = "act version ${ACT_VERSION}" + - run: + name: Stub unsupported non-check actions + command: | + mkdir -p /tmp/noop-action + printf '%s\n' \ + 'name: noop' \ + 'description: disabled service plumbing under act' \ + 'inputs:' \ + ' cache-bin: {required: false}' \ + ' cache-on-failure: {required: false}' \ + ' cache-targets: {required: false}' \ + ' prefix-key: {required: false}' \ + 'runs:' \ + ' using: composite' \ + ' steps:' \ + ' - run: echo "Service plumbing disabled under act"' \ + ' shell: bash' \ + > /tmp/noop-action/action.yml + - run: + name: Run << parameters.wf >> via act + no_output_timeout: 45m + command: | + set -euo pipefail + event_path=/tmp/act-event/github-event.json + event_name="$(jq -er 'if .pull_request then "pull_request" elif .merge_group then "merge_group" elif .ref and .after then "push" else error("unsupported event") end' "$event_path")" + base_ref="$(jq -r '(.pull_request.base.ref // .merge_group.base_ref // "") | sub("^refs/heads/"; "")' "$event_path")" + target_job="<< parameters.job >>" + # Validate documentation without entering Pages deployment on branch pushes. + if [ "<< parameters.wf >>" = "book.yml" ]; then + target_job=build + fi + act_job=() + if [ -n "$target_job" ]; then + case "$target_job" in + crate-checks-*) + partition="${target_job#crate-checks-}" + if [[ ! "$partition" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid crate-checks partition: $partition" >&2 + exit 1 + fi + act_job=(-j crate-checks --matrix "partition:${partition}") + ;; + *) + act_job=(-j "$target_job") + ;; + esac + fi + + act_env=(--env "RUSTC_WRAPPER=" --env "GITHUB_BASE_REF=${base_ref}") + if [ "<< parameters.wf >>" = "unit.yml" ] && [ "$target_job" = "state" ]; then + # EF tests use Rayon internally, so unrestricted nextest processes multiply + # runnable threads and starve valid_blocks past its upstream timeout. + act_env+=(--env "NEXTEST_TEST_THREADS=2") + fi + if [ "$event_name" != "pull_request" ]; then + act_env+=( + --env "GITHUB_REF=refs/heads/${CIRCLE_BRANCH}" + --env "GITHUB_SHA=${CIRCLE_SHA1}" + ) + fi + + act "$event_name" -W ".github/workflows/<< parameters.wf >>" \ + "${act_job[@]}" \ + --eventpath "$event_path" \ + -P "ubuntu-latest=catthehacker/ubuntu@sha256:b839c14c4410998529ec18f951262bdf87a2b23bc1467304d07b491b9455e074" \ + --concurrent-jobs << parameters.concurrent_jobs >> \ + "${act_env[@]}" \ + -s GITHUB_TOKEN="${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + --local-repository "mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba=/tmp/noop-action" \ + --local-repository "Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6=/tmp/noop-action" \ + --local-repository "actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d=/tmp/noop-action" \ + --local-repository "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9=/tmp/noop-action" + +workflows: + reth-act: + jobs: + - prepare-act-event: + context: + - circleci-repo-readonly-authenticated-github-token + - act: + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-<< matrix.wf >>" + requires: + - prepare-act-event + matrix: + parameters: + wf: + - book.yml + - compact.yml + - e2e.yml + - grafana.yml + - integration.yml + - lint-actions.yml + - stage.yml + - act: + wf: lint.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-lint-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - clippy-binaries + - clippy + - wasm + - riscv + - crate-checks-1 + - crate-checks-2 + - crate-checks-3 + - msrv + - docs + - fmt + - udeps + - book + - typos + - check-toml + - grafana + - no-test-deps + - feature-propagation + - cargo-deny: + name: lint-deny + - act: + wf: unit.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-unit-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - test + - state + - doc diff --git a/.circleci/prepare-act-event.sh b/.circleci/prepare-act-event.sh new file mode 100755 index 00000000000..99cccb27f3c --- /dev/null +++ b/.circleci/prepare-act-event.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly OUTPUT_DIR="${ACT_EVENT_OUTPUT_DIR:-/tmp/act-event}" +readonly EVENT_PATH="${OUTPUT_DIR}/github-event.json" + +mkdir -p "$OUTPUT_DIR" +rm -f "$EVENT_PATH" + +: "${CIRCLE_SHA1:?missing CircleCI revision}" +: "${CIRCLE_PROJECT_USERNAME:?missing CircleCI project owner}" +: "${CIRCLE_PROJECT_REPONAME:?missing CircleCI project repository}" + +repository="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + +if [[ "${CIRCLE_BRANCH:-}" == gh-readonly-queue/* ]]; then + event_name="merge_group" +elif [[ -n "${CIRCLE_PULL_REQUEST:-}" ]]; then + event_name="pull_request" +else + event_name="push" +fi + +case "$event_name" in + pull_request) + : "${CIRCLE_PULL_REQUEST:?pull-request trigger must provide CIRCLE_PULL_REQUEST}" + pr_number="${CIRCLE_PULL_REQUEST##*/}" + if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid pull-request URL: ${CIRCLE_PULL_REQUEST}" >&2 + exit 1 + fi + + pull_request_path="${OUTPUT_DIR}/pull-request.json" + curl --fail --silent --show-error --location \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${repository}/pulls/${pr_number}" > "$pull_request_path" + + event_head_sha="$(jq -er '.head.sha' "$pull_request_path")" + if [[ "$event_head_sha" != "$CIRCLE_SHA1" ]]; then + echo "PR head does not match CircleCI revision: ${CIRCLE_SHA1} != ${event_head_sha}" >&2 + exit 1 + fi + + jq '{ + action: "synchronize", + number: .number, + pull_request: ., + repository: .base.repo, + sender: .user + }' "$pull_request_path" > "$EVENT_PATH" + base_ref="$(jq -er '.base.ref | select(length > 0)' "$pull_request_path")" + head_ref="$(jq -er '.head.ref | select(length > 0)' "$pull_request_path")" + echo "act event: pull_request #${pr_number}, ${head_ref} -> ${base_ref}" + ;; + push) + : "${CIRCLE_BRANCH:?push trigger must provide CIRCLE_BRANCH}" + jq -n \ + --arg ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg after "$CIRCLE_SHA1" \ + '{ref: $ref, after: $after}' > "$EVENT_PATH" + echo "act event: push ${CIRCLE_BRANCH} at ${CIRCLE_SHA1}" + ;; + merge_group) + : "${CIRCLE_BRANCH:?merge-queue trigger must provide CIRCLE_BRANCH}" + if [[ ! "$CIRCLE_BRANCH" =~ ^gh-readonly-queue/(.+)/pr-([1-9][0-9]*)-[^/]+$ ]]; then + echo "merge_group requires a gh-readonly-queue//pr-- branch, got: ${CIRCLE_BRANCH}" >&2 + exit 1 + fi + base_ref="${BASH_REMATCH[1]}" + + if ! base_sha="$(git rev-parse "${CIRCLE_SHA1}^1" 2>/dev/null)"; then + git fetch --no-tags --depth=2 origin "$CIRCLE_SHA1" + base_sha="$(git rev-parse "${CIRCLE_SHA1}^1")" + fi + + jq -n \ + --arg head_sha "$CIRCLE_SHA1" \ + --arg head_ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg base_sha "$base_sha" \ + --arg base_ref "refs/heads/${base_ref}" \ + --arg repository "$repository" \ + --arg repository_name "$CIRCLE_PROJECT_REPONAME" \ + --arg repository_owner "$CIRCLE_PROJECT_USERNAME" \ + '{ + action: "checks_requested", + merge_group: { + head_sha: $head_sha, + head_ref: $head_ref, + base_sha: $base_sha, + base_ref: $base_ref + }, + repository: { + full_name: $repository, + name: $repository_name, + owner: {login: $repository_owner} + } + }' > "$EVENT_PATH" + echo "act event: merge_group ${CIRCLE_BRANCH} -> ${base_ref} at ${base_sha}" + ;; +esac diff --git a/.config/nextest.toml b/.config/nextest.toml index 01cb0158619..d07545e2400 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -6,6 +6,12 @@ slow-timeout = { period = "30s", terminate-after = 2 } filter = "test(general_state_tests)" slow-timeout = { period = "1m", terminate-after = 10 } +# valid_blocks covers a full fixture directory, like the grouped state tests. +# Keep its CircleCI runtime bounded by the same ten-minute budget. +[[profile.default.overrides]] +filter = "package(=ef-tests) and test(=valid_blocks)" +slow-timeout = { period = "1m", terminate-after = 10 } + [[profile.default.overrides]] filter = "test(eest_fixtures)" slow-timeout = { period = "2m", terminate-after = 10 } diff --git a/.github/scripts/check_rv32imac.sh b/.github/scripts/check_rv32imac.sh index 4e477cdbed1..988a95a3309 100755 --- a/.github/scripts/check_rv32imac.sh +++ b/.github/scripts/check_rv32imac.sh @@ -32,7 +32,7 @@ trap 'rm -rf -- "$tmpdir"' EXIT INT TERM for crate in "${crates_to_check[@]}"; do outfile="$tmpdir/$crate.log" - if cargo +stable build -p "$crate" --target riscv32imac-unknown-none-elf --no-default-features --color never >"$outfile" 2>&1; then + if cargo build -p "$crate" --target riscv32imac-unknown-none-elf --no-default-features --color never >"$outfile" 2>&1; then echo "✅ $crate" else echo "❌ $crate" diff --git a/.github/scripts/check_wasm.sh b/.github/scripts/check_wasm.sh index 15a8b8122ff..e4d5a8037a8 100755 --- a/.github/scripts/check_wasm.sh +++ b/.github/scripts/check_wasm.sh @@ -110,7 +110,7 @@ for crate in "${crates[@]}"; do fi outfile="$tmpdir/$crate.log" - if cargo +stable build -p "$crate" --target wasm32-wasip1 --no-default-features --color never >"$outfile" 2>&1; then + if cargo build -p "$crate" --target wasm32-wasip1 --no-default-features --color never >"$outfile" 2>&1; then echo "✅ $crate" else echo "❌ $crate" diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index ffdb189146f..ffc7eb13c1e 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -22,6 +22,8 @@ jobs: env: # Include the system trust store containing Aegis's CA for Bun and Node. NODE_EXTRA_CA_CERTS: /etc/ssl/certs/ca-certificates.crt + # Read by build-cargo-docs.sh, which `bun run build` also invokes. + RUST_NIGHTLY: nightly-2026-07-14 steps: - name: Secure runner uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 @@ -47,7 +49,7 @@ jobs: - name: Install Rust nightly uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: nightly + toolchain: ${{ env.RUST_NIGHTLY }} - name: Build docs run: cd docs/vocs && bash scripts/build-cargo-docs.sh diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index edfa4252704..fc6beaf006d 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -1,6 +1,6 @@ # Ensures that `Compact` codec changes are backwards compatible. # -# 1) checkout `main` +# 1) checkout the PR base, or the upstream base of an `op-*` branch for pushes to it # 2) randomly generate and serialize to disk many different type vectors with `Compact` (eg. Header, Transaction, etc) # 3) checkout `pr` # 4) deserialize previously generated test vectors @@ -39,17 +39,39 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: cache-on-failure: true + # An `op-v` branch builds on upstream tag `v`. Pushes to it must decode + # that release's data, not unreleased main formats. + - name: Select base + id: base + env: + BASE_REF: ${{ github.base_ref }} + REF: ${{ github.ref }} + run: | + set -euo pipefail + if [ -n "$BASE_REF" ]; then + base="$BASE_REF" + else + case "$REF" in + refs/heads/op-v*) base="${REF#refs/heads/op-}" ;; + refs/heads/op-*) + echo "no compact base for ${REF}: only op-v branches map to an upstream tag" >&2 + exit 1 + ;; + *) base=main ;; + esac + fi + echo "ref=$base" >> "$GITHUB_OUTPUT" - name: Checkout base uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.base_ref || 'main' }} + ref: ${{ steps.base.outputs.ref }} persist-credentials: false - # On `main` branch, generates test vectors and serializes them to disk using `Compact`. + # On the comparison base, generate test vectors and serialize them to disk using `Compact`. - name: Generate compact vectors run: | ${{ matrix.bin }} -- test-vectors compact --write @@ -60,4 +82,8 @@ jobs: persist-credentials: false # On incoming merge try to read and decode previously generated vectors with `Compact` - name: Read vectors + env: + # act restores the head with source mtimes older than the base build's artifacts. + # Keep the builds separate so Cargo cannot reuse libraries from the comparison base. + CARGO_TARGET_DIR: target/compact-head run: ${{ matrix.bin }} -- test-vectors compact --read diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 74739c415e3..c4bd8b3ab10 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -39,7 +39,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Install nextest uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 @@ -83,7 +83,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Install nextest uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 412d6db76c5..ef5f4619be6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -46,7 +46,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - name: Install Geth run: .github/scripts/install_geth.sh - name: Install nextest @@ -98,7 +98,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - name: Install nextest uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6cd6ab01ae0..556ea80ea04 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,7 +41,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: nightly + toolchain: nightly-2026-07-14 components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -69,7 +69,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: nightly + toolchain: nightly-2026-07-14 components: clippy - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -92,9 +92,10 @@ jobs: with: persist-credentials: false - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 + - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" target: wasm32-wasip1 - name: Install cargo-hack uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 @@ -109,7 +110,7 @@ jobs: - uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 - name: Run Wasm checks run: | - sudo apt update && sudo apt install gcc-multilib + sudo apt update && sudo apt install --yes gcc-multilib .github/scripts/check_wasm.sh riscv: @@ -127,7 +128,7 @@ jobs: - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" target: riscv32imac-unknown-none-elf - name: Install cargo-hack uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 @@ -164,7 +165,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - name: Install cargo-hack uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: @@ -220,7 +221,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: nightly + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -247,7 +248,7 @@ jobs: - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: nightly + toolchain: nightly-2026-07-14 components: rustfmt - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - name: Run fmt @@ -270,7 +271,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: nightly + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -300,7 +301,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: nightly + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -374,7 +375,7 @@ jobs: - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - name: Ensure no arbitrary or proptest dependency on default build run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0 @@ -394,7 +395,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: rui314/setup-mold@10ca16bf91dc22e05ebdc935cad9c75ea248f621 # v1 - uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 0b8af9b96e1..dd2ecce2edc 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -43,7 +43,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index cf325c776c9..f4317224ea6 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -46,7 +46,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: @@ -106,7 +106,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - name: Install nextest uses: tempoxyz/gh-actions/vendor/taiki-e/install-action@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6 with: @@ -138,7 +138,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - toolchain: stable + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: diff --git a/Cargo.lock b/Cargo.lock index 7640e82587a..fee096f1ce6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1051,7 +1051,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1062,7 +1062,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3027,7 +3027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.118", + "syn 1.0.109", ] [[package]] @@ -3512,7 +3512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5254,7 +5254,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5868,9 +5868,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.28" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" dependencies = [ "cc", "libc", @@ -6384,7 +6384,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11574,14 +11574,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -11653,8 +11653,8 @@ dependencies = [ "rustls-webpki", "security-framework", "security-framework-sys", - "webpki-root-certs 1.0.7", - "windows-sys 0.61.2", + "webpki-root-certs 1.0.10", + "windows-sys 0.52.0", ] [[package]] @@ -11665,9 +11665,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -12288,7 +12288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12547,7 +12547,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -12814,9 +12814,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -13753,14 +13753,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" dependencies = [ - "webpki-root-certs 1.0.7", + "webpki-root-certs 1.0.10", ] [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -13771,14 +13771,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -13821,7 +13821,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -14547,9 +14547,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/deny.toml b/deny.toml index be45b22db44..cf907cb3124 100644 --- a/deny.toml +++ b/deny.toml @@ -12,6 +12,13 @@ ignore = [ "RUSTSEC-2023-0089", # https://rustsec.org/advisories/RUSTSEC-2026-0173 proc-macro-error2 is unmaintained "RUSTSEC-2026-0173", + # crypto-bigint 0.5 has no patched release compatible with elliptic-curve 0.13. + "RUSTSEC-2026-0387", + # zstd 0.13 comes from reth-zstd-compressors 0.7; the patched zstd 0.14.3 needs reth-core 0.8 and Rust 1.96. + "RUSTSEC-2026-0391", + # zstd-safe 7 comes from reth-zstd-compressors 0.7; the patched zstd-safe 8.2.0 needs reth-core 0.8 and Rust 1.96. + "RUSTSEC-2026-0398", + "RUSTSEC-2026-0403", ] # This section is considered when running `cargo deny check bans`. diff --git a/docs/vocs/scripts/build-cargo-docs.sh b/docs/vocs/scripts/build-cargo-docs.sh index 634f4acf77b..e03c324e779 100755 --- a/docs/vocs/scripts/build-cargo-docs.sh +++ b/docs/vocs/scripts/build-cargo-docs.sh @@ -10,6 +10,6 @@ echo "Building cargo docs..." # Build the documentation export RUSTDOCFLAGS="--cfg docsrs --show-type-layout --generate-link-to-definition --enable-index-page -Zunstable-options" -cargo +nightly docs --exclude "example-*" +cargo +"${RUST_NIGHTLY:-nightly}" docs --exclude "example-*" echo "Cargo docs built successfully at ./target/doc" From b24c7c265355be78e4f11fd017cac973e817fef0 Mon Sep 17 00:00:00 2001 From: Akosh Farkash Date: Wed, 4 Nov 2026 20:33:19 +0000 Subject: [PATCH 112/133] chore: replace upstream CODEOWNERS with OP owners --- .github/CODEOWNERS | 60 +++++++--------------------------------------- 1 file changed, 9 insertions(+), 51 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6887f88baf4..1eb6e1345fc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,51 +1,9 @@ -* @gakonst -crates/chain-state/ @mattsse -crates/chainspec/ @Rjected @joshieDo @mattsse -crates/cli/ @mattsse @Rjected -crates/config/ @shekhirin @mattsse @Rjected -crates/consensus/ @mattsse @Rjected -crates/e2e-test-utils/ @mattsse @Rjected @klkvr -crates/engine/ @mattsse @Rjected @mediocregopher -crates/era/ @mattsse -crates/era-downloader/ @mattsse -crates/era-utils/ @mattsse -crates/errors/ @mattsse -crates/ethereum/ @mattsse @Rjected -crates/etl/ @joshieDo @shekhirin -crates/evm/ @mattsse @Rjected @klkvr -crates/exex/ @shekhirin -crates/fs-util/ @mattsse -crates/metrics/ @mattsse @Rjected -crates/net/ @mattsse @Rjected -crates/net/downloaders/ @Rjected -crates/node/ @mattsse @Rjected @klkvr -crates/payload/ @mattsse @Rjected -crates/primitives-traits/ @Rjected @mattsse @klkvr -crates/primitives/ @Rjected @mattsse @klkvr -crates/prune/ @shekhirin @joshieDo -crates/ress/ @shekhirin @Rjected -crates/revm/ @mattsse -crates/rpc/ @mattsse @Rjected -crates/stages/ @shekhirin @mediocregopher -crates/static-file/ @joshieDo @shekhirin -crates/stateless/ @mattsse -crates/storage/codecs/ @joshieDo -crates/storage/db-api/ @joshieDo -crates/storage/db-common/ @Rjected -crates/storage/db/ @joshieDo -crates/storage/errors/ @joshieDo -crates/storage/libmdbx-rs/ @shekhirin -crates/storage/nippy-jar/ @joshieDo @shekhirin -crates/storage/provider/ @joshieDo @shekhirin -crates/storage/storage-api/ @joshieDo -crates/storage/storage-overlay/ @joshieDo @shekhirin @mediocregopher -crates/tasks/ @mattsse @DaniPopes -crates/tokio-util/ @mattsse -crates/tracing/ @mattsse @shekhirin -crates/tracing-otlp/ @mattsse @Rjected -crates/transaction-pool/ @mattsse -crates/trie/ @Rjected @shekhirin @mediocregopher -bin/reth/ @mattsse @shekhirin @Rjected -bin/reth-bench-compare/ @mediocregopher @shekhirin -etc/ @Rjected @shekhirin -.github/ @gakonst @DaniPopes +# OP fork of paradigmxyz/reth - default to monorepo-reviewers +# Keep this file on upstream syncs. +* @ethereum-optimism/monorepo-reviewers + +# CI workflows +/.github/workflows/ @ethereum-optimism/cloud-security + +# Must come last to avoid being overridden +/.github/CODEOWNERS @ethereum-optimism/cloud-security From 7ca15f31a42a802482f8134f88e57c74a2c0866c Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Tue, 29 Sep 2026 11:04:31 +0000 Subject: [PATCH 113/133] fix(rpc): time out slow compressed requests (#27530) (cherry picked from commit d994469b9b88897fe0c6f971d16a59f6bf26001e) --- Cargo.lock | 2 + crates/rpc/rpc-layer/Cargo.toml | 4 +- .../rpc/rpc-layer/src/decompression_layer.rs | 82 +++++++++++++++++-- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fee096f1ce6..475ef86c651 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10367,8 +10367,10 @@ version = "2.7.0" dependencies = [ "alloy-rpc-types-engine", "brotli", + "bytes", "flate2", "http", + "http-body", "http-body-util", "jsonrpsee", "jsonrpsee-http-client", diff --git a/crates/rpc/rpc-layer/Cargo.toml b/crates/rpc/rpc-layer/Cargo.toml index b9507fc2add..96c7940544d 100644 --- a/crates/rpc/rpc-layer/Cargo.toml +++ b/crates/rpc/rpc-layer/Cargo.toml @@ -23,8 +23,10 @@ tower-http = { workspace = true, features = ["full"] } tracing.workspace = true [dev-dependencies] +bytes.workspace = true +http-body = "1" reqwest.workspace = true -tokio = { workspace = true, features = ["macros"] } +tokio = { workspace = true, features = ["macros", "test-util"] } jsonrpsee = { workspace = true, features = ["server"] } flate2.workspace = true diff --git a/crates/rpc/rpc-layer/src/decompression_layer.rs b/crates/rpc/rpc-layer/src/decompression_layer.rs index 077003c5e98..3ee2e939a3a 100644 --- a/crates/rpc/rpc-layer/src/decompression_layer.rs +++ b/crates/rpc/rpc-layer/src/decompression_layer.rs @@ -6,6 +6,7 @@ use std::{ pin::Pin, sync::Arc, task::{Context, Poll}, + time::Duration, }; use tokio::sync::Semaphore; use tower::{Layer, Service}; @@ -22,6 +23,9 @@ use tracing::debug; /// being rejected. const MAX_CONCURRENT_DECOMPRESSIONS: usize = 8; +/// Maximum time to read and decompress a compressed request body. +const DECOMPRESSION_BODY_READ_TIMEOUT: Duration = Duration::from_secs(30); + /// This layer is a wrapper around [`tower_http::decompression::RequestDecompressionLayer`] that /// integrates with jsonrpsee's HTTP types. #[expect(missing_debug_implementations)] @@ -140,15 +144,23 @@ where .await .expect("decompression semaphore is never closed"); - let body = match Limited::new(body, max_body_size).collect().await { - Ok(body) => body, - Err(err) if err.is::() => { + let body = match tokio::time::timeout( + DECOMPRESSION_BODY_READ_TIMEOUT, + Limited::new(body, max_body_size).collect(), + ) + .await + { + Ok(Ok(body)) => body, + Ok(Err(err)) if err.is::() => { return Ok(err_response(StatusCode::PAYLOAD_TOO_LARGE, "Payload Too Large")); } - Err(err) => { + Ok(Err(err)) => { debug!(target: "rpc::decompression", %err, "Failed to decompress request body"); return Ok(err_response(StatusCode::BAD_REQUEST, "Invalid compressed body")); } + Err(_) => { + return Ok(err_response(StatusCode::REQUEST_TIMEOUT, "Request body timed out")); + } }; // Decompression is done; release the permit before dispatching the request. @@ -221,10 +233,19 @@ fn err_response(status: StatusCode, msg: &'static str) -> HttpResponse { #[cfg(test)] mod tests { use super::*; + use bytes::Bytes; use http::header::CONTENT_ENCODING; + use http_body::{Body, Frame}; use http_body_util::BodyExt; use jsonrpsee_http_client::{HttpRequest, HttpResponse}; - use std::{convert::Infallible, future::ready, io::Write}; + use std::{ + convert::Infallible, + future::ready, + io::Write, + pin::Pin, + task::{Context, Poll}, + }; + use tokio::sync::Notify; const TEST_DATA: &str = r#"{"method":"test","params":["test data"],"id":1}"#; const DEFAULT_MAX_SIZE: usize = 15 * 1024 * 1024; @@ -234,6 +255,23 @@ mod tests { #[derive(Clone)] struct MockEchoService; + struct PendingBody { + polled: Arc, + } + + impl Body for PendingBody { + type Data = Bytes; + type Error = Infallible; + + fn poll_frame( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + self.polled.notify_one(); + Poll::Pending + } + } + impl Service for MockEchoService { type Response = HttpResponse; type Error = Infallible; @@ -303,6 +341,15 @@ mod tests { .unwrap() } + fn build_pending_compressed_request(encoding: &str) -> (HttpRequest, Arc) { + let polled = Arc::new(Notify::new()); + let request = HttpRequest::builder() + .header(CONTENT_ENCODING, encoding) + .body(HttpBody::new(PendingBody { polled: polled.clone() })) + .unwrap(); + (request, polled) + } + #[tokio::test] async fn configured_algorithms_are_decompressed_without_content_length() { let cases: [(&str, Compressor); 4] = [ @@ -415,6 +462,31 @@ mod tests { } } + #[tokio::test(start_paused = true)] + async fn slow_compressed_body_times_out_and_releases_permit() { + let mut layer = DecompressionLayer::new(&["gzip"], DEFAULT_MAX_SIZE); + layer.decompression_permits = Arc::new(Semaphore::new(1)); + let permits = layer.decompression_permits.clone(); + let service = layer.layer(MockEchoService); + + let (slow_request, slow_body_polled) = build_pending_compressed_request("gzip"); + let mut slow_service = service.clone(); + let slow = tokio::spawn(async move { slow_service.call(slow_request).await.unwrap() }); + slow_body_polled.notified().await; + assert_eq!(permits.available_permits(), 0); + + tokio::time::advance(DECOMPRESSION_BODY_READ_TIMEOUT).await; + assert_eq!(slow.await.unwrap().status(), StatusCode::REQUEST_TIMEOUT); + assert_eq!(permits.available_permits(), 1); + + let mut fast_service = service; + let response = fast_service + .call(build_compressed_request("gzip", compress_gzip(TEST_DATA.as_bytes()))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + #[tokio::test] async fn identity_and_unencoded_bodies_are_not_eagerly_collected() { for encoding in [None, Some("identity"), Some("Identity")] { From 7ef472dfd17bd3816e74ab6b6dd71f2a9dc71a51 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:44:08 +0000 Subject: [PATCH 114/133] fix(storage): use trie changesets for all overlays (#27634) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> (cherry picked from commit 078d026239a1977f5d79eeb67fa44d2087ebf7e5) --- crates/storage/storage-overlay/src/provider.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 0999d125b45..624e0405a05 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -534,14 +534,14 @@ where fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let input = - self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), false)?; + self.build_overlay(TrieInputSorted::from_state(hashed_state.into_sorted()), true)?; Ok(>::overlay_root_from_nodes(self.provider().tx(), input)?) }) } fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { - let input = self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + let input = self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; Ok( as DatabaseStateRoot<_>>::overlay_root_from_nodes( self.provider().tx(), input, @@ -602,7 +602,7 @@ where HashedPostState::from_hashed_storage(hashed_address, hashed_storage) .into_sorted(), ), - false, + true, )?; >::overlay_root(self.provider().tx(), address, input) .map_err(|err| ProviderError::Database(err.into())) @@ -633,7 +633,7 @@ where HashedPostState::from_hashed_storage(hashed_address, hashed_storage) .into_sorted(), ), - false, + true, )?; >::overlay_storage_multiproof( self.provider().tx(), @@ -665,7 +665,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -683,7 +683,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -701,7 +701,7 @@ where ) -> ProviderResult { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let input = TrieInput::new( Arc::unwrap_or_clone(nodes).into(), Arc::unwrap_or_clone(state).into(), @@ -720,7 +720,7 @@ where ) -> ProviderResult> { reth_trie_db::with_adapter!(self.provider(), |A| { let TrieInputSorted { nodes, state, prefix_sets } = - self.build_overlay(TrieInputSorted::from_unsorted(input), false)?; + self.build_overlay(TrieInputSorted::from_unsorted(input), true)?; let witness = TrieWitness::new( InMemoryTrieCursorFactory::new( DatabaseTrieCursorFactory::<_, A>::new(self.provider().tx()), @@ -769,7 +769,7 @@ where return Ok(hashed_state) } - let overlay_state = self.build_overlay(TrieInputSorted::default(), false)?.state; + let overlay_state = self.build_overlay(TrieInputSorted::default(), true)?.state; zero_destroyed_account_storage( &HashedPostStateCursorFactory::new( DatabaseHashedCursorFactory::new(self.provider().tx()), From 272896ca63556d05292e8f6584634a80cf0a647c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 4 Oct 2026 19:29:38 +0000 Subject: [PATCH 115/133] fix(net): never send empty incomplete eth/70 receipts (#27717) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit 42fa3c569ad182914d26db410919d6a10c7b4859) --- crates/net/network/src/eth_requests.rs | 217 ++++++++++++++++++++++++- crates/net/p2p/src/receipts/client.rs | 17 +- 2 files changed, 220 insertions(+), 14 deletions(-) diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index b549b31720e..7900629bd38 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -7,9 +7,10 @@ use crate::{ use alloy_consensus::{constants::KECCAK_EMPTY, BlockHeader, ReceiptWithBloom}; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{Bytes, B256}; -use alloy_rlp::Encodable; +use alloy_rlp::{length_of_length, Encodable}; use futures::StreamExt; use reth_eth_wire::{ + message::MAX_MESSAGE_SIZE, snap::{ AccountData, AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage, GetAccountRangeMessage, GetStorageRangesMessage, SnapProtocolMessage, StorageData, @@ -301,7 +302,11 @@ where /// Handles partial responses for [`GetReceipts70`] queries. /// - /// This will adhere to the soft limit but allow filling the last vec partially. + /// This will adhere to the soft limit but allow filling the last vec partially. A response + /// never ends with an empty incomplete block: if not even the next receipt fits, the response + /// ends after the last complete block, except when it would otherwise be empty, in which case + /// the first receipt is served even if it exceeds the soft limit. If it cannot fit within + /// [`MAX_MESSAGE_SIZE`], including the response framing, the response is empty and complete. fn on_receipts70_request( &self, _peer_id: PeerId, @@ -317,6 +322,11 @@ where let mut total_bytes = 0usize; let mut last_block_incomplete = false; + // The request ID is unavailable here. Reserve the message code, the largest request ID, + // the incomplete flag, and the three RLP list headers to bound the full encoded response. + let max_first_receipt_size = MAX_MESSAGE_SIZE - + (1 + u64::MAX.length() + false.length() + 3 * length_of_length(MAX_MESSAGE_SIZE)); + for (idx, hash) in block_hashes.into_iter().enumerate() { if idx >= MAX_RECEIPTS_SERVE { break @@ -348,18 +358,30 @@ where continue; } + // If nothing has been added to the response yet, serve the first receipt even if it + // exceeds the soft limit, provided the framed response fits within the hard limit. + let always_serve_first = receipts.is_empty(); + let remaining = block_receipts.len(); let mut partial_block = Vec::new(); for receipt in block_receipts { let receipt_size = receipt.length(); - if total_bytes + receipt_size > SOFT_RESPONSE_LIMIT { + if total_bytes + receipt_size > SOFT_RESPONSE_LIMIT && + !(always_serve_first && + partial_block.is_empty() && + receipt_size <= max_first_receipt_size) + { break; } total_bytes += receipt_size; partial_block.push(receipt); } - receipts.push(partial_block); - last_block_incomplete = true; + // An empty partial block delivers nothing, so it is never sent. The requester can + // simply continue with this block in its next request. + if !partial_block.is_empty() { + last_block_incomplete = partial_block.len() < remaining; + receipts.push(partial_block); + } break; } @@ -855,12 +877,14 @@ pub enum IncomingEthRequest { #[cfg(test)] mod tests { use super::*; - use alloy_consensus::constants::EMPTY_ROOT_HASH; + use alloy_consensus::{constants::EMPTY_ROOT_HASH, Header, TxType}; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, }; - use alloy_primitives::{keccak256, Address, TxHash, B128, U256}; + use alloy_primitives::{keccak256, Address, Log, LogData, TxHash, B128, U256}; + use reth_eth_wire::{message::RequestPair, EthMessage, EthStreamInner, EthVersion}; + use reth_ethereum_primitives::Receipt; use reth_network_api::test_utils::PeersHandle; use reth_primitives_traits::Account; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; @@ -1559,4 +1583,183 @@ mod tests { // first missing account instead of skipping it and shifting later positions. assert_eq!(provider.snap_storage_ranges_remaining(), 1); } + + /// Creates a receipt whose single log carries `data_len` bytes of data. + fn receipt_with_data(data_len: usize) -> Receipt { + Receipt { + tx_type: TxType::Legacy, + success: true, + cumulative_gas_used: 21_000, + logs: vec![Log { + address: Address::ZERO, + data: LogData::new_unchecked(Vec::new(), vec![0u8; data_len].into()), + }], + } + } + + /// Stores the receipts as block `number` and returns the block hash. + fn insert_receipts_block( + provider: &MockEthProvider, + number: u64, + receipts: Vec, + ) -> B256 { + let hash = B256::with_last_byte(number as u8); + provider.add_header(hash, Header { number, ..Default::default() }); + provider.add_receipts(number, receipts); + hash + } + + /// Sends a [`GetReceipts70`] request to the handler and returns the response. + async fn get_receipts70( + provider: MockEthProvider, + first_block_receipt_index: u64, + block_hashes: Vec, + ) -> Receipts70 { + let handler = snap_handler(provider); + let (response, rx) = oneshot::channel(); + handler.on_receipts70_request( + PeerId::default(), + GetReceipts70 { first_block_receipt_index, block_hashes }, + response, + ); + rx.await.unwrap().unwrap() + } + + #[tokio::test] + async fn receipts70_serves_oversized_first_receipt() { + let provider = MockEthProvider::default(); + let oversized = receipt_with_data(SOFT_RESPONSE_LIMIT + 1024); + let hash = insert_receipts_block(&provider, 1, vec![oversized.clone()]); + + let resp = get_receipts70(provider, 0, vec![hash]).await; + + // The block holds exactly one receipt, so it is complete. + assert_eq!( + resp, + Receipts70 { last_block_incomplete: false, receipts: vec![vec![oversized]] } + ); + } + + #[tokio::test] + async fn receipts70_serves_only_oversized_first_receipt_of_partial_block() { + let provider = MockEthProvider::default(); + let oversized = receipt_with_data(SOFT_RESPONSE_LIMIT + 1024); + let hash = + insert_receipts_block(&provider, 1, vec![oversized.clone(), receipt_with_data(16)]); + + let resp = get_receipts70(provider, 0, vec![hash]).await; + + // The oversized receipt exhausts the budget, so the rest of the block is left out. + assert_eq!( + resp, + Receipts70 { last_block_incomplete: true, receipts: vec![vec![oversized]] } + ); + } + + #[tokio::test] + async fn receipts70_serves_oversized_receipt_at_first_receipt_index() { + let provider = MockEthProvider::default(); + let oversized = receipt_with_data(SOFT_RESPONSE_LIMIT + 1024); + let hash = insert_receipts_block( + &provider, + 1, + vec![receipt_with_data(16), oversized.clone(), receipt_with_data(16)], + ); + + let resp = get_receipts70(provider, 1, vec![hash]).await; + + assert_eq!( + resp, + Receipts70 { last_block_incomplete: true, receipts: vec![vec![oversized]] } + ); + } + + #[tokio::test] + async fn receipts70_omits_empty_incomplete_block() { + let provider = MockEthProvider::default(); + let first = vec![receipt_with_data(SOFT_RESPONSE_LIMIT - 1024)]; + let first_hash = insert_receipts_block(&provider, 1, first.clone()); + // The first receipt of the second block does not fit into the remaining budget. + let second_hash = insert_receipts_block(&provider, 2, vec![receipt_with_data(4096)]); + + let resp = get_receipts70(provider, 0, vec![first_hash, second_hash]).await; + + assert_eq!(resp, Receipts70 { last_block_incomplete: false, receipts: vec![first] }); + } + + #[tokio::test] + async fn receipts70_serves_partial_block() { + let provider = MockEthProvider::default(); + let receipts = vec![receipt_with_data(16 * 1024); 200]; + let hash = insert_receipts_block(&provider, 1, receipts.clone()); + + let resp = get_receipts70(provider, 0, vec![hash]).await; + + assert!(resp.last_block_incomplete); + assert_eq!(resp.receipts.len(), 1); + let partial = &resp.receipts[0]; + assert!(!partial.is_empty() && partial.len() < receipts.len()); + assert_eq!(partial.as_slice(), &receipts[..partial.len()]); + assert!(partial.iter().map(Encodable::length).sum::() <= SOFT_RESPONSE_LIMIT); + } + + #[tokio::test] + async fn receipts70_omits_receipt_above_hard_limit() { + let provider = MockEthProvider::default(); + let hash = + insert_receipts_block(&provider, 1, vec![receipt_with_data(MAX_MESSAGE_SIZE + 1024)]); + + let resp = get_receipts70(provider, 0, vec![hash]).await; + + assert_eq!(resp, Receipts70 { last_block_incomplete: false, receipts: Vec::new() }); + } + + #[test_case(0, 0; "first receipt at hard limit")] + #[test_case(1, 0; "continuation receipt at hard limit")] + #[test_case(0, 1; "first receipt framing exceeds hard limit")] + #[test_case(1, 1; "continuation receipt framing exceeds hard limit")] + #[tokio::test] + async fn receipts70_enforces_framed_response_limit(first_receipt_index: u64, excess: usize) { + // Measure the actual wire encoding with the largest request ID. At these sizes, changing + // the log data length does not change the width of any RLP length prefix. + let data_len = MAX_MESSAGE_SIZE - 1024; + let encoded = encode_receipts70_response(Receipts70 { + last_block_incomplete: false, + receipts: vec![vec![receipt_with_data(data_len)]], + }); + let receipt = receipt_with_data(data_len + MAX_MESSAGE_SIZE - encoded.len() + excess); + assert!(receipt.length() < MAX_MESSAGE_SIZE); + let expected = + Receipts70 { last_block_incomplete: false, receipts: vec![vec![receipt.clone()]] }; + assert_eq!(encode_receipts70_response(expected.clone()).len(), MAX_MESSAGE_SIZE + excess); + + let provider = MockEthProvider::default(); + let mut receipts = vec![receipt_with_data(16); first_receipt_index as usize]; + receipts.push(receipt); + let hash = insert_receipts_block(&provider, 1, receipts); + + let resp = get_receipts70(provider, first_receipt_index, vec![hash]).await; + + if excess == 0 { + assert_eq!(resp, expected); + } else { + assert_eq!(resp, Receipts70 { last_block_incomplete: false, receipts: Vec::new() }); + } + let encoded = encode_receipts70_response(resp); + assert!(encoded.len() <= MAX_MESSAGE_SIZE); + EthStreamInner::::new(EthVersion::Eth70) + .decode_message(encoded.as_ref().into()) + .unwrap(); + } + + /// Encodes a receipts response with the largest possible request ID. + fn encode_receipts70_response(response: Receipts70) -> Bytes { + EthStreamInner::::new(EthVersion::Eth70) + .encode_message(EthMessage::Receipts70(RequestPair { + request_id: u64::MAX, + message: response, + })) + .unwrap() + .into() + } } diff --git a/crates/net/p2p/src/receipts/client.rs b/crates/net/p2p/src/receipts/client.rs index 54def51bf5f..dcafa7ee166 100644 --- a/crates/net/p2p/src/receipts/client.rs +++ b/crates/net/p2p/src/receipts/client.rs @@ -12,10 +12,14 @@ pub type ReceiptsFut = /// Response from a receipts request. /// -/// **Note for [`ReceiptsClient`] callers:** the network layer handles eth/70 -/// continuation rounds internally, so `last_block_incomplete` is always `false` -/// by the time this response reaches a [`ReceiptsClient`] consumer. The field -/// exists for internal use by the fetcher during multi-round assembly. +/// Eth/70 responses may contain an incomplete last block, indicated by +/// [`last_block_incomplete`](Self::last_block_incomplete). The network layer forwards this flag +/// without fetching the remaining receipts. +/// +/// [`ReceiptsClient`] does not expose a receipt offset. Resuming an incomplete block requires a +/// lower-level eth/70 [`GetReceipts70`](reth_eth_wire_types::GetReceipts70) request with that block +/// first in `block_hashes` and `first_block_receipt_index` set to the number of receipts already +/// received for it. Responses from eth/69 and older peers always set this flag to `false`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ReceiptsResponse { /// Receipts grouped by block, in the same order as the requested hashes. @@ -23,9 +27,8 @@ pub struct ReceiptsResponse { /// When `true`, the **last** block in [`receipts`](Self::receipts) was /// truncated by the remote peer (eth/70 `Receipts70.last_block_incomplete`). /// - /// This is used internally by the fetcher to drive continuation rounds. - /// Responses surfaced through [`ReceiptsClient`] always have this set to - /// `false`. + /// The remaining receipts of that block have to be requested separately. Responses from + /// eth/69 and older peers always have this set to `false`. pub last_block_incomplete: bool, } From 74655bae9116c3a0c58f545670d1a12443f9d86b Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 7 Oct 2026 23:17:37 +0000 Subject: [PATCH 116/133] fix(trie): rewind backward overlay seeks (#27782) (cherry picked from commit 5868ee478841e1eaffc2fae740b60b7d5a920896) --- crates/trie/trie/src/hashed_cursor/post_state.rs | 4 ++++ crates/trie/trie/src/trie_cursor/in_memory.rs | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index ee460545e53..f8625bc1b7b 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -299,6 +299,10 @@ where /// The returned account key is memoized and the cursor remains positioned at that key until /// [`HashedCursor::seek`] or [`HashedCursor::next`] are called. fn seek(&mut self, key: B256) -> Result, DatabaseError> { + // Forward scans can reuse cursor positions; backward seeks and exhausted cursors cannot. + if self.last_key.is_none_or(|last| key < last) { + self.reset(); + } let post_state_entry = self.post_state_cursor.seek(&key).cloned().map(|(k, v)| (k, v.into_option())); diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 1d704cc4abf..8db7853e804 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -238,6 +238,10 @@ impl TrieCursor for InMemoryTrieCursor<'_, C> { &mut self, key: Nibbles, ) -> Result, DatabaseError> { + // Forward scans can reuse cursor positions; backward seeks and exhausted cursors cannot. + if self.last_key.is_none_or(|last| key < last) { + self.reset(); + } let mem_entry = self.in_memory_cursor.seek(&key); if let Some((mem_key, entry_inner)) = mem_entry && @@ -281,6 +285,10 @@ impl TrieCursor for InMemoryTrieCursor<'_, C> { &mut self, key: Nibbles, ) -> Result, DatabaseError> { + // Forward scans can reuse cursor positions; backward seeks and exhausted cursors cannot. + if self.last_key.is_none_or(|last| key < last) { + self.reset(); + } let mem_entry = self.in_memory_cursor.seek(&key); if let Some((mem_key, Some(node))) = mem_entry && From ef70a28608c822ee0c684bc2f3a06a399a91a45f Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:39:09 +0000 Subject: [PATCH 117/133] fix(trie): remove orphaned branches (#27832) Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com> Co-authored-by: Brian Picciano (cherry picked from commit 5e309af68bb064eb594807bf8d2b2e6638b91c27) --- crates/trie/trie/src/trie.rs | 6 + crates/trie/trie/src/walker.rs | 239 ++++++++++++++++++++++++++++++--- 2 files changed, 230 insertions(+), 15 deletions(-) diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index bf3144046de..9ff833dc9ce 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -73,6 +73,9 @@ impl StateRoot { } /// Configures the state root walker to visit all children of changed branch paths. + /// + /// When updates are retained, also removes stored descendants hidden by a changed branch's + /// tree mask. Regenerated branch updates take precedence over these deletions. pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { self.walk_all_changed_branch_children = enabled; self @@ -586,6 +589,9 @@ impl StorageRoot { } /// Configures the storage root walker to visit all children of changed branch paths. + /// + /// When updates are retained, also removes stored descendants hidden by a changed branch's + /// tree mask. Regenerated branch updates take precedence over these deletions. pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { self.walk_all_changed_branch_children = enabled; self diff --git a/crates/trie/trie/src/walker.rs b/crates/trie/trie/src/walker.rs index a8442578575..4702d86b3a0 100644 --- a/crates/trie/trie/src/walker.rs +++ b/crates/trie/trie/src/walker.rs @@ -4,22 +4,10 @@ use crate::{ BranchNodeCompact, Nibbles, }; use alloy_primitives::{map::HashSet, B256}; -use alloy_trie::proof::AddedRemovedKeys; +use alloy_trie::{proof::AddedRemovedKeys, TrieMask}; use reth_storage_errors::db::DatabaseError; use tracing::{instrument, trace}; -#[cfg(test)] -use crate::trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory}; - -#[cfg(test)] -use alloy_primitives::map::B256Map; - -#[cfg(test)] -use alloy_trie::TrieMask; - -#[cfg(test)] -use std::collections::BTreeMap; - #[cfg(feature = "metrics")] use crate::metrics::WalkerMetrics; @@ -126,6 +114,10 @@ impl> TrieWalker { } /// Configures the walker to treat every child of a matching branch path as unskippable. + /// + /// When deletions are retained, also collects stored descendants hidden by a matching + /// branch's tree mask. These keys appear in the deletion set even if traversal never visits + /// them. Root calculations can replace these deletions with regenerated branch updates. pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { self.walk_all_changed_branch_children = enabled; self @@ -354,6 +346,8 @@ impl> TrieWalker { return Ok(()) } + self.remove_orphaned_descendants(key, node.tree_mask)?; + // Create a new CursorSubNode and push it to the stack. let subnode = CursorSubNode::new(key, Some(node)); let position = subnode.position(); @@ -415,13 +409,62 @@ impl> TrieWalker { Ok(()) } + + /// Removes cached descendants hidden by a changed branch's tree mask during recovery. + /// + /// For example, a change under `0x3a` can leave stored `0x3c` and `0x3c4...` entries while + /// parent `0x3` has tree-mask bit `c` clear. The walker skips those entries and therefore never + /// records their keys for deletion. + /// + /// Collect the hidden cache keys without using their hashes; regenerated branch updates take + /// precedence over these deletions. The cursor must be positioned on this parent; restore that + /// position afterwards so traversal continues unchanged. + fn remove_orphaned_descendants( + &mut self, + key: Nibbles, + tree_mask: TrieMask, + ) -> Result<(), DatabaseError> { + let missing_children = !tree_mask; + if self.walk_all_changed_branch_children && + !missing_children.is_empty() && + let Some(removed_keys) = &mut self.removed_keys && + self.changes.contains(&key) + { + for nibble in missing_children.iter() { + let mut prefix = key; + prefix.push(nibble); + let mut entry = self.cursor.seek(prefix)?; + #[cfg(feature = "metrics")] + self.metrics.inc_branch_nodes_seeked(); + while let Some((path, _)) = entry && + path.starts_with(&prefix) + { + removed_keys.insert(path); + entry = self.cursor.next()?; + } + } + self.cursor.seek_exact(key)?; + #[cfg(feature = "metrics")] + self.metrics.inc_branch_nodes_seeked(); + } + Ok(()) + } } #[cfg(test)] mod tests { use super::*; - use crate::prefix_set::PrefixSetMut; - use alloy_primitives::B256; + use crate::{ + mock::KeyVisitType, + prefix_set::PrefixSetMut, + progress::StorageRootProgress, + test_utils::{storage_root_prehashed, TrieTestHarness}, + trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory}, + updates::StorageTrieUpdates, + StorageRoot, + }; + use alloy_primitives::{map::B256Map, B256, U256}; + use std::collections::BTreeMap; fn branch_node(state_mask: u16, tree_mask: u16, hash_mask: u16) -> BranchNodeCompact { let hash_count = hash_mask.count_ones() as usize; @@ -495,4 +538,170 @@ mod tests { assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x4]))); assert!(!walker.can_skip_current_node); } + + #[test] + fn changed_branch_children_remove_orphans() { + for regenerate_branch in [false, true] { + let mut storage = BTreeMap::from([ + (B256::right_padding_from(&[0x3a]), U256::ONE), + (B256::right_padding_from(&[0x3a, 0x01]), U256::from(10)), + (B256::right_padding_from(&[0x3a, 0x10]), U256::from(7)), + (B256::right_padding_from(&[0x3b]), U256::from(2)), + (B256::right_padding_from(&[0x3c, 0x47]), U256::from(3)), + (B256::right_padding_from(&[0x80]), U256::from(4)), + (B256::right_padding_from(&[0x81]), U256::from(5)), + (B256::right_padding_from(&[0x80, 0x10]), U256::from(8)), + ]); + if regenerate_branch { + storage.insert(B256::right_padding_from(&[0x3c, 0x48]), U256::from(6)); + storage.insert(B256::right_padding_from(&[0x3c, 0x47, 0x10]), U256::from(9)); + } + let mut harness = TrieTestHarness::new(storage.clone()); + let expected_nodes = harness + .storage_trie_updates() + .storage_nodes + .iter() + .map(|(key, node)| (*key, node.clone())) + .collect::>(); + let mut stored_nodes = expected_nodes.clone(); + let parent = Nibbles::from_nibbles([0x3]); + stored_nodes.get_mut(&parent).unwrap().tree_mask &= + !TrieMask::new((1 << 0xa) | (1 << 0xc)); + + // The cached branch at 0x3c and its descendants are unreachable through 0x3's mask. + // Include a descendant which should be regenerated and one which should disappear. + let orphan_paths = [ + Nibbles::from_nibbles([0x3, 0xc]), + Nibbles::from_nibbles([0x3, 0xc, 0x4]), + Nibbles::from_nibbles([0x3, 0xc, 0x4, 0xf]), + Nibbles::from_nibbles([0x3, 0xf, 0x1]), + Nibbles::from_nibbles([0x3, 0x1, 0x4]), + ]; + // The valid branch under 0x3a is also hidden by the mask. Its regenerated update can + // be split off before the walker finishes 0x3, so resumption must not delete it again. + assert!(expected_nodes.contains_key(&Nibbles::from_nibbles([0x3, 0xa]))); + for path in orphan_paths { + let mut stale_node = branch_node(0b11, 0, 0b11); + stale_node.hashes = vec![B256::repeat_byte(0xee); 2].into(); + stored_nodes.insert(path, stale_node); + } + harness.set_trie_nodes(stored_nodes.clone()); + + // A change under sibling 0x3a must also clean up 0x3c and state-mask gaps. + let prefix_set = + PrefixSetMut::from([Nibbles::unpack(B256::right_padding_from(&[0x3a]))]).freeze(); + for (enabled, threshold) in [(false, u64::MAX), (true, u64::MAX), (true, 1)] { + // Also rebuild the neighboring branch in checkpointed runs. Resuming a skipped + // branch after a leaf checkpoint currently fails even without orphan cleanup. + let changes = if threshold == 1 { + PrefixSetMut::from([ + Nibbles::unpack(B256::right_padding_from(&[0x3a])), + Nibbles::unpack(B256::right_padding_from(&[0x80])), + ]) + .freeze() + } else { + prefix_set.clone() + }; + let (root, updates) = + storage_root_with_progress(&harness, changes, enabled, threshold); + + let mut resulting_nodes = stored_nodes.clone(); + for (path, node) in updates.into_sorted().storage_nodes { + if let Some(node) = node { + resulting_nodes.insert(path, node); + } else { + resulting_nodes.remove(&path); + } + } + assert_eq!(root, storage_root_prehashed(storage.clone())); + if enabled { + assert_eq!(resulting_nodes, expected_nodes); + } else { + // Normal traversal ignores hidden descendants, but leaves them in the cache. + assert_ne!(resulting_nodes, expected_nodes); + } + } + } + } + + fn storage_root_with_progress( + harness: &TrieTestHarness, + changes: PrefixSet, + enabled: bool, + threshold: u64, + ) -> (B256, StorageTrieUpdates) { + let mut previous_state = None; + let mut updates = StorageTrieUpdates::default(); + let mut checkpoints = 0; + loop { + let progress = StorageRoot::new_hashed( + harness.trie_cursor_factory(), + harness.hashed_cursor_factory(), + harness.hashed_address(), + changes.clone(), + #[cfg(feature = "metrics")] + crate::metrics::TrieRootMetrics::new(crate::TrieType::Storage), + ) + .with_walk_all_changed_branch_children(enabled) + .with_threshold(threshold) + .with_intermediate_state(previous_state) + .root_with_progress() + .unwrap(); + match progress { + StorageRootProgress::Progress(state, _, new_updates) => { + updates.extend(new_updates); + previous_state = Some(*state); + checkpoints += 1; + } + StorageRootProgress::Complete(root, _, new_updates) => { + updates.extend(new_updates); + if threshold == 1 { + assert!(checkpoints > 1); + } + return (root, updates) + } + } + } + } + + #[test] + fn orphan_cleanup_skips_live_subtrees() { + let parent = Nibbles::from_nibbles([0x3]); + let tree_mask = TrieMask::from_nibble(0xa); + let orphans = [ + Nibbles::from_nibbles([0x3, 0xb]), + Nibbles::from_nibbles([0x3, 0xb, 0x5]), + Nibbles::from_nibbles([0x3, 0xb, 0x5, 0xf]), + ]; + let mut nodes = BTreeMap::from([ + (parent, branch_node((1 << 0xa) | (1 << 0xb), tree_mask.get(), 0)), + (Nibbles::from_nibbles([0x3, 0xa]), branch_node(0b11, 0, 0)), + (Nibbles::from_nibbles([0x4]), branch_node(0b11, 0, 0)), + ]); + for index in 0..128 { + nodes.insert( + Nibbles::from_nibbles([0x3, 0xa, index / 16, index % 16]), + branch_node(0b11, 0, 0), + ); + } + nodes.extend(orphans.into_iter().map(|path| (path, branch_node(0b11, 0, 0)))); + let factory = MockTrieCursorFactory::new(nodes, B256Map::default()); + let changes = PrefixSetMut::from([Nibbles::from_nibbles([0x3, 0xa])]).freeze(); + let mut walker = + TrieWalker::<_>::state_trie(factory.account_trie_cursor().unwrap(), changes) + .with_walk_all_changed_branch_children(true) + .with_deletions_retained(true); + walker.cursor.seek_exact(parent).unwrap(); + factory.visited_account_keys().clear(); + walker.remove_orphaned_descendants(parent, tree_mask).unwrap(); + + assert_eq!(walker.take_removed_keys(), HashSet::from_iter(orphans)); + assert_eq!(walker.cursor.current().unwrap(), Some(parent)); + let next_count = factory + .visited_account_keys() + .iter() + .filter(|visit| matches!(visit.visit_type, KeyVisitType::Next)) + .count(); + assert!(next_count <= 4, "cleanup walked {next_count} entries for 3 orphan nodes"); + } } From 85a96e0ad81760c7ce3a4c7f343a92c3befc770e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 19 Oct 2026 11:12:08 +0000 Subject: [PATCH 118/133] fix(engine): report payload job start errors (#28015) Co-authored-by: Sebastian Stammler Co-authored-by: Claude (cherry picked from commit 186af6d36aa49b2c40e96e4e11177816f842cb04) --- crates/engine/primitives/src/error.rs | 5 ++ crates/engine/primitives/src/message.rs | 94 +++++++++++++++----- crates/engine/tree/src/tree/tests.rs | 8 +- crates/ethereum/node/src/engine_ssz_proxy.rs | 8 ++ crates/payload/builder/src/service.rs | 4 + crates/payload/builder/src/traits.rs | 3 + crates/payload/primitives/src/error.rs | 17 ++++ crates/rpc/rpc-engine-api/src/error.rs | 50 +++++++++++ 8 files changed, 163 insertions(+), 26 deletions(-) diff --git a/crates/engine/primitives/src/error.rs b/crates/engine/primitives/src/error.rs index 0062d47c751..692d7e225e0 100644 --- a/crates/engine/primitives/src/error.rs +++ b/crates/engine/primitives/src/error.rs @@ -2,6 +2,7 @@ use alloc::boxed::Box; use alloy_rpc_types_engine::ForkchoiceUpdateError; use reth_errors::{BlockExecutionError, BlockValidationError, ConsensusError, ProviderError}; use reth_execution_errors::InternalBlockExecutionError; +use reth_payload_primitives::PayloadBuilderError; /// Represents all error cases when handling a new payload. /// @@ -57,6 +58,10 @@ pub enum BeaconForkChoiceUpdateError { /// Thrown when the engine task is unavailable/stopped. #[error("beacon consensus engine task stopped")] EngineUnavailable, + /// Thrown when the forkchoice update was applied, but the payload job for the provided payload + /// attributes could not be started. + #[error("failed to start payload job: {0}")] + PayloadJob(#[source] PayloadBuilderError), /// An internal error occurred, not necessarily related to the update. #[error(transparent)] Internal(Box), diff --git a/crates/engine/primitives/src/message.rs b/crates/engine/primitives/src/message.rs index 951f3196421..239fd9e06ad 100644 --- a/crates/engine/primitives/src/message.rs +++ b/crates/engine/primitives/src/message.rs @@ -4,8 +4,8 @@ use crate::{ use alloy_eips::eip4895::Withdrawal; use alloy_primitives::{Bytes, B256}; use alloy_rpc_types_engine::{ - ExecutionData, ForkChoiceUpdateResult, ForkchoiceState, ForkchoiceUpdateError, - ForkchoiceUpdated, PayloadId, PayloadStatus, PayloadStatusEnum, + ExecutionData, ForkchoiceState, ForkchoiceUpdateError, ForkchoiceUpdated, PayloadId, + PayloadStatus, PayloadStatusEnum, }; use core::{ fmt::{self, Display}, @@ -24,9 +24,12 @@ use tokio::sync::{mpsc::UnboundedSender, oneshot}; #[deprecated(note = "Use ConsensusEngineHandle instead")] pub type BeaconConsensusEngineHandle = ConsensusEngineHandle; +/// The result of a forkchoice update, or the error to return to the consensus client. +pub type OnForkChoiceUpdatedResult = Result; + /// Represents the outcome of forkchoice update. /// -/// This is a future that resolves to [`ForkChoiceUpdateResult`] +/// This is a future that resolves to [`OnForkChoiceUpdatedResult`] #[must_use = "futures do nothing unless you `.await` or poll them"] #[derive(Debug)] pub struct OnForkChoiceUpdated { @@ -36,7 +39,7 @@ pub struct OnForkChoiceUpdated { /// depending on the payload attributes, even if the forkchoice update itself is valid. forkchoice_status: ForkchoiceStatus, /// Returns the result of the forkchoice update. - fut: Either, PendingPayloadId>, + fut: Either, PendingPayloadId>, } // === impl OnForkChoiceUpdated === @@ -79,7 +82,9 @@ impl OnForkChoiceUpdated { pub fn invalid_state() -> Self { Self { forkchoice_status: ForkchoiceStatus::Invalid, - fut: Either::Left(futures::future::ready(Err(ForkchoiceUpdateError::InvalidState))), + fut: Either::Left(futures::future::ready(Err( + ForkchoiceUpdateError::InvalidState.into() + ))), } } @@ -88,7 +93,9 @@ impl OnForkChoiceUpdated { pub fn too_deep_reorg() -> Self { Self { forkchoice_status: ForkchoiceStatus::Invalid, - fut: Either::Left(futures::future::ready(Err(ForkchoiceUpdateError::TooDeepReorg))), + fut: Either::Left(futures::future::ready(Err( + ForkchoiceUpdateError::TooDeepReorg.into() + ))), } } @@ -99,12 +106,14 @@ impl OnForkChoiceUpdated { // This is valid because this is only reachable if the state and payload is valid forkchoice_status: ForkchoiceStatus::Valid, fut: Either::Left(futures::future::ready(Err( - ForkchoiceUpdateError::UpdatedInvalidPayloadAttributes, + ForkchoiceUpdateError::UpdatedInvalidPayloadAttributes.into(), ))), } } - /// If the forkchoice update was successful and no payload attributes were provided, this method + /// Creates a new instance of `OnForkChoiceUpdated` that resolves to the id of the payload job + /// started for the payload attributes, or to [`BeaconForkChoiceUpdateError::PayloadJob`] if the + /// job cannot be started. pub const fn updated_with_pending_payload_id( payload_status: PayloadStatus, pending_payload_id: oneshot::Receiver>, @@ -120,7 +129,7 @@ impl OnForkChoiceUpdated { } impl Future for OnForkChoiceUpdated { - type Output = ForkChoiceUpdateResult; + type Output = OnForkChoiceUpdatedResult; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { self.get_mut().fut.poll_unpin(cx) @@ -136,21 +145,17 @@ struct PendingPayloadId { } impl Future for PendingPayloadId { - type Output = ForkChoiceUpdateResult; + type Output = OnForkChoiceUpdatedResult; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); - let res = ready!(this.pending_payload_id.poll_unpin(cx)); - match res { - Ok(Ok(payload_id)) => Poll::Ready(Ok(ForkchoiceUpdated { - payload_status: this.payload_status.take().expect("Polled after completion"), - payload_id: Some(payload_id), - })), - Err(_) | Ok(Err(_)) => { - // failed to initiate a payload build job - Poll::Ready(Err(ForkchoiceUpdateError::UpdatedInvalidPayloadAttributes)) - } - } + let payload_id = ready!(this.pending_payload_id.poll_unpin(cx)) + .unwrap_or_else(|err| Err(err.into())) + .map_err(BeaconForkChoiceUpdateError::PayloadJob)?; + Poll::Ready(Ok(ForkchoiceUpdated { + payload_status: this.payload_status.take().expect("Polled after completion"), + payload_id: Some(payload_id), + })) } } @@ -380,12 +385,11 @@ where state: ForkchoiceState, payload_attrs: Option, ) -> Result { - Ok(self - .send_fork_choice_updated(state, payload_attrs) + self.send_fork_choice_updated(state, payload_attrs) .map_err(|_| BeaconForkChoiceUpdateError::EngineUnavailable) .await? .map_err(BeaconForkChoiceUpdateError::internal)? - .await?) + .await } /// Sends a forkchoice update message to the beacon consensus engine and returns the receiver to @@ -404,3 +408,45 @@ where rx } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Resolves the forkchoice update for a payload job that answers with `result`, or whose sender + /// is dropped if `result` is `None`. + fn on_payload_job( + result: Option>, + ) -> OnForkChoiceUpdatedResult { + let (tx, rx) = oneshot::channel(); + match result { + Some(result) => tx.send(result).unwrap(), + None => drop(tx), + } + OnForkChoiceUpdated::updated_with_pending_payload_id( + PayloadStatus::from_status(PayloadStatusEnum::Valid), + rx, + ) + .now_or_never() + .expect("payload job result is ready") + } + + #[test] + fn payload_job_start_errors_are_returned() { + let id = PayloadId::new([1; 8]); + assert_eq!(on_payload_job(Some(Ok(id))).unwrap().payload_id, Some(id)); + + let err = on_payload_job(Some(Err(PayloadBuilderError::MissingParentHeader(B256::ZERO)))) + .unwrap_err(); + assert!(matches!( + err, + BeaconForkChoiceUpdateError::PayloadJob(PayloadBuilderError::MissingParentHeader(_)) + )); + + let err = on_payload_job(None).unwrap_err(); + assert!(matches!( + err, + BeaconForkChoiceUpdateError::PayloadJob(PayloadBuilderError::ChannelClosed) + )); + } +} diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index de102c4710e..0b889f0a63d 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -25,7 +25,8 @@ use assert_matches::assert_matches; use reth_chain_state::test_utils::TestBlockBuilder; use reth_chainspec::{ChainSpec, HOLESKY, MAINNET}; use reth_engine_primitives::{ - EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook, DEFAULT_BACKFILL_RUN_THRESHOLD, + BeaconForkChoiceUpdateError, EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook, + DEFAULT_BACKFILL_RUN_THRESHOLD, }; use reth_ethereum_consensus::EthBeaconConsensus; use reth_ethereum_engine_primitives::{EthEngineTypes, EthPayloadAttributes}; @@ -1822,7 +1823,10 @@ async fn test_fcu_with_canonical_ancestor_below_finalized_is_rejected() { .await .unwrap_err(); - assert_matches!(err, ForkchoiceUpdateError::TooDeepReorg); + assert_matches!( + err, + BeaconForkChoiceUpdateError::ForkchoiceUpdateError(ForkchoiceUpdateError::TooDeepReorg) + ); } // no payload build is started and the canonical head remains untouched diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index 06bda66b1bb..c564be1c1aa 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -1459,6 +1459,7 @@ mod tests { async fn engine_errors_preserve_validation_semantics() { use alloy_rpc_types_engine::ForkchoiceUpdateError; use reth_engine_primitives::BeaconForkChoiceUpdateError; + use reth_payload_primitives::PayloadBuilderError; for (error, status, kind) in [ (EngineApiError::UnknownPayload, 404, "unknown-payload"), (EngineApiError::BlobRequestTooLarge { len: 129 }, 413, "request-too-large"), @@ -1480,6 +1481,13 @@ mod tests { 422, "invalid-attributes", ), + ( + EngineApiError::ForkChoiceUpdate(BeaconForkChoiceUpdateError::PayloadJob( + PayloadBuilderError::MissingParentHeader(B256::ZERO), + )), + 500, + "internal", + ), ] { let response = engine_error_response(error); assert_eq!(response.status(), status); diff --git a/crates/payload/builder/src/service.rs b/crates/payload/builder/src/service.rs index b9e2abac366..b7669c8734f 100644 --- a/crates/payload/builder/src/service.rs +++ b/crates/payload/builder/src/service.rs @@ -538,6 +538,10 @@ pub enum PayloadServiceCommand { /// /// Carries the caller's [`Span`] so the service can parent payload-building work under the /// originating Engine API trace. + /// + /// The sender receives the id of the started payload job. Answering with + /// [`PayloadBuilderError::InvalidAttributes`] rejects the payload attributes; any other error, + /// or dropping the sender, is reported to the CL as an internal error. BuildNewPayload( Box>, Span, diff --git a/crates/payload/builder/src/traits.rs b/crates/payload/builder/src/traits.rs index 89ae6ecbfa5..39a17ff6fef 100644 --- a/crates/payload/builder/src/traits.rs +++ b/crates/payload/builder/src/traits.rs @@ -112,6 +112,9 @@ pub trait PayloadJobGenerator { /// /// This is called when the CL requests a new payload job via a fork choice update. /// + /// Return [`PayloadBuilderError::InvalidAttributes`] to reject the payload attributes; any + /// other error is reported to the CL as an internal error. + /// /// # Note /// /// This is expected to initially build a new (empty) payload without transactions, so it can be diff --git a/crates/payload/primitives/src/error.rs b/crates/payload/primitives/src/error.rs index 9976bdea97d..a4ec2b78100 100644 --- a/crates/payload/primitives/src/error.rs +++ b/crates/payload/primitives/src/error.rs @@ -31,6 +31,15 @@ pub enum PayloadBuilderError { /// Any other payload building errors. #[error(transparent)] Other(Box), + /// The payload attributes are invalid on top of the parent block, e.g. a transaction they + /// include is invalid. + /// + /// Returned from `PayloadJobGenerator::new_payload_job`, this is the only variant that rejects + /// the attributes; returned later from a running job, it is an internal error like any other. + /// Failures of this node, such as a failed state read during execution, must use another + /// variant. + #[error("invalid payload attributes: {0}")] + InvalidAttributes(Box), } impl PayloadBuilderError { @@ -49,6 +58,14 @@ impl PayloadBuilderError { { Self::Other(Box::new(error)) } + + /// Create a new error that rejects the payload attributes. + pub fn invalid_attributes(error: E) -> Self + where + E: core::error::Error + Send + Sync + 'static, + { + Self::InvalidAttributes(Box::new(error)) + } } impl From for PayloadBuilderError { diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index 7c7dbb3b1ea..de243ba1455 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -214,7 +214,18 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { ); } }, + // Only attributes that the payload builder rejects are invalid payload attributes. + // Any other failure to start the payload job is a failure of this node and must + // not be reported as invalid attributes, which the consensus client would discard. + BeaconForkChoiceUpdateError::PayloadJob( + PayloadBuilderError::InvalidAttributes(_), + ) => jsonrpsee_types::error::ErrorObject::owned( + INVALID_PAYLOAD_ATTRIBUTES_ERROR, + INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG, + Some(ErrorData::new(error)), + ), BeaconForkChoiceUpdateError::EngineUnavailable | + BeaconForkChoiceUpdateError::PayloadJob(_) | BeaconForkChoiceUpdateError::Internal(_) => { jsonrpsee_types::error::ErrorObject::owned( INTERNAL_ERROR_CODE, @@ -241,6 +252,7 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { mod tests { use super::*; use alloy_rpc_types_engine::ForkchoiceUpdateError; + use reth_payload_primitives::InvalidPayloadAttributesError; #[track_caller] fn ensure_engine_rpc_error( code: i32, @@ -363,4 +375,42 @@ mod tests { ), ); } + + fn payload_job_error(err: PayloadBuilderError) -> jsonrpsee_types::error::ErrorObject<'static> { + EngineApiError::ForkChoiceUpdate(BeaconForkChoiceUpdateError::PayloadJob(err)).into() + } + + #[test] + fn payload_job_rejecting_the_attributes_is_invalid_payload_attributes() { + let err = payload_job_error(PayloadBuilderError::invalid_attributes( + InvalidPayloadAttributesError::InvalidTimestamp, + )); + + assert_eq!(err.code(), INVALID_PAYLOAD_ATTRIBUTES_ERROR); + assert_eq!(err.message(), INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG); + assert_eq!( + err.data().unwrap().get(), + r#"{"err":"failed to start payload job: invalid payload attributes: invalid timestamp"}"# + ); + } + + #[test] + fn payload_job_failing_on_this_node_is_an_internal_error() { + let node_local = [ + PayloadBuilderError::MissingParentHeader(B256::ZERO), + PayloadBuilderError::ChannelClosed, + PayloadBuilderError::other(std::io::Error::other("other")), + ]; + + for payload_job in node_local { + let message = payload_job.to_string(); + let err = payload_job_error(payload_job); + assert_eq!(err.code(), INTERNAL_ERROR_CODE, "{message}"); + assert_eq!(err.message(), SERVER_ERROR_MSG, "{message}"); + assert_eq!( + err.data().unwrap().get(), + format!(r#"{{"err":"failed to start payload job: {message}"}}"#) + ); + } + } } From 2158d3b180a6260eb0b76b7ba8310b64bfa6ce16 Mon Sep 17 00:00:00 2001 From: Sebastian Stammler Date: Thu, 1 Oct 2026 10:42:37 +0200 Subject: [PATCH 119/133] test(storage-overlay): cover storage roots over a masked state trie Pin that `OverlayStateProvider::storage_root` reads trie nodes through the overlay when partial persistence leaves database storage-trie nodes older than the persisted hashed storage. OP Stack consensus depends on this: the Isthmus withdrawals root is the L2ToL1MessagePasser storage root, computed through `storage_root` when building and validating blocks. Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit d70cc04dab7d337f96080fad1923ec355750c391) --- .../storage/storage-overlay/src/provider.rs | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/crates/storage/storage-overlay/src/provider.rs b/crates/storage/storage-overlay/src/provider.rs index 624e0405a05..7db1ede333a 100644 --- a/crates/storage/storage-overlay/src/provider.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -1365,4 +1365,184 @@ mod tests { assert_eq!(provider.basic_account(&address).unwrap(), Some(account)); assert_eq!(provider.storage(address, storage_key).unwrap(), Some(storage)); } + + /// Hashed slot with the given first two bytes, so that the storage trie has stored branch + /// nodes at depth one that cache the hashes of their children. + fn hashed_slot(first: u8, second: u8) -> B256 { + let mut slot = alloy_primitives::keccak256([first, second]); + slot.0[0] = first; + slot.0[1] = second; + slot + } + + /// Writes hashed state and trie updates like `save_blocks` does under partial persistence. + fn write_state_trie( + provider: &(impl reth_storage_api::StateWriter + reth_storage_api::TrieWriter), + batch: &[ExecutedBlock], + mask: &[ExecutedBlock], + ) { + provider + .write_hashed_state(&HashedPostStateSorted::disjointed_merge_batch( + &ExecutedBlock::hashed_state_refs(batch), + &ExecutedBlock::hashed_state_refs(mask), + )) + .unwrap(); + provider + .write_trie_updates_sorted(&TrieUpdatesSorted::disjointed_merge_batch( + &ExecutedBlock::trie_updates_refs(batch), + &ExecutedBlock::trie_updates_refs(mask), + )) + .unwrap(); + } + + /// Storage roots over a partially persisted state trie must use the overlay trie nodes: the + /// database trie tables omit nodes overwritten by the masking suffix, so their cached child + /// hashes can be older than the persisted hashed state. + #[test] + fn storage_root_over_masked_state_trie() { + for settings in [reth_provider::StorageSettings::v1(), reth_provider::StorageSettings::v2()] + { + let address = Address::with_last_byte(0x16); + let hashed_address = alloy_primitives::keccak256(address); + + let base = (0..=u8::MAX) + .flat_map(|first| { + [0x00, 0x80].map(|second| (hashed_slot(first, second), U256::from(1))) + }) + .collect::>(); + let changes: [Vec<(B256, U256)>; 8] = [ + vec![], + base, + vec![ + (hashed_slot(0x10, 0x00), U256::from(2)), + (hashed_slot(0x11, 0x42), U256::from(2)), + (hashed_slot(0x20, 0x00), U256::ZERO), + ], + vec![(hashed_slot(0x30, 0x00), U256::from(3))], + vec![(hashed_slot(0x40, 0x80), U256::from(4))], + vec![ + (hashed_slot(0x30, 0x00), U256::from(5)), + (hashed_slot(0x31, 0x00), U256::from(5)), + (hashed_slot(0x50, 0x00), U256::from(5)), + (hashed_slot(0x60, 0x80), U256::ZERO), + ], + vec![ + (hashed_slot(0x1f, 0x80), U256::ZERO), + (hashed_slot(0x50, 0x00), U256::from(6)), + (hashed_slot(0x70, 0x00), U256::from(6)), + ], + vec![ + (hashed_slot(0x50, 0x00), U256::from(7)), + (hashed_slot(0x80, 0x00), U256::ZERO), + ], + ]; + let block_updates = [ + (hashed_slot(0x90, 0x42), U256::from(1)), + (hashed_slot(0xa0, 0x00), U256::from(8)), + (hashed_slot(0xb0, 0x80), U256::ZERO), + ]; + + // Like the engine's, each block's storage trie updates only contain modified nodes. + let reference = create_test_provider_factory(); + reference.set_storage_settings_cache(settings); + let mut storage_nodes = std::collections::BTreeMap::new(); + let blocks = TestBlockBuilder::eth() + .get_executed_blocks(0..changes.len() as u64) + .zip(&changes) + .map(|(block, changes)| { + let mut hashed_state = HashedPostState::default().with_storages([( + hashed_address, + HashedStorage::from_iter(changes.iter().copied()), + )]); + if block.recovered_block().number == 1 { + hashed_state = hashed_state + .with_accounts([(hashed_address, Some(Account::default()))]); + } + let provider = reference.provider().unwrap(); + let (_, trie_updates) = reth_provider::LatestStateProviderRef::new(&provider) + .state_root_with_updates(hashed_state.clone()) + .unwrap(); + drop(provider); + let hashed_state = hashed_state.into_sorted(); + let trie_updates = trie_updates.into_sorted(); + let provider_rw = reference.provider_rw().unwrap(); + reth_storage_api::StateWriter::write_hashed_state(&*provider_rw, &hashed_state) + .unwrap(); + reth_storage_api::TrieWriter::write_trie_updates_sorted( + &*provider_rw, + &trie_updates, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let modified_storage_nodes = trie_updates + .storage_tries_ref() + .get(&hashed_address) + .map(|storage_trie| storage_trie.storage_nodes_ref()) + .unwrap_or_default() + .iter() + .filter(|(path, node)| match node { + Some(node) => { + storage_nodes.insert(*path, node.clone()).as_ref() != Some(node) + } + None => storage_nodes.remove(path).is_some(), + }) + .cloned() + .collect(); + let trie_updates = TrieUpdatesSorted::new( + trie_updates.account_nodes_ref().to_vec(), + alloy_primitives::map::B256Map::from_iter([( + hashed_address, + reth_trie::updates::StorageTrieUpdatesSorted { + storage_nodes: modified_storage_nodes, + }, + )]), + ); + ExecutedBlock::new( + Arc::clone(&block.recovered_block), + Arc::clone(&block.execution_output), + ComputedTrieData::new(Arc::new(hashed_state), Arc::new(trie_updates)), + ) + }) + .collect::>(); + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(settings); + let provider_rw = factory.provider_rw().unwrap(); + for block in &blocks[..=6] { + provider_rw.insert_block(block.recovered_block()).unwrap(); + } + write_state_trie(&*provider_rw, &blocks[1..=1], &[]); + write_state_trie(&*provider_rw, &blocks[2..=4], &blocks[5..=6]); + provider_rw + .save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new(6).with_finish_stage_checkpoint(FinishCheckpoint { + partial_state_trie: Some(4), + }), + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let manager = OverlayManager::default(); + for block in &blocks[5..=7] { + manager.insert_block(block.clone()); + } + let state_provider_factory = OverlayStateProviderFactory::new( + factory, + manager.overlay_builder(blocks[7].recovered_block().hash()), + ); + let provider = state_provider_factory.database_provider_ro().unwrap(); + + let mut expected = std::collections::BTreeMap::new(); + for (slot, value) in changes.iter().flatten().chain(&block_updates) { + expected.insert(*slot, *value); + } + expected.retain(|_, value| !value.is_zero()); + + let storage_root = + provider.storage_root(address, HashedStorage::from_iter(block_updates)).unwrap(); + assert_eq!(storage_root, reth_trie::test_utils::storage_root_prehashed(expected)); + } + } } From f8408b0843a16d816ea9e444c2cfb8e303ed5ddf Mon Sep 17 00:00:00 2001 From: Akosh Farkash Date: Thu, 5 Nov 2026 10:00:17 +0000 Subject: [PATCH 120/133] ci: stub tempoxyz's secure-runner under act --- .circleci/config.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 970fc6f12b5..26c75120147 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,8 @@ version: 2.1 # act's local cache is not reusable across these machines; Pages plumbing is skipped after the # validation build. Lint and unit jobs are distributed; other workflows run up to four jobs. # -# Excluded: pr-title.yml, label-pr.yml, and the privileged cyclops-audit.yml integration. +# Excluded: pr-title.yml, label-pr.yml, the privileged cyclops-audit.yml integration, and +# scan-github-actions.yml, whose reusable workflow needs a GitHub OIDC token. jobs: # Replaces lint.yml's `deny` job. That job calls a reusable workflow from tempoxyz, whose deny job @@ -123,6 +124,27 @@ jobs: ' - run: echo "Service plumbing disabled under act"' \ ' shell: bash' \ > /tmp/noop-action/action.yml + - run: + name: Stub unsupported secure-runner + # Stub tempoxyz's secure-runner: it fetches its policy with a GitHub OIDC token that act + # never receives, and it hardens GitHub-hosted runners, not these machines. act maps whole + # repositories, and this one also serves vendored actions the checks need, so replace only + # secure-runner in a copy at the sha the workflows pin, reusing the no-op action from the + # step above. When re-applying this CI patch, move TEMPOXYZ_GH_ACTIONS_SHA with that pin; + # the guard below names any mismatch. + command: | + readonly TEMPOXYZ_GH_ACTIONS_SHA="ee2960a6cc25d99bda45614135bfe1197d879bac" + if grep -rhoE 'tempoxyz/gh-actions/[^@ ]+@[0-9a-f]{40}' .github/workflows/ \ + | grep -v "@${TEMPOXYZ_GH_ACTIONS_SHA}\$"; then + echo "workflows pin tempoxyz/gh-actions at a sha other than ${TEMPOXYZ_GH_ACTIONS_SHA}" >&2 + exit 1 + fi + git init --quiet /tmp/tempoxyz-gh-actions + git -C /tmp/tempoxyz-gh-actions fetch --quiet --depth 1 \ + https://github.com/tempoxyz/gh-actions "${TEMPOXYZ_GH_ACTIONS_SHA}" + git -C /tmp/tempoxyz-gh-actions checkout --quiet --detach FETCH_HEAD + cp /tmp/noop-action/action.yml /tmp/tempoxyz-gh-actions/actions/secure-runner/action.yml + echo "export TEMPOXYZ_GH_ACTIONS_SHA=${TEMPOXYZ_GH_ACTIONS_SHA}" >> "$BASH_ENV" - run: name: Run << parameters.wf >> via act no_output_timeout: 45m @@ -176,7 +198,8 @@ jobs: --local-repository "mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba=/tmp/noop-action" \ --local-repository "Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6=/tmp/noop-action" \ --local-repository "actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d=/tmp/noop-action" \ - --local-repository "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9=/tmp/noop-action" + --local-repository "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9=/tmp/noop-action" \ + --local-repository "tempoxyz/gh-actions@${TEMPOXYZ_GH_ACTIONS_SHA}=/tmp/tempoxyz-gh-actions" workflows: reth-act: From 68cacdd635255a5bdf4c35857ad96d0b91cb7744 Mon Sep 17 00:00:00 2001 From: Akosh Farkash Date: Thu, 5 Nov 2026 10:12:55 +0000 Subject: [PATCH 121/133] ci: ignore the pin-utils unmaintained advisory RUSTSEC-2026-0441 marks pin-utils unmaintained with no safe upgrade. It is reachable only on wasm targets, through alloy-provider's wasmtimer. --- deny.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deny.toml b/deny.toml index cf907cb3124..cfded2b83d5 100644 --- a/deny.toml +++ b/deny.toml @@ -19,6 +19,9 @@ ignore = [ # zstd-safe 7 comes from reth-zstd-compressors 0.7; the patched zstd-safe 8.2.0 needs reth-core 0.8 and Rust 1.96. "RUSTSEC-2026-0398", "RUSTSEC-2026-0403", + # https://rustsec.org/advisories/RUSTSEC-2026-0441 pin-utils is unmaintained, no safe upgrade; + # only reachable on wasm targets through alloy-provider's wasmtimer + "RUSTSEC-2026-0441", ] # This section is considered when running `cargo deny check bans`. From c6e896681c6c0f3d44535759d4a1e84550537f34 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Sep 2026 14:42:44 +0000 Subject: [PATCH 122/133] test(e2e): wait for pruner in RocksDB pruning tests (#27601) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit f269bccbc0f007d7295ccd0b986ecfd36805ef56) --- Cargo.lock | 1 + crates/e2e-test-utils/Cargo.toml | 3 ++ crates/e2e-test-utils/tests/rocksdb/main.rs | 46 +++++++++++++++++++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 475ef86c651..f1ff609d056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8485,6 +8485,7 @@ dependencies = [ "reth-payload-primitives", "reth-primitives-traits", "reth-provider", + "reth-prune-types", "reth-rpc-api", "reth-rpc-builder", "reth-rpc-eth-api", diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index fcf902070fa..a6ea110ed11 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -70,6 +70,9 @@ serde_json.workspace = true tracing.workspace = true derive_more.workspace = true +[dev-dependencies] +reth-prune-types.workspace = true + [[test]] name = "e2e_testsuite" path = "tests/e2e-testsuite/main.rs" diff --git a/crates/e2e-test-utils/tests/rocksdb/main.rs b/crates/e2e-test-utils/tests/rocksdb/main.rs index 15d5d00b131..c2f6dd12a1b 100644 --- a/crates/e2e-test-utils/tests/rocksdb/main.rs +++ b/crates/e2e-test-utils/tests/rocksdb/main.rs @@ -9,7 +9,8 @@ use reth_chainspec::EthereumHardfork; use reth_db::tables; use reth_e2e_test_utils::{transaction::TransactionTestContext, wait::poll_until, E2ETestSetupExt}; use reth_node_ethereum::EthereumNode; -use reth_provider::RocksDBProviderFactory; +use reth_provider::{PruneCheckpointReader, RocksDBProviderFactory}; +use reth_prune_types::PruneSegment; use std::time::Duration; /// Polls RPC until the given `tx_hash` is visible as pending (not yet mined). @@ -40,6 +41,25 @@ async fn poll_tx_in_rocksdb(provider: &P, tx_hash: B2 .await } +/// Polls until the pruner has pruned `segment` up to at least `block`. +/// +/// The persistence service acknowledges saved blocks before it runs the pruner in a separate +/// commit, so a block being visible in `RocksDB` does not mean the pruner has run for it yet. +async fn wait_for_prune_checkpoint( + provider: &P, + segment: PruneSegment, + block: u64, +) -> Result<()> { + poll_until(format!("{segment} to be pruned up to block {block}"), || async move { + let checkpoint = provider.get_prune_checkpoint(segment)?; + Ok(checkpoint + .and_then(|checkpoint| checkpoint.block_number) + .is_some_and(|n| n >= block) + .then_some(())) + }) + .await +} + /// Smoke test: node boots with `RocksDB` routing enabled. #[tokio::test] async fn test_rocksdb_node_startup() -> Result<()> { @@ -665,12 +685,21 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> { let payload = node.advance_block().await?; assert_eq!(payload.block().number(), nonce + 1); - // Let the persistence cycle (save_blocks → pruner → commit) complete before - // producing the next block, so each cycle processes exactly one block. The - // transaction becomes visible in RocksDB with the commit. + // Wait for the block's save to commit before producing the next block, so each save + // processes exactly one block. The transaction becomes visible in RocksDB with the + // commit, and the persistence service runs the pruner before it handles the next save. poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; } + // The pruner runs after the save of the last block is acknowledged, so wait for it to prune + // the blocks outside the retention window. + wait_for_prune_checkpoint( + &node.inner.provider, + PruneSegment::AccountHistory, + TOTAL_BLOCKS - PRUNE_DISTANCE, + ) + .await?; + // Read the AccountsHistory shard for `sender` directly from RocksDB. // This is the data structure corrupted by the race. let rocksdb = node.inner.provider.rocksdb_provider(); @@ -843,6 +872,15 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> { poll_tx_in_rocksdb(&node.inner.provider, tx_hash).await?; } + // The pruner runs after the save of the last block is acknowledged, so wait for it to prune + // the blocks outside the retention window. + wait_for_prune_checkpoint( + &node.inner.provider, + PruneSegment::StorageHistory, + TOTAL_BLOCKS - PRUNE_DISTANCE, + ) + .await?; + // Read StoragesHistory shard for (contract_address, slot 0) directly from RocksDB let rocksdb = node.inner.provider.rocksdb_provider(); let shards = rocksdb.storage_history_shards(contract_address, storage_slot).unwrap(); From 8ba21dfd7ca5a6e813df8ec5b83fe2f64a1ee053 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 28 Sep 2026 17:51:04 +0200 Subject: [PATCH 123/133] test(e2e): replace fixed sleeps with pool waits (#27500) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit 0fc98b4d82c717e01d99e2c285fd7516c4e19b79) --- Cargo.lock | 1 + crates/e2e-test-utils/Cargo.toml | 1 + crates/e2e-test-utils/src/node.rs | 13 +++++++++---- crates/ethereum/node/tests/e2e/blobs.rs | 14 +++++++++++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1ff609d056..790692cff43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8494,6 +8494,7 @@ dependencies = [ "reth-tasks", "reth-tokio-util", "reth-tracing", + "reth-transaction-pool", "reth-trie", "reth-trie-db", "revm", diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index a6ea110ed11..f314cf93fbd 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -41,6 +41,7 @@ reth-db-common.workspace = true reth-primitives-traits.workspace = true reth-trie.workspace = true reth-trie-db.workspace = true +reth-transaction-pool.workspace = true revm.workspace = true tempfile.workspace = true diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 7eb43c73b7d..ef836698a6a 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -30,6 +30,7 @@ use reth_rpc_api::TestingBuildBlockRequestV1; use reth_rpc_builder::auth::AuthServerHandle; use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt}; use reth_stages_types::StageId; +use reth_transaction_pool::TransactionPool; use std::{pin::Pin, sync::Arc, time::Duration}; use tokio_stream::StreamExt; use url::Url; @@ -344,10 +345,14 @@ where .await .map_err(|_| eyre!("timed out syncing to block {block}"))??; - // Hack to make sure that all components have time to process canonical state update. - // Otherwise, this might result in e.g "nonce too low" errors when advancing chain further, - // making tests flaky. - tokio::time::sleep(Duration::from_millis(1000)).await; + // The transaction pool processes the canonical state update in the background, and + // advancing the chain before it did can fail with e.g. "nonce too low" errors. Blocks + // synced by backfill don't notify the pool, so wait for at most a second. + let _ = tokio::time::timeout( + Duration::from_secs(1), + self.wait_for_pool(|pool| pool.block_info().last_seen_block_hash == block), + ) + .await; Ok(()) } diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index feef4cc18a9..c7f7941f725 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -9,7 +9,7 @@ use reth_node_ethereum::EthereumNode; use reth_transaction_pool::TransactionPool; use std::{ sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::{SystemTime, UNIX_EPOCH}, }; #[tokio::test] @@ -51,7 +51,11 @@ async fn can_handle_blobs() -> eyre::Result<()> { let block_hash = node.submit_payload(payload).await?; node.update_forkchoice(genesis_hash, block_hash).await?; - tokio::time::sleep(std::time::Duration::from_secs(3)).await; + // wait for the pool to process the reorg, which re-injects the blob tx + node.wait_for_pool(|pool| { + pool.block_info().last_seen_block_hash == block_hash && pool.contains(&blob_tx_hash) + }) + .await?; // expects the blob tx to be back in the pool let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; @@ -164,7 +168,11 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); - tokio::time::sleep(Duration::from_secs(6)).await; + // wait for the pool to convert the sidecar ahead of the Osaka activation + node.wait_for_pool(|pool| { + pool.get_blob(blob_tx_hash).ok().flatten().is_some_and(|sidecar| sidecar.is_eip7594()) + }) + .await?; // fetch second blob tx from rpc again let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; From 66a0d0ce2decd8813c6ff89e2e00e18d0b6803d9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Sep 2026 14:14:14 +0000 Subject: [PATCH 124/133] feat(e2e): control the next payload timestamp (#27596) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit e29c3433cfbe4cae25e80dae3086d9dc13652e51) --- crates/e2e-test-utils/src/node.rs | 24 +++++++++++++++ crates/e2e-test-utils/src/payload.rs | 7 +++++ crates/ethereum/node/tests/e2e/blobs.rs | 11 ++++--- .../ethereum/node/tests/e2e/custom_genesis.rs | 29 +++++++++++++++++++ crates/ethereum/node/tests/e2e/p2p.rs | 4 +-- 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index ef836698a6a..824c33111fc 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -169,6 +169,30 @@ where }) } + /// Sets the timestamp of the next payload built by [`Self::new_payload`] and the methods + /// built on it, such as [`Self::advance_block`]. + /// + /// Later payloads continue from this timestamp, one second apart. The attributes generator is + /// called with the new timestamp, so fork-dependent attributes follow it. + /// + /// Returns an error if the timestamp is not greater than the timestamp of the current latest + /// block, which is the parent of the next payload. + pub fn set_next_payload_timestamp(&mut self, timestamp: u64) -> eyre::Result<()> { + let latest = self + .inner + .provider + .sealed_header_by_number_or_tag(BlockNumberOrTag::Latest)? + .ok_or_else(|| eyre!("latest block not found"))?; + ensure!( + timestamp > latest.timestamp(), + "next payload timestamp {timestamp} must be greater than the latest block timestamp {}", + latest.timestamp() + ); + // The payload context increments its timestamp before generating the next attributes. + self.payload.timestamp = timestamp - 1; + Ok(()) + } + /// Creates a new payload from given attributes generator /// expects a payload attribute event and waits until the payload is built. /// diff --git a/crates/e2e-test-utils/src/payload.rs b/crates/e2e-test-utils/src/payload.rs index fe0d4452c9f..c920b1bf065 100644 --- a/crates/e2e-test-utils/src/payload.rs +++ b/crates/e2e-test-utils/src/payload.rs @@ -10,6 +10,13 @@ use tokio_stream::wrappers::BroadcastStream; pub struct PayloadTestContext { pub payload_event_stream: BroadcastStream>, payload_builder: PayloadBuilderHandle, + /// Timestamp of the last generated payload attributes, or the starting point if none were + /// generated yet. + /// + /// [`Self::next_attributes`] increments it before generating the next attributes. Prefer + /// [`NodeTestContext::set_next_payload_timestamp`] over setting it directly. + /// + /// [`NodeTestContext::set_next_payload_timestamp`]: crate::node::NodeTestContext::set_next_payload_timestamp pub timestamp: u64, #[debug(skip)] attributes_generator: Box T::PayloadAttributes + Send + Sync>, diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index c7f7941f725..324c7560312 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -122,8 +122,9 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // build a dummy payload at `current_timestamp` let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.signer(0)).await; node.rpc.inject_tx(raw_tx).await?; - node.payload.timestamp = current_timestamp - 1; - node.advance_block().await?; + node.set_next_payload_timestamp(current_timestamp)?; + let dummy_payload = node.advance_block().await?; + assert_eq!(dummy_payload.block().timestamp, current_timestamp); // build blob txs let first_blob = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(1)).await?; @@ -155,8 +156,9 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { TransactionTestContext::validate_sidecar(envelope); // build last Prague payload - node.payload.timestamp = current_timestamp + 1; + node.set_next_payload_timestamp(current_timestamp + 2)?; let prague_payload = node.new_payload().await?; + assert_eq!(prague_payload.block().timestamp, current_timestamp + 2); assert!(matches!(prague_payload.sidecars(), BlobSidecars::Eip4844(_))); // inject second blob tx to the pool @@ -185,8 +187,9 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { node.update_forkchoice(genesis_hash, node.submit_payload(prague_payload).await?).await?; // Build first Osaka payload - node.payload.timestamp = osaka_timestamp - 1; + node.set_next_payload_timestamp(osaka_timestamp)?; let osaka_payload = node.new_payload().await?; + assert_eq!(osaka_payload.block().timestamp, osaka_timestamp); // Assert that it includes the second blob tx with eip7594 sidecar assert!(osaka_payload.block().body().transactions().any(|tx| *tx.hash() == blob_tx_hash)); diff --git a/crates/ethereum/node/tests/e2e/custom_genesis.rs b/crates/ethereum/node/tests/e2e/custom_genesis.rs index 0b11d6a1e14..12127e9143f 100644 --- a/crates/ethereum/node/tests/e2e/custom_genesis.rs +++ b/crates/ethereum/node/tests/e2e/custom_genesis.rs @@ -92,3 +92,32 @@ async fn can_advance_on_genesis_newer_than_payload_timestamp() -> eyre::Result<( Ok(()) } + +/// Tests that the next payload is built at the timestamp set on the test context, that later +/// payloads continue from it, and that timestamps not after the latest block are rejected. +#[tokio::test] +async fn can_set_next_payload_timestamp() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let genesis_timestamp = 2_000_000_000; + let mut genesis = test_genesis(); + genesis.timestamp = genesis_timestamp; + let chain_spec = + Arc::new(test_chain_spec_builder().genesis(genesis).cancun_activated().build()); + + let (mut node, _) = EthereumNode::test_setup(1, chain_spec).build_single().await?; + + assert_eq!( + node.set_next_payload_timestamp(genesis_timestamp).unwrap_err().to_string(), + "next payload timestamp 2000000000 must be greater than the latest block timestamp 2000000000" + ); + + let timestamp = genesis_timestamp + 100; + node.set_next_payload_timestamp(timestamp)?; + assert_eq!(node.advance_block().await?.block().timestamp, timestamp); + assert_eq!(node.advance_block().await?.block().timestamp, timestamp + 1); + + assert!(node.set_next_payload_timestamp(timestamp + 1).is_err()); + + Ok(()) +} diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index cb076baf906..9aa52c74c78 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -242,7 +242,7 @@ async fn test_long_reorg() -> eyre::Result<()> { second_node.sync_to(head.header.hash).await?; // Produce a fork chain with blocks 21..60 - second_node.payload.timestamp = head.header.timestamp; + second_node.set_next_payload_timestamp(head.header.timestamp + 1)?; advance_with_random_transactions(&mut second_node, 40, &mut rng, true).await?; // Reorg first node from 100th block to new 60th block. @@ -332,7 +332,7 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { second_node.sync_to(head.header.hash).await?; // Produce an unfinalized fork chain with 30 blocks - second_node.payload.timestamp = head.header.timestamp; + second_node.set_next_payload_timestamp(head.header.timestamp + 1)?; advance_with_random_transactions(&mut second_node, 30, &mut rng, false).await?; // Now reorg second node to the finalized canonical head From f5ed4fd708bc24ba8151ae53d346a4fc0ee8e982 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Sep 2026 14:42:21 +0000 Subject: [PATCH 125/133] feat(e2e): wait for the pool to process a new head (#27597) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit cdc4e096f140562f5d2a9f78ead17f3e16bdb3ca) --- crates/e2e-test-utils/src/node.rs | 39 +++++++++++++++++++++---- crates/ethereum/node/tests/e2e/blobs.rs | 8 ++--- crates/ethereum/node/tests/e2e/pool.rs | 31 ++++++++++++++++++++ 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 824c33111fc..1cc977e6d08 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -234,6 +234,14 @@ where Ok(payload) } + /// Advances the node forward one block like [`Self::advance_block`] and waits until the + /// transaction pool processed the new block, see [`Self::wait_for_pool_head`]. + pub async fn advance_block_synced(&mut self) -> eyre::Result { + let payload = self.advance_block().await?; + self.wait_for_pool_head(payload.block().hash()).await?; + Ok(payload) + } + /// Waits for block to be available on node. /// /// Returns an error if the block is not available within [`WAIT_TIMEOUT`]. @@ -296,6 +304,29 @@ where .await } + /// Waits until the transaction pool of the node processed the canonical state update that made + /// the block with the given hash its head. + /// + /// The pool maintenance task processes new heads in the background, so right after e.g. a + /// forkchoice update the pool can still contain the transactions mined in the new block or use + /// outdated sender nonces and pending fees. The pool updates its last seen block together with + /// these, so they are up to date once this returns. Exceptions are reorgs, after which the + /// maintenance task re-injects the transactions of the old chain only afterwards, so wait for + /// them with [`Self::wait_for_pool`], and commits deeper than the maximum update depth of the + /// maintenance task, e.g. after a long sync, which only update the last seen block. + /// + /// Returns an error if the pool does not process the block within [`WAIT_TIMEOUT`], e.g. + /// because the block is not the canonical head, the pool already processed a newer head, or + /// the block was synced by backfill, which does not notify the pool. + pub async fn wait_for_pool_head(&self, hash: B256) -> eyre::Result<()> { + let pool = &self.inner.pool; + poll_until(format!("transaction pool to process block {hash}"), move || { + let ready = pool.block_info().last_seen_block_hash == hash; + async move { Ok(ready.then_some(())) } + }) + .await + } + /// Asserts that a new block has been added to the blockchain and the tx has been included in /// the block, at any position. /// @@ -372,11 +403,7 @@ where // The transaction pool processes the canonical state update in the background, and // advancing the chain before it did can fail with e.g. "nonce too low" errors. Blocks // synced by backfill don't notify the pool, so wait for at most a second. - let _ = tokio::time::timeout( - Duration::from_secs(1), - self.wait_for_pool(|pool| pool.block_info().last_seen_block_hash == block), - ) - .await; + let _ = tokio::time::timeout(Duration::from_secs(1), self.wait_for_pool_head(block)).await; Ok(()) } @@ -560,10 +587,12 @@ mod tests { #[expect(dead_code)] fn test_helper_futures_are_send(node: &mut NodeHelperType) { assert_send(node.advance_block()); + assert_send(node.advance_block_synced()); assert_send(node.inject_and_advance(Bytes::new())); assert_send(node.wait_block(0, B256::ZERO, false)); assert_send(node.wait_unwind(0)); assert_send(node.wait_for_pool(|_| true)); + assert_send(node.wait_for_pool_head(B256::ZERO)); assert_send(node.assert_new_block(B256::ZERO, B256::ZERO, 0)); assert_send(node.sync_to(B256::ZERO)); } diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index 324c7560312..21fc991ecc1 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -51,11 +51,9 @@ async fn can_handle_blobs() -> eyre::Result<()> { let block_hash = node.submit_payload(payload).await?; node.update_forkchoice(genesis_hash, block_hash).await?; - // wait for the pool to process the reorg, which re-injects the blob tx - node.wait_for_pool(|pool| { - pool.block_info().last_seen_block_hash == block_hash && pool.contains(&blob_tx_hash) - }) - .await?; + // wait for the pool to process the reorg, and then re-inject the blob tx + node.wait_for_pool_head(block_hash).await?; + node.wait_for_pool(|pool| pool.contains(&blob_tx_hash)).await?; // expects the blob tx to be back in the pool let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; diff --git a/crates/ethereum/node/tests/e2e/pool.rs b/crates/ethereum/node/tests/e2e/pool.rs index 9b68ff9ff80..56ddcfb703c 100644 --- a/crates/ethereum/node/tests/e2e/pool.rs +++ b/crates/ethereum/node/tests/e2e/pool.rs @@ -289,3 +289,34 @@ async fn maintain_txpool_commit() -> eyre::Result<()> { Ok(()) } + +// Test that the pool processed the new block once `advance_block_synced` returns. +#[tokio::test] +async fn advance_block_synced_waits_for_pool() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut node, wallet) = + EthereumNode::test_setup_for(EthereumHardfork::Cancun).build_single().await?; + + for nonce in 0..3 { + let raw_tx = TransactionTestContext::transfer_tx_bytes_with_nonce( + wallet.chain_id, + wallet.inner.clone(), + nonce, + ) + .await; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; + + let payload = node.advance_block_synced().await?; + let block = payload.block(); + assert!(block.body().transactions().any(|tx| *tx.hash() == tx_hash)); + + // the mined transaction is removed from the pool without waiting any further + let info = node.inner.pool.block_info(); + assert_eq!(info.last_seen_block_hash, block.hash()); + assert_eq!(info.last_seen_block_number, block.header().number); + assert!(node.inner.pool.is_empty()); + } + + Ok(()) +} From 9dffe1fadc0bb4888a1fc4c1f5dddb7c8090168b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 30 Sep 2026 14:43:27 +0000 Subject: [PATCH 126/133] feat(e2e): add import_payload helper (#27599) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> (cherry picked from commit 569c770ca893ac59f13cfcf5c46b0e26efd64fc9) --- crates/e2e-test-utils/src/node.rs | 40 +++++++++++++++++++++---- crates/ethereum/node/tests/e2e/blobs.rs | 14 ++++----- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 1cc977e6d08..43b2b1cc665 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -224,12 +224,12 @@ where Ok(payload) } - /// Advances the node forward one block + /// Advances the node forward one block by building a payload and importing it with + /// [`Self::import_payload`]. pub async fn advance_block(&mut self) -> eyre::Result { - let payload = self.build_and_submit_payload().await?; + let payload = self.new_payload().await?; - // trigger forkchoice update via engine api to commit the block to the blockchain - self.update_forkchoice(payload.block().hash(), payload.block().hash()).await?; + self.import_payload(payload.clone()).await?; Ok(payload) } @@ -446,6 +446,30 @@ where Ok(block_hash) } + /// Submits a payload to the engine and makes its block the canonical head, returning the block + /// hash. + /// + /// The forkchoice update marks the block as head, safe, and finalized block, like + /// [`Self::advance_block`] does, so the node can't reorg to a chain without the block + /// afterwards. The engine only reports the forkchoice update valid after making the head + /// canonical, so the block is the latest block of the node once this returns. The transaction + /// pool processes the new block in the background, see [`Self::wait_for_pool`]. + /// + /// The parent of the payload must be known to the node, e.g. to import a payload built by + /// another node into its peers. Returns an error if the engine does not report the forkchoice + /// update valid, e.g. because the payload is invalid. + pub async fn import_payload(&self, payload: Payload::BuiltPayload) -> eyre::Result { + let block_hash = self.submit_payload(payload).await?; + let updated = self.update_forkchoice(block_hash, block_hash).await?; + ensure!( + updated.is_valid(), + "forkchoice update to block {block_hash} is not valid: {}", + updated.payload_status.status + ); + + Ok(block_hash) + } + /// Returns the RPC URL. pub fn rpc_url(&self) -> Url { let addr = self.inner.rpc_server_handle().http_local_addr().unwrap(); @@ -578,14 +602,17 @@ where mod tests { use super::*; use crate::NodeHelperType; - use reth_node_ethereum::EthereumNode; + use reth_node_ethereum::{EthEngineTypes, EthereumNode}; fn assert_send(_: T) {} /// Tests of downstream nodes await these helpers in spawned tasks, so their futures must be /// `Send`. #[expect(dead_code)] - fn test_helper_futures_are_send(node: &mut NodeHelperType) { + fn test_helper_futures_are_send( + node: &mut NodeHelperType, + payload: ::BuiltPayload, + ) { assert_send(node.advance_block()); assert_send(node.advance_block_synced()); assert_send(node.inject_and_advance(Bytes::new())); @@ -595,5 +622,6 @@ mod tests { assert_send(node.wait_for_pool_head(B256::ZERO)); assert_send(node.assert_new_block(B256::ZERO, B256::ZERO, 0)); assert_send(node.sync_to(B256::ZERO)); + assert_send(node.import_payload(payload)); } } diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index 21fc991ecc1..3882080fd75 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -68,7 +68,6 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let chain_spec = test_chain_spec(EthereumHardfork::Osaka); - let genesis_hash = chain_spec.genesis_hash(); let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec) .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) .build_single() @@ -92,10 +91,8 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { // build a payload let blob_payload = node.new_payload().await?; - // submit the blob payload - let blob_block_hash = node.submit_payload(blob_payload).await?; - - node.update_forkchoice(genesis_hash, blob_block_hash).await?; + // import the blob payload + node.import_payload(blob_payload).await?; Ok(()) } @@ -111,7 +108,6 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { let chain_spec = Arc::new( test_chain_spec_builder().prague_activated().with_osaka_at(osaka_timestamp).build(), ); - let genesis_hash = chain_spec.genesis_hash(); let (mut node, wallet) = EthereumNode::test_setup(1, chain_spec) .with_rpc_modifier(|rpc| rpc.with_force_blob_sidecar_upcasting()) .build_single() @@ -181,8 +177,8 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); - // submit the Prague payload - node.update_forkchoice(genesis_hash, node.submit_payload(prague_payload).await?).await?; + // import the Prague payload + node.import_payload(prague_payload).await?; // Build first Osaka payload node.set_next_payload_timestamp(osaka_timestamp)?; @@ -193,7 +189,7 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { assert!(osaka_payload.block().body().transactions().any(|tx| *tx.hash() == blob_tx_hash)); assert!(matches!(osaka_payload.sidecars(), BlobSidecars::Eip7594(_))); - node.update_forkchoice(genesis_hash, node.submit_payload(osaka_payload).await?).await?; + node.import_payload(osaka_payload).await?; Ok(()) } From b0d9bf8adeb7f71c0a69428b2265e232a50afeaa Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 3 Oct 2026 16:04:23 +0200 Subject: [PATCH 127/133] test: harden Osaka blob conversion e2e (#27687) (cherry picked from commit 7a5c817ab841b8b5ceea6dccb4f100a4d973b9e0) --- crates/ethereum/node/tests/e2e/blobs.rs | 32 ++++++++++--------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index 3882080fd75..02532194737 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -1,4 +1,4 @@ -use alloy_eips::Decodable2718; +use alloy_eips::{merge::SLOT_DURATION_SECS, Decodable2718}; use reth_chainspec::EthereumHardfork; use reth_e2e_test_utils::{ test_chain_spec, test_chain_spec_builder, transaction::TransactionTestContext, E2ETestSetupExt, @@ -7,10 +7,7 @@ use reth_ethereum_engine_primitives::BlobSidecars; use reth_ethereum_primitives::PooledTransactionVariant; use reth_node_ethereum::EthereumNode; use reth_transaction_pool::TransactionPool; -use std::{ - sync::Arc, - time::{SystemTime, UNIX_EPOCH}, -}; +use std::sync::Arc; #[tokio::test] async fn can_handle_blobs() -> eyre::Result<()> { @@ -101,9 +98,10 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { async fn blob_conversion_at_osaka() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let current_timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); - // Osaka activates in 2 slots - let osaka_timestamp = current_timestamp + 24; + // Keep genesis outside the conversion window. Importing the Prague payload below starts + // conversion two slots before Osaka, after all legacy sidecar assertions have completed. + let prague_timestamp = 2 * SLOT_DURATION_SECS; + let osaka_timestamp = prague_timestamp + 2 * SLOT_DURATION_SECS; let chain_spec = Arc::new( test_chain_spec_builder().prague_activated().with_osaka_at(osaka_timestamp).build(), @@ -113,13 +111,6 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { .build_single() .await?; - // build a dummy payload at `current_timestamp` - let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.signer(0)).await; - node.rpc.inject_tx(raw_tx).await?; - node.set_next_payload_timestamp(current_timestamp)?; - let dummy_payload = node.advance_block().await?; - assert_eq!(dummy_payload.block().timestamp, current_timestamp); - // build blob txs let first_blob = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(1)).await?; let second_blob = TransactionTestContext::tx_with_blobs_bytes(1, wallet.signer(2)).await?; @@ -150,9 +141,10 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { TransactionTestContext::validate_sidecar(envelope); // build last Prague payload - node.set_next_payload_timestamp(current_timestamp + 2)?; + node.set_next_payload_timestamp(prague_timestamp)?; let prague_payload = node.new_payload().await?; - assert_eq!(prague_payload.block().timestamp, current_timestamp + 2); + assert_eq!(prague_payload.block().timestamp, prague_timestamp); + assert!(prague_payload.block().body().transactions().any(|tx| *tx.hash() == blob_tx_hash)); assert!(matches!(prague_payload.sidecars(), BlobSidecars::Eip4844(_))); // inject second blob tx to the pool @@ -164,6 +156,9 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); + // Import the Prague payload to trigger conversion only after checking both legacy sidecars. + node.import_payload(prague_payload).await?; + // wait for the pool to convert the sidecar ahead of the Osaka activation node.wait_for_pool(|pool| { pool.get_blob(blob_tx_hash).ok().flatten().is_some_and(|sidecar| sidecar.is_eip7594()) @@ -177,9 +172,6 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> { // validate sidecar TransactionTestContext::validate_sidecar(envelope); - // import the Prague payload - node.import_payload(prague_payload).await?; - // Build first Osaka payload node.set_next_payload_timestamp(osaka_timestamp)?; let osaka_payload = node.new_payload().await?; From 0dcf86d3ff57fa35eb278b5f06981d56b96c2948 Mon Sep 17 00:00:00 2001 From: Akosh Farkash Date: Wed, 11 Nov 2026 18:16:01 +0000 Subject: [PATCH 128/133] ci: pin the docs job's Playwright CLI The docs job ran an unpinned `npx playwright install`, which resolves the newest `playwright` at run time and broke while npm had 1.65.0 of the CLI but not of `playwright-core`. Pin it to 1.64.1, as op-v2.6.0 does since ethereum-optimism/reth#38. Nothing in the docs lockfile depends on a Playwright version. (cherry picked from the book.yml half of commit 806fa3b2aa) --- .github/workflows/book.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index ffc7eb13c1e..a7d070a7a3a 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -44,7 +44,7 @@ jobs: run: | cd docs/vocs/ bun i - npx playwright install --with-deps chromium + npx playwright@1.64.1 install --with-deps chromium - name: Install Rust nightly uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 From 8d320580b075542843f317d68ac141ddc81a7c13 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:51:50 +0000 Subject: [PATCH 129/133] fix(trie): tighten iterator seek boundary (#28264) (cherry picked from commit 8572c88fe04fb2213d86f020ed9e72e612abb097) --- crates/trie/trie/src/node_iter.rs | 90 +++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/crates/trie/trie/src/node_iter.rs b/crates/trie/trie/src/node_iter.rs index c47917adca1..5d7793dec1b 100644 --- a/crates/trie/trie/src/node_iter.rs +++ b/crates/trie/trie/src/node_iter.rs @@ -277,12 +277,10 @@ where // `can_skip_node` is already set when the hash flag is set, so we don't need // to check for the hash flag explicitly. // - // It is possible that the branch node at the key `seek_key` is not stored in - // the database, so the walker will advance to the branch node after it. Because - // of this, we need to check that the current walker key has a prefix of the key - // that we seeked to. + // A deeper stored key may leave unvisited hashed entries between the seek + // prefix and the walker, so only skip the seek at the exact prefix. if can_skip_node && - self.walker.key().is_some_and(|key| key.starts_with(&seek_prefix)) && + self.walker.key().is_some_and(|key| key == &seek_prefix) && self.walker.children_are_in_trie() { trace!( @@ -307,21 +305,26 @@ where #[cfg(test)] mod tests { - use super::{TrieElement, TrieNodeIter}; + use super::*; use crate::{ hashed_cursor::{ - mock::MockHashedCursorFactory, noop::NoopHashedCursor, HashedCursorFactory, - HashedPostStateCursor, + mock::MockHashedCursorFactory, + noop::{NoopHashedCursor, NoopHashedCursorFactory}, + HashedCursorFactory, HashedPostStateCursor, HashedPostStateCursorFactory, }, mock::{KeyVisit, KeyVisitType}, trie_cursor::{ - mock::MockTrieCursorFactory, noop::NoopAccountTrieCursor, TrieCursorFactory, + mock::MockTrieCursorFactory, + noop::{NoopAccountTrieCursor, NoopTrieCursorFactory}, + InMemoryTrieCursorFactory, TrieCursorFactory, }, walker::TrieWalker, + StateRoot, }; use alloy_primitives::{ b256, map::{B256Map, HashMap}, + B256, U256, }; use alloy_trie::{ BranchNodeCompact, HashBuilder, Nibbles, TrieAccount, TrieMask, EMPTY_ROOT_HASH, @@ -329,8 +332,8 @@ mod tests { use itertools::Itertools; use reth_primitives_traits::Account; use reth_trie_common::{ - prefix_set::PrefixSetMut, updates::TrieUpdates, BranchNode, HashedPostState, LeafNode, - RlpNode, + prefix_set::PrefixSetMut, updates::TrieUpdates, BranchNode, HashedPostState, HashedStorage, + LeafNode, RlpNode, }; use std::collections::BTreeMap; @@ -380,6 +383,71 @@ mod tests { trie_updates.account_nodes } + #[test] + fn inserted_leaf_before_deeper_stored_branch_is_included() { + fn slot(prefix: &[u8]) -> B256 { + let mut bytes = [0; 32]; + bytes[..prefix.len()].copy_from_slice(prefix); + B256::from(bytes) + } + + let address = B256::repeat_byte(0xaa); + let account = || Account { nonce: 1, ..Default::default() }; + let old_slots = [ + slot(&[0x00]), + slot(&[0x0f, 0x00]), + slot(&[0x0f, 0x10]), + slot(&[0x10, 0xa0, 0x00]), + slot(&[0x10, 0xa0, 0x01]), + slot(&[0x10, 0xa0, 0x10]), + slot(&[0x10, 0xa1]), + ]; + let inserted = slot(&[0x10, 0x00]); + let old_state = + HashedPostState::default().with_accounts([(address, Some(account()))]).with_storages([ + (address, HashedStorage::from_iter(old_slots.map(|slot| (slot, U256::from(1))))), + ]); + let old_sorted = old_state.clone().into_sorted(); + let (old_root, old_updates) = StateRoot::new( + NoopTrieCursorFactory::default(), + HashedPostStateCursorFactory::new(NoopHashedCursorFactory::default(), &old_sorted), + ) + .with_prefix_sets(old_state.construct_prefix_sets().freeze()) + .root_with_updates() + .unwrap(); + + let new_state = HashedPostState::default() + .with_accounts([(address, Some(account()))]) + .with_storages([( + address, + HashedStorage::from_iter( + old_slots.into_iter().chain([inserted]).map(|slot| (slot, U256::from(1))), + ), + )]); + let delta = HashedPostState::default() + .with_accounts([(address, Some(account()))]) + .with_storages([(address, HashedStorage::from_iter([(inserted, U256::from(1))]))]); + let new_sorted = new_state.clone().into_sorted(); + let rebuilt_root = StateRoot::new( + NoopTrieCursorFactory::default(), + HashedPostStateCursorFactory::new(NoopHashedCursorFactory::default(), &new_sorted), + ) + .with_prefix_sets(new_state.construct_prefix_sets().freeze()) + .root() + .unwrap(); + let old_trie = old_updates.into_sorted(); + let incremental_root = StateRoot::new( + InMemoryTrieCursorFactory::new(NoopTrieCursorFactory::default(), &old_trie), + HashedPostStateCursorFactory::new(NoopHashedCursorFactory::default(), &new_sorted), + ) + .with_prefix_sets(delta.construct_prefix_sets().freeze()) + .root() + .unwrap(); + + assert_ne!(old_root, rebuilt_root); + assert_eq!(incremental_root, rebuilt_root); + } + #[test] fn test_trie_node_iter() { fn empty_leaf_rlp_for_key(key: Nibbles) -> RlpNode { From f4a13ba686feb3b528eb58dc2b91b4b847006dba Mon Sep 17 00:00:00 2001 From: einar-oplabs Date: Thu, 5 Nov 2026 23:33:20 -0600 Subject: [PATCH 130/133] rpc: replay block transactions through the block executor (backport of #28266) (#37) Backports paradigmxyz/reth#28266 to op-v2.6.0. Every replay of a block's own transactions, canonical, raw or bad, runs through the chain's block executor instead of a bare EVM, so traces, replayed state diffs, `debug_accountAt` and `debug_intermediateRoots` include the per-transaction changes the executor applies on top of the EVM result. `Call` gains `block_executor` and `replay_transactions_until_with_executor`; `Trace::inspect_transaction_in_block` takes the target transaction's index instead of a transaction environment and rejects an out-of-range index; `TracingCtx` borrows the result and owns the inspector, as upstream. Raw and bad blocks the executor rejects error instead of returning pre-refund traces. Differences from upstream, both forced by alloy-evm 0.39: the inspector is a type parameter of the EVM, so tracing paths build an inspected executor and enable the inspector only for the target transaction instead of calling `inspect_transaction_without_commit`; and there is no `set_block_access_index`, so the cached-BAL path still executes the target transaction on a bare EVM. Ports upstream's end-to-end test: a node whose block executor credits a settlement to every sender shows that prestate diffs, block traces and intermediate roots agree with committed state. Co-authored-by: Claude Fable 5.1 (cherry picked from commit 58c6608d33acce0460c7f0aba253566b41b2a3eb) --- .../node/tests/e2e/executor_replay.rs | 363 ++++++++++++++++++ crates/ethereum/node/tests/e2e/main.rs | 1 + crates/rpc/rpc-eth-api/src/helpers/call.rs | 95 +++-- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 247 ++++++++---- crates/rpc/rpc/src/debug.rs | 75 ++-- crates/rpc/rpc/src/eth/helpers/trace.rs | 39 ++ crates/rpc/rpc/src/otterscan.rs | 2 +- crates/rpc/rpc/src/trace.rs | 17 +- 8 files changed, 663 insertions(+), 176 deletions(-) create mode 100644 crates/ethereum/node/tests/e2e/executor_replay.rs diff --git a/crates/ethereum/node/tests/e2e/executor_replay.rs b/crates/ethereum/node/tests/e2e/executor_replay.rs new file mode 100644 index 00000000000..02040831527 --- /dev/null +++ b/crates/ethereum/node/tests/e2e/executor_replay.rs @@ -0,0 +1,363 @@ +//! Replays of a block's transactions run through the chain's block executor. + +use alloy_consensus::Header; +use alloy_eips::BlockNumberOrTag; +use alloy_network::eip2718::Encodable2718; +use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; +use alloy_provider::{ext::DebugApi, Provider, ProviderBuilder}; +use alloy_rpc_types_engine::ExecutionData; +use alloy_rpc_types_eth::{TransactionReceipt, TransactionRequest}; +use alloy_rpc_types_trace::geth::{ + DiffMode, GethDebugTracingOptions, GethTrace, PreStateConfig, PreStateFrame, TraceResult, +}; +use jsonrpsee_core::client::ClientT; +use reth_chainspec::{ChainSpec, EthereumHardfork}; +use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet::Wallet, E2ETestSetupExt}; +use reth_ethereum_primitives::{Block, EthPrimitives, Receipt, TransactionSigned, TxType}; +use reth_evm::{ + block::{ + BlockExecutionError, BlockExecutionResult, BlockExecutor, BlockExecutorFactory, + ExecutableTx, GasOutput, StateDB, + }, + eth::{ + EthBlockExecutionCtx, EthBlockExecutor, EthBlockExecutorFactory, EthEvmContext, EthTxResult, + }, + ConfigureEngineEvm, ConfigureEvm, EvmEnvFor, EvmFactory, ExecutableTxIterator, ExecutionCtxFor, + NextBlockEnvAttributes, RecoveredTx, +}; +use reth_evm_ethereum::{ + factory::RethEvmFactory, EthBlockAssembler, EthEvmConfig, RethReceiptBuilder, +}; +use reth_node_builder::{ + components::{BasicPayloadServiceBuilder, ComponentsBuilder, ExecutorBuilder}, + node::{FullNodeTypes, NodeTypes}, + BuilderContext, Node, NodeAdapter, +}; +use reth_node_ethereum::{ + EthEngineTypes, EthereumAddOns, EthereumConsensusBuilder, EthereumEngineValidatorBuilder, + EthereumEthApiBuilder, EthereumNetworkBuilder, EthereumNode, EthereumPayloadBuilder, + EthereumPoolBuilder, +}; +use reth_primitives_traits::{SealedBlock, SealedHeader}; +use reth_provider::EthStorage; +use revm::{context::result::HaltReason, Inspector}; +use std::sync::Arc; + +/// Amount the test executor credits to the sender of every transaction. +const SETTLEMENT: U256 = U256::from_limbs([1_000_000_000, 0, 0, 0]); + +/// Traces and intermediate roots of a block's transactions include the changes the block executor +/// applies on top of the EVM result, so they agree with the committed state. +#[tokio::test] +async fn replays_include_block_executor_settlement() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut node, wallet) = + SettlingNode::test_setup_for(EthereumHardfork::Prague).build_single().await?; + let provider = ProviderBuilder::new().connect_http(node.rpc_url()); + let client = node.rpc_client().expect("the HTTP RPC server is enabled"); + let sender = wallet.inner.address(); + let recipient = Address::with_last_byte(0x77); + let values = [U256::from(1), U256::from(2)]; + + let mut tx_hashes = Vec::new(); + for (nonce, value) in values.into_iter().enumerate() { + let raw_tx = transfer(&wallet, recipient, value, nonce as u64).await; + tx_hashes.push(node.rpc.inject_tx(raw_tx).await?); + } + let payload = node.advance_block().await?; + let block_hash = payload.block().hash(); + let block_number = payload.block().number; + let state_root = payload.block().state_root; + node.assert_new_block(tx_hashes[0], block_hash, block_number).await?; + + // The executor's settlement is part of the committed state. + let mut balance = provider.get_balance(sender).number(0).await?; + let mut expected_diffs = Vec::new(); + for (tx_hash, value) in tx_hashes.into_iter().zip(values) { + let receipt = provider + .get_transaction_receipt(tx_hash) + .await? + .ok_or_else(|| eyre::eyre!("missing receipt for {tx_hash}"))?; + let pre = balance; + balance = pre - value - fee(&receipt) + SETTLEMENT; + expected_diffs.push((tx_hash, pre, balance)); + } + assert_eq!(provider.get_balance(sender).number(1).await?, balance); + + let opts = GethDebugTracingOptions::prestate_tracer(PreStateConfig { + diff_mode: Some(true), + ..Default::default() + }); + let mut diffs = Vec::new(); + for (tx_hash, pre, post) in expected_diffs { + let trace = provider.debug_trace_transaction(tx_hash, opts.clone()).await?; + let diff = prestate_diff(trace)?; + assert_eq!(diff.pre[&sender].balance, Some(pre)); + assert_eq!(diff.post[&sender].balance, Some(post)); + diffs.push(diff); + } + + let block_traces = + provider.debug_trace_block_by_number(BlockNumberOrTag::Number(1), opts).await?; + let block_diffs = block_traces + .into_iter() + .map(|result| match result { + TraceResult::Success { result, .. } => prestate_diff(result), + TraceResult::Error { error, .. } => Err(eyre::eyre!("trace failed: {error}")), + }) + .collect::>>()?; + assert_eq!(block_diffs, diffs); + + let roots = + client.request::, _>("debug_intermediateRoots", (block_hash, None::<()>)).await?; + assert_eq!(roots.len(), 2); + assert_eq!(roots.last(), Some(&state_root)); + + Ok(()) +} + +/// Signs a transfer of `value` from the wallet's account to `recipient`, encoded for +/// `eth_sendRawTransaction`. +async fn transfer(wallet: &Wallet, recipient: Address, value: U256, nonce: u64) -> Bytes { + let request = TransactionRequest { + chain_id: Some(wallet.chain_id), + gas: Some(21_000), + max_fee_per_gas: Some(20_000_000_000), + max_priority_fee_per_gas: Some(1_000_000_000), + nonce: Some(nonce), + to: Some(TxKind::Call(recipient)), + value: Some(value), + ..Default::default() + }; + TransactionTestContext::sign_tx(wallet.inner.clone(), request).await.encoded_2718().into() +} + +fn fee(receipt: &TransactionReceipt) -> U256 { + U256::from(receipt.gas_used) * U256::from(receipt.effective_gas_price) +} + +fn prestate_diff(trace: GethTrace) -> eyre::Result { + match trace { + GethTrace::PreStateTracer(PreStateFrame::Diff(diff)) => Ok(diff), + trace => Err(eyre::eyre!("expected a prestate diff, got {trace:?}")), + } +} + +/// Ethereum node whose block executor credits `SETTLEMENT` to every transaction's sender. +#[derive(Debug, Default, Clone, Copy)] +struct SettlingNode; + +impl NodeTypes for SettlingNode { + type Primitives = EthPrimitives; + type ChainSpec = ChainSpec; + type Storage = EthStorage; + type Payload = EthEngineTypes; +} + +impl Node for SettlingNode +where + N: FullNodeTypes, +{ + type ComponentsBuilder = ComponentsBuilder< + N, + EthereumPoolBuilder, + BasicPayloadServiceBuilder, + EthereumNetworkBuilder, + SettlingExecutorBuilder, + EthereumConsensusBuilder, + >; + + type AddOns = + EthereumAddOns, EthereumEthApiBuilder, EthereumEngineValidatorBuilder>; + + fn components_builder(&self) -> Self::ComponentsBuilder { + EthereumNode::components().executor(SettlingExecutorBuilder) + } + + fn add_ons(&self) -> Self::AddOns { + EthereumAddOns::default() + } +} + +#[derive(Debug, Default, Clone, Copy)] +struct SettlingExecutorBuilder; + +impl ExecutorBuilder for SettlingExecutorBuilder +where + N: FullNodeTypes, +{ + type EVM = SettlingEvmConfig; + + async fn build_evm(self, ctx: &BuilderContext) -> eyre::Result { + let inner = EthEvmConfig::new_with_evm_factory(ctx.chain_spec(), RethEvmFactory::default()); + let executor_factory = SettlingExecutorFactory { inner: inner.executor_factory.clone() }; + Ok(SettlingEvmConfig { inner, executor_factory }) + } +} + +#[derive(Debug, Clone)] +struct SettlingEvmConfig { + inner: EthEvmConfig, + executor_factory: SettlingExecutorFactory, +} + +impl ConfigureEvm for SettlingEvmConfig { + type Primitives = EthPrimitives; + type Error = as ConfigureEvm>::Error; + type NextBlockEnvCtx = NextBlockEnvAttributes; + type BlockExecutorFactory = SettlingExecutorFactory; + type BlockAssembler = EthBlockAssembler; + + fn block_executor_factory(&self) -> &Self::BlockExecutorFactory { + &self.executor_factory + } + + fn block_assembler(&self) -> &Self::BlockAssembler { + self.inner.block_assembler() + } + + fn evm_env(&self, header: &Header) -> Result, Self::Error> { + self.inner.evm_env(header) + } + + fn next_evm_env( + &self, + parent: &Header, + attributes: &NextBlockEnvAttributes, + ) -> Result, Self::Error> { + self.inner.next_evm_env(parent, attributes) + } + + fn context_for_block<'a>( + &self, + block: &'a SealedBlock, + ) -> Result, Self::Error> { + self.inner.context_for_block(block) + } + + fn context_for_next_block( + &self, + parent: &SealedHeader, + attributes: Self::NextBlockEnvCtx, + ) -> Result, Self::Error> { + self.inner.context_for_next_block(parent, attributes) + } +} + +impl ConfigureEngineEvm for SettlingEvmConfig { + fn evm_env_for_payload(&self, payload: &ExecutionData) -> Result, Self::Error> { + self.inner.evm_env_for_payload(payload) + } + + fn context_for_payload<'a>( + &self, + payload: &'a ExecutionData, + ) -> Result, Self::Error> { + self.inner.context_for_payload(payload) + } + + fn tx_iterator_for_payload( + &self, + payload: &ExecutionData, + ) -> Result, Self::Error> { + self.inner.tx_iterator_for_payload(payload) + } +} + +#[derive(Debug, Clone)] +struct SettlingExecutorFactory { + inner: EthBlockExecutorFactory, RethEvmFactory>, +} + +type SettlingEvm = ::Evm; + +type SettlingTxResult = EthTxResult; + +impl BlockExecutorFactory for SettlingExecutorFactory { + type EvmFactory = RethEvmFactory; + type TxExecutionResult = SettlingTxResult; + type ExecutionCtx<'a> = EthBlockExecutionCtx<'a>; + type Transaction = TransactionSigned; + type Receipt = Receipt; + type Executor<'a, DB: StateDB, I: Inspector>> = SettlingExecutor<'a, DB, I>; + + fn evm_factory(&self) -> &Self::EvmFactory { + self.inner.evm_factory() + } + + fn create_executor<'a, DB, I>( + &'a self, + evm: SettlingEvm, + ctx: Self::ExecutionCtx<'a>, + ) -> Self::Executor<'a, DB, I> + where + DB: StateDB, + I: Inspector>, + { + SettlingExecutor { + inner: EthBlockExecutor::new(evm, ctx, self.inner.spec(), self.inner.receipt_builder()), + } + } +} + +type InnerExecutor<'a, DB, I> = + EthBlockExecutor<'a, SettlingEvm, &'a Arc, &'a RethReceiptBuilder>; + +/// Ethereum block executor that credits `SETTLEMENT` to the sender in each transaction's +/// uncommitted state, like a chain settling a per-transaction refund outside the EVM. +struct SettlingExecutor<'a, DB: StateDB, I: Inspector>> { + inner: InnerExecutor<'a, DB, I>, +} + +impl BlockExecutor for SettlingExecutor<'_, DB, I> +where + DB: StateDB, + I: Inspector>, +{ + type Transaction = TransactionSigned; + type Receipt = Receipt; + type Evm = SettlingEvm; + type Result = SettlingTxResult; + + fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> { + self.inner.apply_pre_execution_changes() + } + + fn execute_transaction_without_commit( + &mut self, + tx: impl ExecutableTx, + ) -> Result { + let (tx_env, tx) = tx.into_parts(); + let sender = *tx.signer(); + + let mut output = self.inner.execute_transaction_without_commit((tx_env, tx))?; + let account = + output.result.state.get_mut(&sender).expect("every transaction loads its sender"); + account.info.balance += SETTLEMENT; + + Ok(output) + } + + fn commit_transaction(&mut self, output: Self::Result) -> GasOutput { + self.inner.commit_transaction(output) + } + + fn finish( + self, + ) -> Result<(Self::Evm, BlockExecutionResult), BlockExecutionError> { + self.inner.finish() + } + + fn evm_mut(&mut self) -> &mut Self::Evm { + self.inner.evm_mut() + } + + fn evm(&self) -> &Self::Evm { + self.inner.evm() + } + + fn receipts(&self) -> &[Self::Receipt] { + self.inner.receipts() + } +} diff --git a/crates/ethereum/node/tests/e2e/main.rs b/crates/ethereum/node/tests/e2e/main.rs index 7272b4fade8..c43552a56f7 100644 --- a/crates/ethereum/node/tests/e2e/main.rs +++ b/crates/ethereum/node/tests/e2e/main.rs @@ -5,6 +5,7 @@ mod custom_genesis; mod dev; mod eip6780; mod eth; +mod executor_replay; mod forkchoice; mod invalid_payload; mod p2p; diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index ce47b596fd0..818b44d9af9 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -7,7 +7,7 @@ use super::{LoadBlock, LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocki use crate::{ helpers::estimate::EstimateCall, FromEvmError, FullEthApiTypes, RpcBlock, RpcNodeCore, }; -use alloy_consensus::{transaction::TxHashRef, BlockHeader}; +use alloy_consensus::BlockHeader; use alloy_eips::eip2930::AccessListResult; use alloy_evm::overrides::{apply_block_overrides, apply_state_overrides, OverrideBlockHashes}; use alloy_network::TransactionBuilder; @@ -21,11 +21,14 @@ use futures::Future; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_errors::{ProviderError, RethError}; use reth_evm::{ - block::BlockExecutor, env::BlockEnvironment, execute::BlockBuilder, ConfigureEvm, Evm, - EvmEnvFor, EvmFor, HaltReasonFor, InspectorFor, TransactionEnvMut, TxEnvFor, + block::{BlockExecutor, TxResult}, + env::BlockEnvironment, + execute::BlockBuilder, + BlockExecutorForEvm, ConfigureEvm, Evm, EvmEnvFor, HaltReasonFor, InspectorFor, + TransactionEnvMut, TxEnvFor, }; use reth_node_api::BlockBody; -use reth_primitives_traits::Recovered; +use reth_primitives_traits::{BlockTy, Recovered, RecoveredBlock}; use reth_revm::{ database::StateProviderDatabase, db::{ @@ -40,7 +43,9 @@ use reth_rpc_eth_types::{ simulate::{self, EthSimulateError}, EthApiError, StateCacheDb, }; -use reth_storage_api::{BlockIdReader, ProviderTx, StateProvider}; +use reth_storage_api::{ + BlockIdReader, EvmStateProviderAdapter, ProviderTx, StateProvider, StateProviderBox, +}; use reth_tasks::CancelOnDrop; use revm::{ context::Block, @@ -701,7 +706,7 @@ pub trait Call: /// /// Before the transaction is executed, the state is positioned right before the transaction, /// either by attaching the block's cached BAL or by executing all previous transactions in the - /// block. + /// block through the block executor, which then also executes the transaction. /// The callback `f` is invoked with the [`ResultAndState`] after the transaction was executed /// and the database that points to the beginning of the transaction. The database may have /// the block's BAL attached and must only be used for reads, because an attached BAL takes @@ -747,25 +752,16 @@ pub trait Call: return f(tx_info, res, db) } - let block_txs = block.transactions_recovered(); - - let mut executor = RpcNodeCore::evm_config(&this) - .executor_for_block(&mut db, block.sealed_block()) - .map_err(RethError::other) - .map_err(Self::Error::from_eth_err)?; - executor.apply_pre_execution_changes().map_err(Self::Error::from_eth_err)?; - - // replay all transactions prior to the targeted transaction - for block_tx in block_txs { - if block_tx.tx_hash() == tx.tx_hash() { - break; - } - executor.execute_transaction(block_tx).map_err(Self::Error::from_eth_err)?; - } - - let tx_env = RpcNodeCore::evm_config(&this).tx_env(tx); - - let res = executor.evm_mut().transact(tx_env).map_err(Self::Error::from_evm_err)?; + let mut executor = this.block_executor(&block, &mut db)?; + this.replay_transactions_until_with_executor( + &mut executor, + block.transactions_recovered(), + tx_info.index.expect("included transaction has an index") as usize, + )?; + let res = executor + .execute_transaction_without_commit(tx) + .map_err(Self::Error::from_eth_err)? + .into_result(); drop(executor); f(tx_info, res, db) }) @@ -774,34 +770,49 @@ pub trait Call: } } - /// Replays all transactions before the target transaction index on the given EVM. + /// Creates a block executor for `block` on `db` and applies the block's pre-execution + /// changes. /// - /// This executes on a caller provided EVM, so the target transaction can then be run on the - /// same EVM, keeping any block-scoped EVM state intact. The EVM's inspector configuration is - /// left untouched; see - /// [`Trace::inspect_transaction_in_block`] to replay without inspection and trace the target. + /// Replaying a block's own transactions through the executor rather than a bare EVM includes + /// the changes the executor applies on top of each EVM result, so replayed results agree with + /// the block's receipts and state. + fn block_executor<'a>( + &'a self, + block: &'a RecoveredBlock>, + db: &'a mut StateCacheDb, + ) -> Result< + BlockExecutorForEvm< + 'a, + Self::Evm, + StateProviderDatabase>, + >, + Self::Error, + > { + let mut executor = self + .evm_config() + .executor_for_block(db, block.sealed_block()) + .map_err(RethError::other) + .map_err(Self::Error::from_eth_err)?; + executor.apply_pre_execution_changes().map_err(Self::Error::from_eth_err)?; + Ok(executor) + } + + /// Replays all transactions before the target transaction index through the given executor. /// /// If the target index is greater than or equal to the iterator length, all transactions are /// replayed. - fn replay_transactions_until_with_evm<'a, DB, I, Txs>( + fn replay_transactions_until_with_executor<'a, E, Txs>( &self, - evm: &mut EvmFor, + executor: &mut E, transactions: Txs, target_tx_index: usize, ) -> Result<(), Self::Error> where - DB: Database> + DatabaseCommit + core::fmt::Debug, - I: InspectorFor, + E: BlockExecutor>, Txs: IntoIterator>>, { - for (index, tx) in transactions.into_iter().enumerate() { - if index == target_tx_index { - // reached the target transaction - break - } - - let tx_env = self.evm_config().tx_env(tx); - evm.transact_commit(tx_env).map_err(Self::Error::from_evm_err)?; + for tx in transactions.into_iter().take(target_tx_index) { + executor.execute_transaction(tx).map_err(Self::Error::from_eth_err)?; } Ok(()) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index d51921713e5..010c153f195 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -9,12 +9,15 @@ use alloy_rpc_types_eth::{BlockId, TransactionInfo}; use futures::Future; use reth_errors::RethError; use reth_evm::{ - block::BlockExecutor, evm::EvmFactoryExt, tracing::TracingCtx, ConfigureEvm, Evm, EvmEnvFor, - EvmFor, HaltReasonFor, InspectorFor, IntoTxEnv, TxEnvFor, + block::{BlockExecutor, BlockExecutorFor, TxResult}, + ConfigureEvm, Evm, EvmEnvFor, EvmFor, HaltReasonFor, InspectorFor, IntoTxEnv, TxEnvFor, }; -use reth_primitives_traits::{BlockBody, BlockTy, Recovered, RecoveredBlock}; -use reth_rpc_eth_types::cache::db::{attach_bal_before_tx, StateCacheDb}; -use reth_storage_api::{ProviderBlock, ProviderTx}; +use reth_primitives_traits::{BlockBody, BlockTy, RecoveredBlock}; +use reth_rpc_eth_types::{ + cache::db::{attach_bal_before_tx, StateCacheDb}, + EthApiError, +}; +use reth_storage_api::ProviderBlock; use revm::{context::Block, context_interface::result::ResultAndState, state::bal::Bal as RevmBal}; use revm_inspectors::tracing::{TracingInspector, TracingInspectorConfig}; use std::sync::Arc; @@ -40,7 +43,7 @@ pub trait Trace: LoadState> + Call { /// /// Before the transaction is traced, the state is positioned right before the transaction, /// either by attaching the block's cached BAL or by executing all previous transactions in - /// the block. + /// the block, see [`Trace::inspect_transaction_in_block`]. /// The callback `f` is invoked with the [`ResultAndState`] after the transaction was executed /// and the database that points to the beginning of the transaction. The database may have /// the block's BAL attached and must only be used for reads, because an attached BAL takes @@ -73,7 +76,7 @@ pub trait Trace: LoadState> + Call { /// /// Before the transaction is traced, the state is positioned right before the transaction, /// either by attaching the block's cached BAL or by executing all previous transactions in - /// the block. + /// the block, see [`Trace::inspect_transaction_in_block`]. /// The callback `f` is invoked with the [`ResultAndState`] after the transaction was executed /// and the database that points to the beginning of the transaction. The database may have /// the block's BAL attached and must only be used for reads, because an attached BAL takes @@ -106,7 +109,7 @@ pub trait Trace: LoadState> + Call { None => return Ok(None), Some(res) => res, }; - let (tx, tx_info) = transaction.split(); + let (_, tx_info) = transaction.split(); // we need to get the state of the parent block because we're essentially replaying the // block the transaction is included in @@ -121,7 +124,6 @@ pub trait Trace: LoadState> + Call { // returns transactions included in a block tx_info.index.expect("transaction_and_block only returns block transactions") as usize, - tx, bal.as_deref(), )?; f(tx_info, inspector, res, db) @@ -134,9 +136,10 @@ pub trait Trace: LoadState> + Call { /// Positions the state of `db` right before the transaction at the target index. /// /// If the block's cached BAL is given, it is attached to the database at the target index and - /// no transactions are executed, see [`attach_bal_before_tx`]. Otherwise all transactions - /// before the target transaction are executed and their changes are written to the - /// _runtime_ db ([`StateCacheDb`]). + /// no transactions are executed, see [`attach_bal_before_tx`]. Otherwise the block's + /// pre-execution changes and all transactions before the target transaction are executed + /// through the block executor, and their changes are written to the _runtime_ db + /// ([`StateCacheDb`]). /// /// If the target index is greater than or equal to the block's transaction count, all /// transactions are replayed. @@ -152,23 +155,26 @@ pub trait Trace: LoadState> + Call { return Ok(()) } - self.apply_pre_execution_changes(block, db)?; - - let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; - let mut evm = self.evm_config().evm_with_env(db, evm_env); - self.replay_transactions_until_with_evm( - &mut evm, + let mut executor = self.block_executor(block, db)?; + self.replay_transactions_until_with_executor( + &mut executor, block.transactions_recovered(), target_tx_index, ) } - /// Executes the target transaction with the configured inspector on the state right before - /// the transaction. + /// Executes the block's transaction at the target index with the configured inspector on the + /// state right before the transaction. /// /// If the block's cached BAL is given, the state is positioned by attaching the BAL at the - /// target index, see [`attach_bal_before_tx`]. Otherwise all transactions before the target - /// transaction are replayed without inspection first. + /// target index, see [`attach_bal_before_tx`], and the transaction is executed on its own, so + /// per-transaction changes the block executor applies are skipped. Otherwise all transactions + /// before the target transaction are replayed without inspection through the block executor, + /// see [`Call::block_executor`], which then executes the target transaction, so the result + /// includes any per-transaction changes the executor applies. The result is not committed + /// to `db`. + /// + /// Returns [`EthApiError::InvalidParams`] if the block has no transaction at that index. #[expect(clippy::type_complexity)] fn inspect_transaction_in_block<'a>( &self, @@ -176,36 +182,81 @@ pub trait Trace: LoadState> + Call { db: &'a mut StateCacheDb, inspector: impl InspectorFor, target_tx_index: usize, - target_tx_env: impl IntoTxEnv>, bal: Option<&DecodedBal>>, ) -> Result<(ResultAndState>, EvmEnvFor), Self::Error> { + let Some(target_tx) = block.transactions_recovered().nth(target_tx_index) else { + return Err(Self::Error::from_eth_err(EthApiError::InvalidParams(format!( + "tx_index {target_tx_index} out of bounds for block with {} transactions", + block.transaction_count() + )))) + }; + let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; + if let Some(bal) = bal { // the BAL also covers the block's pre-execution changes attach_bal_before_tx(db, bal, target_tx_index); - } else { - self.apply_pre_execution_changes(block, db)?; + let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env, inspector); + let res = evm + .transact(self.evm_config().tx_env(target_tx)) + .map_err(Self::Error::from_evm_err)?; + let (_, evm_env) = evm.finish(); + return Ok((res, evm_env)) } - let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; - let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env, inspector); - - if bal.is_none() { - evm.disable_inspector(); - self.replay_transactions_until_with_evm( - &mut evm, - block.transactions_recovered(), - target_tx_index, - )?; - evm.enable_inspector(); - } - - let res = evm.transact(target_tx_env).map_err(Self::Error::from_evm_err)?; + let mut executor = self.inspected_block_executor(block, db, evm_env.clone(), inspector)?; + self.replay_transactions_until_with_executor( + &mut executor, + block.transactions_recovered(), + target_tx_index, + )?; - let (_, evm_env) = evm.finish(); + executor.evm_mut().enable_inspector(); + let res = executor + .execute_transaction_without_commit(target_tx) + .map_err(Self::Error::from_eth_err)? + .into_result(); Ok((res, evm_env)) } + /// Creates a block executor for `block` whose EVM runs the given inspector, and applies the + /// block's pre-execution changes. The inspector is left disabled. + /// + /// The uninspected counterpart is [`Call::block_executor`]; this one exists because the EVM + /// takes its inspector as a type parameter, so a tracer has to be in place when the executor + /// is created. + #[expect(clippy::type_complexity, reason = "the executor type is spelled out by its factory")] + fn inspected_block_executor<'a, 'db, I>( + &'a self, + block: &'a RecoveredBlock>, + db: &'db mut StateCacheDb, + evm_env: EvmEnvFor, + inspector: I, + ) -> Result< + BlockExecutorFor< + 'a, + ::BlockExecutorFactory, + &'db mut StateCacheDb, + I, + >, + Self::Error, + > + where + I: InspectorFor, + { + let evm_config = self.evm_config(); + let ctx = evm_config + .context_for_block(block.sealed_block()) + .map_err(RethError::other) + .map_err(Self::Error::from_eth_err)?; + let mut evm = evm_config.evm_with_env_and_inspector(db, evm_env, inspector); + evm.disable_inspector(); + + let mut executor = evm_config.create_executor_with_state(evm, ctx); + executor.apply_pre_execution_changes().map_err(Self::Error::from_eth_err)?; + Ok(executor) + } + /// Executes all transactions of a block up to a given index. /// /// If a `highest_index` is given, this will only execute the first `highest_index` @@ -226,7 +277,7 @@ pub trait Trace: LoadState> + Call { TransactionInfo, TracingCtx< '_, - Recovered<&ProviderTx>, + TracingInspector, EvmFor, >, ) -> Result @@ -265,16 +316,12 @@ pub trait Trace: LoadState> + Call { Self: LoadBlock, F: Fn( TransactionInfo, - TracingCtx< - '_, - Recovered<&ProviderTx>, - EvmFor, - >, + TracingCtx<'_, Insp, EvmFor>, ) -> Result + Send + 'static, Setup: FnMut() -> Insp + Send + 'static, - Insp: Clone + for<'a> InspectorFor, + Insp: for<'a> InspectorFor, R: Send + 'static, { async move { @@ -299,38 +346,49 @@ pub trait Trace: LoadState> + Call { let block_timestamp = evm_env.block_env.timestamp().saturating_to(); let base_fee = evm_env.block_env.basefee(); - this.apply_pre_execution_changes(&block, &mut db)?; - // prepare transactions, we do everything upfront to reduce time spent with open // state - let max_transactions = highest_index.map_or_else( - || block.body().transaction_count(), - |highest| { - // we need + 1 because the index is 0-based - highest as usize + 1 - }, - ); - - let mut idx = 0; - - let results = this - .evm_config() - .evm_factory() - .create_tracer(&mut db, evm_env, inspector_setup()) - .try_trace_many(block.transactions_recovered().take(max_transactions), |ctx| { - let tx_info = TransactionInfo { - hash: Some(*ctx.tx.tx_hash()), - index: Some(idx), - block_hash: Some(block_hash), - block_number: Some(block_number), - block_timestamp: Some(block_timestamp), - base_fee: Some(base_fee), - }; - idx += 1; - - f(tx_info, ctx) - }) - .collect::>()?; + let max_transactions = highest_index + .map_or_else( + || block.body().transaction_count(), + |highest| { + // we need + 1 because the index is 0-based + (highest as usize).saturating_add(1) + }, + ) + .min(block.body().transaction_count()); + + let mut executor = + this.inspected_block_executor(&block, &mut db, evm_env, inspector_setup())?; + executor.evm_mut().enable_inspector(); + + let mut results = Vec::with_capacity(max_transactions); + let mut transactions = + block.transactions_recovered().take(max_transactions).enumerate().peekable(); + while let Some((idx, tx)) = transactions.next() { + let output = executor + .execute_transaction_without_commit(tx) + .map_err(Self::Error::from_eth_err)?; + + let tx_info = TransactionInfo { + hash: Some(*tx.tx_hash()), + index: Some(idx as u64), + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp: Some(block_timestamp), + base_fee: Some(base_fee), + }; + let (db, inspector, _) = executor.evm_mut().components_mut(); + let inspector = core::mem::replace(inspector, inspector_setup()); + results.push(f( + tx_info, + TracingCtx { result: output.result(), db: &mut *db, inspector }, + )?); + // The state of the last traced transaction is not needed. + if transactions.peek().is_some() { + executor.commit_transaction(output); + } + } Ok(Some(results)) }) @@ -363,7 +421,7 @@ pub trait Trace: LoadState> + Call { TransactionInfo, TracingCtx< '_, - Recovered<&ProviderTx>, + TracingInspector, EvmFor, >, ) -> Result @@ -401,16 +459,12 @@ pub trait Trace: LoadState> + Call { // state and db F: Fn( TransactionInfo, - TracingCtx< - '_, - Recovered<&ProviderTx>, - EvmFor, - >, + TracingCtx<'_, Insp, EvmFor>, ) -> Result + Send + 'static, Setup: FnMut() -> Insp + Send + 'static, - Insp: Clone + for<'a> InspectorFor, + Insp: for<'a> InspectorFor, R: Send + 'static, { self.trace_block_until_with_inspector(block_id, block, None, insp_setup, f) @@ -435,3 +489,28 @@ pub trait Trace: LoadState> + Call { Ok(()) } } + +/// Context passed to per-transaction trace callbacks. +/// +/// Explicit callback annotations must import this type instead of alloy-evm's tracing context. +pub struct TracingCtx<'a, Insp, E: Evm> { + /// Execution result and state changes for the transaction. + pub result: &'a ResultAndState, + /// Database pointing to the transaction pre-state. + pub db: &'a mut E::DB, + /// Inspector used while executing the transaction. + pub inspector: Insp, +} + +impl core::fmt::Debug for TracingCtx<'_, Insp, E> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TracingCtx").finish_non_exhaustive() + } +} + +impl TracingCtx<'_, Insp, E> { + /// Consumes this context and returns the inspector. + pub fn take_inspector(self) -> Insp { + self.inspector + } +} diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 3c137519b83..35a9ee16b2a 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -20,7 +20,11 @@ use parking_lot::RwLock; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; use reth_engine_primitives::ConsensusEngineEvent; use reth_errors::RethError; -use reth_evm::{block::BlockExecutor, execute::Executor, ConfigureEvm, EvmEnvFor}; +use reth_evm::{ + block::{BlockExecutor, TxResult}, + execute::Executor, + ConfigureEvm, EvmEnvFor, +}; use reth_primitives_traits::{ Block as BlockTrait, BlockBody, BlockTy, ReceiptWithBloom, RecoveredBlock, }; @@ -29,7 +33,7 @@ use reth_rpc_api::DebugApiServer; use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ helpers::{EthTransactions, TraceExt}, - AsEthApiError, FromEthApiError, FromEvmError, RpcConvert, RpcNodeCore, + AsEthApiError, FromEthApiError, RpcConvert, RpcNodeCore, }; use reth_rpc_eth_types::{EthApiError, StateCacheDb}; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; @@ -113,7 +117,9 @@ where self.inner.blocking_task_guard.clone().acquire_owned().await } - /// Trace the entire block asynchronously + /// Traces every transaction of a block through the block executor, see + /// [`Call::block_executor`](reth_rpc_eth_api::helpers::Call::block_executor), so the traces + /// include the per-transaction changes the executor applies. async fn trace_block( &self, block: Arc>>, @@ -124,20 +130,22 @@ where .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { let mut results = Vec::with_capacity(block.body().transactions().len()); - eth_api.apply_pre_execution_changes(&block, &mut db)?; - let block_env = evm_env.block_env.clone(); - let mut transactions = block.transactions_recovered().enumerate().peekable(); let inspector = DebugInspector::new(opts).map_err(Eth::Error::from_eth_err)?; - let mut evm = - eth_api.evm_config().evm_with_env_and_inspector(&mut db, evm_env, inspector); + let mut executor = + eth_api.inspected_block_executor(&block, &mut db, evm_env, inspector)?; + executor.evm_mut().enable_inspector(); + + let mut transactions = block.transactions_recovered().enumerate().peekable(); while let Some((index, tx)) = transactions.next() { let tx_env = eth_api.evm_config().tx_env(tx); - let res = evm.transact(tx_env.clone()).map_err(Eth::Error::from_evm_err)?; + let output = executor + .execute_transaction_without_commit(tx) + .map_err(Eth::Error::from_eth_err)?; - let (db, inspector, _) = evm.components_mut(); + let (db, inspector, _) = executor.evm_mut().components_mut(); let result = inspector .get_result( Some(TransactionContext { @@ -147,7 +155,7 @@ where }), &tx_env, &block_env, - &res, + output.result(), db, ) .map_err(Eth::Error::from_eth_err)?; @@ -157,7 +165,7 @@ where inspector.fuse().map_err(Eth::Error::from_eth_err)?; // need to apply the state changes of this transaction before executing the // next transaction - db.commit(res.state) + executor.commit_transaction(output); } } @@ -180,8 +188,6 @@ where let tx_count = block.body().transactions().len(); let mut results = Vec::with_capacity(tx_count); - eth_api.apply_pre_execution_changes(&block, &mut db)?; - let block_env = evm_env.block_env.clone(); let mut transactions = block.transactions_recovered().enumerate().peekable(); let inspector = match DebugInspector::new(opts) { @@ -197,14 +203,15 @@ where return Ok(results) } }; - let mut evm = - eth_api.evm_config().evm_with_env_and_inspector(&mut db, evm_env, inspector); + let mut executor = + eth_api.inspected_block_executor(&block, &mut db, evm_env, inspector)?; + executor.evm_mut().enable_inspector(); while let Some((index, tx)) = transactions.next() { let tx_hash = *tx.tx_hash(); let tx_env = eth_api.evm_config().tx_env(tx); - let res = match evm.transact(tx_env.clone()) { - Ok(res) => res, + let output = match executor.execute_transaction_without_commit(tx) { + Ok(output) => output, Err(err) => { results.push(Some(TraceResult::Error { error: err.to_string(), @@ -214,7 +221,7 @@ where } }; - let (db, inspector, _) = evm.components_mut(); + let (db, inspector, _) = executor.evm_mut().components_mut(); let result = match inspector.get_result( Some(TransactionContext { block_hash: Some(block.hash()), @@ -223,7 +230,7 @@ where }), &tx_env, &block_env, - &res, + output.result(), db, ) { Ok(result) => result, @@ -245,7 +252,7 @@ where })); break } - db.commit(res.state); + executor.commit_transaction(output); } } @@ -340,7 +347,6 @@ where &mut db, &mut inspector, index, - tx_env.clone(), bal.as_deref(), )?; @@ -703,16 +709,12 @@ where self.eth_api() .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { - let mut executor = eth_api - .evm_config() - .executor_for_block(&mut db, block.sealed_block()) - .map_err(RethError::other) - .map_err(Eth::Error::from_eth_err)?; - executor.apply_pre_execution_changes().map_err(Eth::Error::from_eth_err)?; - - for tx in block.transactions_recovered().take(tx_index + 1) { - executor.execute_transaction(tx).map_err(Eth::Error::from_eth_err)?; - } + let mut executor = eth_api.block_executor(&block, &mut db)?; + eth_api.replay_transactions_until_with_executor( + &mut executor, + block.transactions_recovered(), + tx_index + 1, + )?; drop(executor); f(&mut db) @@ -812,22 +814,19 @@ where .recovered_block(block_hash.into()) .await? .ok_or(EthApiError::HeaderNotFound(block_hash.into()))?; - let evm_env = self.eth_api().evm_env_for_header(block.sealed_block().sealed_header())?; self.eth_api() .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { // Enable transition tracking so that merge_transitions works db.transition_state = Some(Default::default()); - eth_api.apply_pre_execution_changes(&block, &mut db)?; + let mut executor = eth_api.block_executor(&block, &mut db)?; let mut roots = Vec::with_capacity(block.body().transactions().len()); - let mut evm = eth_api.evm_config().evm_with_env(&mut db, evm_env); for tx in block.transactions_recovered() { - let tx_env = eth_api.evm_config().tx_env(tx); - evm.transact_commit(tx_env).map_err(Eth::Error::from_evm_err)?; + executor.execute_transaction(tx).map_err(Eth::Error::from_eth_err)?; - let state = evm.db_mut(); + let state = executor.evm_mut().db_mut(); // Merge transitions into cumulative bundle_state state.merge_transitions(BundleRetention::PlainState); // Compute state root from the accumulated state changes diff --git a/crates/rpc/rpc/src/eth/helpers/trace.rs b/crates/rpc/rpc/src/eth/helpers/trace.rs index 55b1604eeae..ddfaea5c66b 100644 --- a/crates/rpc/rpc/src/eth/helpers/trace.rs +++ b/crates/rpc/rpc/src/eth/helpers/trace.rs @@ -13,3 +13,42 @@ where Rpc: RpcConvert, { } + +#[cfg(test)] +mod tests { + use super::*; + use crate::eth::helpers::types::EthRpcConverter; + use alloy_eips::BlockId; + use reth_chainspec::ChainSpec; + use reth_ethereum_primitives::Block; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_primitives_traits::RecoveredBlock; + use reth_provider::test_utils::NoopProvider; + use reth_rpc_eth_api::helpers::Call; + use reth_transaction_pool::test_utils::testing_pool; + use revm_inspectors::tracing::{TracingInspector, TracingInspectorConfig}; + + #[tokio::test] + async fn inspect_transaction_in_block_out_of_bounds_is_invalid_params() { + let eth_api = EthApi::<_, EthRpcConverter>::builder( + NoopProvider::default(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::mainnet(), + ) + .build(); + let block = RecoveredBlock::new_unhashed(Block::default(), vec![]); + + let err = eth_api + .spawn_with_state_at_block(BlockId::latest(), move |this, mut db| { + let mut inspector = TracingInspector::new(TracingInspectorConfig::default_parity()); + this.inspect_transaction_in_block(&block, &mut db, &mut inspector, 0, None) + .map(|_| ()) + }) + .await + .unwrap_err(); + + assert_eq!(err.to_string(), "tx_index 0 out of bounds for block with 0 transactions"); + } +} diff --git a/crates/rpc/rpc/src/otterscan.rs b/crates/rpc/rpc/src/otterscan.rs index 54355746a60..34a00ef3853 100644 --- a/crates/rpc/rpc/src/otterscan.rs +++ b/crates/rpc/rpc/src/otterscan.rs @@ -333,7 +333,7 @@ where num.into(), None, TracingInspectorConfig::default_parity(), - |tx_info, mut ctx| { + |tx_info, ctx| { Ok(ctx .take_inspector() .into_parity_builder() diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 4aa98ac83fe..19fbfc6570c 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -453,7 +453,7 @@ where Some(block.clone()), None, TracingInspectorConfig::default_parity(), - move |tx_info, mut ctx| { + move |tx_info, ctx| { // Keep the block replay permit inside the spawned replay task. let _block_replay_permit = &permit; let mut traces = ctx @@ -536,7 +536,7 @@ where block_id, Some(block.clone()), TracingInspectorConfig::default_parity(), - |tx_info, mut ctx| { + |tx_info, ctx| { let traces = ctx .take_inspector() .into_parity_builder() @@ -572,16 +572,11 @@ where block_id, None, TracingInspectorConfig::from_parity_config(&trace_types), - move |tx_info, mut ctx| { + move |tx_info, ctx| { let full_trace = ctx - .take_inspector() + .inspector .into_parity_builder() - .into_trace_results_with_state_parts( - &ctx.result, - ctx.state, - &trace_types, - &ctx.db, - ) + .into_trace_results_with_state(ctx.result, &trace_types, &**ctx.db) .map_err(Eth::Error::from_eth_err)?; let trace = TraceResultsWithTransactionHash { @@ -648,7 +643,7 @@ where block_id, Some(block.clone()), StorageInspector::default, - move |tx_info, mut ctx| { + move |tx_info, ctx| { let unique_loads = ctx.inspector.unique_loads(); let warm_loads = ctx.inspector.warm_loads(); let trace = TransactionStorageAccess { From df85687cdbf9a2ded413c09ac5081f77c897735d Mon Sep 17 00:00:00 2001 From: Akosh Farkash Date: Tue, 10 Nov 2026 20:23:57 +0000 Subject: [PATCH 131/133] rpc: name the database the replay block executor runs on clippy's type_complexity rejects the return type of Call::block_executor since v2.7.0 wraps the provider in EvmStateProviderAdapter. Name the database StateCacheDb reads through instead. --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 818b44d9af9..0e20e1d39b2 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -59,6 +59,9 @@ use tracing::{debug, trace}; /// Result type for `eth_simulateV1` RPC method. pub type SimulatedBlocksResult = Result>>, E>; +/// The database a [`StateCacheDb`] reads through, which [`Call::block_executor`] executes on. +type StateCacheProviderDb = StateProviderDatabase>; + /// Execution related functions for the [`EthApiServer`](crate::EthApiServer) trait in /// the `eth_` namespace. pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthApiTypes { @@ -780,14 +783,7 @@ pub trait Call: &'a self, block: &'a RecoveredBlock>, db: &'a mut StateCacheDb, - ) -> Result< - BlockExecutorForEvm< - 'a, - Self::Evm, - StateProviderDatabase>, - >, - Self::Error, - > { + ) -> Result, Self::Error> { let mut executor = self .evm_config() .executor_for_block(db, block.sealed_block()) From d36f93b8046534d21d7286ba7e212b62a7e33045 Mon Sep 17 00:00:00 2001 From: StreamingFast Chain Update Bot Date: Mon, 16 Nov 2026 18:00:33 +0000 Subject: [PATCH 132/133] Build the Firehose engine path against reth v2.7.0 The traced executor takes an EvmStateProvider like upstream's execute_block, and the engine-tree Firehose tests still need serde_json. --- Cargo.lock | 1 + crates/engine/tree/Cargo.toml | 2 ++ crates/engine/tree/src/tree/payload_validator.rs | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 9a5314772ba..837748dac93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8688,6 +8688,7 @@ dependencies = [ "reth-trie-sparse", "revm", "schnellru", + "serde_json", "thiserror 2.0.20", "tokio", "tracing", diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index f4c2d9a57cf..295cb4097cd 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -100,6 +100,8 @@ eyre.workspace = true proptest.workspace = true rand.workspace = true rand_08.workspace = true +# Firehose tests parse their prestate fixtures as JSON. +serde_json.workspace = true [features] test-utils = [ diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index da6c6a09a6b..285efbf4f4e 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -1245,7 +1245,7 @@ where InsertBlockErrorKind, > where - S: StateProvider + Send, + S: EvmStateProvider + Send, Err: core::error::Error + Send + Sync + 'static, V: PayloadValidator, T: PayloadTypes>, From f965b380132b5a770e2a6cfaf3e8d9f68076f380 Mon Sep 17 00:00:00 2001 From: StreamingFast Chain Update Bot Date: Mon, 16 Nov 2026 18:00:57 +0000 Subject: [PATCH 133/133] Add the op-reth-v2.9.0-fh3.1 changelog entry --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0325ae5b67..f59795fcd5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,18 @@ This changelog covers Firehose-specific changes only. For upstream reth changes, The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## op-reth-v2.8.1-fh3.1-1 +## op-reth-v2.9.0-fh3.1 + +### Changed + +- Merged ethereum-optimism/reth `op-v2.7.0-0` (reth v2.7.0 plus OP commits), the reth pin of + op-reth v2.9.0. The previous pin was `op-v2.6.0-2` (reth v2.6.0). revm moves to 43.0.3; + alloy-evm stays at 0.39 and the `alloy-evm` patch stays on `streamingfast/evm` `v0.39.0-sf`, + now pinned by commit. The Firehose engine path follows upstream's new `EvmStateProvider` + bound; the traced block is still printed only after post-execution validation passes. +- RPC replays (`debug_trace*`, `trace_*`) now run through the chain's block executor upstream. + They build it with `executor_for_block`, which the Firehose tracer does not hook, so they never + print Firehose blocks. ### Fixed