Skip to content
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ This changelog covers Firehose-specific changes only. For upstream reth changes,

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

## base-v2.5.2.11-fh3.1

### Changed

- Rebase on Base's reth fork at `base-v2.5.2.11`, the reth version pinned by Base `v1.5.4`. It
backports the `eth_getLogs` range scan fixes (blocking IO permits, window by window scans), a
trie iterator seek fix, and RocksDB indices for the full snapshot download preset. Firehose code
and the block model are unchanged.

## base-v2.5.2.8-fh3.1-2

### Fixed
Expand Down
182 changes: 178 additions & 4 deletions crates/cli/commands/src/download/config_gen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ pub fn write_config(config: &Config, data_dir: &Path) -> eyre::Result<bool> {
pub(crate) fn write_prune_checkpoints_tx<Tx>(
tx: &Tx,
config: &Config,
selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
snapshot_block: u64,
) -> eyre::Result<()>
where
Expand Down Expand Up @@ -72,8 +73,36 @@ where
tx.get::<tables::BlockBodyIndices>(snapshot_block)?.map(|indices| indices.last_tx_num());

for (segment, prune_mode) in &checkpoints {
let (block_number, tx_number) = match segment {
// Record the history actually selected, not the future pruning policy, which may
// retain a minimum distance even when no changesets were downloaded.
PruneSegment::AccountHistory | PruneSegment::StorageHistory => {
let component = if *segment == PruneSegment::AccountHistory {
SnapshotComponentType::AccountChangesets
} else {
SnapshotComponentType::StorageChangesets
};
let target = match selections.get(&component) {
Some(ComponentSelection::All) => None,
Some(ComponentSelection::Distance(distance)) => {
snapshot_block.checked_sub(*distance)
}
Some(ComponentSelection::Since(block)) => {
block.checked_sub(1).map(|target| target.min(snapshot_block))
}
Some(ComponentSelection::None) | None => Some(snapshot_block),
};
let Some(target) = target else { continue };
// Downloading a wider range cannot restore history already pruned by the source.
let existing = tx
.get::<tables::PruneCheckpoints>(*segment)?
.and_then(|checkpoint| checkpoint.block_number);
(existing.map_or(target, |existing| existing.max(target)), None)
}
_ => (snapshot_block, tx_number),
};
let checkpoint = PruneCheckpoint {
block_number: Some(snapshot_block),
block_number: Some(block_number),
tx_number,
prune_mode: *prune_mode,
};
Expand All @@ -82,7 +111,7 @@ where

info!(target: "reth::cli",
segment = %segment,
block = snapshot_block,
block = block_number,
tx = ?tx_number,
mode = ?prune_mode,
"Set prune checkpoint"
Expand Down Expand Up @@ -328,7 +357,7 @@ mod tests {

{
let tx = db.tx_mut().unwrap();
write_prune_checkpoints_tx(&tx, &config, snapshot_block).unwrap();
write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap();
tx.commit().unwrap();
}

Expand All @@ -352,6 +381,40 @@ mod tests {
}
}

/// Writes prune checkpoints for state history retained over `distance` blocks.
fn history_checkpoint(snapshot_block: u64, distance: u64) -> Option<PruneCheckpoint> {
let dir = tempfile::tempdir().unwrap();
let db = reth_db::init_db(dir.path(), reth_db::mdbx::DatabaseArguments::default()).unwrap();
let selections = BTreeMap::from([(
SnapshotComponentType::AccountChangesets,
ComponentSelection::Distance(distance),
)]);
let config = config_for_selections(
&selections,
&empty_manifest(),
None,
None::<&reth_chainspec::ChainSpec>,
);

let tx = db.tx_mut().unwrap();
write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap();
tx.commit().unwrap();

db.tx().unwrap().get::<tables::PruneCheckpoints>(PruneSegment::AccountHistory).unwrap()
}

#[test]
fn write_prune_checkpoints_sets_history_to_retained_window_start() {
let checkpoint = history_checkpoint(21_000_000, 500_000).unwrap();
assert_eq!(checkpoint.block_number, Some(20_500_000));
assert_eq!(checkpoint.prune_mode, PruneMode::Distance(500_000));
}

#[test]
fn write_prune_checkpoints_skips_history_shorter_than_distance() {
assert_eq!(history_checkpoint(100_000, 500_000), None);
}

#[test]
fn write_prune_checkpoints_archive_no_checkpoints() {
let dir = tempfile::tempdir().unwrap();
Expand All @@ -371,7 +434,7 @@ mod tests {

{
let tx = db.tx_mut().unwrap();
write_prune_checkpoints_tx(&tx, &config, 21_000_000).unwrap();
write_prune_checkpoints_tx(&tx, &config, &selections, 21_000_000).unwrap();
tx.commit().unwrap();
}

Expand Down Expand Up @@ -658,4 +721,115 @@ mod tests {
.expect("sender prune checkpoint should exist");
assert_eq!(sender_prune_checkpoint.block_number, Some(tip_checkpoint.block_number));
}

#[test]
fn history_checkpoints_do_not_advertise_undownloaded_changesets() {
let snapshot_block = 21_000_000;
for (selection, expected) in [
(None, Some(snapshot_block)),
(Some(ComponentSelection::None), Some(snapshot_block)),
(Some(ComponentSelection::Distance(1)), Some(snapshot_block - 1)),
(Some(ComponentSelection::Since(snapshot_block)), Some(snapshot_block - 1)),
(Some(ComponentSelection::Since(0)), None),
(Some(ComponentSelection::All), None),
] {
let dir = tempfile::tempdir().unwrap();
let db =
reth_db::init_db(dir.path(), reth_db::mdbx::DatabaseArguments::default()).unwrap();
let mut selections = BTreeMap::from([
(SnapshotComponentType::State, ComponentSelection::All),
(SnapshotComponentType::Headers, ComponentSelection::All),
(SnapshotComponentType::RocksdbIndices, ComponentSelection::All),
]);
if let Some(selection) = selection {
selections.insert(SnapshotComponentType::AccountChangesets, selection);
selections.insert(SnapshotComponentType::StorageChangesets, selection);
}
let config = config_for_selections(
&selections,
&empty_manifest(),
None,
None::<&reth_chainspec::ChainSpec>,
);
assert!(!super::super::should_reset_index_stage_checkpoints(&selections));
let tx = db.tx_mut().unwrap();
write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap();
let mut checkpoints = Vec::new();
for segment in [PruneSegment::AccountHistory, PruneSegment::StorageHistory] {
let checkpoint = tx.get::<tables::PruneCheckpoints>(segment).unwrap();
assert_eq!(checkpoint.and_then(|checkpoint| checkpoint.block_number), expected);
if let Some(checkpoint) = checkpoint {
checkpoints.push((segment, checkpoint));
}
}
if let Some(boundary) = expected {
assert!(config
.prune
.segments
.ensure_unwind_target_unpruned(snapshot_block, boundary - 1, &checkpoints,)
.is_err());
assert!(config
.prune
.segments
.ensure_unwind_target_unpruned(snapshot_block, boundary, &checkpoints,)
.is_ok());
}
}
}

#[test]
fn history_checkpoints_preserve_source_pruning_boundary() {
let snapshot_block = 21_000_000;
let selected_boundary = 20_500_000;
for selection in [
ComponentSelection::Distance(snapshot_block - selected_boundary),
ComponentSelection::Since(selected_boundary + 1),
] {
for existing_boundary in [None, Some(20_000_000), Some(20_750_000)] {
let dir = tempfile::tempdir().unwrap();
let db = reth_db::init_db(dir.path(), reth_db::mdbx::DatabaseArguments::default())
.unwrap();
let selections = BTreeMap::from([
(SnapshotComponentType::AccountChangesets, selection),
(SnapshotComponentType::StorageChangesets, selection),
(SnapshotComponentType::RocksdbIndices, ComponentSelection::All),
]);
let config = config_for_selections(
&selections,
&empty_manifest(),
None,
None::<&reth_chainspec::ChainSpec>,
);
let tx = db.tx_mut().unwrap();
for segment in [PruneSegment::AccountHistory, PruneSegment::StorageHistory] {
tx.put::<tables::PruneCheckpoints>(
segment,
PruneCheckpoint {
block_number: existing_boundary,
tx_number: Some(100),
prune_mode: PruneMode::Full,
},
)
.unwrap();
}

write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap();

let expected = existing_boundary.unwrap_or_default().max(selected_boundary);
for (segment, mode) in [
(PruneSegment::AccountHistory, config.prune.segments.account_history),
(PruneSegment::StorageHistory, config.prune.segments.storage_history),
] {
assert_eq!(
tx.get::<tables::PruneCheckpoints>(segment).unwrap(),
Some(PruneCheckpoint {
block_number: Some(expected),
tx_number: None,
prune_mode: mode.unwrap(),
}),
);
}
}
}
}
}
68 changes: 64 additions & 4 deletions crates/cli/commands/src/download/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,10 +179,14 @@ pub struct DownloadDefaults {
pub snapshot_api_url: Cow<'static, str>,
/// Optional custom long help text that overrides the generated help
pub long_help: Option<String>,
/// Whether the full preset also downloads the snapshot's RocksDB indices instead of
/// rebuilding them.
pub full_preset_includes_rocksdb_indices: bool,
}

impl DownloadDefaults {
/// Initialize the global download defaults with this configuration
#[expect(clippy::result_large_err, reason = "returns the rejected defaults to the caller")]
pub fn try_init(self) -> Result<(), Self> {
DOWNLOAD_DEFAULTS.set(self)
}
Expand All @@ -203,6 +207,7 @@ impl DownloadDefaults {
default_chain_aware_base_url: None,
snapshot_api_url: Cow::Borrowed(RETH_SNAPSHOTS_API_URL),
long_help: None,
full_preset_includes_rocksdb_indices: false,
}
}

Expand Down Expand Up @@ -312,6 +317,12 @@ impl DownloadDefaults {
self
}

/// Builder: Make the full preset also download the snapshot's RocksDB indices.
pub const fn with_full_preset_rocksdb_indices(mut self) -> Self {
self.full_preset_includes_rocksdb_indices = true;
self
}

/// Builder: Set custom long help text, overriding the generated help
pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
self.long_help = Some(help.into());
Expand Down Expand Up @@ -419,7 +430,8 @@ pub struct DownloadCommand<C: ChainSpecParser> {

/// Skip optional RocksDB indices even when archive components are selected.
///
/// This affects `--archive`/`--all` and TUI archive preset (`a`).
/// This affects `--archive`/`--all`, TUI archive preset (`a`), and the full preset when it
/// includes RocksDB indices.
#[arg(long, conflicts_with_all = ["url", "with_rocksdb"])]
without_rocksdb: bool,

Expand Down Expand Up @@ -612,8 +624,17 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCo
let manifest = self.load_manifest(chain_id).await?;
let ResolvedComponents { mut selections, preset } = self.resolve_components(&manifest)?;

if matches!(preset, Some(SelectionPreset::Archive)) {
inject_archive_only_components(&mut selections, &manifest, !self.without_rocksdb);
match preset {
Some(SelectionPreset::Archive) => {
inject_archive_only_components(&mut selections, &manifest, !self.without_rocksdb)
}
Some(SelectionPreset::Full)
if DownloadDefaults::get_global().full_preset_includes_rocksdb_indices &&
!self.without_rocksdb =>
{
inject_rocksdb_indices(&mut selections, &manifest)
}
_ => {}
}

let planned = collect_planned_archives(&manifest, &selections)?;
Expand Down Expand Up @@ -667,7 +688,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCo
let tx = db.tx_mut()?;

if should_write_prune {
config_gen::write_prune_checkpoints_tx(&tx, &config, manifest.block)?;
config_gen::write_prune_checkpoints_tx(&tx, &config, selections, manifest.block)?;
}

if should_reset_indices {
Expand Down Expand Up @@ -1023,6 +1044,16 @@ fn inject_archive_only_components(
}
}

/// Selects the manifest's RocksDB indices, if it publishes them.
fn inject_rocksdb_indices(
selections: &mut BTreeMap<SnapshotComponentType, ComponentSelection>,
manifest: &SnapshotManifest,
) {
if manifest.component(SnapshotComponentType::RocksdbIndices).is_some() {
selections.insert(SnapshotComponentType::RocksdbIndices, ComponentSelection::All);
}
}

/// Returns `true` when RocksDB-backed index stages should be reset after download.
fn should_reset_index_stage_checkpoints(
selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
Expand Down Expand Up @@ -1539,6 +1570,35 @@ mod tests {
assert_eq!(selections.get(&SnapshotComponentType::RocksdbIndices), None);
}

#[test]
fn inject_rocksdb_indices_keeps_index_stages() {
let manifest = manifest_with_archive_only_components();
let mut selections = BTreeMap::new();
selections
.insert(SnapshotComponentType::AccountChangesets, ComponentSelection::Distance(10));

inject_rocksdb_indices(&mut selections, &manifest);

assert_eq!(
selections.get(&SnapshotComponentType::RocksdbIndices),
Some(&ComponentSelection::All)
);
assert_eq!(selections.get(&SnapshotComponentType::TransactionSenders), None);
assert!(!should_reset_index_stage_checkpoints(&selections));
}

#[test]
fn inject_rocksdb_indices_skips_manifest_without_indices() {
let mut manifest = manifest_with_archive_only_components();
manifest.components.clear();
let mut selections = BTreeMap::new();

inject_rocksdb_indices(&mut selections, &manifest);

assert!(selections.is_empty());
assert!(should_reset_index_stage_checkpoints(&selections));
}

#[test]
fn should_reset_index_stage_checkpoints_without_rocksdb_indices() {
let mut selections = BTreeMap::new();
Expand Down
Loading
Loading