Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# ZeroDB CI — the CONTRIBUTING.md check battery, automated.
#
# Per push / PR: fmt, clippy, tests (x86-64 and aarch64), an MSRV check, miri
# on the core, loom on the reader table, the fuzz gate and the crash smoke.
# on the core, loom on the reader table, the free-list save without pool
# draws, the fuzz gate and the crash smoke.
# Nightly: the long fuzz, the full crash run and the reader stress.
# The loom, fuzz, crash and stress jobs run the justfile recipes of the same
# name, so `just <recipe>` locally runs exactly what CI runs.
Expand Down Expand Up @@ -115,6 +116,22 @@ jobs:
- run: cargo install just --version 1.58.0 --locked
- run: just loom

save-no-draw:
name: free-list save without pool draws
if: github.event_name != 'schedule'
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master (2026-10-01)
with:
toolchain: stable
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- run: cargo install just --version 1.58.0 --locked
- run: just save-no-draw

fuzz-quick:
name: fuzz gate (${{ matrix.os }})
if: github.event_name != 'schedule'
Expand Down
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,13 @@ Ratios are ZeroDB time ÷ LMDB time on Graviton4 unless stated; details in

### Fixed

- **A commit no longer fails when the free-list save converges slowly**
(ADR-0044). The save loop was limited to 64 rounds, a number its
termination proof did not give; past it the commit panicked, and a retry
of the same batch was likely to panic again. After 8 rounds the save now
stops reusing free pages and extends the file by a few pages, which ends
it. The remaining guard is computed from the proof (SPEC 05 GC-13,
GC-13a). No test, crash or stress run has needed more than 3 rounds.
- `LeafRef::new` and `BranchRef::new` reject a buffer shorter than the page
size, and `LeafMut::lookup` bounds-checks its reads. Both could read out of
bounds from safe code when called directly; the engine always passed whole
Expand Down
1 change: 1 addition & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ cargo +nightly miri test -p zerodb-core # non-mmap logic must pass miri
just fuzz-quick # differential fuzz against LMDB
just crash-test-quick # crash-consistency smoke
just loom # reader-table model check, when touching readers or nested txns
just save-no-draw # free-list save without pool draws, when touching the allocator or the save
just stress # 180 s reader/writer stress, before an integration gate
```

Expand Down
8 changes: 6 additions & 2 deletions crates/zerodb-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,9 @@ proptest.workspace = true
zerodb-io = { path = "../zerodb-io", features = ["test-backing"] }

[lints.rust]
# `cfg(loom)` is set externally by the `just loom` recipe (ADR-0006).
unexpected_cfgs = { level = "warn", check-cfg = ["cfg(loom)"] }
# `cfg(loom)` is set externally by the `just loom` recipe (ADR-0006), and
# `cfg(zerodb_save_draw_rounds)` by the `just save-no-draw` recipe (ADR-0044).
unexpected_cfgs = { level = "warn", check-cfg = [
"cfg(loom)",
'cfg(zerodb_save_draw_rounds, values("0", "1"))',
] }
47 changes: 30 additions & 17 deletions crates/zerodb-core/src/rwtxn/alloc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,15 @@ pub(super) enum AllocMode {
/// loose pages, the already-loaded drain pool, the carried annex pool or
/// extend only (GC-12). Every request is a single page: GC records are
/// always inline (GC-5), so the save never allocates an overflow run.
/// Set once at `freelist_save` entry and never cleared: the commit
/// consumes the txn.
/// Set at `freelist_save` entry; the commit consumes the txn.
GcSave,
/// Inside `freelist_save` once its drawing rounds are spent (GC-13a):
/// loose pages or extend. Neither pool is drawn from, so no record
/// already written can come to list a handed-out page. The one
/// exception is a full map: a save the pools can serve must not fail
/// with `MapFull`, so the pools are drawn from when the file cannot be
/// extended.
GcSaveExtend,
}

impl<'env> RwTxn<'env> {
Expand Down Expand Up @@ -312,22 +318,16 @@ impl<'env> RwTxn<'env> {
}
}
AllocMode::GcSave => {
// GC-12 (ADR-0005): never *read* the GC tree in-save, but
// draw from the already-loaded drain pool, which keeps file
// growth bounded (`freelist_save` gives the argument). Each
// entry touched here is rewritten again before the fixed
// point (`save_touched`), so no rewritten entry lists a
// handed-out page.
if let Some(start) = self.save_pool_draw(n) {
if let Some(start) = self.save_draw(n) {
return Ok(start);
}
// The carried annex pool (SPEC 05 §2a GC-30): gate-checked
// like every annex draw; the step-(c) placement re-reads
// `annex.live()` after every put, so a draw here can never
// leave a handed-out id in the persisted set — the GC-12
// pool argument.
if let Some(start) = self.annex_draw(n) {
return Ok(start);
}
AllocMode::GcSaveExtend => {
let mp = map_pages(self.env.inner().map_size(), self.psize);
if is_map_full(self.next_pgno, n, mp) {
if let Some(start) = self.save_draw(n) {
return Ok(start);
}
}
}
}
Expand All @@ -351,6 +351,19 @@ impl<'env> RwTxn<'env> {
}
}

/// The in-save draws (GC-12, ADR-0005): the GC tree is never *read*
/// in-save, but the already-loaded drain pool is drawn from, which keeps
/// file growth bounded (`freelist_save` gives the argument). Each entry
/// touched is rewritten again before the fixed point (`save_touched`),
/// so no rewritten entry lists a handed-out page. Then the carried annex
/// pool (SPEC 05 §2a GC-30), gate-checked like every annex draw: the
/// step-(c) placement re-reads `annex.live()` after every put, so a draw
/// can never leave a handed-out id in the persisted set.
#[inline(never)]
fn save_draw(&mut self, n: u64) -> Option<u64> {
self.save_pool_draw(n).or_else(|| self.annex_draw(n))
}

/// In-save draw from the **already-loaded drain pool** (GC-12;
/// see [`RwTxn::allocate`]). Deterministic like GC-19: smallest
/// reclaimable record first (`drains` is a `BTreeMap` in key order),
Expand All @@ -361,7 +374,7 @@ impl<'env> RwTxn<'env> {
/// pages are absent.
#[inline(never)]
pub(super) fn save_pool_draw(&mut self, n: u64) -> Option<u64> {
debug_assert!(self.alloc_mode == AllocMode::GcSave);
debug_assert!(self.alloc_mode != AllocMode::Normal);
let mut hit: Option<(GcKey, u64)> = None;
for (key, remaining) in &work!(self).drains {
if let Some(start) = find_run(remaining.live(), n) {
Expand Down
66 changes: 52 additions & 14 deletions crates/zerodb-core/src/rwtxn/freelist.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ use super::*;
struct SaveLoop {
/// GC records awaiting their GC-20 rewrite.
pending: BTreeSet<GcKey>,
/// Rounds of the outer loop so far, and their bound, which release
/// builds enforce too (GC-13 guard). Every round that does not exit
/// consumes a pool id, a committed GC page, or a loose fold; folds follow
/// record deletes, which follow pool draws or the drained records. The
/// budget counts each of those quantities generously.
outer_iters: usize,
outer_budget: usize,
/// Rewrites done so far, and their bound, which release builds enforce
/// too (GC-13 guard):
/// each rewrite consumes one pending entry, and an entry only re-enters
Expand Down Expand Up @@ -56,9 +63,22 @@ pub(super) fn merge_sorted_unique_into(a: &[u64], b: &[u64], out: &mut Vec<u64>)
debug_assert!(out.windows(2).all(|w| w[0] < w[1]), "merge not ascending");
}

/// GC-13 guard: `freelist_save`'s loop provably terminates; a bug that
/// breaks the fixed point should panic, not hang.
pub(super) const FREELIST_SAVE_MAX_ITERS: usize = 64;
/// Rounds of [`RwTxn::freelist_save`] whose allocations may draw from the
/// drain pool and the annex pool (GC-12). Later rounds take loose pages or
/// extend the file, which removes the only way one round makes the records
/// of the previous one stale, so a save that converges slowly ends with a
/// slightly larger file instead of failing the commit (GC-13a, ADR-0044).
///
/// `--cfg zerodb_save_draw_rounds="0"` or `="1"` (the `just save-no-draw`
/// recipe) lowers it for a test run, so that every save, or every save of
/// two rounds or more, takes the path that is otherwise almost never
/// reached.
#[cfg(not(any(zerodb_save_draw_rounds = "0", zerodb_save_draw_rounds = "1")))]
pub(super) const FREELIST_SAVE_DRAW_ROUNDS: usize = 8;
#[cfg(zerodb_save_draw_rounds = "0")]
pub(super) const FREELIST_SAVE_DRAW_ROUNDS: usize = 0;
#[cfg(zerodb_save_draw_rounds = "1")]
pub(super) const FREELIST_SAVE_DRAW_ROUNDS: usize = 1;

impl<'env> RwTxn<'env> {
/// GC-10 trailing shrink: loose pages that are the highest-numbered pages
Expand Down Expand Up @@ -115,8 +135,9 @@ impl<'env> RwTxn<'env> {
/// the old `free_db` root whose entry `F` lists that page as free and
/// whose trees do not reference it (`F ≤ oldest_reader() ≤ N−1`, TXN-62).
///
/// **Allocation restriction (GC-12):** `alloc_mode = GcSave` for the
/// whole procedure — `allocate` never **reads** the GC tree; it draws
/// **Allocation restriction (GC-12):** `alloc_mode` is `GcSave`, then
/// `GcSaveExtend`, never `Normal` — `allocate` never **reads** the GC
/// tree; it draws
/// loose pages, pages from the **already-loaded drain pool**
/// ([`RwTxn::save_pool_draw`]), the carried annex pool, or extends. Every
/// request is for one page: GC records are always inline (GC-5).
Expand All @@ -125,7 +146,10 @@ impl<'env> RwTxn<'env> {
/// references it, TXN-62), so without them every commit would extend
/// the file. An entry touched by a pool draw goes back on the `pending`
/// rewrite set, and the loop only exits when no rewrite is pending, so
/// the final tree state never lists a handed-out page.
/// the final tree state never lists a handed-out page. After
/// [`FREELIST_SAVE_DRAW_ROUNDS`] rounds `alloc_mode` becomes
/// `GcSaveExtend` and the pools are no longer drawn from, unless the
/// map is full (GC-13a).
///
/// **Termination (GC-13):** each iteration that does not exit strictly
/// shrinks a finite pool (a drain-pool or annex-pool draw), grows `freed`
Expand All @@ -135,8 +159,8 @@ impl<'env> RwTxn<'env> {
/// after a record is deleted, which needs a record emptied by pool draws:
/// so loose folds are bounded by the pool draws too. Nothing else frees
/// a page: a put into a dirty leaf rewrites it in place or splits it
/// (one fresh page, nothing freed). `FREELIST_SAVE_MAX_ITERS` turns a
/// violation into a panic.
/// (one fresh page, nothing freed). The round budget computed at entry
/// from those quantities turns a violation into a panic.
pub(super) fn freelist_save(&mut self) -> Result<()> {
debug_assert!(self.alloc_mode == AllocMode::Normal);
self.alloc_mode = AllocMode::GcSave;
Expand All @@ -162,10 +186,20 @@ impl<'env> RwTxn<'env> {
// Every entry the txn's operations drew from needs its GC-20
// rewrite at least once.
let pending: BTreeSet<GcKey> = work!(self).drains.keys().copied().collect();
let drain_ids = work!(self).drains.values().map(Drain::len).sum::<usize>();
let committed_gc_pages = usize::try_from(
self.free_db
.branch_pages
.saturating_add(self.free_db.leaf_pages),
)
.unwrap_or(usize::MAX);
let mut st = SaveLoop {
inner_budget: pending.len()
+ work!(self).drains.values().map(Drain::len).sum::<usize>()
+ 16,
outer_iters: 0,
outer_budget: (pending.len() + drain_ids + work!(self).annex.len())
.saturating_mul(3)
.saturating_add(committed_gc_pages)
.saturating_add(64),
inner_budget: pending.len() + drain_ids + 16,
pending,
inner_iters: 0,
own: BTreeMap::new(),
Expand All @@ -179,13 +213,15 @@ impl<'env> RwTxn<'env> {
self.release_trailing_loose();
let work = work_mut!(self);
work.freed.extend(std::mem::take(&mut work.loose));
let mut iters = 0usize;
loop {
iters += 1;
st.outer_iters += 1;
assert!(
iters <= FREELIST_SAVE_MAX_ITERS,
st.outer_iters <= st.outer_budget,
"freelist_save failed to reach a fixed point (GC-13)"
);
if st.outer_iters > FREELIST_SAVE_DRAW_ROUNDS {
self.alloc_mode = AllocMode::GcSaveExtend;
}
// (a) apply_drains (GC-11a/GC-20).
self.rewrite_pending_drains(&mut st)?;
// (c) this txn's own list: the meta annex when it fits (GC-32),
Expand Down Expand Up @@ -436,6 +472,8 @@ mod tests {
let cap = gc_chunk_cap(PS) as u64;
let mut st = SaveLoop {
pending: BTreeSet::new(),
outer_iters: 0,
outer_budget: 0,
inner_iters: 0,
inner_budget: 0,
own: BTreeMap::new(),
Expand Down
11 changes: 6 additions & 5 deletions crates/zerodb-core/src/rwtxn/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@
//! C1, `RwTxn::freelist_save` rewrites drained entries, releases trailing
//! loose pages (GC-10), and writes this txn's freed set under
//! `BE(writer_txnid)` in a fixed-point loop (GC-11..13) during which
//! `allocate` is restricted to loose pages / extend only (GC-12,
//! `AllocMode::GcSave`). The GC tree is mutated by the **same** split/COW
//! `allocate` never reads the GC tree: it takes loose pages, draws from the
//! pools already loaded, or extends (GC-12, GC-13a; `AllocMode::GcSave`,
//! then `AllocMode::GcSaveExtend`). The GC tree is mutated by the **same** split/COW
//! code as the main tree, selected by `TreeId`.
//!
//! ## Commit (SPEC 04 §9, SPEC 06 §2)
Expand Down Expand Up @@ -221,9 +222,9 @@ pub(crate) struct WriterCarry {
/// The base meta's free-list annex pool (SPEC 05 §2a, ADR-0022): the
/// pages freed by txn `base.txnid`, parsed (and `validate_pil_ids`-
/// checked, GC-33) at begin when `base.free_annex > 0`. Drawn by
/// [`RwTxn::annex_draw`] (allocate step 2b, gated per GC-31); whatever
/// remains is folded into `freed` at the top of `freelist_save` (the
/// GC-30 carry) — by then the pool is empty, so GcSave never sees it.
/// [`RwTxn::annex_draw`] (allocate step 2b, gated per GC-31), during
/// ops and inside `freelist_save`; whatever remains at the save's fixed
/// point is carried into this txn's own entry (GC-30).
annex: Drain,
/// Entries of `drains` whose `remaining` shrank via an **in-save pool
/// draw** (GC-12 — see `freelist_save`): each must be
Expand Down
1 change: 0 additions & 1 deletion crates/zerodb-tools/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ const DEFAULT_MAP_SIZE: u64 = 256 << 20;
/// Parse `argv` (excluding the program name) and run the selected subcommand.
/// Returns the process exit code: `0` success, `1` = `check` found violations,
/// `2` = usage/other error.
#[must_use]
pub fn run(args: &[String]) -> ExitCode {
match dispatch(args) {
Ok(code) => code,
Expand Down
87 changes: 87 additions & 0 deletions crates/zerodb/tests/save_at_map_limit.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
//! Commits when the file is within a few pages of the map size and holds
//! more free pages than the meta annex lists (SPEC 05 GC-13a, GC-17): once
//! the free-list save can no longer extend the file, every page it needs
//! must come from the free pages it already holds, and the commit must
//! still succeed. The case matters when the save has stopped drawing from
//! its pools, which `just save-no-draw` forces on every commit.

mod common;
use common::TempDir;

use zerodb::{check, EnvOpenOptions};

const PS: u32 = 4096;
const MAP_PAGES: u64 = 1024;
/// Pages left between the file's end and the map's end.
const HEADROOM: u64 = 6;

fn key(i: u32) -> Vec<u8> {
format!("key{i:05}").into_bytes()
}

#[test]
fn commits_succeed_when_the_file_cannot_grow() {
let dir = TempDir::new();
let mut opts = EnvOpenOptions::new();
opts.map_size((MAP_PAGES * u64::from(PS)) as usize);
opts.page_size(PS);
// SAFETY: only ZeroDB writes this environment's files.
let env = unsafe { opts.open(dir.path()) }.expect("open env");
let db = {
let mut w = env.write_txn().unwrap();
let db = env.create_database(&mut w, None).unwrap();
for i in 0..60 {
db.put(&mut w, &key(i), &[0xA5u8; 500]).unwrap();
}
w.commit().unwrap();
db
};

// One value on an overflow run takes the file to `HEADROOM` pages from
// the map's end; deleting it leaves the file that long and the run free.
// The run is longer than the meta annex, so it is listed in free-page
// records, which later commits drain and rewrite.
let used = env.info().last_pgno + 1;
let run = MAP_PAGES - used - HEADROOM - 4;
{
let mut w = env.write_txn().unwrap();
db.put(
&mut w,
b"big",
&vec![0x5Au8; (run * u64::from(PS)) as usize - 64],
)
.unwrap();
w.commit().unwrap();
}
{
let mut w = env.write_txn().unwrap();
assert!(db.delete(&mut w, b"big").unwrap());
w.commit().unwrap();
}
assert!(
env.info().last_pgno + 1 + HEADROOM + 4 >= MAP_PAGES,
"the file is not near the map's end: last page {}",
env.info().last_pgno
);

// Each commit copies tree pages and rewrites free-page records. A save
// that extends the file uses up the headroom within a few commits.
for round in 0u8..80 {
let mut w = env.write_txn().unwrap();
for i in (0..60).filter(|i| i % 4 == u32::from(round) % 4) {
db.put(&mut w, &key(i), &[round; 500]).unwrap();
}
w.commit().unwrap_or_else(|e| panic!("round {round}: {e}"));
}

let rtxn = env.read_txn().unwrap();
assert_eq!(db.len(&rtxn).unwrap(), 60);
drop(rtxn);
drop(env);
let bytes = std::fs::read(dir.path().join(zerodb::DATA_FILE_NAME)).unwrap();
let violations = check::check_image(&bytes, PS);
assert!(
violations.is_empty(),
"invariant violations: {violations:#?}"
);
}
Loading