refactor(storage)!: remove the LMDB chunk store, and never refuse a start over what it left behind

.github/workflows/ci.yml

@@ -50,46 +50,13 @@ jobs:
           version: ${{ env.FOUNDRY_VERSION }}
       - name: Run unit tests
         run: cargo test --lib --features test-utils
-      # Before the e2e suite, deliberately. These are fast and deterministic, and the e2e
-      # suite flakes on hosted runners for transport reasons that have nothing to do with
-      # storage. A failing step aborts the job, so anything sequenced after a flaky one
-      # never reports, which is how these ran on no platform at all for a whole run.
-      - name: Prove the migration returns disk to the filesystem
+      - name: Kill a node mid-write and check what survived
         shell: bash
         run: |
           set -euo pipefail
-          cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log
-          # A target whose required features are not passed is skipped with a
-          # warning and a zero exit, so a harness can stop running without anyone
-          # noticing. This is what makes that loud.
-          grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \
-            || { echo 'reclaims_disk ran no tests'; exit 1; }
-      - name: Kill a node mid-migration and check what survived
-        shell: bash
-        run: |
-          set -euo pipefail
-          cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log
-          # A target whose required features are not passed is skipped with a
-          # warning and a zero exit, so a harness can stop running without anyone
-          # noticing. This is what makes that loud.
+          cargo test --test chunk_store_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log
           grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \
-            || { echo 'crash_safety ran no tests'; exit 1; }
-      - name: Several nodes migrating on one disk
-        shell: bash
-        run: |
-          set -euo pipefail
-          cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log
-          # A target whose required features are not passed is skipped with a
-          # warning and a zero exit, so a harness can stop running without anyone
-          # noticing. This is what makes that loud.
-          grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \
-            || { echo 'shared_volume ran no tests'; exit 1; }
-      # Linux only. This one plants a hundred thousand files to measure what a restart
-      # costs, and the answer it is after is a fleet answer, where every node is Linux.
-      # The scan itself reads names and nothing else, which is not a platform-specific
-      # path, and opening a store is covered on all three by the unit tests. Planting that
-      # many files on the Windows runner would cost minutes of every run to re-measure
-      # something no node will ever do there.
+            || { echo 'chunk_store_crash_safety ran no tests'; exit 1; }
       - name: Startup scan, index memory and inode cost at scale
         if: runner.os == 'Linux'
         shell: bash
@@ -109,15 +76,7 @@ jobs:
         run: cargo test --test poc_audit_handler_live --features test-utils
       - name: Run bootstrap-stall PoC regression marker
         run: cargo test --test poc_bootstrap_stall --features test-utils
-      - name: Shutdown waits for writes whose caller has gone
-        run: cargo test --test poc_shutdown_lmdb_drain --features test-utils
 
-  # Runs the storage tests against real ext4, XFS and btrfs rather than whatever the
-  # runner provides. Deliberately NOT named durability: killing a process and reopening
-  # the same mounted filesystem keeps the page cache, so this exercises each filesystem's
-  # syscall, locking, rename and delete behaviour, not its behaviour under power loss.
-  # That still needs block-device fault injection or a real machine, and remains a fleet
-  # gate.
   filesystems:
     name: Storage on ${{ matrix.fs }}
     runs-on: ubuntu-latest
@@ -139,52 +98,32 @@ jobs:
           # on whatever the runner happens to give us. ext4 is what most of the fleet is
           # on; XFS and btrfs are the two the design reasons about separately, btrfs
           # because it has been observed reordering writes around a rename.
-          # 3 GiB is ample: these tests use tens of MiB. The scale harness, which is
-          # the one that needs room, is not in this job.
+          # 3 GiB is ample: these tests use tens of MiB.
           truncate -s 3G /tmp/${{ matrix.fs }}.img
           mkfs.${{ matrix.fs }} -q /tmp/${{ matrix.fs }}.img
           sudo mkdir -p /mnt/antfs
           sudo mount -o loop /tmp/${{ matrix.fs }}.img /mnt/antfs
           sudo chown "$USER" /mnt/antfs
           df -hT /mnt/antfs
-      # TMPDIR is what `TempDir::new` uses, so this is what puts the test data on the
-      # mounted filesystem rather than on the runner's root.
-      - name: The migration returns disk on ${{ matrix.fs }}
+      # TMPDIR is what `TempDir::new` uses, so this puts every temporary store these tests
+      # build on the mounted filesystem rather than on the runner's root.
+      #
+      # These used to be the migration harnesses. The migration is gone, and what is left
+      # worth asking of a filesystem is what the store itself does on it: publish a chunk
+      # through a temporary and a rename, flush the directory, unlink it again, and rebuild
+      # an index from the names afterwards. The storage tests do all of that, and running
+      # them here is what keeps ext4, XFS and btrfs covered now that the harnesses that used
+      # to cover them have been deleted. They do not measure free space before and after;
+      # the harness that did was about the migration and went with it.
+      - name: Storage behaviour on ${{ matrix.fs }}
         env:
           TMPDIR: /mnt/antfs
         shell: bash
         run: |
           set -euo pipefail
-          cargo test --test migration_reclaims_disk --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/reclaims_disk.log
-          # A target whose required features are not passed is skipped with a
-          # warning and a zero exit, so a harness can stop running without anyone
-          # noticing. This is what makes that loud.
-          grep -qE 'test result: ok\. [1-9]' /tmp/reclaims_disk.log \
-            || { echo 'reclaims_disk ran no tests'; exit 1; }
-      - name: A node killed mid-write on ${{ matrix.fs }} loses nothing
-        env:
-          TMPDIR: /mnt/antfs
-        shell: bash
-        run: |
-          set -euo pipefail
-          cargo test --test migration_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log
-          # A target whose required features are not passed is skipped with a
-          # warning and a zero exit, so a harness can stop running without anyone
-          # noticing. This is what makes that loud.
-          grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \
-            || { echo 'crash_safety ran no tests'; exit 1; }
-      - name: Several nodes on one ${{ matrix.fs }} volume
-        env:
-          TMPDIR: /mnt/antfs
-        shell: bash
-        run: |
-          set -euo pipefail
-          cargo test --test migration_shared_volume --features test-utils 2>&1 | tee /tmp/shared_volume.log
-          # A target whose required features are not passed is skipped with a
-          # warning and a zero exit, so a harness can stop running without anyone
-          # noticing. This is what makes that loud.
-          grep -qE 'test result: ok\. [1-9]' /tmp/shared_volume.log \
-            || { echo 'shared_volume ran no tests'; exit 1; }
+          cargo test --lib --features test-utils storage:: 2>&1 | tee /tmp/storage.log
+          grep -qE 'test result: ok\. [1-9]' /tmp/storage.log \
+            || { echo 'the storage tests ran nothing'; exit 1; }
 
   doc:
     name: Documentation
@@ -212,7 +151,8 @@ jobs:
       - name: Build release (no logging)
         run: cargo build --release --no-default-features
       # The crash harness drives the store through a failpoint that parks the process
-      # forever on an environment variable. It is compiled only under `test-utils`, which
+      # forever on an environment variable, and the store's own tests use the same one. It
+      # is compiled only under `test-utils`, which
       # is not a default feature and is not passed by the release workflow, so a shipped
       # binary does not contain it. This proves that rather than trusting it: the variable
       # name is a string literal, so it survives into the binary whenever the code that

Cargo.toml

@@ -56,7 +56,8 @@ xor_name = "5"
 lru = "0.16.3"
 parking_lot = "0.12"  # Efficient mutex for cache
 
-# Storage - LMDB via heed for content-addressed chunk store
+# LMDB via heed for the paid-key list. The chunk store is one file per chunk and does
+# not use it.
 heed = "0.22"
 
 blake3 = "1"
@@ -109,15 +110,18 @@ sha2 = "0.10"
 # Cross-platform file locking for upgrade caches
 fs2 = "0.4"
 
-# System page size (for LMDB map alignment during resize)
-page_size = "0.6"
 
 # Protocol serialization
 postcard = { version = "1.1.3", features = ["use-std"] }
 bao = "0.13.1"
 
 [target.'cfg(unix)'.dependencies]
 libc = "0.2"
+# Safe wrappers for `openat`/`unlinkat`, so the storage-migration cleanup can delete relative
+# to a directory handle it opened `O_NOFOLLOW` rather than by re-resolving a path it has
+# already checked. Adds nothing to the build: `tempfile`, a direct dependency, already pulls
+# this exact crate and version in with the `fs` feature on.
+rustix = { version = "1", features = ["fs"] }
 
 [target.'cfg(windows)'.dependencies]
 self-replace = "1"
@@ -132,32 +136,18 @@ proptest = "1"
 alloy = { version = "1", features = ["node-bindings"] }
 serial_test = "3"
 
-# Proves the migration returns disk to the filesystem, which is the claim the whole
-# change exists to make good. Needs the test-only migration-state accessor.
-[[test]]
-name = "migration_reclaims_disk"
-path = "tests/migration_reclaims_disk.rs"
-required-features = ["test-utils"]
-
-# Kills a real child process part-way through writing and migrating, then checks what
-# survived. The automatable half of the power-loss gate.
-[[test]]
-name = "migration_crash_safety"
-path = "tests/migration_crash_safety.rs"
-required-features = ["test-utils"]
-
 # Startup scan time, index memory and inode cost at scale. Regression gates, not
 # benchmarks; ANT_SCALE_KEYS raises the count for a deliberate larger run.
 [[test]]
 name = "storage_scale"
 path = "tests/storage_scale.rs"
 required-features = ["test-utils"]
 
-# Several nodes migrating on one disk: the volume lock, and that each finishes with its
-# own chunks and only its own.
+# A process killed part-way through a write: the store is whole or absent, never
+# half-indexed, and what the interrupted write left behind is swept.
 [[test]]
-name = "migration_shared_volume"
-path = "tests/migration_shared_volume.rs"
+name = "chunk_store_crash_safety"
+path = "tests/chunk_store_crash_safety.rs"
 required-features = ["test-utils"]
 
 # E2E test infrastructure (run with --features test-utils)
@@ -176,7 +166,7 @@ path = "tests/poc_commitment_audit_attacks.rs"
 required-features = ["test-utils"]
 
 # Live responder-handler tests for the v12 audit. Use
-# LmdbStorageConfig::test_default(), gated on test-utils.
+# ChunkStoreConfig::test_default(), gated on test-utils.
 [[test]]
 name = "poc_audit_handler_live"
 path = "tests/poc_audit_handler_live.rs"
@@ -197,14 +187,6 @@ name = "poc_price_floor_live"
 path = "tests/poc_price_floor_live.rs"
 required-features = ["test-utils"]
 
-# Shutdown/LMDB-drain regression: `ReplicationEngine::shutdown()` must not
-# return while a detached LMDB blocking op is still running. Uses the
-# test-only storage put gate, so it requires the test-utils feature.
-[[test]]
-name = "poc_shutdown_lmdb_drain"
-path = "tests/poc_shutdown_lmdb_drain.rs"
-required-features = ["test-utils"]
-
 [features]
 default = ["logging"]
 # Enable tracing/logging infrastructure.

config/production.toml

@@ -46,62 +46,14 @@ enabled = true
 # Verify content hash on read
 verify_on_read = true
 
-# Maximum size in GiB of the legacy LMDB store, while one still exists
-# (0 = derive it from available disk). Retired along with LMDB itself.
-db_size_gb = 0
-
-# --- Moving off the legacy LMDB chunk store ---
-#
-# Chunks are now one file each, under {root_dir}/chunks/. A node that still has a
-# chunks.mdb copies it into files in the background, then deletes it whole, which is the
-# only moment LMDB's disk comes back.
-#
-# The two release-level switches (whether to delete the old store, and whether audits
-# still penalise) belong to the build, not to this file, so they are deliberately absent.
-[storage.migration]
-# Run the copier. Turning this off leaves both stores in place forever and never
-# returns the old store's disk.
-enabled = true
-
-# Also write new chunks to the legacy store while it exists, so a fleet rollback to an
-# older build cannot lose a chunk uploaded during the migration.
-dual_write_legacy = true
-
-# Allow a node that cannot fit its chunks to give up the ones it is furthest from.
-#
-# Whatever this is set to, a chunk is only ever given up when the node is near the back of
-# its group for it, its close group has received the node's reduced commitment, AND all but
-# one of that group has cryptographically proven it holds a copy. A node that cannot show
-# all three keeps both stores and asks for more disk. Turn this off if you would rather add
-# disk than have the node give anything up at all.
-allow_shed = true
-
-# Hours after this build first starts before a node may give anything up, so peers on
-# older builds have upgraded and stopped penalising it for doing so.
-shed_hold_hours = 72
-
-# Hours between one migration wave opening and the next.
-#
-# A close group is split into waves so only two of its members give chunks up at a time.
-# If all seven went together none could prove to the others that a copy survived, and the
-# group would deadlock waiting on each other. A node with room to copy everything does not
-# wait for a wave: it is never unable to serve, so it is not part of that problem.
-wave_hours = 24
-
-# Hours between a node committing to what it will keep and deleting the old store.
-# Never shorter than 4: that is what the answerability window needs.
-retire_delay_hours = 4
-
-# Free space, in MiB, the copier leaves untouched on top of disk_reserve_mb.
-copier_slack_mb = 2048
-
-# Copy rate ceiling, in MiB/s. Keep it modest: an unthrottled copier competing with the
-# audit responder for disk turns a storage migration into an audit incident.
-copier_throttle_mib_per_sec = 32
-
 # --- Upgrade ---
 [upgrade]
-enabled = false
+# There is no `enabled` setting, and there never was one. This file used to carry
+# `enabled = false`, which serde ignored: `UpgradeConfig` has no such field, so every node
+# reading this was self-upgrading while its own configuration appeared to say otherwise.
+# Removed rather than annotated, because a line that does nothing is worse than an absent one
+# — and this one said the opposite of the truth about the mechanism a release rollback would
+# have to go through.
 channel = "stable"
 check_interval_hours = 1
 github_repo = "WithAutonomi/ant-node"

deploy/scripts/spawn-nodes.sh

@@ -68,15 +68,6 @@ fi
 # Create directories
 mkdir -p "$BASE_DIR" "$LOG_DIR"
 
-# The per-volume migration lock. Every node on this host shares it and nothing else, so
-# they can serialise their copies off LMDB without being able to reach each other's data.
-# It needs its own directory because PrivateTmp=true below gives each unit a /tmp of its
-# own, and the node's default lock location is in there: without this every node takes a
-# lock nobody else can see, all of them start copying at once, and the host runs out of
-# space with several half-finished migrations on it.
-LOCK_DIR="${BASE_DIR%/*}/migration"
-mkdir -p "$LOCK_DIR"
-
 # Create ant user if not exists
 if ! id -u ant &>/dev/null; then
     useradd -r -s /bin/false ant || true
@@ -99,8 +90,6 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do
     # Create node directory
     mkdir -p "$NODE_DIR"
     chown ant:ant "$NODE_DIR"
-    chown ant:ant "$LOCK_DIR"
-    chmod 0750 "$LOCK_DIR"
 
     # Create systemd service
     cat > "/etc/systemd/system/$SERVICE_NAME.service" <<EOF
@@ -131,10 +120,6 @@ NoNewPrivileges=true
 ProtectSystem=strict
 ProtectHome=true
 ReadWritePaths=$NODE_DIR
-# Only the lock lives here. Granting write access to the shared node directory instead
-# would let every node write into every other node's data.
-ReadWritePaths=$LOCK_DIR
-Environment=ANT_MIGRATION_LOCK_DIR=$LOCK_DIR
 PrivateTmp=true
 ProtectKernelTunables=true
 ProtectKernelModules=true