diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 364e8fc..2555f0d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ name: BIMRI tests on: push: - branches: [main] + branches: [main, "agent/**"] pull_request: permissions: @@ -29,6 +29,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Compile Python sources - run: python -m py_compile bimri-engine.py tests/test_bimri_engine.py tests/test_v503_release.py tests/test_v510_lifecycle.py tests/crash_worker.py + run: python -m py_compile bimri-engine.py tests/test_bimri_engine.py tests/test_v503_release.py tests/test_v510_lifecycle.py tests/test_v511_performance.py tests/crash_worker.py - name: Run unit tests run: python -m unittest discover -s tests -v + - name: Benchmark smoke with canonical gates + run: python tests/benchmark_v511.py --init --samples 5 --gate default diff --git a/AGENTS.md b/AGENTS.md index e137e5e..100301e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md -## BIMRI Engine v5.1.0 | Authority Format v5.1.0 | Hot Grammar v5.0.2 +## BIMRI Engine v5.1.1 | Authority Format v5.1.0 | Hot Grammar v5.0.2 This project uses BIMRI portable memory. `bimri.md` is the small, readable current state. Full evidence and history live under `.bimri/`. diff --git a/BIMRI-AGENT-BLOCK.md b/BIMRI-AGENT-BLOCK.md index 24f6512..c64fbd0 100644 --- a/BIMRI-AGENT-BLOCK.md +++ b/BIMRI-AGENT-BLOCK.md @@ -1,4 +1,4 @@ -## BIMRI Engine v5.1.0 | Authority Format v5.1.0 | Hot Grammar v5.0.2 +## BIMRI Engine v5.1.1 | Authority Format v5.1.0 | Hot Grammar v5.0.2 This project uses BIMRI portable memory. `bimri.md` is the small, readable current state. Full evidence and history live under `.bimri/`. diff --git a/BIMRI-PROTOCOL.md b/BIMRI-PROTOCOL.md index bf08057..e813ea2 100644 --- a/BIMRI-PROTOCOL.md +++ b/BIMRI-PROTOCOL.md @@ -6,11 +6,11 @@ This document is the normative protocol for a portable, human-governed BIMRI memory folder. `AGENTS.md` is the short runtime adapter. `bimri-engine.py` is the reference implementation. -The engine, mutable state, and new authority-record format are v5.1.0. The -readable `bimri.md` line grammar and header remain v5.0.2 because v5.1 changes -subject lifecycle and residency rather than the visible line syntax. Frozen -v5.0-v5.0.2 artifacts retain their original version and are validated against -their original contract. +The reference engine is v5.1.1. The mutable state and new authority-record +format remain v5.1.0, and the readable `bimri.md` line grammar and header remain +v5.0.2 because v5.1 changes subject lifecycle and residency rather than the +visible line syntax. Frozen v5.0-v5.0.2 artifacts retain their original version +and are validated against their original contract. The words MUST, MUST NOT, SHOULD, SHOULD NOT, and MAY describe interoperability requirements. @@ -75,6 +75,12 @@ bimri.md generated view of accepted memory state.json head pointer, counters, active runs, cold-current engine.lock local cross-process lock index.tsv rebuildable retrieval index + audit-witness.json compact reference-engine audit checkpoint + audit-manifest.json detailed reference-engine audit evidence + audit-manifests/ retained manifest generations for audit evidence + audit-transition.json write-ahead marker for a checkpoint change + audit-drift/ bounded rolling receipts of unexplained divergence + audit-blocked.json owner-repair baseline while a quarantine is open log/R000001.md append-only log for one run revisions/V000000.md immutable shared-memory snapshots proposals/R000001-Q001.json immutable agent proposals @@ -113,6 +119,31 @@ durable records. `index.tsv` is a derived, non-authoritative cache. It MAY be deleted and rebuilt from canonical hot and cold memory, logs, and archives; an index failure MUST NOT alter the outcome of a memory mutation. +Reference-engine note (non-normative): engine v5.1.1 may maintain a compact +`audit-witness.json` checkpoint and separate `audit-manifest.json` path-and-hash +evidence for a successful full integrity audit. Current-only reads may validate +the checkpoint, accepted head, current state, and one selected cold binding +without traversing historical authority. Authority-changing writes and +explicit audit, review, search, and historical retrieval perform the full +check. The checkpoint is a derived cache: divergence from it is a cache miss +that forces the full semantic audit, never a stored verdict of its own. When +that audit passes over divergence the engine cannot attribute to its own +recorded operation, it first durably records a sealed drift receipt +(bounded to the newest 200, each sealing the diverging paths with prior and +current hashes up to a documented per-section bound, with the complete +delta pinned in a validated attachment when truncated) under +`.bimri/audit-drift/`, then continues; when the audit fails, the store +refuses into damaged-authority recovery exactly as it would without a +checkpoint and invalidates the checkpoint (its bytes remain as the prior +baseline; its epoch binding advances), so later warm reads and starts +re-prove the store and report the recovery condition rather than serving +past it. +`audit-blocked.json` appears only while an owner-approved quarantine holds its +pre-repair baseline; restoration, or a clean doctor pass after it, clears it. +The witness-protected roots are flat; an unexpected subdirectory prevents a +valid full audit. These derived files never change the authority-store +contract described here. + ## 4. Identifiers and Stable Keys New v5 identifiers have fixed-width forms: @@ -276,17 +307,16 @@ Under the engine lock, `start` MUST: 4. exclusively create `.bimri/log/.md`; 5. add only that run to `active_runs`; 6. persist state atomically; -7. print the BIMRI brief and explicit run handle; and -8. attempt to rebuild the derived index. +7. print the BIMRI brief and explicit run handle. The normal brief MUST be quiet about open review records. `start` and `hook-start` MUST NOT print conflict IDs, choices, questions, open-review counts, or `HUMAN DECISION NEEDED`. They MUST continue to print authority recovery warnings because those indicate that shared writes are unsafe. -Failure to rebuild the index after step 7 MUST be reported as a warning, not -as failure to create the run. The run handle is already durable and the index -can be rebuilt independently. +`start` MUST NOT rebuild the derived index while holding the engine lock. The +run handle is already durable and the non-authoritative index can be rebuilt +independently with `index` or as part of scheduled maintenance. `--session ` MAY bind a harness session to a run. Starting the same actor and session resumes its active run instead of allocating a @@ -357,11 +387,15 @@ unchanged cold subject appear absent or stale. A true same-key generation change still requires `sync` before a new proposal is created. An admitted proposal binds `base_revision` to that current accepted head and -`base_hash` to the exact keyed line hash, or literal `absent`. It also carries -one optional backward-readable `preflight_receipt` containing engine release -v5.1.0, accepted-head revision and hash, and observed key hash. The receipt -MUST validate against the named immutable revision before the proposal may -create a new concurrent conflict. Proposal records remain immutable. +`base_hash` to the exact keyed line hash, or literal `absent`. It MAY also carry +one backward-readable `preflight_receipt` containing the engine release that +created it, accepted-head revision and hash, and observed key hash. The v5.1.1 +engine writes this receipt on every new proposal and stamps v5.1.1; it also +accepts earlier v5.1.0 receipts. Older v5.1.0 engines reject v5.1.1-authored +receipts. When present, the receipt MUST validate against the named immutable +revision before the proposal may create a new concurrent conflict. A +receipt-less legacy proposal cannot create a new concurrent conflict and must +be synced and restaged instead. Proposal records remain immutable. Every v5.1 proposal carries an explicit Boolean `new_subject`. A proposal based on cold-current memory additionally records `base_storage: cold` and binds the @@ -470,8 +504,11 @@ section. A conforming commit performs these steps: 13. attempt to regenerate `bimri.md`, warning if the durable state has committed but the generated view cannot yet be refreshed; 14. finalize the proposal decision; -15. attempt to rebuild the derived index; and -16. release the lock. +15. release the lock. + +The commit path MUST NOT rebuild the derived index while holding the engine +lock. A stale or missing index does not change accepted memory and can be +rebuilt independently with `index` or as part of scheduled maintenance. Durable replacement SHOULD use a temporary file in the destination directory, flush and `fsync` it, and atomically replace the destination. On POSIX, the @@ -649,15 +686,18 @@ Recovery validates the exact precommitted historical effect for proposal, NOT strand an applying resolution, rewrite its historical choice, or roll back newer current memory. -While a proposal-choice resolution has an uncommitted intended effect, -unrelated subjects remain writable and a later same-key intent MUST be durably -held as `owner-resolution-in-progress` rather than overwrite the pending owner -choice or become another owner conflict. Recovery silently finalizes an exact -effect already proven committed. If the intended effect did not commit, the -owner must re-attest the exact choice before recovery may rebind it to the next -revision. `current` and `dismiss` choices remain bound to their precommitted -historical snapshot even if later same-key work proceeds. Recovery MUST -preserve any later accepted current generation. +While a proposal-choice resolution remains `applying` and has an uncommitted +intended effect, unrelated subjects remain writable and a later same-key intent +MUST be durably held as `owner-resolution-in-progress` rather than overwrite +the pending owner choice or become another owner conflict. Recovery silently +finalizes an exact effect already proven committed. If resolution processing +records `failed`, ordinary retrieval and shared-memory writes MUST pause as an +authority-recovery condition; only an explicit retry of that exact conflict and +choice may proceed. If the intended effect did not commit, the owner must +re-attest the exact choice before recovery may rebind it to the next revision. +`current` and `dismiss` choices remain bound to their precommitted historical +snapshot even if later same-key work proceeds. Recovery MUST preserve any +later accepted current generation. An immutable revision file whose number was predeclared by an applying resolution but whose state pointer never committed is an orphan, not accepted @@ -839,10 +879,19 @@ its key: bimri-engine.py recall --query "checkout retries" ``` -Current generations rank ahead of replaced and closed history. Retrieval is -read-only and MUST NOT silently rehydrate or touch a subject. Repeating the -same query MUST NOT game residency. Because the index is derived, corruption -or deletion of the index is repaired with: +An exact-key request without `--history` returns only the accepted current +generation. Held candidates, replaced generations, and closed generations are +not current; an agent requests them explicitly with `--history` or uses the +review workflow. Task-language discovery retains its current-and-historical +scope. Current generations rank ahead of replaced and closed history. + +Retrieval is read-only and MUST NOT silently rehydrate or touch a subject. +Repeating the same query MUST NOT game residency. A derived audit checkpoint +MAY short-circuit historical re-verification, but it never supplies memory, +conflict, held-candidate, trust, archive, or recovery truth itself. The +returned current entry MUST still come from the accepted head or the exact +selected cold-current binding. Because the index is derived, corruption or deletion of +the index is repaired with: ```text bimri-engine.py index @@ -879,6 +928,28 @@ candidate hashes, resolution authority and revision effects, quarantine evidence, restore receipts, manual-edit evidence, active-run logs, pointer containment, and index shape. +The reference engine performs a full protected-tree audit before an +authority-changing write and for explicit doctor, review, task-language search, +and historical retrieval. When intact prior path-and-hash evidence disagrees +with that audit, the engine MUST NOT silently bless the new bytes: it MUST +re-prove the complete semantic authority graph, and only a passing proof may +adopt the new inventory as its baseline. Recording a durable, validated +drift receipt is a precondition of that adoption: the receipt preserves the +diverging paths with their prior and current hashes, inline up to the +engine's documented per-section bound with any remainder counted +explicitly, and a truncated receipt MUST reference a hash-and-size-pinned +attachment carrying the complete delta, retained while the receipt is +retained and validated with it — as is every other referenced attachment. +A receipt that cannot be durably recorded and validated MUST prevent the +rebaseline and surface as an error; receipts MUST be seal-validated before +their content is trusted or reported; and a sealed witness whose +referenced manifest evidence is unavailable is damaged evidence that MUST +refuse rebaseline rather than adopt new bytes blind. +A failing proof MUST refuse into damaged-authority recovery with the prior +evidence retained. A normal current-only exact read is not required +to traverse unrelated historical authority; every non-engine filesystem writer +therefore remains outside the supported cooperative lock protocol. + Run-log `[PROPOSE:]` references and the monotonic conflict counter are deletion anchors. A referenced missing proposal, a required missing decision, or a conflict gap at or below the durable counter MUST be a governance error. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec3746..d6f8703 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,133 @@ This file records the public BIMRI architecture history. Historical instruction files are preserved under [`legacy/`](legacy/) and are not current installers. +## 5.1.1 + +- Kept the performance work as an engine-only patch. The authority and + mutable-state formats remain v5.1.0, and the readable hot-memory grammar + remains v5.0.2. +- Proposal preflight receipts now stamp engine release v5.1.1. This engine + accepts receipts written by v5.1.0, but an older v5.1.0 engine rejects a + store after its first v5.1.1 proposal is staged. Upgrades therefore require + a quiescent, complete copy of `bimri.md` and `.bimri/` outside the project; + that copy is the rollback boundary. +- Added a compact engine-managed audit checkpoint bound to the v5.1.0 current + authority, with the detailed protected path-and-hash inventory stored as + separate audit evidence. Warm exact-current reads and ordinary lifecycle + bookkeeping no longer enumerate or hash the historical authority tree. + Behavior change: warm reads defer whole-tree verification to + authority-changing writes and explicit audit, review, search, and + historical-recall boundaries, so an out-of-engine edit to unrelated history + is seen at the next such boundary rather than on every read. +- The checkpoint is a derived cache, never an authority record. Divergence + from it forces the full semantic audit; when that audit passes over changes + the engine cannot attribute to its own recorded operation, the engine + first durably records a sealed drift receipt under `.bimri/audit-drift/` + — the diverging paths with prior and current hashes (inline up to 2,000 + entries per section with any remainder counted, the complete delta + pinned in a hash-and-size-validated attachment when truncated), + unbounded monotonic sequence, bounded to the newest 200 — and only then + publishes the new baseline. The written receipt is re-read and fully + validated (seal, filename binding, attachments, post-prune existence) + before the write counts as success, and deduplication only ever reuses a + receipt that validates. A receipt that cannot be recorded keeps the + prior checkpoint and surfaces as an error; `doctor` validates every + receipt and its attachments and reports damaged evidence instead of + trusting it; attachments cited by retained receipts outlive the + unreferenced-attachment bound; and a checkpoint whose referenced + manifest evidence is missing refuses rebaselining instead of adopting + new bytes with no recorded delta. A failed semantic audit + refuses into the existing damaged-authority recovery lane and + invalidates the checkpoint by advancing the state's audit epoch + (owner-ruled 2026-09-02), so the next `start` prints + `AUTHORITY RECOVERY NEEDED` and exact reads refuse until the store is + repaired, as in v5.1.0. The checkpoint bytes stay on disk as the prior + baseline for receipts, quarantine and restore, the same + retained-but-invalid shape a failed resolution leaves; a blocked receipt + sink, an open quarantine and a strict restore comparison keep their + prior checkpoint readable. Interrupted + operations self-heal exactly as in v5.1.0; no drift or crash state ever + requires hand deletion of derived files. `audit-blocked.json` now appears + only while an owner-approved quarantine holds its pre-repair baseline, and + restoration clears it. Owner-ruled 2026-08-27: this receipts contract + replaces the earlier normative fail-closed rule in the protocol, and the + current-only exact recall plus deferred warm-read verification below are + confirmed semantics. +- An obstruction on the audit-transition marker path that is not a regular + file or symlink is a hard error on every surface; doctor never reports + health past it, and the checkpoint is left untouched until the owner + removes the obstruction. +- During interrupted-transition recovery an archive month counts as the + operation's own effect only when its change is byte-provably the prior + witnessed content plus appended rows stamped by the operation's scope. + Any other archive change is preserved as drift evidence. +- Read-only doctor and warm exact reads enforce the same legacy-lineage + refusals as writable load: marker and state must claim each other, and + unclaimed legacy root files refuse the read. +- Authority-changing writes still re-verify the complete evidence inventory: + propose 1.31 s, sync 1.26 s and authority close 1.24 s measured on the + ~500-run live-size store. This exceeds the design brief's one-second + entry-point target and is recorded as a known, owner-accepted deviation + (2026-08-27); incremental authenticated manifests are the planned v5.2 + fix. Reads and lifecycle bookkeeping are unaffected. +- Unknown files inside witnessed roots — crash-orphaned engine temp files + included — are never deleted or blocked on. They enter the audited + inventory, cost at most one full re-audit when they first appear, and stay + visible through drift receipts and doctor litter reporting. The engine + cannot prove a temp-named file is its own, so it preserves it. +- Added a direct current-key path for `get --key` and `recall --key` without + `--history`. Behavior change: it returns only the accepted current + generation (hot or cold-current); held candidates and history remain + reachable through `--history`, `--query`, and `review`. It validates at + most the bounded hot head plus one selected cold archive month. +- Removed automatic derived-index rebuilds from start, commit, and resolution + hot paths. `index`, `maintain`, `doctor`, installation, and migration remain + the explicit rebuild points; the index remains non-authoritative. +- Increased the example Claude hook timeout to 90 seconds so a cold first audit + on an established store has enough time to complete and seed its witness. + Dimension this against your store: the cold first audit measured 32.4 s on a + ~200-revision store and 77.5 s on a 10x synthetic store (2026-08-23, Windows + 11 desktop), so a very large store may need a larger hook timeout for its + first run after installing this release. +- Made a `failed` owner-resolution record fail closed as an explicit recovery + condition. Ordinary retrieval and shared-memory writes pause until the owner + explicitly retries the recorded conflict choice; a failed attempt can no + longer be mistaken for a healthy audited store. +- Reworked the public README around BIMRI's open-source, local-first persistent + memory protocol, cross-session retrieval, human-governed provenance, agent + runtime support, and explicit same-lock-domain concurrency boundary. +- Measured before and after on the live development store (~198 revisions, + 433 run logs, Windows 11 desktop, 2026-08-23): `recall --key` warm + engine-side p50 11 ms (was ~21.3 s), warm CLI end-to-end p50 282 ms; + `start` 0.36 s (was 51.8 s); one-line `journal` ~0.30 s (was ~23.4 s); + `close` 0.35 s (was 43.8 s). On a 10x synthetic store, warm reads held + p50 11 ms and p99 18 ms. +- Console output is UTF-8 on every host. Memory text is UTF-8 on disk, and + on Windows a piped or redirected stdout defaulted to the ANSI code page, + so `get`, `recall` and `search` died with a codec error on any entry + carrying a character outside it (reproduced 2026-09-02; present since + v5.1.0). The engine now reconfigures stdout and stderr to UTF-8 at + startup. +- Stated behaviour after an interrupted authority write: while a proposal + decision or an owner resolution that the next command's recovery pass + cannot settle on its own remains `applying` or `failed` (a `sync` killed + mid-batch is the reproduced case), a passing audit still refuses to + publish a fresh checkpoint, so `start` and exact reads take the + full-audit path until that run's own `sync` or `close` settles the + records, or the owner authorizes `recover-run`. `doctor` passes meanwhile + and lists each unfinished applying decision with the remedy. The first + command after a killed `propose` may refuse with + `an earlier audit transition is incomplete; run doctor before retrying`; + the following command, or `doctor`, completes the recovery. No hand + deletion is ever required. +- Stated boundary: run logs under `.bimri/log/` are not part of the + witnessed inventory. The seven witnessed roots are proposals, decisions, + conflicts, resolutions, revisions, archive and recovery, as in the + original checkpoint design; an out-of-engine append to a closed run log + is not detected by any audit boundary in v5.1.0 or v5.1.1. +- The missing-lock refusal on read-only paths now names the lifecycle + commands that recreate the lock file. + ## 5.1.0 - Restored normal Tier 1 authoring and confirmed-memory updates. Direct diff --git a/CLAUDE.md b/CLAUDE.md index bb31c5a..2ab28bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,8 +2,8 @@ ## Claude Code -Use BIMRI engine and authority format v5.1.0. The readable hot-memory grammar -remains v5.0.2. +Use BIMRI engine v5.1.1 and authority format v5.1.0. The readable hot-memory +grammar remains v5.0.2. `BIMRI-PROTOCOL.md` is the normative memory protocol. `` in BIMRI instructions means the exact absolute Python 3.8+ diff --git a/INSTALL.md b/INSTALL.md index 089c75e..8c7e049 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -64,10 +64,17 @@ two paths to its ignore rules explicitly. ## Agent Installation Contract 1. Obtain this repository in a temporary local folder. -2. Inspect the target for BIMRI v1-v4 or v5 files. Before every update, stop - every process executing the old engine and keep the folder quiescent until - installation completes. The lock serializes commands; it cannot fence an - already-loaded old process waiting to write after the installer exits. +2. Inspect the target for BIMRI v1-v4 or v5 files. Before every update or + migration, stop every process executing the old engine, keep the folder + quiescent until installation completes, and copy every existing BIMRI + artifact somewhere outside the project. For a v5 target, that snapshot MUST + include `bimri.md` plus the entire `.bimri/` tree. The lock serializes + commands; it cannot fence an already-loaded old process waiting to write + after the installer exits. For v5.1.0 to v5.1.1, that complete external + snapshot is the only rollback after the first v5.1.1 proposal is staged: an + older v5.1.0 engine rejects the newer proposal receipt. A store the new + engine has only started and closed, with no v5.1.1 proposal, remains readable + by v5.1.0. 3. If Claude Cowork Global Instructions contain a v1-v3 BIMRI block, ask the owner to disable or remove it before installation. Those instructions directly rewrite the old hot-memory file and must not run alongside v5. @@ -204,7 +211,7 @@ and then runs the new read-only audit. Success is reported only with an explicit receipt equivalent to: ```text -BIMRI 5.1.0 installed. +BIMRI 5.1.1 installed. Existing v5.0.2 hot memory preserved; authority state activated at v5.1.0. Accepted head unchanged: V...... . Memory preservation: PASSED (bimri.md and immutable evidence unchanged). diff --git a/MIGRATION.md b/MIGRATION.md index c214b94..a3ca755 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,7 +1,19 @@ # BIMRI Migration and the v5.1.0 Lifecycle Upgrade -Engine v5.1.0 uses authority format v5.1.0 while retaining the readable v5.0.2 -hot-memory grammar. It automatically migrates explicitly versioned v1-v3 +Engine v5.1.1 uses authority format v5.1.0 while retaining the readable v5.0.2 +hot-memory grammar. The upgrade from v5.1.0 is one-way: v5.1.1 stamps its +own engine release into proposal preflight receipts, and every proposal — +pending or decided — remains an immutable authority record that a v5.1.0 +engine keeps validating. Once any v5.1.1 proposal has been staged, the +v5.1.0 engine's `status`, `doctor`, `recall` and `sync` refuse the store +permanently, and its `start` opens only a degraded run behind the +`AUTHORITY RECOVERY NEEDED` banner (writing its run log and state entry, as +the protocol's degraded-run lane allows). Safe rollback exists only by +restoring the complete pre-update backup taken before the first v5.1.1 +proposal. A store the v5.1.1 engine has only started and closed, with no +proposal, remains fully readable by v5.1.0, which ignores the derived +`audit-*` files. The +engine automatically migrates explicitly versioned v1-v3 tiered Markdown and the engine-based v4 format. This canonical repository publicly distributed the original v1 and streamlined v3 instructions; the parser also accepts a valid v2 header. Migration preserves old material before creating v5 @@ -26,9 +38,13 @@ Make the project quiescent across every runtime boundary: 1. finish or stop every agent using the project; 2. wait for every BIMRI command to finish; 3. pause any synchronization or copy operation affecting the folder; -4. if v1-v3 was installed in Claude Cowork Global Instructions, disable or +4. take a complete copy of the whole project folder (`bimri.md` plus the + entire `.bimri/` tree) somewhere outside it — this snapshot is the only + rollback that exists, because the first v5.1.1 proposal is an + intentional one-way boundary that no older engine can read past; +5. if v1-v3 was installed in Claude Cowork Global Instructions, disable or remove that BIMRI block; and -5. keep the folder quiescent until migration and `doctor` complete. +6. keep the folder quiescent until migration and `doctor` complete. The old v1-v3 Global Instructions directly edit the hot-memory file. v4 uses an older engine contract. None of those writers can participate in the v5 diff --git a/README.md b/README.md index 803efb0..c43cf8c 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,317 @@ -# BIMRI: Portable Memory for Local Agents +# BIMRI: Persistent Memory for AI Agents, in Files You Own -**Brief Interaction Memory and Retrieval Intelligence, engine and authority -format v5.1.0. The readable hot-memory grammar remains v5.0.2.** +[![Tests](https://github.com/EvolutionUnleashed/bimri/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/EvolutionUnleashed/bimri/actions/workflows/tests.yml) +![Python 3.8 or newer](https://img.shields.io/badge/python-3.8%2B-blue) +![Dependencies: none](https://img.shields.io/badge/dependencies-none-lightgrey) +![License: MIT](https://img.shields.io/badge/license-MIT-green) -BIMRI gives a project one durable memory that Claude, Codex, and other local -agents can share without a server, database, account, or model-specific -service. The memory travels with the folder. Its readable current state stays -small, while its evidence, decisions, and revision history remain on disk. +**BIMRI gives local AI agents persistent project memory across sessions.** +Installed project instructions tell each agent to read the current accepted +memory at the start of a session, so your decisions, preferences and the state +of the work survive the handoff. The memory lives in your project folder as +plain files you can open and read. It works with Claude Code, OpenAI Codex and +any local agent that can run a Python script. Free and open source under the +MIT license, with no database, no BIMRI account and no API key. -Installation is deliberately simple: point your agent at the repo and ask it -to install BIMRI. +Setup is one sentence to your agent: -> Point your agent at the repo and ask it to install BIMRI. +```text +Install BIMRI in this project from https://github.com/EvolutionUnleashed/bimri. +Follow INSTALL.md, preserve my existing instructions and memory, and run the +self-check. +``` + +**Who this is for.** Business owners, consultants and agencies who run AI +agents on real work and are tired of re-explaining the business every morning. +And developers who want durable, inspectable, multi-agent memory without adding +another service to run. The plain-language overview lives at +[agentguru.ai/bimri](https://agentguru.ai/bimri); everything technical is +below. + +## AI Agent Memory Should Outlast the Runtime + +Claude Code can keep editable, machine-local project memory. OpenAI Codex can +generate local memory files you can inspect. Both give one runtime useful +continuity under that runtime's schema, loading rules and lifecycle. + +The gap appears when several supported agents need one accepted memory they can +all use and take with the project. A shared notes file travels, but by itself it +does not label sources, preserve accepted history, govern conflicting writes or +recover a half-finished change. BIMRI does. + +## Your Agent's Memory Lives on Your Machine + +BIMRI keeps accepted memory in a folder on your machine and uses an engine to +protect its history. Accepted facts, decisions, preferences, rules and current +work record their source and trust level. Learned patterns record their +evidence, confidence and what would prove them wrong. When two agents +concurrently propose incompatible changes to the same stable key from the same +accepted base, BIMRI records one conflict and leaves the choice to you. + +When BIMRI is idle, copy the project folder and rerun the installer on the new +machine. Its accepted memory, evidence and history move with it. The model and +runtime can change; what BIMRI has preserved about your business stays with the +project. + +| What you get | What it means day to day | +| --- | --- | +| **Portable** | When BIMRI is idle, move the project to another machine or supported agent and its accepted memory, evidence and history go with it. | +| **Readable by you** | Plain Markdown you can open in any editor. Read exactly what BIMRI currently carries, then tell the agent to correct it through the engine if it is wrong. | +| **Safe with many agents** | On one machine and one filesystem lock domain, several agents can work at once without silently overwriting one another. | +| **Governed by you** | An incompatible concurrent change to the same keyed subject stops for your decision, and the choice is recorded. | +| **Provenance built in** | Accepted facts and current work record source and trust. Learned patterns record evidence, confidence and what would prove them wrong. | +| **Long-term memory that stays small in context** | The agent loads a bounded working set of roughly 12,000 tokens. Everything else stays on disk and can be recalled when needed. | +| **No hosted dependency** | MIT licensed, with no BIMRI account or subscription. The memory stays in files you control. | + +## What One Memory Looks Like + +```text +[R000012-E004] [K:offer.spring-promotion] [fact] [T:confirmed] [SRC:user] [offer,pricing] The spring promotion runs to 30 April at 20% off the annual plan. -> .bimri/log/R000012.md +``` + +- `[K:offer.spring-promotion]` is a stable key, so a newer memory on the same + subject replaces this one instead of piling up beside it. +- `[SRC:user]` says the owner directly supplied the claim. Agent inference uses + `[SRC:agent]`; outside material uses `[SRC:external]`. +- `[T:confirmed]` records how much to trust it, so agents weigh what they read + instead of swallowing it whole. +- `-> .bimri/log/R000012.md` points at the journal where the full detail and + the reasoning live. + +The file this line sits in, `bimri.md`, is generated from an immutable revision +and is never edited by hand. Under `.bimri/` sit the run journals, every +accepted revision, every immutable proposal, its decision record once decided, +every recorded BIMRI conflict and every recorded owner resolution. Markdown +holds the memory; small JSON and TSV files hold the bookkeeping, all of it in +the project folder. + +## Works With Claude Code, OpenAI Codex and Local Agents + +| Agent runtime | Integration | +| --- | --- | +| Claude Code | `CLAUDE.md`, `AGENTS.md`, and optional session hooks that open and close a run automatically | +| OpenAI Codex | `AGENTS.md` and explicit engine commands | +| Other local coding agents | Supported when they can follow the instruction block and execute the verified Python runtime | + +The runtime is one Python 3.8 or newer standard-library script plus ordinary +local files. There are no packages to install. The BIMRI engine has no network +client, server, account or API key; it reads and writes local project files +only. Your agent runtime may still send prompts, relevant file excerpts or tool +results to its model provider under that runtime's own data controls. Several +agents can share one memory folder as long as they share one operating system +and filesystem lock domain; see the concurrency boundary in the reference +section for the exact rule. + +## How It Works, in One Minute + +Each agent asks the engine for its own run handle, receives a short brief, and +then reads the accepted current memory from `bimri.md`. It journals detail as it +works. Anything that should shape future sessions is proposed under a stable, +lowercase key such as `checkout.next-step`. A sync commits accepted proposals +as a new immutable revision and regenerates `bimri.md`. Only concurrent +incompatible changes to the same key from the same accepted base become human +conflicts. Independent-key changes can commit without a human decision, while +exact compatible same-key effects become no-ops. Other refusals stay with the +agent instead of becoming owner conflicts. + +The readable view stays small on purpose. Tier 1 holds durable facts, +decisions, preferences and rules. Tier 2 holds current work, risks and next +actions. Tier 3 holds patterns only when evidence and a falsifier exist. When +an incoming change would make the view too large, the engine moves eligible +Tier 2 subjects into keyed cold storage and retries the change. Those subjects +stay current and recallable. Earlier generations remain in immutable history +rather than being deleted. + +```text + bimri-engine.py start --actor codex + bimri-engine.py journal --run R000042 --importance 3 --text "Checkout retries must use the existing idempotency key." + bimri-engine.py propose --run R000042 --tier 2 --new-subject --key checkout.next-step --text "Verify retry behavior under concurrent requests." + bimri-engine.py sync --run R000042 + bimri-engine.py recall --key checkout.next-step + bimri-engine.py close --run R000042 --outcome success --summary "Retry behavior verified." +``` + +`` stands for the absolute Python executable the installer +verified on your machine; the installer records it in +`.bimri/runtime.local.json`, and agents read it from there rather than guessing +a PATH name. The [quick start](#quick-start-store-and-retrieve-project-memory) +below walks through every command. + +## Fast Enough to Run on Every Session + +Measured on the development store used by agents running Stu Jordan's business +(about 500 runs and 216 revisions), on Windows 11 on 2026-09-02: + +| Operation | Time | +| --- | ---: | +| Exact recall of one memory, warm, end to end | 0.33 s | +| Session start, warm | 0.41 s | +| Journal one line | 0.39 s | +| Propose or sync an authority change | 1.3 to 1.6 s | +| Cold full audit to seed or rebuild the checkpoint | about 33 s per audit | + +Warm exact-current reads, starts and journals use the checkpoint while it +remains valid. `doctor`, review, search and historical recall deliberately run +the full audit, and a missing or invalid checkpoint is rebuilt. Before an +authority-changing write, the engine rechecks the complete protected record +and runs a semantic audit if anything changed. 255 tests cover concurrency, +crash recovery, migration, integrity and safety, and run in public CI on Linux +and Windows under Python 3.8 and 3.12 on every change. + +## BIMRI and Compounding Intelligence + +BIMRI is the memory layer inside the compounding intelligence engine Stu Jordan +uses to run agents in his own business. Continuity, feedback learning and a +results ledger handle the other parts. BIMRI makes improvement durable by +preserving the accepted decisions, corrections and business context those +systems need, and it is the part released as open source. + +What the business teaches the agent can survive the session, the model and the +runtime. + +If you want the finished agent, [Agent in a Box](https://agentguru.ai/agent-in-a-box) +is one agent for one named job, built on this architecture. [The Starter +Engine](https://agentguru.ai/start) is a complete scheduled agent you can run +free. + +## Frequently Asked Questions + +### What is BIMRI? + +BIMRI is a free, open-source persistent memory system for Claude Code, OpenAI +Codex and other local agents. It stores accepted project knowledge in +human-readable files, preserves its provenance and verified history, and lets +supported agents share one governed memory across sessions. + +### What is AI agent memory? + +AI agent memory is the project knowledge an agent carries from one session to +the next: decisions, preferences and the current state of the work. Each +runtime has its own way of carrying or rebuilding that picture. BIMRI keeps one +accepted record in the project, bounds what loads into each session and lets +the agent recall the rest when needed. -The agent follows [`INSTALL.md`](INSTALL.md), merges BIMRI into the project's -existing instructions, preserves existing BIMRI memory, and runs a self-check. -The runtime is one Python 3.8+ standard-library script plus ordinary local -files. There are no packages to install. Commands below use -`` as a placeholder for the absolute Python 3.8+ executable -that the installer has executed and verified on the current machine. A name -such as `python3` is never assumed to work. Multiline examples use POSIX -continuation syntax; on Windows, run them on one line or adapt them to the -active shell and path format. +### Does Claude Code remember between sessions? + +Yes. Claude Code reads your `CLAUDE.md` instructions and can keep editable, +machine-local [project memory](https://code.claude.com/docs/en/memory) between +sessions. That gives one runtime useful continuity. By default, its automatic +notes are Claude Code-specific and stored per repository on one machine, and +they do not require provenance on each fact. BIMRI adds a governed project +memory that supported agents in the folder can share, with stable keys, +provenance, immutable history and bounded loading. When the rendered hooks are +enabled, Claude Code opens and closes a BIMRI run automatically. + +### Does OpenAI Codex remember between sessions? + +Yes, when local memories are enabled. Codex can generate local memory files +from eligible prior chats, store them under the Codex home directory +(`$CODEX_HOME/memories/`, normally `~/.codex/memories/`), and use them in later +sessions. OpenAI documents these as +[inspectable generated state](https://learn.chatgpt.com/docs/customization/memories); +the feature is off by default and updates in the background rather than after +every chat. BIMRI can run alongside it for project-owned memory that supported +agents can share, with stable keys, provenance, immutable history and bounded +loading. Keep rules that must always apply in `AGENTS.md`. + +### What does BIMRI stand for? + +Brief Interaction Memory and Retrieval Intelligence: brief because the working +memory an agent loads stays small, retrieval because the long tail is recalled +on demand rather than stuffed into every session. + +### Is BIMRI free? + +Yes. BIMRI is MIT licensed and free for personal and commercial use. The source +and full documentation are in this public GitHub repository. + +### Does BIMRI need a database, embeddings or an API key? + +No. It uses ordinary local files, exact stable-key retrieval and lexical +task-language search. There is no database, no package to install, no embedding +model, no cloud service and no API key. + +### Does BIMRI send my memory anywhere? + +The BIMRI engine has no network client, server, account or API key; it reads +and writes local project files only. Your agent runtime may send prompts, +relevant file excerpts or tool results to its model provider under that +runtime's own data controls. BIMRI does not change that boundary. + +### Which agents does it work with? + +Claude Code and OpenAI Codex today, and any local agent that can run a Python +script and follow written instructions. Claude Code can open and close a run +automatically through session hooks. + +### How is this different from the memory built into my AI app? + +Built-in memory is useful, and runtimes differ. Local Codex can generate +inspectable files, while Claude Code can keep editable project memory. BIMRI +adds a project-owned layer that supported local agents can share: stable keys, +source and trust labels on Tier 1 and Tier 2 claims, immutable revisions, +recorded conflict decisions and a defined one-lock-domain concurrency +boundary. + +### Can several agents share one memory? + +Yes, when every agent shares one machine and operating-system/filesystem lock +domain. Each agent has its own log. Independent changes can commit without a +human decision; concurrent incompatible changes to the same key from the same +accepted base stop for yours. Simultaneous writes from different machines or +synchronized copies are outside the guarantee. + +### What happens if an agent crashes mid-write? + +BIMRI records the intended memory change before replacing accepted state. After +an interruption, recovery either proves the change committed, keeps the prior +accepted memory authoritative, or reports the exact next action. That may be +the original run finishing its write or the owner re-approving the same conflict +choice. Accepted memory is never left half-written, and recovery does not +require deleting files by hand. + +### Does it run on Windows? + +Yes. The suite runs on Linux and Windows in public CI, and the development +store that produced the numbers above lives on Windows 11. + +--- + +## Reference + +Everything below is the documentation of record for people installing, +operating or building on BIMRI. The normative protocol is +[`BIMRI-PROTOCOL.md`](BIMRI-PROTOCOL.md). Questions and problems go to +[GitHub Issues](https://github.com/EvolutionUnleashed/bimri/issues). + +| I want to... | Go to... | +| --- | --- | +| Install BIMRI in a project | [Install BIMRI](#install-bimri) or [`INSTALL.md`](INSTALL.md) | +| Understand the memory architecture | [How BIMRI persistent memory works](#how-bimri-persistent-memory-works) | +| Store and retrieve agent memory | [Quick start](#quick-start-store-and-retrieve-project-memory) | +| Understand exact recall and integrity checks | [Exact recall and integrity performance](#exact-recall-and-integrity-performance) | +| Read the normative memory protocol | [`BIMRI-PROTOCOL.md`](BIMRI-PROTOCOL.md) | +| Migrate an older BIMRI store | [`MIGRATION.md`](MIGRATION.md) | +| Review releases and architecture changes | [`CHANGELOG.md`](CHANGELOG.md) | + +The current engine is v5.1.1, the authority format is v5.1.0, and the readable +hot-memory grammar remains v5.0.2. + +### Supported AI Agent Runtimes + +BIMRI is agent-independent. Its canonical interface is a dependency-free +Python 3.8+ command-line engine plus the universal instructions in `AGENTS.md`. + +The concurrency guarantee applies to agents that share the same operating- +system/filesystem lock domain. Simultaneous writes from different machines, +independently copied folders, or unverified mounted filesystems are not a v5 +guarantee. + +Commands in this document use `` as a placeholder for the +absolute Python 3.8+ executable that the installer has executed and verified on +the current machine. A name such as `python3` is never assumed to work. +Multiline examples use POSIX continuation syntax; on Windows, run them on one +line or adapt them to the active shell and path format. Machine bindings are not portable memory. The installer writes `.bimri/runtime.local.json` with the verified runtime argv prefix and @@ -30,7 +320,7 @@ commit either file or copy its absolute paths into shared instructions. After moving the project or replacing Python, rerun installation to regenerate both files before an agent uses BIMRI. -## What v5 Is +### How BIMRI Persistent Memory Works `bimri.md` is a generated view of the latest accepted memory revision. Agents do not compete to rewrite it. Each agent instead receives its own run handle, @@ -59,7 +349,7 @@ This design gives BIMRI four useful properties: - **Human-governed:** exceptional concurrent choices are shown as actions and consequences, with a durable resolution record for an explicit owner choice. -## Active Memory, Not a Diary +### Long-Term Memory Without Context Bloat Flat-file memory tends to become a transcript of completed work. Every session adds another summary, old context stays visible, and useful signal is buried in @@ -76,7 +366,7 @@ useful action, backed by a durable long tail. Run logs, immutable revisions, decisions, resolutions, archives, and migration backups remain available even when they are outside the generated hot view. -## Install +### Install BIMRI Give a coding agent this repository URL and say: @@ -92,6 +382,13 @@ For a fresh target or v1-v4 migration, the installer command is: bimri-engine.py install --target /absolute/path/to/the/project ``` +Before any upgrade, make the target quiescent and copy `bimri.md` plus the +entire `.bimri/` tree somewhere outside the project. For v5.1.0 to v5.1.1, +that complete snapshot is the only rollback after the first v5.1.1 proposal is +staged: an older v5.1.0 engine rejects the newer proposal receipt. A store the +new engine has only started and closed, with no v5.1.1 proposal, remains +readable by v5.1.0. + For any existing v5 store, stop every old engine process first and attest that external handoff explicitly: @@ -109,18 +406,18 @@ profile expands to the v5.0.1 profile; custom values remain custom and become soft curation targets. Earlier state bytes are backed up, and accepted revisions are preserved rather than rewritten. -Updating an existing v5.0.2 store to engine v5.1.0 uses a dedicated lossless -authority-activation operation. Stop every process running the old engine, then -invoke install with the mandatory `--quiescent` handoff attestation. The -updater audits the accepted head without healing or rebuilding memory, -records every pre-existing path, and verifies that `bimri.md`, the accepted -head, and every immutable evidence/history path remain byte-identical. It backs -up the exact old mutable state, then activates v5.1 state last so an older -engine fails closed immediately. The readable hot-memory grammar and all +Updating an existing v5.0.2 store to the v5.1 authority format uses a dedicated +lossless authority-activation operation. Stop every process running the old +engine, then invoke install with the mandatory `--quiescent` handoff +attestation. The updater audits the accepted head without healing or rebuilding +memory, records every pre-existing path, and verifies that `bimri.md`, the +accepted head, and every immutable evidence/history path remain byte-identical. +It backs up the exact old mutable state, then activates v5.1 state last so an +older engine fails closed immediately. The readable hot-memory grammar and all legacy evidence remain unchanged. Engine v5.0.3 deliberately kept its persisted state at v5.0.2, so this is also -the normal upgrade path from a public v5.0.3 installation. +the normal authority-upgrade path from a public v5.0.3 installation. The v5 installer serializes with other v5 engine commands in the same lock domain. Before upgrading any earlier version, disable the old Claude Cowork @@ -140,7 +437,7 @@ for recovery and inspection. The installer packages them into the target as inert rollback references; they must never be activated or pasted into Claude Global Instructions while v5 is active. -## Quick Start +### Quick Start: Store and Retrieve Project Memory An agent starts by requesting its own run handle: @@ -148,8 +445,23 @@ An agent starts by requesting its own run handle: bimri-engine.py start --actor codex ``` -The engine prints a brief and a handle such as `R000042`. The agent reads -`bimri.md`, then journals durable detail as work happens: +The engine prints a brief and a handle such as `R000042`. The agent reads the +bounded working set in `bimri.md`. When it needs detail from the durable long +tail, it retrieves an exact stable key or searches in task language: + +```text + bimri-engine.py recall --key checkout.next-step + bimri-engine.py recall --query "checkout retries" +``` + +Exact-key recall returns the current subject by default; add `--history` when +prior generations are relevant. Task-language recall searches current and +historical memory. Retrieval is read-only and does not silently move a subject +back into the hot working set. v5.1.x uses deterministic exact-key and lexical +task-language retrieval; it does not require embeddings or claim semantic +vector search. + +The agent journals durable detail as work happens: ```text bimri-engine.py journal --run R000042 --importance 3 \ @@ -215,7 +527,90 @@ confirms that it should close, an agent can recover it with: The outcome defaults to `partial`; `--outcome` accepts the normal close outcomes. -## Memory and Trust +### Exact Recall and Integrity Performance + +Engine v5.1.1 adds a validated fast path for current exact-key retrieval. A +`get --key` or `recall --key` command without `--history` resolves the accepted +hot revision and keyed cold-current storage directly and returns only the +accepted current generation. Held candidates and superseded generations remain +available through `--history` and the review workflow. The current lookup does +not construct one combined collection from every historical generation. + +The fast path is gated by `.bimri/audit-witness.json`, an engine-managed, +non-authoritative integrity checkpoint. The compact witness binds the engine, +memory format, validation policy, accepted head, and canonical current-memory +state. Detailed path-and-SHA-256 evidence for the last full audit lives +separately in `.bimri/audit-manifest.json`, so a warm exact lookup does not +parse or hash the historical inventory. Neither file stores memory, conflicts, +or held candidates. + +With a valid checkpoint, a hot exact lookup reads the bounded accepted head. A +cold exact lookup additionally validates only the selected subject's archive +month. It does not enumerate run logs, revisions, proposals, decisions, +conflicts, resolutions, recovery evidence, or unrelated archives. Normal start +and journal commands likewise avoid historical traversal. + +Full verification still happens before authority-changing writes and during +explicit audit, historical recall, task-language search, and review. Those +checks compare the live protected inventory with the prior manifest. Protected +roots are flat; an unexpected subdirectory or redirected path prevents a valid +audit. Divergence from the prior manifest is a cache miss, never a verdict: +the full semantic audit decides. When it passes over changes the engine cannot +attribute to its own recorded operation, a sealed drift receipt is durably +recorded and validated under `.bimri/audit-drift/` before the new baseline can +publish. The receipt carries the diverging paths with prior and current hashes, +inline up to a documented per-section bound with any remainder counted, and +pins the complete delta in a validated attachment when truncated. `doctor` +validates every receipt (seal, filename binding, and each referenced +attachment's existence, size and hash) before trusting it. A receipt that +cannot be written keeps the prior checkpoint as the baseline and surfaces as an +error, and a checkpoint whose referenced manifest evidence is missing refuses +rebaselining instead of adopting new bytes blind. A failed semantic audit +refuses into damaged-authority recovery and invalidates the checkpoint by +advancing the audit epoch, so the next `start` prints +`AUTHORITY RECOVERY NEEDED` and exact reads refuse until the store is repaired, +exactly as v5.1.0 behaved. The checkpoint bytes stay on disk as the prior +baseline for receipts, quarantine and restore; only a blocked receipt sink, an +open quarantine, or a strict restore comparison keeps the prior checkpoint +readable. This detects and records standalone or accidental edits; it is not a +defense against a coordinated writer editing history and derived evidence +together, which no local store can prove from its own bytes. + +#### Support envelope + +v5.1.1 is a bounded, single-store performance release, validated on a store +shaped like its own development project: roughly 500 runs, a few hundred +revisions and current subjects, one machine, one lock domain. Inside that +shape, warm exact reads run in the low hundreds of milliseconds end to end, +start and journal likewise, authority-changing writes about 1.3 seconds, +and the full audit that seeds the checkpoint runs once at about 30 seconds. +Outside that shape the documented ceilings apply until the planned v5.2 +work: exact reads scale with total current-state size, a selected cold key +scans its whole archive month, every operation serializes behind one +exclusive lock, task-language `recall --query` is unranked ASCII substring +matching on the fully audited path, and authority writes rescan retained +history. Larger current-state sets, high reader concurrency, ranked or +non-ASCII retrieval, multi-machine fleets, and indefinite-lifetime storage +are explicitly not claims of this release. + +This is a cooperative local integrity model, not a claim that each read takes a +filesystem snapshot. An out-of-engine edit to unrelated protected history can +remain unseen by current-only recall until the next full-audit boundary; +changes to state, the accepted head, or the selected cold binding invalidate +that lookup immediately. Every writer must therefore use the engine and shared +lock. Run logs under `.bimri/log/` sit outside the witnessed inventory +altogether: the audit validates active-run logs and the run facts that bear on +authority, and it does not hash closed journals. A missing or unreadable +checkpoint can still require a full audit before the fast path is +re-established. While an interrupted authority write leaves a decision or +resolution `applying` that the next command's recovery pass cannot settle on +its own, the engine withholds the checkpoint and every start and exact read +takes the full-audit path until that run's own sync or close settles it; +`doctor` passes meanwhile and lists each unfinished applying decision. BIMRI +does not promise one universal latency across filesystems, security scanners, +or unbounded current state. + +### Memory Tiers, Provenance, and Trust Hot memory has three tiers with soft curation targets: @@ -279,7 +674,7 @@ Tier 1 admission is preserved quietly as a held candidate. If the owner directly adopts that change, the agent submits the exact owner statement as a normal `--source user --trust confirmed` update. -## Pull Review and Human Resolution +### Human Review and Conflict Resolution `start` and `hook-start` never replay open review records. Ask for actionable concurrent choices explicitly: @@ -318,7 +713,7 @@ later accepted revision contains a historical candidate's exact normalized effect, BIMRI derives that it is satisfied without rewriting the old proposal, decision, conflict, or resolution history. -## File Map +### Local-First File and Storage Map Repository files: @@ -355,6 +750,12 @@ Runtime files: | `.bimri/conflicts/` | Open and historical questions for the human. | | `.bimri/resolutions/` | Durable human choices. | | `.bimri/index.tsv` | Rebuildable, non-authoritative retrieval index. | +| `.bimri/audit-witness.json` | Compact, rebuildable checkpoint for the last successful full integrity audit. | +| `.bimri/audit-manifest.json` | Detailed, rebuildable path-and-hash evidence behind that checkpoint. | +| `.bimri/audit-manifests/` | Retained manifest generations referenced by live audit evidence. | +| `.bimri/audit-transition.json` | Write-ahead marker while a checkpoint change is in flight. | +| `.bimri/audit-drift/` | Bounded rolling receipts (newest 200) sealing diverging paths with prior and current hashes; truncated receipts pin their complete delta in a validated attachment. | +| `.bimri/audit-blocked.json` | Owner-repair baseline held while a quarantine is open; cleared by restoration. | | `.bimri/archive/` | Cooled-current, replaced, and closed generations with provenance. | | `.bimri/backups/` | Migration and pre-change safety copies. | | `.bimri/recovery/` | Direct edits, damaged authority evidence, and restore receipts. | @@ -365,7 +766,7 @@ Runtime files: Markdown carries the human-readable memory and evidence. Small JSON and TSV files carry transparent bookkeeping. All of it stays in the project folder. -## Operational Commands +### BIMRI Command-Line Operations ```text bimri-engine.py status @@ -402,7 +803,7 @@ reports the event, and restores the accepted generated view. Repeating the same edit reuses the same exact recovery bytes. An exact byte copy of a referenced immutable revision is already preserved and is simply healed as a stale view. -## Authority Damage and Recovery +### Authority Damage and Recovery Proposals, decisions, conflicts, and resolutions form the authority graph. If one of those JSON records is unreadable or invalid while state and the accepted @@ -413,6 +814,10 @@ the warning for health. Shared-memory commits, conflict resolution, maintenance, migration, and index rebuilding remain paused. Isolated run journals and proposals can remain staged until recovery completes. +A resolution recorded as `failed` is also an explicit recovery condition. +Ordinary retrieval and shared-memory writes fail closed until the owner +re-attests and retries that exact conflict choice with `resolve`. + After the owner reviews the damaged record, preserve it with: ```text @@ -450,7 +855,7 @@ final restore and resume only after it passes. For both recovery commands, `--human-approved` is an attestation of the owner's choice, not authentication of the caller. -## Concurrency and Portability Boundary +### Concurrency and Portability Boundary The v5 concurrency guarantee covers processes that access the same folder through one shared operating-system/filesystem lock domain. Every writer must @@ -466,10 +871,10 @@ An agent harness running inside a VM or sandbox against a mounted host folder is also outside the guarantee unless that exact mount has been verified to share both lock and atomic-rename semantics with host processes. Containers do not all behave alike, so the boundary must be verified rather than inferred -from the word “container.” The safe default is one runtime boundary active at +from the word "container." The safe default is one runtime boundary active at a time, with a quiescent handoff before another boundary uses the folder. -## Privacy +### Privacy Memory may contain private project context. Decide deliberately whether it belongs in version control. For local-only memory: @@ -491,9 +896,9 @@ host-bound adapter records explicitly: **Stu Jordan**, Agent Architect +- Product site: [agentguru.ai](https://agentguru.ai) - Community: [Evolution Unleashed](https://evolutionunleashed.com) - Patreon: [Evolution Unleashed VIP](https://www.patreon.com/evolutionunleashedvip) -- Web: [evolutionunleashed.com](https://evolutionunleashed.com) ## License diff --git a/bimri-engine.py b/bimri-engine.py index 0043580..b2fc6f3 100644 --- a/bimri-engine.py +++ b/bimri-engine.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -BIMRI Engine v5.1.0 (authority store v5.1.0; hot grammar v5.0.2) +BIMRI Engine v5.1.1 (authority store v5.1.0; hot grammar v5.0.2) Portable, human-governed memory for local agents. The shared memory is a generated Markdown view. Agents work in independent @@ -53,9 +53,21 @@ msvcrt = None -ENGINE_VERSION = "5.1.0" +ENGINE_VERSION = "5.1.1" +V5_1_0_ENGINE_VERSION = "5.1.0" MEMORY_FORMAT_VERSION = "5.1.0" HOT_FORMAT_VERSION = "5.0.2" +AUDIT_WITNESS_SCHEMA = 1 +AUDIT_MANIFEST_SCHEMA = 1 +AUDIT_BLOCKED_SCHEMA = 1 +AUDIT_DRIFT_SCHEMA = 1 +AUDIT_TRANSITION_SCHEMA = 1 +AUDIT_DRIFT_KEEP = 200 +AUDIT_DRIFT_DELTA_CAP = 2000 +AUDIT_DRIFT_BLOB_KEEP = 20 +PRIOR_EVIDENCE_INVALID_PREFIX = "prior audit manifest evidence is invalid" +AUTHORITY_POLICY_VERSION = "5.1.1-authority-1" +HOOK_TIMEOUT_SECONDS = 90 PREVIOUS_V5_VERSION = "5.0" V5_0_1_VERSION = "5.0.1" V5_0_2_VERSION = "5.0.2" @@ -384,6 +396,12 @@ def __init__(self, root): self.hot = self.root / "bimri.md" self.state = self.bdir / "state.json" self.index = self.bdir / "index.tsv" + self.audit_witness = self.bdir / "audit-witness.json" + self.audit_manifest = self.bdir / "audit-manifest.json" + self.audit_manifests = self.bdir / "audit-manifests" + self.audit_blocked = self.bdir / "audit-blocked.json" + self.audit_drift = self.bdir / "audit-drift" + self.audit_transition = self.bdir / "audit-transition.json" self.lock = self.bdir / "engine.lock" self.logs = self.bdir / "log" self.proposals = self.bdir / "proposals" @@ -396,6 +414,18 @@ def __init__(self, root): self.backups = self.bdir / "backups" self.recovery = self.bdir / "recovery" self.migrations = self.bdir / "migrations" + # Per-process evidence for a command that has already checked the + # witness while holding this store's lock. It is never persisted as + # authority and callers must not reuse it after a mutation. + self.validated_audit_witness = None + self.pending_checkpoint_witness = None + self.authority_write_mode = False + self.authority_write_audit_healthy = False + self.full_audit_manifest = None + self.full_audit_run_facts = None + # Drift observed by the most recent full audit in this process, for + # read-only reporting surfaces that must not write a receipt. + self.last_audit_drift = [] @property def legacy_active(self): @@ -411,6 +441,7 @@ def dirs(self): self.bdir, self.logs, self.proposals, self.decisions, self.revisions, self.conflicts, self.resolutions, self.archive, self.inbox, self.backups, self.recovery, self.migrations, + self.audit_manifests, ) @@ -821,8 +852,10 @@ def existing_store_lock(paths, timeout=10.0): raise BimriError("existing .bimri directory is missing or unsafe.") if path_is_redirected(paths.lock) or not paths.lock.is_file(): raise BimriError( - "existing .bimri/engine.lock is missing or unsafe; the code-only " - "updater will not create it." + "existing .bimri/engine.lock is missing or unsafe. A lifecycle " + "command such as start, status or doctor recreates a missing " + "lock file; read-only commands and the code-only updater never " + "create it." ) with _held_lock(paths.lock.open("r+b"), timeout): yield @@ -936,6 +969,7 @@ def validate_state(state, accepted_versions=None): raise BimriError(f"unsupported BIMRI state version: {state.get('bimri_version')}") if state.get("legacy_migration") not in (None, "legacy-to-v5"): raise BimriError("state legacy_migration has an unsupported value.") + state_audit_epoch(state) if not isinstance(state.get("active_runs"), dict): raise BimriError("state active_runs must be an object.") if not isinstance(state.get("session_runs"), dict): @@ -1015,7 +1049,7 @@ def current_run_date_references(paths, state): return referenced -def save_state(paths, state): +def prepare_state_for_save(paths, state): validate_state(state) if len(state.get("run_dates", {})) > 500: keys = sorted( @@ -1040,6 +1074,12 @@ def save_state(paths, state): key: state["run_dates"][key] for key in keys if key in retained } + validate_state(state) + return state + + +def save_state(paths, state): + prepare_state_for_save(paths, state) atomic_write_json(paths.state, state) @@ -1778,6 +1818,31 @@ def verify_all(after_view): fsync_directory(source.parent) +def require_read_only_legacy_lineage(paths, state): + """Pure legacy-lineage refusal for read paths; never mutates anything. + + Read-only surfaces must be exactly as strict as writable load about + competing or forged legacy authority: the migration marker and the v5 + state must claim each other, a present marker must validate, and + unclaimed legacy root files refuse the read. + """ + marker_path = paths.migrations / "legacy-to-v5.json" + if not (marker_path.exists() or marker_path.is_symlink()): + if state.get("legacy_migration") == "legacy-to-v5": + raise BimriError( + "v5 state requires legacy-to-v5.json, but the durable migration " + "marker is missing. Restore it from recovery before continuing." + ) + else: + if state.get("legacy_migration") != "legacy-to-v5": + raise BimriError( + "legacy-to-v5.json exists, but v5 state does not claim that migration." + ) + marker = read_json_strict(marker_path, marker_path.name) + _validate_legacy_marker(paths, marker, state) + reject_unclaimed_legacy_roots(paths) + + def finalize_legacy_migration(paths, state): marker_path = paths.migrations / "legacy-to-v5.json" if not marker_path.exists(): @@ -2685,6 +2750,22 @@ def load_or_initialize(paths): merged = fresh_state() merged.update(raw) state = validate_state(merged) + state = reconcile_audit_transition(paths, state) + witness = load_valid_audit_witness(paths, state) + if witness is not None and not load_audit_blocked_issues(paths): + validate_checkpoint_head(paths, state) + # Legacy-root and legacy-marker refusal are cheap root checks that + # must hold on every load, warm reads included. + finalize_legacy_migration(paths, state) + reject_unclaimed_legacy_roots(paths) + record_migration_receipt( + paths, + "verified", + source_version=MEMORY_FORMAT_VERSION, + limits=limits_profile(state), + metadata_revision=None, + ) + return state validate_current_residency(paths, state) finalize_legacy_migration(paths, state) reject_unclaimed_legacy_roots(paths) @@ -2694,6 +2775,10 @@ def load_or_initialize(paths): metadata_revision = None if not recovery_gate_issues: metadata_revision = finalize_current_v5_metadata(paths, state) + if authority_has_recoverable_applying(paths): + enter_authority_write_after_audit( + paths, state, operation="recover-interrupted-authority" + ) recover_interrupted_authority(paths, state) record_migration_receipt( paths, @@ -3225,6 +3310,14 @@ def sync_generated_view(paths, state): except OSError: continue if current_hash not in known_hashes and hot_exists: + if not paths.authority_write_mode: + begin_authority_write( + paths, + state, + allow_degraded=True, + operation="recover-manual-hot-edit", + scope={"hot_hash": current_hash}, + ) try: current_bytes.decode("utf-8") suffix = ".md" @@ -3252,6 +3345,8 @@ def sync_generated_view(paths, state): else: conflict = None write_generated_view(paths, expected) + if conflict is not None: + refresh_audit_witness_after_trusted_write(paths, state) return conflict @@ -3495,10 +3590,11 @@ def validate_quarantine_stub(paths, path, stub, kind, record_id): return stub -def logged_proposal_records(paths, state): +def logged_proposal_records(paths, state, include_run_status=False): """Return durable proposal references and processing expectations.""" records = [] issues = [] + closed_by_run = {} for log in sorted(paths.logs.glob("R*.md")): if not RUN_RE.fullmatch(log.stem): continue @@ -3518,6 +3614,7 @@ def logged_proposal_records(paths, state): content, re.MULTILINE, ) is not None + closed_by_run[run_id] = closed for proposal_id in dict.fromkeys(proposal_ids): if not proposal_id.startswith(f"{run_id}-Q"): issues.append( @@ -3526,6 +3623,8 @@ def logged_proposal_records(paths, state): ) continue records.append((proposal_id, run_id, closed)) + if include_run_status: + return records, issues, closed_by_run return records, issues @@ -3902,133 +4001,2871 @@ def scan_open_conflicts(paths, state): return items, issues -def governance_snapshot(paths, state, allow_recoverable_applying=False): - conflicts, conflict_issues = scan_open_conflicts(paths, state) - issues = list(dict.fromkeys( - authority_storage_issues( - paths, - state, - allow_recoverable_applying=allow_recoverable_applying, - ) - + conflict_issues - )) - return conflicts, issues +def audit_witness_roots(paths): + """Stable authority and history roots covered by the audit witness.""" + return ( + paths.proposals, + paths.decisions, + paths.conflicts, + paths.resolutions, + paths.revisions, + paths.archive, + paths.recovery, + ) -def require_governance_healthy(paths, state): - conflicts, issues = governance_snapshot(paths, state) +def audit_witness_manifest(paths): + """Hash a closed inventory without following redirected directory entries.""" + manifest = [] + for root in audit_witness_roots(paths): + if path_is_redirected(root) or not root.is_dir(): + raise BimriError( + f"audit witness root {root.relative_to(paths.root)} is missing " + "or unsafe." + ) + try: + children = sorted(root.iterdir(), key=lambda item: item.name) + except OSError as exc: + raise BimriError( + f"audit witness could not inventory " + f"{root.relative_to(paths.root)}: {exc}" + ) from exc + for path in children: + relative = path.relative_to(paths.root).as_posix() + if path_is_redirected(path): + raise BimriError( + f"audit witness refused redirected path {relative}." + ) + if path.is_dir(): + raise BimriError( + f"audit witness refused unexpected subdirectory {relative}." + ) + if not path.is_file(): + raise BimriError( + f"audit witness refused non-file path {relative}." + ) + try: + digest = sha256_bytes(path.read_bytes()) + except OSError as exc: + raise BimriError( + f"audit witness could not read {relative}: {exc}" + ) from exc + manifest.append({"path": relative, "sha256": digest}) + manifest.sort(key=lambda item: item["path"]) + return manifest + + +def audit_witness_run_facts(paths, state): + """Return only governance-relevant facts from high-churn run logs.""" + for log in sorted(paths.logs.glob("R*.md")): + if RUN_RE.fullmatch(log.stem) and ( + path_is_redirected(log) or not log.is_file() + ): + raise BimriError(f"run log is missing or unsafe: {log.name}") + records, issues, closed_by_run = logged_proposal_records( + paths, state, include_run_status=True + ) if issues: - raise BimriError( - "authority recovery is required; shared-memory writes are paused: " - + " | ".join(issues[:3]) - ) - return conflicts + raise BimriError(" | ".join(issues[:3])) + marker_by_proposal = { + proposal_id: (run_id, closed) + for proposal_id, run_id, closed in records + } + proposal_files = { + path.stem + for path in paths.proposals.glob("R*-Q*.json") + if PROPOSAL_RE.fullmatch(path.stem) + } + proposal_ids = sorted(set(marker_by_proposal) | proposal_files) + facts = [] + for proposal_id in proposal_ids: + run_id = proposal_id.split("-Q", 1)[0] + marker = marker_by_proposal.get(proposal_id) + facts.append({ + "proposal_id": proposal_id, + "run_id": run_id, + "file_present": proposal_id in proposal_files, + "marker_present": bool(marker and marker[0] == run_id), + "closed": bool(closed_by_run.get(run_id, False)), + }) + return facts -def open_conflicts(paths, state): - return require_governance_healthy(paths, state) +def audit_witness_proposal_runs(run_facts, state): + """Bind bases only for proposal-bearing runs that remain active.""" + active = state.get("active_runs", {}) + return sorted({ + record["run_id"] for record in run_facts + if record["run_id"] in active + }) -def allocate_conflict_id(paths, state): - existing = [ - int(path.stem[1:]) - for path in paths.conflicts.glob("C*.json") - if CONFLICT_RE.fullmatch(path.stem) - ] - number = max([state["conflict_count"]] + existing + [0]) + 1 - if number > 999999: - raise BimriError("BIMRI has exhausted its six-digit conflict ID space.") - while conflict_path(paths, f"C{number:06d}").exists(): - number += 1 - if number > 999999: - raise BimriError("BIMRI has exhausted its six-digit conflict ID space.") - state["conflict_count"] = number - return f"C{number:06d}" +def audit_witness_state_payload(state, proposal_runs): + """Select current authority state while excluding lifecycle-only churn.""" + active_bases = { + run_id: state.get("active_runs", {}).get(run_id, {}).get("base_revision") + for run_id in proposal_runs + } + return { + "bimri_version": state["bimri_version"], + "project_id": state["project_id"], + "head_revision": state["head_revision"], + "head_hash": state["head_hash"], + "conflict_count": state["conflict_count"], + "pattern_count": state["pattern_count"], + "cold_current": state["cold_current"], + "prune_policy": state["prune_policy"], + "flag_threshold": state["flag_threshold"], + "limits": limits_profile(state), + "active_proposal_bases": active_bases, + } -def proposal_file_hash(paths, proposal_id): - path = proposal_path(paths, proposal_id) - if not path.exists() or path.is_symlink(): - raise BimriError(f"proposal file is missing or unsafe: {proposal_id}") - data = read_json_strict(path, path.name) - if is_quarantine_stub(data): - validate_quarantine_stub( - paths, path, data, "proposal", proposal_id +def audit_witness_state_hash(state, proposal_runs): + return audit_witness_digest( + audit_witness_state_payload(state, proposal_runs) + ) + + +def state_audit_epoch(state): + """Return the optional v5.1.1 cache epoch (absent in v5.1.0 stores).""" + value = state.get("_audit_epoch", 0) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise BimriError("state _audit_epoch must be a non-negative integer.") + return value + + +def audit_witness_write_state_payload(state): + """Bind every state input that can influence an authority transition.""" + payload = copy.deepcopy(state) + # These timestamps are display/liveness bookkeeping. They never select, + # authorize, age, or render shared memory and may change on a journal-only + # lifecycle write without forcing a checkpoint rewrite. + payload.pop("last_started_at", None) + payload.pop("last_closed_at", None) + payload.pop("_audit_epoch", None) + active_runs = payload.get("active_runs", {}) + if isinstance(active_runs, dict): + for metadata in active_runs.values(): + if isinstance(metadata, dict): + metadata.pop("last_activity_at", None) + return payload + + +def audit_witness_write_state_hash(state): + return audit_witness_digest(audit_witness_write_state_payload(state)) + + +def audit_witness_digest(value): + encoded = json.dumps( + value, separators=(",", ":"), sort_keys=True + ).encode("utf-8") + return sha256_bytes(encoded) + + +def audit_record_seal(record, seal_field): + """Detect standalone cache edits; coordinated forgery is outside scope.""" + payload = { + key: value for key, value in record.items() if key != seal_field + } + return audit_witness_digest(payload) + + +def discard_audit_witness(paths): + """Remove the derived witness without following an unsafe replacement.""" + paths.validated_audit_witness = None + path = paths.audit_witness + try: + if path.is_symlink() or (path.exists() and not path.is_dir()): + path.unlink() + fsync_directory(path.parent) + except OSError: + # A derived cache must never turn a valid authority operation into a + # failure. An unreadable witness simply leaves the slow audit active. + pass + + +def normalize_audit_manifest(manifest): + if not isinstance(manifest, list): + raise BimriError("audit manifest must be a list.") + normalized = [] + seen_paths = set() + for item in manifest: + if not isinstance(item, dict) or set(item) != {"path", "sha256"}: + raise BimriError("audit manifest row fields are invalid.") + relative = clean_scalar(item.get("path"), "audit manifest path", 1000) + digest = item.get("sha256") + if ( + relative in seen_paths + or not isinstance(digest, str) + or not HASH_RE.fullmatch(digest) + ): + raise BimriError("audit manifest row is invalid or duplicated.") + seen_paths.add(relative) + normalized.append({"path": relative, "sha256": digest}) + if normalized != sorted(normalized, key=lambda item: item["path"]): + raise BimriError("audit manifest paths are not sorted.") + return normalized + + +def validate_sealed_audit_witness_record(witness): + """Validate a compact checkpoint record from live or blocked evidence.""" + try: + expected_fields = { + "witness_schema", "engine_version", "memory_format_version", + "policy_version", "created_at", "head_revision", "head_hash", + "state_hash", "write_state_hash", "audit_epoch", + "run_authority_hash", "manifest_hash", "manifest_count", + "proposal_runs", "witness_hash", + } + if set(witness) != expected_fields: + return None + if witness.get("witness_hash") != audit_record_seal( + witness, "witness_hash" + ): + return None + if ( + witness.get("witness_schema") != AUDIT_WITNESS_SCHEMA + or witness.get("engine_version") != ENGINE_VERSION + or witness.get("memory_format_version") != MEMORY_FORMAT_VERSION + or witness.get("policy_version") != AUTHORITY_POLICY_VERSION + ): + return None + parse_timestamp(witness.get("created_at"), "audit witness timestamp") + validate_revision_number( + witness.get("head_revision"), "audit witness head revision" ) - raise BimriError(f"proposal {proposal_id} is quarantined.") - return sha256_bytes(path.read_bytes()) + for field in ( + "head_hash", "state_hash", "write_state_hash", + "run_authority_hash", "manifest_hash" + ): + value = witness.get(field) + if not isinstance(value, str) or not HASH_RE.fullmatch(value): + return None + count = witness.get("manifest_count") + if isinstance(count, bool) or not isinstance(count, int) or count < 0: + return None + epoch = witness.get("audit_epoch") + if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch < 0: + return None + proposal_runs = witness.get("proposal_runs") + if ( + not isinstance(proposal_runs, list) + or proposal_runs != sorted(set(proposal_runs)) + ): + return None + for run_id in proposal_runs: + validate_fixed_id(run_id, RUN_RE, "audit witness proposal run ID") + return witness + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + return None -def validate_proposal_id_list(value, name): - if not isinstance(value, list): - raise BimriError(f"{name} must be a list.") - proposal_ids = [] - for item in value: - proposal_id = validate_fixed_id( - item, PROPOSAL_RE, f"{name} entry" +def reconcile_engine_checkpoint_for_exact_read(paths): + """Finish only a sealed engine transition under the existing read lock.""" + if not (paths.audit_transition.exists() or paths.audit_transition.is_symlink()): + return False + marker = load_audit_transition(paths) + if marker is None or marker["kind"] not in {"lifecycle", "authority"}: + return False + if path_is_redirected(paths.state) or not paths.state.is_file(): + raise BimriError("lifecycle checkpoint state is missing or unsafe.") + raw = read_json_strict(paths.state, "state.json") + if raw.get("bimri_version") != MEMORY_FORMAT_VERSION: + raise BimriError("lifecycle checkpoint requires current-format state.") + require_complete_v5_state(raw) + state = fresh_state() + state.update(raw) + validate_state(state) + reconcile_audit_transition(paths, state) + return not ( + paths.audit_transition.exists() or paths.audit_transition.is_symlink() + ) + + +def load_sealed_audit_witness(paths): + """Load the compact checkpoint without consulting authority/history files.""" + path = paths.audit_witness + if path_is_redirected(path) or not path.is_file(): + return None + try: + witness = read_json_strict(path, "audit witness") + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + return None + return validate_sealed_audit_witness_record(witness) + + +def load_valid_audit_witness(paths, state, state_hash=None): + """Validate a fixed-cost read checkpoint against state and accepted head.""" + paths.validated_audit_witness = None + # A durable transition is the write-ahead marker for a checkpoint change. + # Even an otherwise valid old/new witness is non-readable until that + # marker has been reconciled while holding the engine lock. + if paths.audit_transition.exists() or paths.audit_transition.is_symlink(): + return None + witness = load_sealed_audit_witness(paths) + if witness is None: + return None + del state_hash # Compatibility with the transitional private call surface. + try: + current_state_hash = audit_witness_state_hash( + state, witness["proposal_runs"] ) - if proposal_id != item: - raise BimriError(f"{name} entries must already be normalized.") - proposal_ids.append(proposal_id) - if len(proposal_ids) != len(set(proposal_ids)): - raise BimriError(f"{name} contains duplicate proposal IDs.") - return proposal_ids + except (BimriError, KeyError, TypeError, ValueError): + return None + if ( + witness["head_revision"] != state["head_revision"] + or witness["head_hash"] != state["head_hash"] + or witness["state_hash"] != current_state_hash + or witness["audit_epoch"] != state_audit_epoch(state) + ): + return None + paths.validated_audit_witness = witness + return witness -def validate_revision_number(value, name): +def audit_manifest_generation_path(paths, manifest_hash): + if not isinstance(manifest_hash, str) or not HASH_RE.fullmatch(manifest_hash): + raise BimriError("audit manifest generation hash is invalid.") + return paths.audit_manifests / f"{manifest_hash}.json" + + +def load_audit_manifest_evidence(paths, witness, manifest_override=None): + if manifest_override is not None: + manifest = normalize_audit_manifest(manifest_override) + if ( + witness.get("manifest_hash") != audit_witness_digest(manifest) + or witness.get("manifest_count") != len(manifest) + ): + raise BimriError( + "preserved audit manifest does not match its checkpoint." + ) + return manifest + generation = audit_manifest_generation_path( + paths, witness.get("manifest_hash") + ) + # Transitional v5.1.1 installs may only have the fixed current alias. + # Once read successfully, the next publication creates the generation. + path = generation if generation.is_file() else paths.audit_manifest + if path_is_redirected(path) or not path.is_file(): + raise BimriError("audit manifest evidence is missing or unsafe.") + record = read_json_strict(path, "audit manifest") + expected = { + "manifest_schema", "created_at", "manifest", "manifest_hash", + "manifest_count", "manifest_file_hash", + } + if set(record) != expected: + raise BimriError("audit manifest evidence fields are invalid.") + if record.get("manifest_schema") != AUDIT_MANIFEST_SCHEMA: + raise BimriError("audit manifest evidence schema is invalid.") + parse_timestamp(record.get("created_at"), "audit manifest timestamp") + if record.get("manifest_file_hash") != audit_record_seal( + record, "manifest_file_hash" + ): + raise BimriError("audit manifest evidence seal is invalid.") + manifest = normalize_audit_manifest(record.get("manifest")) + manifest_hash = audit_witness_digest(manifest) if ( - isinstance(value, bool) - or not isinstance(value, int) - or value < 0 - or value > 999999 + record.get("manifest_hash") != manifest_hash + or record.get("manifest_count") != len(manifest) + or witness.get("manifest_hash") != manifest_hash + or witness.get("manifest_count") != len(manifest) ): - raise BimriError(f"{name} must be a valid revision number.") - return value + raise BimriError("audit manifest evidence does not match its checkpoint.") + return manifest -def validate_conflict_record( - paths, conflict, expected_conflict_id=None, verify_candidates=True -): - if not isinstance(conflict, dict): - raise BimriError("conflict must be a JSON object.") - artifact_version = conflict.get("bimri_version") - if artifact_version not in COMPATIBLE_ARTIFACT_VERSIONS: - raise BimriError("conflict BIMRI version is invalid.") - conflict_id = validate_fixed_id( - conflict.get("conflict_id"), CONFLICT_RE, "conflict ID" +def write_audit_manifest_generation(paths, manifest_record): + generation = audit_manifest_generation_path( + paths, manifest_record["manifest_hash"] ) - if expected_conflict_id and conflict_id != expected_conflict_id: - raise BimriError("conflict filename does not match its ID.") - conflict_type = conflict.get("type") - if conflict_type not in CONFLICT_TYPES: - raise BimriError("conflict type is invalid.") - key = clean_key(conflict.get("key")) - if key != conflict.get("key"): - raise BimriError("conflict key must already be normalized.") - parse_timestamp(conflict.get("created_at"), "conflict timestamp") - proposal_ids = validate_proposal_id_list( - conflict.get("proposal_ids"), "conflict proposal_ids" + if path_is_redirected(paths.audit_manifests): + raise BimriError("audit manifest generation directory is redirected.") + ensure_directory_durable(paths.audit_manifests) + encoded = canonical_json_bytes(manifest_record) + if generation.exists() or generation.is_symlink(): + if path_is_redirected(generation) or not generation.is_file(): + raise BimriError("audit manifest generation conflicts with evidence.") + existing = read_json_strict(generation, "audit manifest generation") + existing_manifest = normalize_audit_manifest(existing.get("manifest")) + if ( + set(existing) != { + "manifest_schema", "created_at", "manifest", "manifest_hash", + "manifest_count", "manifest_file_hash", + } + or existing.get("manifest_schema") != AUDIT_MANIFEST_SCHEMA + or existing.get("manifest_file_hash") + != audit_record_seal(existing, "manifest_file_hash") + or existing.get("manifest_hash") != manifest_record["manifest_hash"] + or existing.get("manifest_count") != manifest_record["manifest_count"] + or existing_manifest != manifest_record["manifest"] + ): + raise BimriError("audit manifest generation conflicts with evidence.") + else: + exclusive_write_bytes(generation, encoded) + return generation + + +def ensure_audit_manifest_generation(paths, witness, manifest=None): + """Make the manifest named by a prior checkpoint crash-addressable.""" + generation = audit_manifest_generation_path( + paths, witness["manifest_hash"] ) - if conflict_type == "manual-edit": - if proposal_ids: - raise BimriError("manual-edit conflicts cannot contain proposals.") - elif not proposal_ids: - raise BimriError("proposal conflicts require at least one candidate.") - hashes = conflict.get("proposal_hashes") - if not isinstance(hashes, dict) or set(hashes) != set(proposal_ids): - raise BimriError( - "conflict proposal hashes must exactly match its candidate IDs." - ) - for proposal_id in proposal_ids: - expected = hashes.get(proposal_id) - if not isinstance(expected, str) or not HASH_RE.fullmatch(expected): - raise BimriError( - f"conflict is missing a valid hash for {proposal_id}." - ) - if verify_candidates and proposal_file_hash(paths, proposal_id) != expected: - raise BimriError( - f"proposal {proposal_id} changed after the conflict was raised. " - "BIMRI stopped so the owner can review it again." + if manifest is None and generation.is_file() and not path_is_redirected( + generation + ): + record = read_json_strict(generation, "audit manifest generation") + candidate = normalize_audit_manifest(record.get("manifest")) + if ( + set(record) == { + "manifest_schema", "created_at", "manifest", "manifest_hash", + "manifest_count", "manifest_file_hash", + } + and record.get("manifest_schema") == AUDIT_MANIFEST_SCHEMA + and record.get("manifest_file_hash") + == audit_record_seal(record, "manifest_file_hash") + and record.get("manifest_hash") == witness["manifest_hash"] + and record.get("manifest_count") == witness["manifest_count"] + and audit_witness_digest(candidate) == witness["manifest_hash"] + ): + return candidate + raise BimriError("audit manifest generation conflicts with its checkpoint.") + manifest = load_audit_manifest_evidence( + paths, witness, manifest_override=manifest + ) + record = { + "manifest_schema": AUDIT_MANIFEST_SCHEMA, + "created_at": witness["created_at"], + "manifest": manifest, + "manifest_hash": witness["manifest_hash"], + "manifest_count": witness["manifest_count"], + } + record["manifest_file_hash"] = audit_record_seal( + record, "manifest_file_hash" + ) + write_audit_manifest_generation(paths, record) + return manifest + + +def clear_audit_transition(paths): + path = paths.audit_transition + if path_is_redirected(path): + raise BimriError("audit transition marker is redirected.") + if path.exists(): + if not path.is_file(): + raise BimriError("audit transition marker is not a regular file.") + path.unlink() + fsync_directory(path.parent) + + +def load_audit_transition(paths): + path = paths.audit_transition + if not (path.exists() or path.is_symlink()): + return None + if path_is_redirected(path) or not path.is_file(): + raise BimriError("audit transition marker is missing or unsafe.") + marker = read_json_strict(path, "audit transition marker") + expected = { + "transition_schema", "created_at", "kind", "operation", "run_id", + "scope", "prior_witness", "prior_witness_hash", + "prior_manifest_hash", "epoch_before", "epoch_after", + "pre_write_state", "pre_write_state_hash", "post_write_state_hash", "log_path", + "pre_log_hash", "post_log_hash", "log_append", "transition_hash", + } + if set(marker) != expected: + raise BimriError("audit transition marker fields are invalid.") + if marker.get("transition_schema") != AUDIT_TRANSITION_SCHEMA: + raise BimriError("audit transition marker schema is invalid.") + parse_timestamp(marker.get("created_at"), "audit transition timestamp") + if marker.get("kind") not in {"authority", "lifecycle"}: + raise BimriError("audit transition kind is invalid.") + clean_scalar(marker.get("operation"), "audit transition operation", 80) + run_id = marker.get("run_id") + if run_id is not None: + validate_fixed_id(run_id, RUN_RE, "audit transition run ID") + if not isinstance(marker.get("scope"), dict): + raise BimriError("audit transition scope is invalid.") + if marker.get("kind") == "authority": + validate_frozen_authority_transition_scope(marker["scope"]) + prior = validate_sealed_audit_witness_record(marker.get("prior_witness")) + if ( + prior is None + or marker.get("prior_witness_hash") != prior.get("witness_hash") + or marker.get("prior_manifest_hash") != prior.get("manifest_hash") + ): + raise BimriError("audit transition prior checkpoint is invalid.") + for field in ("epoch_before", "epoch_after"): + value = marker.get(field) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise BimriError(f"audit transition {field} is invalid.") + for field in ("pre_write_state_hash", "post_write_state_hash"): + value = marker.get(field) + if value is not None and ( + not isinstance(value, str) or not HASH_RE.fullmatch(value) + ): + raise BimriError(f"audit transition {field} is invalid.") + pre_write_state = marker.get("pre_write_state") + if ( + not isinstance(pre_write_state, dict) + or audit_witness_digest(pre_write_state) + != marker.get("pre_write_state_hash") + ): + raise BimriError("audit transition pre-write state is invalid.") + for field in ("pre_log_hash", "post_log_hash"): + value = marker.get(field) + if value is not None and value != "absent" and ( + not isinstance(value, str) or not HASH_RE.fullmatch(value) + ): + raise BimriError(f"audit transition {field} is invalid.") + log_path = marker.get("log_path") + log_append = marker.get("log_append") + if marker["kind"] == "lifecycle": + if ( + not isinstance(log_path, str) + or not isinstance(log_append, str) + or marker.get("post_write_state_hash") is None + or marker.get("pre_log_hash") is None + or marker.get("post_log_hash") is None + or marker["epoch_before"] != marker["epoch_after"] + ): + raise BimriError("lifecycle audit transition fields are invalid.") + pure = PurePosixPath(log_path) + if ( + pure.is_absolute() + or ".." in pure.parts + or tuple(pure.parts[:2]) != (".bimri", "log") + or len(pure.parts) != 3 + or not RUN_RE.fullmatch(PurePosixPath(log_path).stem) + ): + raise BimriError("lifecycle audit transition log path is invalid.") + if len(log_append) > 25000: + raise BimriError("lifecycle audit transition log append is too large.") + elif any( + marker.get(field) is not None + for field in ( + "post_write_state_hash", "log_path", "pre_log_hash", + "post_log_hash", "log_append", + ) + ) or marker["epoch_after"] != marker["epoch_before"] + 1: + raise BimriError("authority audit transition fields are invalid.") + if marker.get("transition_hash") != audit_record_seal( + marker, "transition_hash" + ): + raise BimriError("audit transition marker seal is invalid.") + return marker + + +def write_audit_transition(paths, marker): + if paths.audit_transition.exists() or paths.audit_transition.is_symlink(): + raise BimriError( + "an earlier audit transition is incomplete; run doctor before retrying." + ) + marker = dict(marker) + marker["transition_hash"] = audit_record_seal( + marker, "transition_hash" + ) + if path_is_redirected(paths.audit_transition): + raise BimriError("audit transition destination is redirected.") + atomic_write_json(paths.audit_transition, marker) + return load_audit_transition(paths) + + +def referenced_audit_manifest_hashes(paths): + hashes = set() + witness = load_sealed_audit_witness(paths) + if witness is not None: + hashes.add(witness["manifest_hash"]) + try: + blocked = load_audit_blocked_record(paths) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + blocked = None + if blocked and blocked.get("prior_witness"): + hashes.add(blocked["prior_witness"]["manifest_hash"]) + try: + transition = load_audit_transition(paths) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + transition = None + if transition: + hashes.add(transition["prior_manifest_hash"]) + return hashes + + +def prune_audit_manifest_generations(paths): + """Bound derived evidence after every completed publication.""" + keep = referenced_audit_manifest_hashes(paths) + try: + children = list(paths.audit_manifests.iterdir()) + except (FileNotFoundError, OSError): + return + for path in children: + if ( + path_is_redirected(path) + or not path.is_file() + or not HASH_RE.fullmatch(path.stem) + or path.suffix != ".json" + or path.stem in keep + ): + continue + try: + path.unlink() + except OSError: + continue + fsync_directory(paths.audit_manifests) + + +def audit_checkpoint_drift_delta( + prior_manifest, manifest, allowed_manifest_paths=None +): + """Complete per-path divergence between two witnessed inventories.""" + allowed = set(allowed_manifest_paths or ()) + prior_by_path = {item["path"]: item["sha256"] for item in prior_manifest} + current_by_path = {item["path"]: item["sha256"] for item in manifest} + changed = [ + { + "path": path, + "prior_sha256": prior_by_path[path], + "sha256": current_by_path[path], + } + for path in sorted(prior_by_path.keys() & current_by_path.keys()) + if prior_by_path[path] != current_by_path[path] + and path not in allowed + ] + added = [ + {"path": path, "sha256": current_by_path[path]} + for path in sorted(current_by_path.keys() - prior_by_path.keys()) + if path not in allowed + ] + deleted = [ + {"path": path, "prior_sha256": prior_by_path[path]} + for path in sorted(prior_by_path.keys() - current_by_path.keys()) + if path not in allowed + ] + return {"changed": changed, "added": added, "deleted": deleted} + + +def next_audit_drift_sequence(paths): + """One past the highest sequence any D-named file has ever claimed.""" + last = 0 + try: + children = list(paths.audit_drift.iterdir()) + except (FileNotFoundError, OSError): + return 1 + for path in children: + match = re.match(r"D(\d+)-", path.name) + if match: + last = max(last, int(match.group(1))) + return last + 1 + + +def write_audit_drift_attachment(paths, sequence, payload): + """Persist one hash-and-size-pinned evidence attachment. + + Returns its descriptor {path, sha256, bytes} or None on failure. + """ + try: + digest = sha256_bytes(payload) + ensure_directory_durable(paths.audit_drift) + target = paths.audit_drift / f"A{sequence:06d}-{digest[:12]}.bin" + if not target.exists() and not target.is_symlink(): + exclusive_write_bytes(target, payload) + if target.is_symlink() or not target.is_file(): + return None + raw = target.read_bytes() + if sha256_bytes(raw) != digest: + return None + return { + "path": target.relative_to(paths.root).as_posix(), + "sha256": digest, + "bytes": len(payload), + } + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + return None + + +def write_audit_drift_receipt( + paths, reasons, marker=None, prior_witness=None, state=None, delta=None, + attachments=(), +): + """Preserve unexplained-drift evidence as a rebaseline precondition. + + Drift that a passing full audit could not condemn is recorded as sealed + derived evidence: the diverging paths with their prior and current + hashes, inline up to AUDIT_DRIFT_DELTA_CAP entries per section. When + the inline delta truncates, the complete delta is persisted first as a + hash-and-size-pinned attachment the receipt references, retained while + the receipt is retained and validated with it — as is every other + attachment (preserved corrupt-marker bytes included). Retention is a + bounded rolling window of the newest AUDIT_DRIFT_KEEP receipts in + numeric sequence order. The written receipt is re-read and fully + validated, attachments and post-prune existence included, before this + function reports success. Returning None means no durable, valid + receipt exists — callers MUST treat that as a refusal to proceed, + never as success. + """ + reasons = [str(reason) for reason in reasons if reason][:40] + attachments = [ + dict(item) for item in attachments if isinstance(item, dict) + ] + if not reasons and not delta: + return None + try: + if path_is_redirected(paths.audit_drift): + return None + sequence = next_audit_drift_sequence(paths) + truncated = {} + if isinstance(delta, dict): + complete = { + section: list(delta.get(section) or ()) + for section in ("changed", "added", "deleted") + } + bounded = {} + for section, items in complete.items(): + if len(items) > AUDIT_DRIFT_DELTA_CAP: + truncated[section] = len(items) - AUDIT_DRIFT_DELTA_CAP + bounded[section] = items[:AUDIT_DRIFT_DELTA_CAP] + else: + bounded[section] = items + delta = bounded + if not any(delta.values()) and not reasons: + return None + if truncated: + # The complete delta is durable evidence in its own right, + # pinned before the receipt that references it. + descriptor = write_audit_drift_attachment( + paths, + sequence, + canonical_json_bytes({"complete_delta": complete}), + ) + if descriptor is None: + return None + descriptor["role"] = "complete-delta" + attachments.append(descriptor) + event_binding = { + "prior_witness_hash": ( + prior_witness.get("witness_hash") + if isinstance(prior_witness, dict) else None + ), + "prior_manifest_hash": ( + prior_witness.get("manifest_hash") + if isinstance(prior_witness, dict) else None + ), + "head_revision": state.get("head_revision") if state else None, + "head_hash": state.get("head_hash") if state else None, + "audit_epoch": state_audit_epoch(state) if state else None, + "transition_marker": marker, + } + newest = list_audit_drift_receipts(paths)[-1:] + if newest: + try: + existing = read_json_strict(newest[0], newest[0].name) + except (BimriError, OSError, UnicodeError): + existing = None + if ( + existing is not None + and validate_audit_drift_receipt( + paths, newest[0], existing + ) is None + and existing.get("reasons") == reasons + and existing.get("delta") == delta + and [ + item.get("sha256") + for item in existing.get("attachments") or () + ] == [item.get("sha256") for item in attachments] + and all( + existing.get(field) == value + for field, value in event_binding.items() + ) + ): + # A repeated attempt against the same divergence OF THE + # SAME EVENT adds no evidence; a receipt from a different + # prior baseline, head, epoch or transition never counts. + return newest[0] + record = { + "drift_schema": AUDIT_DRIFT_SCHEMA, + "sequence": sequence, + "created_at": now_iso(), + "reasons": reasons, + "delta": delta, + "truncated": truncated, + "attachments": attachments, + } + record.update(event_binding) + record["receipt_hash"] = audit_record_seal(record, "receipt_hash") + ensure_directory_durable(paths.audit_drift) + digest = sha256_bytes(canonical_json_bytes(record))[:12] + path = paths.audit_drift / ( + f"D{record['sequence']:06d}-{digest}.json" + ) + if not path.exists() and not path.is_symlink(): + atomic_write_json(path, record) + try: + on_disk = read_json_strict(path, path.name) + except (BimriError, OSError, UnicodeError): + return None + if validate_audit_drift_receipt(paths, path, on_disk) is not None: + return None + prune_audit_drift_receipts(paths) + if not path.is_file(): + return None + return path + except (BimriError, OSError, UnicodeError, ValueError, TypeError, KeyError): + return None + + +def parse_audit_drift_sequence(name): + """Sequence for a conforming receipt filename, else None.""" + match = re.fullmatch(r"D(\d{6,})-[0-9a-f]{12}\.json", name) + return int(match.group(1)) if match else None + + +def list_audit_drift_receipts(paths): + """Conforming receipts in numeric sequence order (never lexical).""" + entries = [] + try: + children = list(paths.audit_drift.iterdir()) + except (FileNotFoundError, OSError): + return [] + for path in children: + if path.is_symlink() or not path.is_file(): + continue + sequence = parse_audit_drift_sequence(path.name) + if sequence is not None: + entries.append((sequence, path.name, path)) + return [entry[2] for entry in sorted(entries)] + + + + +def prune_audit_drift_receipts(paths): + """Bound drift evidence to the newest receipts plus cited attachments. + + An attachment cited by a retained receipt outlives the + unreferenced-attachment bound, so no retained receipt ever cites + pruned bytes. + """ + for path in list_audit_drift_receipts(paths)[:-AUDIT_DRIFT_KEEP]: + with contextlib.suppress(OSError): + path.unlink() + referenced = retained_drift_receipt_attachment_paths(paths) + try: + attachments = sorted( + ( + path for path in paths.audit_drift.iterdir() + if not path.is_symlink() + and path.is_file() + and ( + path.name.startswith("corrupt-transition-") + or re.match(r"A\d{6,}-", path.name) + ) + and path.relative_to(paths.root).as_posix() not in referenced + ), + key=lambda item: item.stat().st_mtime, + ) + except (FileNotFoundError, OSError): + return + for path in attachments[:-AUDIT_DRIFT_BLOB_KEEP]: + with contextlib.suppress(OSError): + path.unlink() + + +def validate_audit_drift_receipt(paths, path, record): + """Return a problem string for an untrustworthy receipt, else None. + + Full validation covers schema, filename-to-sequence binding, + filename-to-content digest, the recomputed seal, delta shape, and every + referenced attachment's existence, size, and hash. A receipt is only as + trustworthy as the evidence it cites. + """ + if not isinstance(record, dict): + return "receipt is not an object" + if record.get("drift_schema") != AUDIT_DRIFT_SCHEMA: + return "unsupported drift schema" + match = re.fullmatch(r"D(\d{6,})-([0-9a-f]{12})\.json", path.name) + if not match: + return "receipt filename is not bound to a sequence" + if record.get("sequence") != int(match.group(1)): + return "receipt sequence does not match its filename" + if record.get("receipt_hash") != audit_record_seal(record, "receipt_hash"): + return "receipt seal is invalid" + if sha256_bytes(canonical_json_bytes(record))[:12] != match.group(2): + return "receipt content does not match its filename digest" + delta = record.get("delta") + if delta is not None: + if not isinstance(delta, dict): + return "receipt delta is invalid" + for section, fields in ( + ("changed", ("path", "prior_sha256", "sha256")), + ("added", ("path", "sha256")), + ("deleted", ("path", "prior_sha256")), + ): + items = delta.get(section) + if not isinstance(items, list): + return "receipt delta is invalid" + for item in items: + if not isinstance(item, dict) or set(item) != set(fields): + return "receipt delta is invalid" + for field in fields: + value = item.get(field) + if field == "path": + if not isinstance(value, str) or not value: + return "receipt delta is invalid" + elif not ( + isinstance(value, str) and HASH_RE.fullmatch(value) + ): + return "receipt delta is invalid" + if record.get("truncated") and not any( + item.get("role") == "complete-delta" + for item in record.get("attachments") or () + if isinstance(item, dict) + ): + return "truncated receipt lacks its complete-delta attachment" + for item in record.get("attachments") or (): + if not isinstance(item, dict): + return "receipt attachment descriptor is invalid" + relative = item.get("path") + digest = item.get("sha256") + size = item.get("bytes") + if ( + not isinstance(relative, str) + or not relative.startswith(".bimri/audit-drift/") + or "/../" in relative + or not isinstance(digest, str) + or not HASH_RE.fullmatch(digest) + or isinstance(size, bool) + or not isinstance(size, int) + or size < 0 + ): + return "receipt attachment descriptor is invalid" + target = paths.root.joinpath(*PurePosixPath(relative).parts) + if target.is_symlink() or not target.is_file(): + return f"receipt attachment is missing: {relative}" + try: + raw = target.read_bytes() + except OSError: + return f"receipt attachment is unreadable: {relative}" + if len(raw) != size or sha256_bytes(raw) != digest: + return f"receipt attachment does not match its pin: {relative}" + return None + + +def retained_drift_receipt_attachment_paths(paths): + """Attachment paths cited by retained, seal-intact receipts. + + Retention uses the seal alone so a receipt whose attachment vanished + still protects its remaining attachments from pruning while doctor + reports it damaged. + """ + referenced = set() + for path in list_audit_drift_receipts(paths): + try: + record = read_json_strict(path, path.name) + except (BimriError, OSError, UnicodeError): + continue + if not isinstance(record, dict): + continue + if record.get("receipt_hash") != audit_record_seal( + record, "receipt_hash" + ): + continue + for item in record.get("attachments") or (): + if isinstance(item, dict) and isinstance(item.get("path"), str): + referenced.add(item["path"]) + return referenced + + +def audit_drift_summary(paths, limit=3): + """Validated receipt count, newest valid receipts, damaged names.""" + children = list_audit_drift_receipts(paths) + valid = [] + damaged = [] + for path in children: + try: + record = read_json_strict(path, path.name) + except (BimriError, OSError, UnicodeError): + damaged.append(path.name) + continue + problem = validate_audit_drift_receipt(paths, path, record) + if problem: + damaged.append(path.name) + continue + record["receipt_name"] = path.name + valid.append(record) + newest = list(reversed(valid[-max(0, int(limit)):])) + return len(children), newest, sorted(damaged) + + +def write_audit_witness(paths, state, conflicts=None, manifest=None, run_facts=None): + """Publish detailed audit evidence, then its compact read checkpoint.""" + del conflicts # Read projections never come from the checkpoint. + try: + if authority_has_unsettled_records(paths): + paths.validated_audit_witness = None + if not ( + paths.audit_transition.exists() + or paths.audit_transition.is_symlink() + ): + discard_audit_witness(paths) + return False + if run_facts is None: + run_facts = audit_witness_run_facts(paths, state) + if manifest is None: + manifest = audit_witness_manifest(paths) + manifest = normalize_audit_manifest(manifest) + manifest_hash = audit_witness_digest(manifest) + transition = load_audit_transition(paths) + frozen_completion = ( + transition.get("scope", {}).get("completion") + if transition is not None and transition["kind"] == "authority" + else None + ) + created_at = ( + frozen_completion["witness"]["created_at"] + if frozen_completion is not None + else now_iso() + ) + manifest_record = { + "manifest_schema": AUDIT_MANIFEST_SCHEMA, + "created_at": created_at, + "manifest": manifest, + "manifest_hash": manifest_hash, + "manifest_count": len(manifest), + } + manifest_record["manifest_file_hash"] = audit_record_seal( + manifest_record, "manifest_file_hash" + ) + proposal_runs = audit_witness_proposal_runs(run_facts, state) + witness = { + "witness_schema": AUDIT_WITNESS_SCHEMA, + "engine_version": ENGINE_VERSION, + "memory_format_version": MEMORY_FORMAT_VERSION, + "policy_version": AUTHORITY_POLICY_VERSION, + "created_at": created_at, + "head_revision": state["head_revision"], + "head_hash": state["head_hash"], + "state_hash": audit_witness_state_hash(state, proposal_runs), + "write_state_hash": audit_witness_write_state_hash(state), + "audit_epoch": state_audit_epoch(state), + "run_authority_hash": audit_witness_digest(run_facts), + "manifest_hash": manifest_hash, + "manifest_count": len(manifest), + "proposal_runs": proposal_runs, + } + witness["witness_hash"] = audit_record_seal(witness, "witness_hash") + if ( + frozen_completion is not None + and witness != frozen_completion["witness"] + ): + raise BimriError( + "live authority no longer matches its frozen completed checkpoint." + ) + # Freeze the exact M1/run/state/head closure before either M1 or W1 is + # published. Recovery compares against this record and never expands + # scope from the post-crash tree. + seal_authority_transition_completion(paths, state, witness) + if path_is_redirected(paths.audit_manifest): + raise BimriError("audit manifest destination is redirected.") + if paths.audit_manifest.exists() and paths.audit_manifest.is_dir(): + raise BimriError("audit manifest destination is a directory.") + # Publish immutable evidence first. The fixed alias remains for + # operators and transitional installs; checkpoints resolve the + # content-addressed generation so replacing the alias cannot strand + # an older witness after a crash. + write_audit_manifest_generation(paths, manifest_record) + atomic_write_json(paths.audit_manifest, manifest_record) + if path_is_redirected(paths.audit_witness): + raise BimriError("audit witness destination is redirected.") + if paths.audit_witness.exists() and paths.audit_witness.is_dir(): + raise BimriError("audit witness destination is a directory.") + atomic_write_json(paths.audit_witness, witness) + paths.validated_audit_witness = witness + return True + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + paths.validated_audit_witness = None + if not ( + paths.audit_transition.exists() + or paths.audit_transition.is_symlink() + ): + discard_audit_witness(paths) + return False + + +def sealed_witness_matches_state(witness, state, include_write_state=True): + try: + if ( + witness["head_revision"] != state["head_revision"] + or witness["head_hash"] != state["head_hash"] + or witness["audit_epoch"] != state_audit_epoch(state) + or witness["state_hash"] != audit_witness_state_hash( + state, witness["proposal_runs"] + ) + ): + return False + if include_write_state and witness["write_state_hash"] != ( + audit_witness_write_state_hash(state) + ): + return False + return True + except (BimriError, KeyError, TypeError, ValueError): + return False + + +def frozen_authority_transition_scope(paths, operation, run_id, scope): + """Freeze immutable authority inputs before the first transition write.""" + frozen = copy.deepcopy(scope or {}) + proposal_ids = set() + for value in frozen.get("proposal_ids", []): + proposal_ids.add(validate_fixed_id( + value, PROPOSAL_RE, "audit transition proposal ID" + )) + if operation in {"sync", "close-authority-run"} and run_id: + proposal_ids.update( + path.stem + for path in paths.proposals.glob(f"{run_id}-Q*.json") + if PROPOSAL_RE.fullmatch(path.stem) + ) + conflict_id = frozen.get("conflict_id") + if operation == "resolve" and conflict_id: + conflict_id = validate_fixed_id( + conflict_id, CONFLICT_RE, "audit transition conflict ID" + ) + cpath = conflict_path(paths, conflict_id) + if path_is_redirected(cpath) or not cpath.is_file(): + raise BimriError("cannot freeze a missing or unsafe conflict.") + conflict = validate_conflict_record( + paths, + read_json_strict(cpath, cpath.name), + expected_conflict_id=conflict_id, + verify_candidates=False, + ) + proposal_ids.update(conflict["proposal_ids"]) + proposal_ids = sorted(proposal_ids) + provided_hashes = frozen.get("proposal_hashes", {}) + if not isinstance(provided_hashes, dict): + raise BimriError("audit transition proposal hashes are invalid.") + proposal_hashes = {} + for proposal_id in proposal_ids: + path = proposal_path(paths, proposal_id) + if path_is_redirected(path): + raise BimriError( + f"cannot freeze redirected proposal {proposal_id}." + ) + if path.is_file(): + digest = sha256_bytes(path.read_bytes()) + supplied = provided_hashes.get(proposal_id) + if supplied is not None and supplied != digest: + raise BimriError( + f"audit transition proposal hash changed for {proposal_id}." + ) + proposal_hashes[proposal_id] = digest + else: + digest = provided_hashes.get(proposal_id) + if not isinstance(digest, str) or not HASH_RE.fullmatch(digest): + raise BimriError( + f"new proposal {proposal_id} lacks its exact intended hash." + ) + proposal_hashes[proposal_id] = digest + if set(provided_hashes) - set(proposal_ids): + raise BimriError("audit transition contains an unscoped proposal hash.") + frozen["proposal_ids"] = proposal_ids + frozen["proposal_hashes"] = proposal_hashes + # The exact post-effect checkpoint is sealed after all engine effects are + # durable but before manifest/witness publication. A crash before that seal + # remains an incomplete transition and is never inferred from live files. + frozen["completion"] = None + return frozen + + +def validate_frozen_authority_transition_scope(scope): + if not isinstance(scope, dict): + raise BimriError("audit transition scope is invalid.") + proposal_ids = scope.get("proposal_ids") + proposal_hashes = scope.get("proposal_hashes") + if ( + not isinstance(proposal_ids, list) + or proposal_ids != sorted(set(proposal_ids)) + or not isinstance(proposal_hashes, dict) + or set(proposal_hashes) != set(proposal_ids) + ): + raise BimriError("audit transition frozen proposal scope is invalid.") + for proposal_id in proposal_ids: + validate_fixed_id( + proposal_id, PROPOSAL_RE, "audit transition proposal ID" + ) + digest = proposal_hashes[proposal_id] + if not isinstance(digest, str) or not HASH_RE.fullmatch(digest): + raise BimriError("audit transition proposal hash is invalid.") + completion = scope.get("completion") + if completion is not None: + if set(completion) != {"state_file_hash", "witness"}: + raise BimriError("audit transition completion fields are invalid.") + state_file_hash = completion.get("state_file_hash") + if ( + not isinstance(state_file_hash, str) + or not HASH_RE.fullmatch(state_file_hash) + or validate_sealed_audit_witness_record(completion.get("witness")) + is None + ): + raise BimriError("audit transition completion is invalid.") + return scope + + +def seal_authority_transition_completion(paths, state, witness): + """Bind the exact completed state/inventory/run facts before W1 publish.""" + marker = load_audit_transition(paths) + if marker is None or marker["kind"] != "authority": + return + if path_is_redirected(paths.state) or not paths.state.is_file(): + raise BimriError("cannot seal authority completion without safe state.") + validate_frozen_authority_transition_scope(marker["scope"]) + for proposal_id, expected_hash in marker["scope"]["proposal_hashes"].items(): + path = proposal_path(paths, proposal_id) + if ( + path_is_redirected(path) + or not path.is_file() + or sha256_bytes(path.read_bytes()) != expected_hash + ): + raise BimriError( + f"authority transition proposal changed outside scope: {proposal_id}." + ) + completion = { + "state_file_hash": sha256_bytes(paths.state.read_bytes()), + "witness": witness, + } + existing = marker["scope"].get("completion") + if existing is not None and existing != completion: + raise BimriError("authority transition completed bytes changed before publish.") + if existing is None: + marker["scope"]["completion"] = completion + marker["transition_hash"] = audit_record_seal(marker, "transition_hash") + atomic_write_json(paths.audit_transition, marker) + # Re-read the durable marker so malformed or truncated evidence cannot + # be followed by checkpoint publication. + load_audit_transition(paths) + + +def begin_authority_checkpoint_transition( + paths, state, operation, run_id=None, scope=None +): + """Persist W0/M0 and dirty the state epoch before authority mutation.""" + prior = paths.validated_audit_witness or load_sealed_audit_witness(paths) + if prior is None or not sealed_witness_matches_state(prior, state): + raise BimriError( + "cannot begin an authority transition without an intact checkpoint." + ) + ensure_audit_manifest_generation(paths, prior) + before = state_audit_epoch(state) + frozen_scope = frozen_authority_transition_scope( + paths, operation, run_id, scope + ) + marker = write_audit_transition(paths, { + "transition_schema": AUDIT_TRANSITION_SCHEMA, + "created_at": now_iso(), + "kind": "authority", + "operation": clean_scalar(operation, "audit transition operation", 80), + "run_id": run_id, + "scope": frozen_scope, + "prior_witness": prior, + "prior_witness_hash": prior["witness_hash"], + "prior_manifest_hash": prior["manifest_hash"], + "epoch_before": before, + "epoch_after": before + 1, + "pre_write_state_hash": audit_witness_write_state_hash(state), + "pre_write_state": audit_witness_write_state_payload(state), + "post_write_state_hash": None, + "log_path": None, + "pre_log_hash": None, + "post_log_hash": None, + "log_append": None, + }) + state["_audit_epoch"] = marker["epoch_after"] + # The epoch is the durable dirty bit. A crash after this replace makes W0 + # unreadable while W0 and its content-addressed M0 remain recoverable. + save_state(paths, state) + paths.validated_audit_witness = None + return marker + + +def complete_authority_checkpoint_transition(paths): + marker = load_audit_transition(paths) + if marker is None: + return + if marker["kind"] != "authority": + raise BimriError("cannot complete authority through a lifecycle marker.") + clear_audit_transition(paths) + prune_audit_manifest_generations(paths) + + +def begin_lifecycle_checkpoint_transition( + paths, state, post_state, operation, run_id, log_path, post_log_text +): + """Write ahead one bounded start/close state+log transition.""" + prior = paths.validated_audit_witness + if prior is None or not sealed_witness_matches_state(prior, state): + return None + validate_state(post_state) + ensure_audit_manifest_generation(paths, prior) + relative_log = log_path.relative_to(paths.root).as_posix() + if path_is_redirected(log_path): + raise BimriError("lifecycle run log is redirected.") + if log_path.exists(): + if not log_path.is_file(): + raise BimriError("lifecycle run log is not a regular file.") + pre_bytes = log_path.read_bytes() + pre_hash = sha256_bytes(pre_bytes) + try: + pre_text = pre_bytes.decode("utf-8") + except UnicodeDecodeError as exc: + raise BimriError("lifecycle run log is not UTF-8.") from exc + else: + pre_text = "" + pre_hash = "absent" + if not post_log_text.startswith(pre_text): + raise BimriError("lifecycle log update is not an exact append.") + append = post_log_text[len(pre_text):] + marker = write_audit_transition(paths, { + "transition_schema": AUDIT_TRANSITION_SCHEMA, + "created_at": now_iso(), + "kind": "lifecycle", + "operation": clean_scalar(operation, "audit transition operation", 80), + "run_id": validate_fixed_id(run_id, RUN_RE, "run ID"), + "scope": {}, + "prior_witness": prior, + "prior_witness_hash": prior["witness_hash"], + "prior_manifest_hash": prior["manifest_hash"], + "epoch_before": state_audit_epoch(state), + "epoch_after": state_audit_epoch(post_state), + "pre_write_state_hash": audit_witness_write_state_hash(state), + "pre_write_state": audit_witness_write_state_payload(state), + "post_write_state_hash": audit_witness_write_state_hash(post_state), + "log_path": relative_log, + "pre_log_hash": pre_hash, + "post_log_hash": sha256_text(post_log_text), + "log_append": append, + }) + paths.pending_checkpoint_witness = prior + return marker + + +def lifecycle_log_hash(path): + if path_is_redirected(path): + raise BimriError("lifecycle run log is redirected.") + if not path.exists(): + return "absent" + if not path.is_file(): + raise BimriError("lifecycle run log is not a regular file.") + return sha256_bytes(path.read_bytes()) + + +def publish_lifecycle_checkpoint(paths, state, marker): + prior = marker["prior_witness"] + if ( + state_audit_epoch(state) != marker["epoch_after"] + or audit_witness_write_state_hash(state) + != marker["post_write_state_hash"] + or state["head_revision"] != prior["head_revision"] + or state["head_hash"] != prior["head_hash"] + ): + raise BimriError("lifecycle state does not match its durable transition.") + witness = dict(prior) + witness.update({ + "state_hash": audit_witness_state_hash( + state, witness["proposal_runs"] + ), + "write_state_hash": audit_witness_write_state_hash(state), + "audit_epoch": state_audit_epoch(state), + }) + witness["witness_hash"] = audit_record_seal(witness, "witness_hash") + ensure_audit_manifest_generation(paths, prior) + atomic_write_json(paths.audit_witness, witness) + paths.validated_audit_witness = witness + clear_audit_transition(paths) + paths.pending_checkpoint_witness = None + prune_audit_manifest_generations(paths) + return True + + +def finish_lifecycle_log(paths, marker): + log_path = paths.root.joinpath(*PurePosixPath(marker["log_path"]).parts) + current_hash = lifecycle_log_hash(log_path) + if current_hash == marker["post_log_hash"]: + return + if current_hash != marker["pre_log_hash"]: + raise BimriError("lifecycle run log changed outside its transition.") + if current_hash == "absent": + exclusive_write_text(log_path, marker["log_append"]) + else: + current = log_path.read_text(encoding="utf-8") + atomic_write_text(log_path, current + marker["log_append"]) + if lifecycle_log_hash(log_path) != marker["post_log_hash"]: + raise BimriError("lifecycle run log did not reach its intended bytes.") + + +def authority_transition_state_issues(marker, state): + """Reject state deltas outside the engine operation recorded pre-write.""" + before = copy.deepcopy(marker["pre_write_state"]) + current = audit_witness_write_state_payload(state) + operation = marker["operation"] + allowed_top = set() + if operation in { + "sync", "close-authority-run", "resolve", + "recover-interrupted-authority", "authority-recovery", + }: + allowed_top.update({ + "head_revision", "head_hash", "conflict_count", "pattern_count", + "cold_current", "last_revision_reason", + }) + elif operation == "recover-manual-hot-edit": + allowed_top.add("conflict_count") + for field in allowed_top: + before.pop(field, None) + current.pop(field, None) + run_id = marker.get("run_id") + if operation in {"sync", "close-authority-run"} and run_id: + before_active = before.get("active_runs", {}) + current_active = current.get("active_runs", {}) + if operation == "sync": + before_meta = before_active.get(run_id) + current_meta = current_active.get(run_id) + if isinstance(before_meta, dict) and isinstance(current_meta, dict): + before_meta.pop("base_revision", None) + current_meta.pop("base_revision", None) + else: + before_active.pop(run_id, None) + current_active.pop(run_id, None) + before_sessions = before.get("session_runs", {}) + current_sessions = current.get("session_runs", {}) + for key, value in list(before_sessions.items()): + if value == run_id: + before_sessions.pop(key, None) + for key, value in list(current_sessions.items()): + if value == run_id: + current_sessions.pop(key, None) + if before != current: + return [ + "interrupted authority transition contains an out-of-scope state change" + ] + return [] + + +def authority_transition_proposal_ids(paths, marker): + del paths + validate_frozen_authority_transition_scope(marker.get("scope")) + return set(marker["scope"]["proposal_ids"]) + + +def authority_transition_allowed_manifest_paths(paths, state, marker): + """Return only graph paths provably tied to the recorded operation scope.""" + allowed = set() + proposal_ids = authority_transition_proposal_ids(paths, marker) + for proposal_id in proposal_ids: + if marker["operation"] == "propose": + allowed.add(f".bimri/proposals/{proposal_id}.json") + dpath = decision_path(paths, proposal_id) + if dpath.is_file() and not path_is_redirected(dpath): + try: + decision = validate_decision( + read_json_strict(dpath, dpath.name), proposal_id + ) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + continue + allowed.add(f".bimri/decisions/{proposal_id}.json") + for field in ("revision", "revision_before"): + revision = decision.get(field) + if isinstance(revision, int) and not isinstance(revision, bool): + allowed.add(f".bimri/revisions/V{revision:06d}.md") + conflict_id = decision.get("conflict_id") or decision.get( + "resolution_id" + ) + if isinstance(conflict_id, str) and CONFLICT_RE.fullmatch(conflict_id): + allowed.add(f".bimri/conflicts/{conflict_id}.json") + allowed.add(f".bimri/resolutions/{conflict_id}.json") + conflict_id = marker.get("scope", {}).get("conflict_id") + if isinstance(conflict_id, str) and CONFLICT_RE.fullmatch(conflict_id): + allowed.add(f".bimri/resolutions/{conflict_id}.json") + if marker["operation"] == "recover-manual-hot-edit": + hot_hash = marker.get("scope", {}).get("hot_hash") + if isinstance(hot_hash, str) and HASH_RE.fullmatch(hot_hash): + allowed.update({ + f".bimri/recovery/manual-hot-{hot_hash}.md", + f".bimri/recovery/manual-hot-{hot_hash}.bin", + }) + for path in paths.conflicts.glob("C*.json"): + if CONFLICT_RE.fullmatch(path.stem): + allowed.add(f".bimri/conflicts/{path.name}") + return allowed + + +def archive_paths_explained_by_transition(paths, marker): + """Archive months whose only change is appends stamped by the scoped op. + + Proof, not exemption: the current bytes must reduce to the prior + witnessed hash by stripping whole trailing rows, and every stripped row + must carry a [BY:] stamp from the transition's own scope. A new + month must consist entirely of such rows. Deleted months are never + explained, and a torn or foreign tail leaves the month as drift. + """ + explained = set() + scoped_ids = set(authority_transition_proposal_ids(paths, marker)) + conflict_id = marker.get("scope", {}).get("conflict_id") + if isinstance(conflict_id, str) and CONFLICT_RE.fullmatch(conflict_id): + scoped_ids.add(conflict_id) + if not scoped_ids: + return explained + try: + prior_manifest = load_audit_manifest_evidence( + paths, marker["prior_witness"] + ) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + return explained + prior_by_path = { + item["path"]: item["sha256"] for item in prior_manifest + } + stamps = tuple( + f"[BY:{scoped}]".encode("utf-8") for scoped in sorted(scoped_ids) + ) + try: + children = sorted(paths.archive.glob("*.md")) + except OSError: + return explained + for path in children: + if path.is_symlink() or not path.is_file(): + continue + relative = path.relative_to(paths.root).as_posix() + try: + raw = path.read_bytes() + except OSError: + continue + prior_hash = prior_by_path.get(relative) + if prior_hash is not None and sha256_bytes(raw) == prior_hash: + continue + remainder = raw + proven = False + for _ in range(64): + if prior_hash is None: + if remainder == b"": + proven = True + break + elif sha256_bytes(remainder) == prior_hash: + proven = True + break + if not remainder.endswith(b"\n"): + break + cut = remainder.rfind(b"\n", 0, len(remainder) - 1) + line = remainder[cut + 1:] + if not any(stamp in line for stamp in stamps): + break + remainder = remainder[:cut + 1] + if proven: + explained.add(relative) + return explained + + +def authority_transition_completion_issues( + paths, state, marker, manifest, run_facts +): + completion = marker.get("scope", {}).get("completion") + if completion is not None: + try: + validate_frozen_authority_transition_scope(marker["scope"]) + expected = completion["witness"] + exact_issues = [] + if audit_witness_digest(manifest) != expected["manifest_hash"]: + exact_issues.append( + "completed authority inventory differs from its frozen closure" + ) + if len(manifest) != expected["manifest_count"]: + exact_issues.append( + "completed authority inventory count differs from its frozen closure" + ) + if audit_witness_digest(run_facts) != expected["run_authority_hash"]: + exact_issues.append( + "completed authority run facts differ from their frozen closure" + ) + if ( + path_is_redirected(paths.state) + or not paths.state.is_file() + or sha256_bytes(paths.state.read_bytes()) + != completion["state_file_hash"] + ): + exact_issues.append( + "completed authority state bytes differ from their frozen closure" + ) + if not sealed_witness_matches_state(expected, state): + exact_issues.append( + "completed authority state/head differs from its frozen closure" + ) + for proposal_id, expected_hash in marker["scope"][ + "proposal_hashes" + ].items(): + path = proposal_path(paths, proposal_id) + if ( + path_is_redirected(path) + or not path.is_file() + or sha256_bytes(path.read_bytes()) != expected_hash + ): + exact_issues.append( + "completed authority proposal differs from frozen intent: " + + proposal_id + ) + conflicts, conflict_issues = scan_open_conflicts(paths, state) + del conflicts + semantic_issues = authority_storage_issues( + paths, state, allow_recoverable_applying=True + ) + if authority_has_unsettled_records(paths): + semantic_issues.append( + "interrupted authority transition still has applying or failed records" + ) + return list(dict.fromkeys( + exact_issues + semantic_issues + conflict_issues + )) + except (BimriError, OSError, UnicodeError, ValueError, TypeError) as exc: + return [f"frozen authority completion is invalid: {exc}"] + + allowed = authority_transition_allowed_manifest_paths(paths, state, marker) + # Accepted replacements append displaced entries to cold archive months + # as a routine engine effect. A month is transition-explained only when + # its change is byte-provably the prior content plus appended rows + # stamped by this operation's own scope; anything else stays drift. + allowed |= archive_paths_explained_by_transition(paths, marker) + mismatch = audit_checkpoint_mismatch_issues( + paths, + state, + marker["prior_witness"], + manifest, + run_facts, + allowed_manifest_paths=allowed, + allow_audit_epoch_advance=True, + ) + # Head/current/run facts are expected to move inside a scoped transition; + # their *content* is re-proven below. Inventory paths and state scope are + # the independent checks that prevent unrelated drift from riding along. + mismatch = [ + issue for issue in mismatch + if not issue.startswith(( + "current authority state differs", + "authority-influencing state differs", + "accepted head differs", + "governance-relevant run facts differ", + "audit epoch differs", + )) + ] + state_issues = authority_transition_state_issues(marker, state) + conflicts, conflict_issues = scan_open_conflicts(paths, state) + del conflicts + semantic_issues = authority_storage_issues( + paths, state, allow_recoverable_applying=True + ) + unsettled = authority_has_unsettled_records(paths) + if unsettled: + semantic_issues.append( + "interrupted authority transition still has applying or failed records" + ) + return list(dict.fromkeys( + ["interrupted authority transition has no frozen post-effect closure"] + + mismatch + state_issues + semantic_issues + conflict_issues + )) + + +def preserve_corrupt_audit_transition(paths): + """Copy an unreadable transition marker aside as verified evidence. + + The original marker is NOT removed here: the caller retires it only + after the receipt citing this attachment is durably recorded, so a + death in between leaves the marker in place for the next attempt. + """ + path = paths.audit_transition + try: + if path.is_symlink(): + raw = os.fsencode(os.readlink(path)) + elif path.is_file(): + raw = path.read_bytes() + else: + return None + ensure_directory_durable(paths.audit_drift) + digest = sha256_bytes(raw) + target = paths.audit_drift / f"corrupt-transition-{digest[:16]}.bin" + if not target.exists() and not target.is_symlink(): + exclusive_write_bytes(target, raw) + if target.is_symlink() or not target.is_file(): + return None + written = target.read_bytes() + if len(written) != len(raw) or sha256_bytes(written) != digest: + return None + return { + "path": target.relative_to(paths.root).as_posix(), + "sha256": digest, + "bytes": len(raw), + } + except (OSError, UnicodeError): + return None + + +def reconcile_audit_transition(paths, state): + """Complete a proven lifecycle cutpoint or preserve an authority baseline.""" + try: + marker = load_audit_transition(paths) + except (BimriError, OSError, UnicodeError, ValueError, TypeError) as exc: + # A corrupt write-ahead marker must never wedge recovery itself: + # preserve its exact bytes as drift evidence, then rebuild trust + # through the full audit. An obstruction that cannot be preserved + # and retired (for example a directory squatting on the marker + # path) is a hard error — no command may report health while the + # marker path is unusable, and the witness stays untouched. + preserved = preserve_corrupt_audit_transition(paths) + if preserved is None and ( + paths.audit_transition.exists() + or paths.audit_transition.is_symlink() + ): + raise BimriError( + "the audit transition marker is invalid and could not be " + f"preserved for recovery: {exc}. Clear any obstruction on " + ".bimri/audit-drift or .bimri/audit-transition.json, then " + "run doctor." + ) from exc + reasons = [f"audit transition marker was invalid: {exc}"] + attachments = [] + if preserved: + reasons.append( + f"marker bytes preserved at {preserved['path']}" + ) + preserved["role"] = "corrupt-transition-marker" + attachments.append(preserved) + if write_audit_drift_receipt( + paths, + reasons, + state=state, + attachments=attachments, + ) is None: + raise BimriError( + "the invalid transition marker's drift evidence could not " + "be durably recorded; the marker remains in place. Repair " + ".bimri/audit-drift, then retry." + ) from exc + # Only now, with the attachment verified and the receipt durable, + # is the original marker retired. A failure here just repeats the + # (deduplicated) preservation on the next load. + with contextlib.suppress(OSError): + paths.audit_transition.unlink() + fsync_directory(paths.audit_transition.parent) + discard_audit_witness(paths) + return state + if marker is None: + return state + prior = marker["prior_witness"] + with contextlib.suppress( + BimriError, OSError, UnicodeError, ValueError, TypeError + ): + ensure_audit_manifest_generation(paths, prior) + if marker["kind"] == "lifecycle": + current_hash = audit_witness_write_state_hash(state) + log_path = paths.root.joinpath(*PurePosixPath(marker["log_path"]).parts) + current_log_hash = lifecycle_log_hash(log_path) + if ( + current_hash == marker["pre_write_state_hash"] + and state_audit_epoch(state) == marker["epoch_before"] + and current_log_hash == marker["pre_log_hash"] + ): + # Marker durable, first state replace not reached. + atomic_write_json(paths.audit_witness, prior) + clear_audit_transition(paths) + paths.validated_audit_witness = prior + prune_audit_manifest_generations(paths) + return state + if ( + current_hash == marker["post_write_state_hash"] + and state_audit_epoch(state) == marker["epoch_after"] + and current_log_hash in { + marker["pre_log_hash"], marker["post_log_hash"] + } + ): + finish_lifecycle_log(paths, marker) + publish_lifecycle_checkpoint(paths, state, marker) + return state + reasons = [ + f"incomplete {marker['operation']} lifecycle transition for " + f"{marker['run_id']} does not match its intended state/log bytes", + "lifecycle state/log divergence; no inventory delta applies", + ] + # Cache-miss semantics: preserve the evidence, retire the reconciled + # marker, and let the full semantic audit decide the store's fate. + if write_audit_drift_receipt( + paths, reasons, marker=marker, prior_witness=prior, state=state + ) is None: + raise BimriError( + "interrupted-transition drift evidence could not be durably " + "recorded; the transition marker and prior checkpoint " + "remain in place. Repair .bimri/audit-drift, then retry." + ) + discard_audit_witness(paths) + clear_audit_transition(paths) + prune_audit_manifest_generations(paths) + return state + + # If W1 reached disk and only marker cleanup was interrupted, its complete + # state binding is enough to finish the transaction without graph replay. + live = load_sealed_audit_witness(paths) + if live is not None and sealed_witness_matches_state(live, state): + try: + load_audit_manifest_evidence(paths, live) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + pass + else: + paths.validated_audit_witness = live + clear_audit_transition(paths) + prune_audit_manifest_generations(paths) + return state + + manifest = None + run_facts = None + reasons = [] + try: + manifest = audit_witness_manifest(paths) + run_facts = audit_witness_run_facts(paths, state) + except (BimriError, OSError, UnicodeError, ValueError, TypeError) as exc: + reasons.append(f"interrupted authority transition audit failed: {exc}") + if manifest is not None and run_facts is not None: + reasons.extend(authority_transition_completion_issues( + paths, state, marker, manifest, run_facts + )) + if not reasons: + # The full graph and exact scoped state/inventory delta prove either + # the pre-mutation cutpoint or a completely accepted engine effect. + if write_audit_witness( + paths, state, manifest=manifest, run_facts=run_facts + ): + clear_audit_transition(paths) + prune_audit_manifest_generations(paths) + return state + reasons.append("interrupted authority checkpoint could not be republished") + unsettled_reason = ( + "interrupted authority transition still has applying or failed records" + ) + incomplete_reason = ( + "interrupted authority transition has no frozen post-effect closure" + ) + if set(reasons) <= {incomplete_reason, unsettled_reason}: + # A consistent applying record is the engine's durable intent. The + # existing recovery/retry path may finish it under this same marker. + return state + if marker["operation"] == "resolve": + conflict_id = marker.get("scope", {}).get("conflict_id") + target_prefix = f"resolution {conflict_id} (" + retry_marker = "resolution status is failed; explicit retry is required." + non_retry_reasons = [ + reason for reason in reasons + if not ( + reason in {incomplete_reason, unsettled_reason} + or ( + isinstance(conflict_id, str) + and reason.startswith(target_prefix) + and retry_marker in reason + ) + ) + ] + if not non_retry_reasons: + # Keep W0 invalid and retain the transition, but do not manufacture + # a general block: only an explicit retry of this exact conflict is + # permitted by require_governance_for_resolution_retry. + return state + drift_delta = None + if manifest is not None: + try: + prior_m = load_audit_manifest_evidence(paths, prior) + allowed_recovery = authority_transition_allowed_manifest_paths( + paths, state, marker + ) | archive_paths_explained_by_transition(paths, marker) + drift_delta = audit_checkpoint_drift_delta( + prior_m, manifest, allowed_recovery + ) + except ( + BimriError, OSError, UnicodeError, ValueError, TypeError, + ) as exc: + # Retiring the marker and prior checkpoint without the per-path + # delta would be a blind rebaseline. Missing referenced + # evidence refuses, exactly as it does on every other surface. + raise BimriError( + PRIOR_EVIDENCE_INVALID_PREFIX + f": {exc}; the transition " + "marker and prior checkpoint remain in place. Restore the " + ".bimri/audit-manifests evidence from backup, or remove " + ".bimri/audit-witness.json and the marker to rebuild trust " + "from the full audit." + ) from exc + if write_audit_drift_receipt( + paths, + reasons, + marker=marker, + prior_witness=prior, + state=state, + delta=drift_delta, + ) is None: + raise BimriError( + "interrupted-transition drift evidence could not be durably " + "recorded; the transition marker and prior checkpoint remain in " + "place. Repair .bimri/audit-drift, then retry." + ) + discard_audit_witness(paths) + clear_audit_transition(paths) + prune_audit_manifest_generations(paths) + return state + + +def refresh_audit_checkpoint_after_state_write(paths, state): + """Retain a verdict when a lifecycle write changed no authority state.""" + try: + marker = load_audit_transition(paths) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + marker = None + if marker is not None: + if marker["kind"] != "lifecycle": + return False + return publish_lifecycle_checkpoint(paths, state, marker) + witness = paths.pending_checkpoint_witness or paths.validated_audit_witness + paths.pending_checkpoint_witness = None + if witness is None: + return False + try: + if ( + witness["head_revision"] != state["head_revision"] + or witness["head_hash"] != state["head_hash"] + or witness["state_hash"] != audit_witness_state_hash( + state, witness["proposal_runs"] + ) + or witness["write_state_hash"] + != audit_witness_write_state_hash(state) + or witness["audit_epoch"] != state_audit_epoch(state) + ): + paths.validated_audit_witness = None + return False + paths.validated_audit_witness = witness + return True + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + discard_audit_witness(paths) + return False + + +def prepare_audit_checkpoint_state_write(paths): + """Remember a checkpoint across a lifecycle-only state mutation.""" + witness = paths.validated_audit_witness + paths.pending_checkpoint_witness = None + if witness is None: + return False + paths.pending_checkpoint_witness = witness + return True + + +def invalidate_audit_witness_before_authority_write(paths): + """Strict crash marker: no authority byte changes while a verdict survives.""" + paths.validated_audit_witness = None + path = paths.audit_witness + if path_is_redirected(path): + raise BimriError("audit witness is redirected and cannot be invalidated.") + if path.exists(): + if not path.is_file(): + raise BimriError("audit witness is not a regular file.") + path.unlink() + fsync_directory(path.parent) + if path.exists() or path.is_symlink(): + raise BimriError("audit witness invalidation did not complete.") + + +def authority_has_recoverable_applying(paths): + """Detect the only case where the recovery audit is weaker than normal.""" + for directory, field, value in ( + (paths.decisions, "outcome", "applying"), + (paths.resolutions, "status", "applying"), + ): + for path in sorted(directory.glob("*.json")): + if path_is_redirected(path) or not path.is_file(): + continue + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + continue + if isinstance(data, dict) and data.get(field) == value: + return True + return False + + +def authority_has_unsettled_records(paths): + """Return true while authority needs recovery or completion, not caching.""" + for directory, field, values in ( + (paths.decisions, "outcome", {"applying"}), + (paths.resolutions, "status", {"applying", "failed"}), + ): + for path in sorted(directory.glob("*.json")): + if path_is_redirected(path) or not path.is_file(): + return True + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return True + if isinstance(data, dict) and data.get(field) in values: + return True + return False + + +def load_audit_blocked_record(paths): + path = paths.audit_blocked + if not (path.exists() or path.is_symlink()): + return None + if path_is_redirected(path) or not path.is_file(): + raise BimriError("audit checkpoint is blocked by an unsafe marker") + record = read_json_strict(path, "audit blocked marker") + expected = { + "blocked_schema", "created_at", "reasons", "prior_witness", + "prior_witness_hash", "prior_manifest", "marker_hash", + } + if set(record) != expected: + raise BimriError("audit blocked marker fields are invalid.") + if record.get("blocked_schema") != AUDIT_BLOCKED_SCHEMA: + raise BimriError("audit blocked marker schema is invalid.") + parse_timestamp(record.get("created_at"), "audit blocked timestamp") + reasons = record.get("reasons") + if ( + not isinstance(reasons, list) + or not reasons + or not all(isinstance(reason, str) and reason for reason in reasons) + ): + raise BimriError("audit blocked reasons are invalid.") + prior_hash = record.get("prior_witness_hash") + prior_witness = record.get("prior_witness") + if prior_witness is None: + if prior_hash is not None or record.get("prior_manifest") is not None: + raise BimriError("audit blocked prior witness hash is orphaned.") + else: + validated = validate_sealed_audit_witness_record(prior_witness) + if ( + validated is None + or prior_hash != validated.get("witness_hash") + ): + raise BimriError("audit blocked prior witness evidence is invalid.") + prior_manifest = record.get("prior_manifest") + if prior_manifest is not None: + load_audit_manifest_evidence( + paths, validated, manifest_override=prior_manifest + ) + if record.get("marker_hash") != audit_record_seal( + record, "marker_hash" + ): + raise BimriError("audit blocked marker seal is invalid.") + return record + + +def load_audit_blocked_issues(paths): + try: + record = load_audit_blocked_record(paths) + if record is None: + return [] + return [ + f"audit checkpoint blocked: {reason}" + for reason in record["reasons"] + ] + except (BimriError, OSError, UnicodeError, ValueError, TypeError) as exc: + return [f"audit checkpoint blocked marker is invalid: {exc}"] + + +def load_audit_blocked_prior_witness(paths): + try: + record = load_audit_blocked_record(paths) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + return None + return record.get("prior_witness") if record is not None else None + + +def load_audit_blocked_prior_manifest(paths): + try: + record = load_audit_blocked_record(paths) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + return None + return record.get("prior_manifest") if record is not None else None + + +def record_audit_blocked(paths, witness, reasons, prior_manifest=None): + reasons = list(dict.fromkeys(str(reason) for reason in reasons if reason)) + if witness is not None and prior_manifest is None: + try: + existing = load_audit_blocked_record(paths) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + existing = None + if ( + existing is not None + and existing.get("prior_witness_hash") == witness.get("witness_hash") + ): + prior_manifest = existing.get("prior_manifest") + if prior_manifest is None: + try: + prior_manifest = load_audit_manifest_evidence(paths, witness) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + prior_manifest = None + record = { + "blocked_schema": AUDIT_BLOCKED_SCHEMA, + "created_at": now_iso(), + "reasons": reasons[:20] or ["protected audit evidence changed"], + "prior_witness": witness, + "prior_witness_hash": ( + witness.get("witness_hash") if witness is not None else None + ), + "prior_manifest": prior_manifest, + } + record["marker_hash"] = audit_record_seal(record, "marker_hash") + if path_is_redirected(paths.audit_blocked): + raise BimriError("audit blocked marker destination is redirected.") + atomic_write_json(paths.audit_blocked, record) + + +def clear_audit_blocked(paths): + path = paths.audit_blocked + if path_is_redirected(path): + raise BimriError("audit blocked marker is redirected.") + if path.exists(): + if not path.is_file(): + raise BimriError("audit blocked marker is not a regular file.") + path.unlink() + fsync_directory(path.parent) + + +def audit_checkpoint_mismatch_issues( + paths, + state, + witness, + manifest, + run_facts, + allowed_manifest_paths=None, + prior_manifest_override=None, + allow_audit_epoch_advance=False, +): + if witness is None: + return [] + issues = [] + try: + prior_manifest = load_audit_manifest_evidence( + paths, witness, manifest_override=prior_manifest_override + ) + except (BimriError, OSError, UnicodeError) as exc: + return [ + PRIOR_EVIDENCE_INVALID_PREFIX + f": {exc}; restore the " + ".bimri/audit-manifests evidence from backup, or remove " + ".bimri/audit-witness.json to rebuild trust from the full audit" + ] + if prior_manifest != manifest: + allowed_manifest_paths = set(allowed_manifest_paths or ()) + prior_by_path = { + item["path"]: item["sha256"] for item in prior_manifest + } + current_by_path = { + item["path"]: item["sha256"] for item in manifest + } + changed = sorted( + path for path in prior_by_path.keys() & current_by_path.keys() + if ( + prior_by_path[path] != current_by_path[path] + and path not in allowed_manifest_paths + ) + ) + added = sorted( + path for path in current_by_path.keys() - prior_by_path.keys() + if path not in allowed_manifest_paths + ) + deleted = sorted( + path for path in prior_by_path.keys() - current_by_path.keys() + if path not in allowed_manifest_paths + ) + details = [] + for label, values in ( + ("changed", changed), ("added", added), ("deleted", deleted) + ): + if values: + sample = ", ".join(values[:3]) + suffix = f", +{len(values) - 3} more" if len(values) > 3 else "" + details.append(f"{label} {len(values)}: {sample}{suffix}") + if details: + issues.append( + "protected path/hash inventory differs from the prior audit witness" + + " (" + "; ".join(details) + ")" + ) + try: + state_hash = audit_witness_state_hash( + state, witness.get("proposal_runs", []) + ) + except (BimriError, KeyError, TypeError, ValueError) as exc: + issues.append(f"state could not be compared with prior audit: {exc}") + else: + if witness.get("state_hash") != state_hash: + issues.append("current authority state differs from the prior audit witness") + try: + write_state_hash = audit_witness_write_state_hash(state) + except (BimriError, KeyError, TypeError, ValueError) as exc: + issues.append( + f"authority-influencing state could not be compared with prior audit: {exc}" + ) + else: + if witness.get("write_state_hash") != write_state_hash: + issues.append( + "authority-influencing state differs from the prior audit witness" + ) + try: + current_epoch = state_audit_epoch(state) + prior_epoch = witness.get("audit_epoch") + epoch_matches = current_epoch == prior_epoch + if allow_audit_epoch_advance: + epoch_matches = ( + isinstance(prior_epoch, int) and current_epoch >= prior_epoch + ) + if not epoch_matches: + issues.append("audit epoch differs from the prior audit witness") + except (BimriError, TypeError, ValueError) as exc: + issues.append(f"audit epoch could not be compared with prior audit: {exc}") + if ( + witness.get("head_revision") != state.get("head_revision") + or witness.get("head_hash") != state.get("head_hash") + ): + issues.append("accepted head differs from the prior audit witness") + if witness.get("run_authority_hash") != audit_witness_digest(run_facts): + issues.append("governance-relevant run facts differ from the prior audit witness") + return issues + + +def refresh_audit_witness_after_trusted_write(paths, state): + """Publish a new verdict only after a pre-audited authority mutation.""" + if not paths.authority_write_mode: + return False + try: + if not paths.authority_write_audit_healthy: + return False + if authority_has_unsettled_records(paths): + # W0 is retained as crash/recovery evidence. The advanced state + # epoch plus live transition makes it unreadable; deleting it + # would only discard the baseline needed by an explicit retry. + paths.validated_audit_witness = None + return False + published = write_audit_witness(paths, state) + if published: + complete_authority_checkpoint_transition(paths) + return published + finally: + paths.authority_write_mode = False + paths.authority_write_audit_healthy = False + paths.full_audit_manifest = None + paths.full_audit_run_facts = None + + +def condemn_audit_checkpoint(paths, state): + """Stop a still-valid checkpoint from shielding the warm path. + + Owner-ruled 2026-09-02. A full audit that condemns authority must not + leave the earlier checkpoint readable: otherwise the next warm start + prints a clean brief and exact reads keep serving a store whose writes + are paused, against BIMRI-PROTOCOL 9.5. The witness bytes stay on disk + as the prior baseline for drift receipts, quarantine and restore; only + the state's audit epoch advances, the same retained-but-invalid shape a + failed resolution leaves. The next command re-proves the store and + refuses or degrades exactly as v5.1.0 did, until an audit passes and + publishes a fresh checkpoint. + """ + paths.validated_audit_witness = None + witness = load_sealed_audit_witness(paths) + if witness is None: + return False + if witness.get("audit_epoch") != state_audit_epoch(state): + return False + try: + state["_audit_epoch"] = state_audit_epoch(state) + 1 + save_state(paths, state) + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + # The epoch could not be persisted. Removing the derived file keeps + # the warm path from trusting it; the baseline is lost only here. + discard_audit_witness(paths) + return True + + +def governance_snapshot( + paths, + state, + allow_recoverable_applying=False, + use_witness=True, + write_witness=True, + strict_prior_comparison=False, + audit_blocked=False, + recover_blocked=False, + prior_witness_override=None, + prior_manifest_override=None, + allowed_manifest_paths=None, + allow_audit_epoch_advance=False, +): + blocked_issues = load_audit_blocked_issues(paths) + if blocked_issues and not audit_blocked: + return [], blocked_issues + if paths.authority_write_mode: + return scan_open_conflicts(paths, state) + if use_witness: + if paths.validated_audit_witness is not None: + return [], [] + witness = load_valid_audit_witness(paths, state) + if witness is not None: + return [], [] + live_prior_witness = load_sealed_audit_witness(paths) + blocked_prior_witness = load_audit_blocked_prior_witness(paths) + blocked_prior_manifest = load_audit_blocked_prior_manifest(paths) + if prior_witness_override is not None: + prior_witness = prior_witness_override + if ( + prior_manifest_override is None + and blocked_prior_witness is not None + and blocked_prior_witness.get("witness_hash") + == prior_witness.get("witness_hash") + ): + prior_manifest_override = blocked_prior_manifest + elif live_prior_witness is not None: + prior_witness = live_prior_witness + else: + prior_witness = blocked_prior_witness + prior_manifest_override = blocked_prior_manifest + manifest = None + run_facts = None + inventory_issues = [] + try: + manifest = audit_witness_manifest(paths) + except (BimriError, OSError, UnicodeError) as exc: + inventory_issues.append(f"protected inventory audit failed: {exc}") + try: + run_facts = audit_witness_run_facts(paths, state) + except (BimriError, OSError, UnicodeError) as exc: + inventory_issues.append(f"run authority audit failed: {exc}") + conflicts, conflict_issues = scan_open_conflicts(paths, state) + mismatch_issues = [] + if manifest is not None and run_facts is not None: + mismatch_issues = audit_checkpoint_mismatch_issues( + paths, + state, + prior_witness, + manifest, + run_facts, + allowed_manifest_paths=allowed_manifest_paths, + prior_manifest_override=prior_manifest_override, + allow_audit_epoch_advance=allow_audit_epoch_advance, + ) + if not strict_prior_comparison: + # Checkpoint divergence is a cache miss, never an issue: the full + # semantic audit is the authority on this store. Unexplained + # divergence is preserved as a drift receipt at publication. The + # one exception is a sealed witness whose referenced manifest + # evidence is unavailable: that is damaged evidence, and it + # refuses instead of rebaselining blind. + paths.last_audit_drift = [ + issue for issue in mismatch_issues + if not issue.startswith(PRIOR_EVIDENCE_INVALID_PREFIX) + ] + issues = list(dict.fromkeys( + authority_storage_issues( + paths, + state, + allow_recoverable_applying=allow_recoverable_applying, + ) + + conflict_issues + + inventory_issues + + ( + mismatch_issues if strict_prior_comparison else [ + issue for issue in mismatch_issues + if issue.startswith(PRIOR_EVIDENCE_INVALID_PREFIX) + ] + ) + )) + clear_block_after_publish = False + if blocked_issues: + if recover_blocked and prior_witness is not None and not issues: + clear_block_after_publish = True + elif not issues: + issues = blocked_issues + paths.full_audit_manifest = manifest + paths.full_audit_run_facts = run_facts + if issues: + condemned = [ + issue for issue in issues + if issue not in blocked_issues + and not issue.startswith(PRIOR_EVIDENCE_INVALID_PREFIX) + ] + if write_witness and prior_witness is None: + discard_audit_witness(paths) + elif write_witness and condemned and not strict_prior_comparison: + condemn_audit_checkpoint(paths, state) + elif write_witness and ( + not allow_recoverable_applying + or not authority_has_recoverable_applying(paths) + ): + # The ruled contract makes a durable receipt a transactional + # precondition of rebaselining over unexplained divergence: the + # receipt is recorded before the replacement checkpoint can + # publish, and a receipt that cannot be recorded keeps the prior + # checkpoint as the baseline and surfaces as an issue rather than + # a silent pass. + receipt_blocked = False + if not strict_prior_comparison and mismatch_issues: + drift_delta = None + delta_reasons = list(mismatch_issues) + if prior_witness is not None and manifest is not None: + try: + prior_m = load_audit_manifest_evidence( + paths, + prior_witness, + manifest_override=prior_manifest_override, + ) + drift_delta = audit_checkpoint_drift_delta( + prior_m, manifest, allowed_manifest_paths + ) + except ( + BimriError, OSError, UnicodeError, ValueError, + TypeError, + ) as exc: + drift_delta = None + delta_reasons.append( + "prior manifest evidence could not be loaded; " + f"per-path delta unavailable: {exc}" + ) + receipt = write_audit_drift_receipt( + paths, + delta_reasons, + prior_witness=prior_witness, + state=state, + delta=drift_delta, + ) + receipt_blocked = receipt is None + if receipt_blocked: + issues.append( + "unexplained divergence cannot be rebaselined because its " + "drift receipt could not be durably recorded; the prior " + "checkpoint remains the baseline." + ) + else: + published = write_audit_witness( + paths, + state, + conflicts, + manifest=manifest, + run_facts=run_facts, + ) + if published and clear_block_after_publish: + clear_audit_blocked(paths) + try: + transition = load_audit_transition(paths) + except ( + BimriError, OSError, UnicodeError, ValueError, TypeError, + ): + transition = None + if transition is not None and transition["kind"] == "authority": + clear_audit_transition(paths) + prune_audit_manifest_generations(paths) + return conflicts, issues + + +def verify_intact_audit_evidence(paths, state, prior_witness): + """Compare live bytes with an intact prior verdict without semantic replay.""" + inventory_issues = [] + manifest = None + run_facts = None + try: + manifest = audit_witness_manifest(paths) + except (BimriError, OSError, UnicodeError) as exc: + inventory_issues.append(f"protected inventory audit failed: {exc}") + try: + run_facts = audit_witness_run_facts(paths, state) + except (BimriError, OSError, UnicodeError) as exc: + inventory_issues.append(f"run authority audit failed: {exc}") + mismatch_issues = [] + if manifest is not None and run_facts is not None: + mismatch_issues = audit_checkpoint_mismatch_issues( + paths, state, prior_witness, manifest, run_facts + ) + return manifest, run_facts, list(dict.fromkeys( + inventory_issues + mismatch_issues + )) + + +def begin_authority_write( + paths, + state, + allow_degraded=False, + operation="authority-write", + run_id=None, + scope=None, +): + """Verify prior evidence, then durably invalidate before mutation.""" + blocked_issues = load_audit_blocked_issues(paths) + if blocked_issues: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(blocked_issues[:3]) + ) + existing_transition = load_audit_transition(paths) + if paths.authority_write_mode and existing_transition is not None: + if existing_transition["kind"] != "authority": + raise BimriError("cannot resume authority through a lifecycle marker.") + return [], [] + prior_witness = load_sealed_audit_witness(paths) + if prior_witness is not None: + manifest, run_facts, mismatch = verify_intact_audit_evidence( + paths, state, prior_witness + ) + if not mismatch: + paths.full_audit_manifest = manifest + paths.full_audit_run_facts = run_facts + begin_authority_checkpoint_transition( + paths, state, operation, run_id=run_id, scope=scope + ) + paths.authority_write_mode = True + paths.authority_write_audit_healthy = True + return [], [] + # Divergence from the prior verdict is a cache miss, never a latch. + # Preserve it as drift evidence, then rebuild trust below with the + # complete semantic authority audit. The stale witness file stays on + # disk: warm reads keep serving inside the documented boundary, and + # a failed audit leaves W0 as the recovery baseline for quarantine. + drift_delta = None + if manifest is not None: + try: + prior_m = load_audit_manifest_evidence(paths, prior_witness) + drift_delta = audit_checkpoint_drift_delta(prior_m, manifest) + except ( + BimriError, OSError, UnicodeError, ValueError, TypeError, + ): + drift_delta = None + if write_audit_drift_receipt( + paths, + mismatch, + prior_witness=prior_witness, + state=state, + delta=drift_delta, + ) is None: + raise BimriError( + "authority recovery is required; the divergence evidence " + "receipt could not be durably recorded before rebuilding " + "trust. Repair .bimri/audit-drift, then retry." + ) + + # Missing, corrupt or diverged derived evidence has no prior verdict to + # reuse. Rebuild trust once with the complete semantic authority audit. + conflicts, issues = governance_snapshot( + paths, + state, + use_witness=False, + write_witness=False, + ) + if issues and not allow_degraded: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(issues[:3]) + ) + if not issues: + if write_audit_witness( + paths, + state, + conflicts, + manifest=paths.full_audit_manifest, + run_facts=paths.full_audit_run_facts, + ): + begin_authority_checkpoint_transition( + paths, state, operation, run_id=run_id, scope=scope + ) + else: + # Unsettled records legitimately refuse a fresh checkpoint even + # though the audit passed. Invalidate durably and proceed; the + # post-write refresh publishes once the store settles. + invalidate_audit_witness_before_authority_write(paths) + else: + # Degraded proposal staging has no healthy verdict to preserve. It is + # already recovery-gated and cannot publish a new checkpoint. + invalidate_audit_witness_before_authority_write(paths) + paths.authority_write_mode = True + paths.authority_write_audit_healthy = not issues + return conflicts, issues + + +def require_governance_healthy(paths, state): + conflicts, issues = governance_snapshot(paths, state) + if issues: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(issues[:3]) + ) + return conflicts + + +def require_full_governance_healthy(paths, state): + """Audit authority and prior inventory evidence at an explicit boundary.""" + conflicts, issues = governance_snapshot( + paths, + state, + use_witness=False, + ) + if issues: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(issues[:3]) + ) + return conflicts + + +def require_governance_for_resolution_retry( + paths, state, conflict_id +): + """Permit only the failed target resolution through its repair command.""" + blocked_issues = load_audit_blocked_issues(paths) + if blocked_issues: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(blocked_issues[:3]) + ) + transition = load_audit_transition(paths) + if ( + transition is not None + and transition["kind"] == "authority" + and transition["operation"] == "resolve" + and transition.get("scope", {}).get("conflict_id") == conflict_id + ): + prior = transition["prior_witness"] + manifest = None + run_facts = None + try: + manifest = audit_witness_manifest(paths) + run_facts = audit_witness_run_facts(paths, state) + issues = authority_transition_completion_issues( + paths, state, transition, manifest, run_facts + ) + except (BimriError, OSError, UnicodeError, ValueError, TypeError) as exc: + issues = [f"resolution retry transition audit failed: {exc}"] + retry_marker = "resolution status is failed; explicit retry is required." + target_prefix = f"resolution {conflict_id} (" + unsettled_reason = ( + "interrupted authority transition still has applying or failed records" + ) + incomplete_reason = ( + "interrupted authority transition has no frozen post-effect closure" + ) + remaining = [ + issue for issue in issues + if not ( + issue in {incomplete_reason, unsettled_reason} + or ( + issue.startswith(target_prefix) + and retry_marker in issue + ) + ) + ] + if not remaining: + conflicts, conflict_issues = scan_open_conflicts(paths, state) + if conflict_issues: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(conflict_issues[:3]) + ) + return conflicts + # The retry window was disturbed beyond its own frozen closure. + # Preserve the divergence with its complete per-path delta, then + # let the full semantic audit below decide whether this exact + # retry may still proceed. + retry_delta = None + if manifest is not None: + try: + prior_m = load_audit_manifest_evidence(paths, prior) + allowed_retry = authority_transition_allowed_manifest_paths( + paths, state, transition + ) | archive_paths_explained_by_transition(paths, transition) + retry_delta = audit_checkpoint_drift_delta( + prior_m, manifest, allowed_retry + ) + except ( + BimriError, OSError, UnicodeError, ValueError, TypeError, + ): + retry_delta = None + if write_audit_drift_receipt( + paths, + remaining, + marker=transition, + prior_witness=prior, + state=state, + delta=retry_delta, + ) is None: + raise BimriError( + "authority recovery is required; the retry-window drift " + "receipt could not be durably recorded. Repair " + ".bimri/audit-drift, then retry." + ) + prior_witness = load_sealed_audit_witness(paths) + if prior_witness is not None: + manifest, run_facts, mismatch = verify_intact_audit_evidence( + paths, state, prior_witness + ) + if not mismatch: + paths.full_audit_manifest = manifest + paths.full_audit_run_facts = run_facts + return [] + drift_delta = None + if manifest is not None: + try: + prior_m = load_audit_manifest_evidence(paths, prior_witness) + drift_delta = audit_checkpoint_drift_delta(prior_m, manifest) + except ( + BimriError, OSError, UnicodeError, ValueError, TypeError, + ): + drift_delta = None + if write_audit_drift_receipt( + paths, + mismatch, + prior_witness=prior_witness, + state=state, + delta=drift_delta, + ) is None: + raise BimriError( + "authority recovery is required; the divergence evidence " + "receipt could not be durably recorded before rebuilding " + "trust. Repair .bimri/audit-drift, then retry." + ) + conflicts, issues = governance_snapshot( + paths, + state, + use_witness=False, + write_witness=False, + ) + retry_marker = "resolution status is failed; explicit retry is required." + target_prefix = f"resolution {conflict_id} (" + remaining = [ + issue for issue in issues + if not ( + issue.startswith(target_prefix) + and retry_marker in issue + ) + ] + if remaining: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(remaining[:3]) + ) + return conflicts + + +def enter_authority_write_after_audit( + paths, state=None, operation="authority-recovery", scope=None +): + """Durably mark a mutation after its full audit and semantic preflight.""" + existing = load_audit_transition(paths) + if existing is None and state is not None: + prior = paths.validated_audit_witness or load_sealed_audit_witness(paths) + if prior is not None and sealed_witness_matches_state(prior, state): + begin_authority_checkpoint_transition( + paths, state, operation, scope=scope + ) + else: + invalidate_audit_witness_before_authority_write(paths) + elif existing is None: + invalidate_audit_witness_before_authority_write(paths) + paths.authority_write_mode = True + paths.authority_write_audit_healthy = True + + +def open_conflicts(paths, state): + return require_governance_healthy(paths, state) + + +def allocate_conflict_id(paths, state): + existing = [ + int(path.stem[1:]) + for path in paths.conflicts.glob("C*.json") + if CONFLICT_RE.fullmatch(path.stem) + ] + number = max([state["conflict_count"]] + existing + [0]) + 1 + if number > 999999: + raise BimriError("BIMRI has exhausted its six-digit conflict ID space.") + while conflict_path(paths, f"C{number:06d}").exists(): + number += 1 + if number > 999999: + raise BimriError("BIMRI has exhausted its six-digit conflict ID space.") + state["conflict_count"] = number + return f"C{number:06d}" + + +def proposal_file_hash(paths, proposal_id): + path = proposal_path(paths, proposal_id) + if not path.exists() or path.is_symlink(): + raise BimriError(f"proposal file is missing or unsafe: {proposal_id}") + data = read_json_strict(path, path.name) + if is_quarantine_stub(data): + validate_quarantine_stub( + paths, path, data, "proposal", proposal_id + ) + raise BimriError(f"proposal {proposal_id} is quarantined.") + return sha256_bytes(path.read_bytes()) + + +def validate_proposal_id_list(value, name): + if not isinstance(value, list): + raise BimriError(f"{name} must be a list.") + proposal_ids = [] + for item in value: + proposal_id = validate_fixed_id( + item, PROPOSAL_RE, f"{name} entry" + ) + if proposal_id != item: + raise BimriError(f"{name} entries must already be normalized.") + proposal_ids.append(proposal_id) + if len(proposal_ids) != len(set(proposal_ids)): + raise BimriError(f"{name} contains duplicate proposal IDs.") + return proposal_ids + + +def validate_revision_number(value, name): + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + or value > 999999 + ): + raise BimriError(f"{name} must be a valid revision number.") + return value + + +def validate_conflict_record( + paths, conflict, expected_conflict_id=None, verify_candidates=True +): + if not isinstance(conflict, dict): + raise BimriError("conflict must be a JSON object.") + artifact_version = conflict.get("bimri_version") + if artifact_version not in COMPATIBLE_ARTIFACT_VERSIONS: + raise BimriError("conflict BIMRI version is invalid.") + conflict_id = validate_fixed_id( + conflict.get("conflict_id"), CONFLICT_RE, "conflict ID" + ) + if expected_conflict_id and conflict_id != expected_conflict_id: + raise BimriError("conflict filename does not match its ID.") + conflict_type = conflict.get("type") + if conflict_type not in CONFLICT_TYPES: + raise BimriError("conflict type is invalid.") + key = clean_key(conflict.get("key")) + if key != conflict.get("key"): + raise BimriError("conflict key must already be normalized.") + parse_timestamp(conflict.get("created_at"), "conflict timestamp") + proposal_ids = validate_proposal_id_list( + conflict.get("proposal_ids"), "conflict proposal_ids" + ) + if conflict_type == "manual-edit": + if proposal_ids: + raise BimriError("manual-edit conflicts cannot contain proposals.") + elif not proposal_ids: + raise BimriError("proposal conflicts require at least one candidate.") + hashes = conflict.get("proposal_hashes") + if not isinstance(hashes, dict) or set(hashes) != set(proposal_ids): + raise BimriError( + "conflict proposal hashes must exactly match its candidate IDs." + ) + for proposal_id in proposal_ids: + expected = hashes.get(proposal_id) + if not isinstance(expected, str) or not HASH_RE.fullmatch(expected): + raise BimriError( + f"conflict is missing a valid hash for {proposal_id}." + ) + if verify_candidates and proposal_file_hash(paths, proposal_id) != expected: + raise BimriError( + f"proposal {proposal_id} changed after the conflict was raised. " + "BIMRI stopped so the owner can review it again." ) current_line = conflict.get("current_line") current_hash = conflict.get("current_hash") @@ -4654,6 +7491,7 @@ def exact_effect_reflected_at_head(paths, state, proposal, current): state, base_entry["raw"], state["head_revision"], + skip_validation_id=proposal["proposal_id"], ) ) if proposal["operation"] == "touch": @@ -5055,7 +7893,10 @@ def validate_proposal( "observed_key_hash", }: raise BimriError("proposal preflight receipt fields are invalid.") - accepted_receipt_engines = {ENGINE_VERSION} + accepted_receipt_engines = { + ENGINE_VERSION, + V5_1_0_ENGINE_VERSION, + } if proposal.get("bimri_version") == V5_0_2_VERSION: accepted_receipt_engines.add("5.0.3") if receipt.get("engine_release") not in accepted_receipt_engines: @@ -6258,11 +9099,123 @@ def validate_current_residency(paths, state, hot_entries=None): pointer_error = pointer_validation_error(paths, entry) if pointer_error: raise BimriError( - f"cold-current subject {key} pointer is invalid: {pointer_error}" + f"cold-current subject {key} pointer is invalid: {pointer_error}" + ) + if head_bytes is not None and sha256_bytes(head_bytes) != state["head_hash"]: + raise BimriError("state head hash does not match the accepted head revision.") + return True + + +def validate_selected_cold_archive_binding(paths, key, cold): + """Prove one cold-current key against only its recorded archive month.""" + month = cold["archived_on"][:7] + target = paths.archive / f"{month}.md" + if path_is_redirected(target) or not target.is_file(): + raise BimriError( + f"cold-current archive month {month} is missing or unsafe." + ) + exact = 0 + for number, line in enumerate( + target.read_text(encoding="utf-8").splitlines(), 1 + ): + if not line.startswith("[ARCHIVED:"): + continue + try: + record = parse_archive_record(line) + except BimriError as exc: + raise BimriError( + f"cold-current archive {target.name}:{number} is invalid: {exc}" + ) from exc + if ( + record["proposal_id"] == cold["archived_by"] + and record["reason"] == "cooled" + and record["raw_line"] == cold["raw_line"] + ): + exact += 1 + if exact != 1: + raise BimriError( + f"cold-current subject {key} is not bound to exactly one " + "immutable cooled archive record." + ) + return True + + +def validate_checkpoint_head(paths, state): + """Validate bounded accepted-head truth without traversing cold history.""" + head = revision_path(paths, state["head_revision"]) + if path_is_redirected(head) or not head.is_file(): + raise BimriError("accepted head is missing or unsafe.") + try: + head_bytes = head.read_bytes() + content = head_bytes.decode("utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise BimriError(f"accepted head is unreadable: {exc}") from exc + _, entries, errors, _ = validate_hot_content( + content, state, allow_legacy_overflow=True + ) + if errors: + raise BimriError( + "accepted head memory grammar is invalid: " + "; ".join(errors) + ) + if sha256_bytes(head_bytes) != state["head_hash"]: + raise BimriError("state head hash does not match the accepted head revision.") + return entries + + +def load_exact_checkpoint_state(paths, key): + """Load a warm exact read from fixed files plus one selected cold binding.""" + if path_is_redirected(paths.state) or not paths.state.is_file(): + return None + try: + state_bytes = paths.state.read_bytes() + raw = json.loads(state_bytes.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + if not isinstance(raw, dict) or raw.get("bimri_version") != MEMORY_FORMAT_VERSION: + return None + try: + require_complete_v5_state(raw) + state = fresh_state() + state.update(raw) + if ( + isinstance(state.get("head_revision"), bool) + or not isinstance(state.get("head_revision"), int) + or state["head_revision"] < 0 + or not isinstance(state.get("head_hash"), str) + or not HASH_RE.fullmatch(state["head_hash"]) + ): + return None + # Warm reads are exactly as strict as every other surface about + # competing legacy authority; a refusal here degrades to the slow + # path, which raises the same lineage error. + require_read_only_legacy_lineage(paths, state) + witness = load_valid_audit_witness(paths, state) + if witness is None or load_audit_blocked_issues(paths): + return None + entries = validate_checkpoint_head(paths, state) + hot = [entry for entry in entries if entry.get("key") == key] + cold_record = state.get("cold_current", {}).get(key) + if hot and cold_record is not None: + raise BimriError( + f"memory key {key} exists in both hot and cold-current state." ) - if head_bytes is not None and sha256_bytes(head_bytes) != state["head_hash"]: - raise BimriError("state head hash does not match the accepted head revision.") - return True + if cold_record is not None: + validate_cold_current({key: cold_record}) + validate_selected_cold_archive_binding(paths, key, cold_record) + cold_entry_value = parse_entry_line(cold_record["raw_line"]) + if any(entry["id"] == cold_entry_value["id"] for entry in entries): + raise BimriError( + f"current memory ID {cold_entry_value['id']} is duplicated " + "across residency." + ) + pointer_error = pointer_validation_error(paths, cold_entry_value) + if pointer_error: + raise BimriError( + f"cold-current subject {key} pointer is invalid: {pointer_error}" + ) + return state, entries, witness + except (BimriError, OSError, UnicodeError, ValueError, TypeError): + return None def exact_archive_effect(paths, raw_line, reason="closed", proposal_id=None): @@ -6585,6 +9538,14 @@ def process_run_proposals(paths, state, run_id): return results +def run_has_authority_proposals(paths, run_id, log_content): + """Use the checkpoint plus one run log to classify a close operation.""" + witness = paths.validated_audit_witness + if witness is not None and run_id in witness.get("proposal_runs", []): + return True + return LOG_PROPOSAL_RE.search(log_content) is not None + + def build_index(paths, state): content = revision_path(paths, state["head_revision"]).read_text(encoding="utf-8") _, entries, errors = parse_hot(content) @@ -6656,19 +9617,6 @@ def build_index(paths, state): return len(safe_rows) -def rebuild_index_best_effort(paths, state): - try: - return build_index(paths, state) - except Exception as exc: - print( - "BIMRI WARNING: the durable operation succeeded, but the derived " - f"index could not be rebuilt: {exc}. Run `doctor` after repairing " - "the reported file.", - file=sys.stderr, - ) - return None - - def print_authority_recovery(issues): if not issues: return @@ -7111,23 +10059,6 @@ def print_brief( stale.append(rid) except (KeyError, ValueError): stale.append(rid) - for log in sorted(paths.logs.glob("R*.md")): - rid = log.stem - if rid in state["active_runs"] or not re.fullmatch(r"R\d+", rid): - continue - if log.is_symlink(): - stale.append(rid) - continue - try: - log_text = log.read_text(encoding="utf-8") - except (UnicodeDecodeError, OSError): - stale.append(rid) - continue - if not any( - line.strip().startswith("[CLOSED:") - for line in log_text.splitlines() - ): - stale.append(rid) stale = sorted(set(stale)) if stale: print("ORPHAN CANDIDATES (never auto-closed): " + ", ".join(stale)) @@ -7161,45 +10092,68 @@ def cmd_start(paths, actor, session=None): authority_issues=authority_issues, ) return existing - existing_numbers = [ - int(path.stem[1:]) for path in paths.logs.glob("R*.md") - if RUN_RE.fullmatch(path.stem) - ] + existing_numbers = [] + if paths.validated_audit_witness is None: + existing_numbers = [ + int(path.stem[1:]) for path in paths.logs.glob("R*.md") + if RUN_RE.fullmatch(path.stem) + ] number = max([state["run_count"]] + existing_numbers + [0]) + 1 while True: if number > 999999: raise BimriError("BIMRI has exhausted its six-digit run ID space.") run_id = f"R{number:06d}" log = run_log_path(paths, run_id) - try: - exclusive_write_text( - log, - f"# Run {run_id} | {now_iso()} | actor:{actor} | " - f"base:V{state['head_revision']:06d}\n\n" - "## Journal\n\n" - "\n\n" - "## Proposals\n\n" - "## Outcome\n\n", - ) - break - except FileExistsError: + if log.exists() or log.is_symlink(): number += 1 - state["run_count"] = number - state["run_dates"][run_id] = today() - state["active_runs"][run_id] = { + continue + break + started_at = now_iso() + log_text = ( + f"# Run {run_id} | {started_at} | actor:{actor} | " + f"base:V{state['head_revision']:06d}\n\n" + "## Journal\n\n" + "\n\n" + "## Proposals\n\n" + "## Outcome\n\n" + ) + post_state = copy.deepcopy(state) + post_state["run_count"] = number + post_state["run_dates"][run_id] = today() + post_state["active_runs"][run_id] = { "actor": actor, "session_key": skey, - "started_at": now_iso(), - "last_activity_at": now_iso(), + "started_at": started_at, + "last_activity_at": started_at, "base_revision": state["head_revision"], } if skey: - state["session_runs"][skey] = run_id - state["last_started_at"] = now_iso() - save_state(paths, state) + post_state["session_runs"][skey] = run_id + post_state["last_started_at"] = started_at + prepare_state_for_save(paths, post_state) + marker = begin_lifecycle_checkpoint_transition( + paths, + state, + post_state, + "start", + run_id, + log, + log_text, + ) + if marker is None: + # Recovery-degraded starts have no healthy verdict to carry. + exclusive_write_text(log, log_text) + save_state(paths, post_state) + else: + # State first: the durable marker can recreate the exact log if + # the process dies before its exclusive create. + save_state(paths, post_state) + exclusive_write_text(log, log_text) + state = post_state + refresh_audit_checkpoint_after_state_write(paths, state) print(f"BIMRI RUN HANDLE: {run_id}", flush=True) print_brief( paths, @@ -7208,7 +10162,6 @@ def cmd_start(paths, actor, session=None): conflicts=conflicts, authority_issues=authority_issues, ) - rebuild_index_best_effort(paths, state) return run_id @@ -7230,9 +10183,32 @@ def cmd_journal(paths, run_id, text, importance=3): state = load_or_initialize(paths) log = require_active_run(paths, state, run_id) entry_id = next_entry_id(paths, run_id) - append_line(log, f"[ID:{entry_id}] [I:{importance}] {text}") - state["active_runs"][run_id]["last_activity_at"] = now_iso() - save_state(paths, state) + line = f"[ID:{entry_id}] [I:{importance}] {text}" + post_state = copy.deepcopy(state) + post_state["active_runs"][run_id]["last_activity_at"] = now_iso() + prepare_state_for_save(paths, post_state) + marker = None + if audit_witness_write_state_hash(post_state) != ( + audit_witness_write_state_hash(state) + ): + current_log = log.read_text(encoding="utf-8") + marker = begin_lifecycle_checkpoint_transition( + paths, + state, + post_state, + "journal-state-maintenance", + run_id, + log, + current_log + line.rstrip() + "\n", + ) + if marker is None: + append_line(log, line) + save_state(paths, post_state) + else: + save_state(paths, post_state) + finish_lifecycle_log(paths, marker) + state = post_state + refresh_audit_checkpoint_after_state_write(paths, state) print(entry_id) return entry_id @@ -7458,6 +10434,21 @@ def cmd_propose(paths, args): # Preflight validates the complete current-head binding, exact effect, # authority policy, and deterministic residency before any durable write. if proposal is not None: + begin_authority_write( + paths, + state, + allow_degraded=True, + operation="propose", + run_id=run_id, + scope={ + "proposal_ids": [proposal_id], + "proposal_hashes": { + proposal_id: sha256_text( + json.dumps(proposal, indent=2, sort_keys=True) + "\n" + ) + }, + }, + ) append_line(log, f"[ID:{entry_id}] [I:{importance}] {rationale}") exclusive_write_text( proposal_path(paths, proposal_id), @@ -7470,6 +10461,7 @@ def cmd_propose(paths, args): ) state["active_runs"][run_id]["last_activity_at"] = now_iso() save_state(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) print(proposal_id) return proposal_id @@ -7510,13 +10502,19 @@ def cmd_sync(paths, run_id): with engine_lock(paths): state = load_or_initialize(paths) sync_generated_view(paths, state) - require_governance_healthy(paths, state) require_active_run(paths, state, run_id) + begin_authority_write( + paths, + state, + operation="sync", + run_id=run_id, + scope={"run_id": run_id}, + ) results = process_run_proposals(paths, state, run_id) state["active_runs"][run_id]["base_revision"] = state["head_revision"] state["active_runs"][run_id]["last_activity_at"] = now_iso() save_state(paths, state) - rebuild_index_best_effort(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) notices = new_conflict_notices(paths, state, results) held_notices = new_held_notices(results) for notice in notices: @@ -7545,7 +10543,6 @@ def cmd_close(paths, run_id=None, actor=None, session=None, with engine_lock(paths): state = load_or_initialize(paths) sync_generated_view(paths, state) - require_governance_healthy(paths, state) rid = resolve_run_from_args( state, run_id, @@ -7556,21 +10553,72 @@ def cmd_close(paths, run_id=None, actor=None, session=None, if rid is None: return None log = require_active_run(paths, state, rid) - results = process_run_proposals(paths, state, rid) text = log.read_text(encoding="utf-8") - if not any( + _conflicts, authority_issues = governance_snapshot(paths, state) + if authority_issues: + raise BimriError( + "authority recovery is required; shared-memory writes are paused: " + + " | ".join(authority_issues[:3]) + ) + authority_close = run_has_authority_proposals( + paths, rid, text + ) + if authority_close: + begin_authority_write( + paths, + state, + operation="close-authority-run", + run_id=rid, + scope={"run_id": rid}, + ) + results = process_run_proposals(paths, state, rid) + else: + results = [] + needs_close_marker = not any( line.strip().startswith("[CLOSED:") for line in text.splitlines() - ): + ) + close_at = now_iso() + if authority_close and needs_close_marker: append_line(log, f"[OUTCOME:{outcome}] {summary}") - append_line(log, f"[CLOSED:{rid} {now_iso()}]") - meta = state["active_runs"].pop(rid) + append_line(log, f"[CLOSED:{rid} {close_at}]") + post_state = copy.deepcopy(state) + meta = post_state["active_runs"].pop(rid) skey = meta.get("session_key") - if skey and state["session_runs"].get(skey) == rid: - del state["session_runs"][skey] - state["last_closed_at"] = now_iso() - save_state(paths, state) - rebuild_index_best_effort(paths, state) + if skey and post_state["session_runs"].get(skey) == rid: + del post_state["session_runs"][skey] + post_state["last_closed_at"] = close_at + prepare_state_for_save(paths, post_state) + if authority_close: + state = post_state + save_state(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) + else: + log_append = "" + if needs_close_marker: + log_append = ( + f"[OUTCOME:{outcome}] {summary}\n" + f"[CLOSED:{rid} {close_at}]\n" + ) + marker = begin_lifecycle_checkpoint_transition( + paths, + state, + post_state, + "close", + rid, + log, + text + log_append, + ) + if marker is None: + if needs_close_marker: + append_line(log, f"[OUTCOME:{outcome}] {summary}") + append_line(log, f"[CLOSED:{rid} {close_at}]") + save_state(paths, post_state) + else: + save_state(paths, post_state) + finish_lifecycle_log(paths, marker) + state = post_state + refresh_audit_checkpoint_after_state_write(paths, state) notices = new_conflict_notices(paths, state, results) held_notices = new_held_notices(results) for notice in notices: @@ -7766,7 +10814,7 @@ def recover_interrupted_authority(paths, state): validate_decision_effect(paths, state, accepted) changed = True if changed: - rebuild_index_best_effort(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) return changed @@ -7840,6 +10888,12 @@ def accepted_archive_effect_by_revision( or proposal["key"] != archived_entry.get("key") ): continue + if proposal_id == skip_validation_id: + # The decision under replay is still `applying` by definition. + # The archived row bound to its exact base line, plus the line + # already being absent from current residency, is the + # precommitted effect the replay finalizes. + return True path = decision_path(paths, proposal_id) if path.is_symlink() or not path.is_file(): continue @@ -8212,7 +11266,9 @@ def cmd_resolve(paths, conflict_id, choice, human_approved=False): with engine_lock(paths): state = load_or_initialize(paths) sync_generated_view(paths, state) - require_governance_healthy(paths, state) + require_governance_for_resolution_retry( + paths, state, conflict_id + ) cpath = conflict_path(paths, conflict_id) if not cpath.exists(): raise BimriError(f"unknown conflict: {conflict_id}") @@ -8245,8 +11301,11 @@ def cmd_resolve(paths, conflict_id, choice, human_approved=False): validate_conflict_candidate_decisions(paths, conflict, existing) if existing and existing["status"] == "resolved": validate_resolution_effect(paths, state, conflict, existing) + enter_authority_write_after_audit( + paths, state, operation="resolve", scope={"conflict_id": conflict_id} + ) finalize_conflict_decisions(paths, existing) - rebuild_index_best_effort(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) print( f"BIMRI: {conflict_id} already resolved as " f"{existing['choice']}." @@ -8328,9 +11387,12 @@ def cmd_resolve(paths, conflict_id, choice, human_approved=False): expected_conflict_id=conflict_id, ) validate_resolution_effect(paths, state, conflict, resolution) + enter_authority_write_after_audit( + paths, state, operation="resolve", scope={"conflict_id": conflict_id} + ) atomic_write_json(resolution_path, resolution) finalize_conflict_decisions(paths, resolution) - rebuild_index_best_effort(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) print(f"BIMRI: {conflict_id} resolved with {choice}.") return resolution if choice in proposal_ids: @@ -8407,9 +11469,15 @@ def cmd_resolve(paths, conflict_id, choice, human_approved=False): validate_resolution_effect( paths, state, conflict, resolution ) + enter_authority_write_after_audit( + paths, + state, + operation="resolve", + scope={"conflict_id": conflict_id}, + ) atomic_write_json(resolution_path, resolution) finalize_conflict_decisions(paths, resolution) - rebuild_index_best_effort(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) print(f"BIMRI: {conflict_id} resolved with {choice}.") return resolution @@ -8457,6 +11525,9 @@ def cmd_resolve(paths, conflict_id, choice, human_approved=False): expected_conflict_id=conflict_id, ) validate_resolution_state_bounds(paths, state, resolution) + enter_authority_write_after_audit( + paths, state, operation="resolve", scope={"conflict_id": conflict_id} + ) atomic_write_json(resolution_path, resolution) try: if choice in {"current", "dismiss"}: @@ -8487,6 +11558,11 @@ def cmd_resolve(paths, conflict_id, choice, human_approved=False): ) validate_resolution_state_bounds(paths, state, resolution) atomic_write_json(resolution_path, resolution) + # Retain W0/M0 as evidence. The epoch and still-live resolve + # transition make W0 unreadable until this exact conflict retries. + paths.validated_audit_witness = None + paths.authority_write_mode = False + paths.authority_write_audit_healthy = False raise resolution.update({ "status": "resolved", @@ -8502,7 +11578,7 @@ def cmd_resolve(paths, conflict_id, choice, human_approved=False): validate_resolution_effect(paths, state, conflict, resolution) atomic_write_json(resolution_path, resolution) finalize_conflict_decisions(paths, resolution) - rebuild_index_best_effort(paths, state) + refresh_audit_witness_after_trusted_write(paths, state) print(f"BIMRI: {conflict_id} resolved with {choice}.") return resolution @@ -8579,6 +11655,10 @@ def validate_authority_record_data( data, expected_conflict_id=record_id, ) + if resolution["status"] == "failed": + raise BimriError( + "resolution status is failed; explicit retry is required." + ) validate_resolution_state_bounds(paths, state, resolution) return resolution cpath = conflict_path(paths, record_id) @@ -8595,6 +11675,10 @@ def validate_authority_record_data( conflict=conflict, expected_conflict_id=record_id, ) + if resolution["status"] == "failed": + raise BimriError( + "resolution status is failed; explicit retry is required." + ) validate_resolution_effect(paths, state, conflict, resolution) if verify_dependencies: validate_conflict_candidate_decisions( @@ -8617,6 +11701,18 @@ def cmd_quarantine_authority( ) with engine_lock(paths): state = load_or_initialize(paths) + blocked_prior = load_audit_blocked_prior_witness(paths) + recovery_prior = blocked_prior or load_sealed_audit_witness(paths) + recovery_prior_manifest = ( + load_audit_blocked_prior_manifest(paths) + if blocked_prior is not None + else None + ) + recovery_baseline_issues = [] + if recovery_prior is not None and blocked_prior is None: + _manifest, _run_facts, recovery_baseline_issues = ( + verify_intact_audit_evidence(paths, state, recovery_prior) + ) path = authority_record_path(paths, kind, record_id) reviewed_link_target = None original_type = "file" @@ -8693,6 +11789,23 @@ def cmd_quarantine_authority( "authority record is valid; BIMRI refuses " "to quarantine it as corruption." ) + # Owner-authorized quarantine is itself the recovery boundary. Preserve + # W0/M0 before the first recovery/evidence byte changes, including when + # this command is the first operation to discover unrelated drift. + if recovery_prior is not None: + record_audit_blocked( + paths, + recovery_prior, + recovery_baseline_issues or [ + f"owner-authorized quarantine pending for {kind} {record_id}" + ], + prior_manifest=recovery_prior_manifest, + ) + state["_audit_epoch"] = state_audit_epoch(state) + 1 + save_state(paths, state) + paths.validated_audit_witness = None + else: + discard_audit_witness(paths) digest = sha256_bytes(raw) recovery = paths.recovery / ( f"authority-{kind}-{record_id}-{digest}.json" @@ -8772,6 +11885,7 @@ def validate_authority_replacement_graph( "proposals", "decisions", "revisions", + "archive", "conflicts", "resolutions", "recovery", @@ -8792,7 +11906,12 @@ def validate_authority_replacement_graph( shadow_paths, kind, record_id ) atomic_write_json(shadow_target, replacement_data) - _, issues = governance_snapshot(shadow_paths, state) + _, issues = governance_snapshot( + shadow_paths, + state, + use_witness=False, + write_witness=False, + ) semantic_issues = [ issue for issue in issues if "quarantined" not in issue.lower() ] @@ -8880,6 +11999,43 @@ def matching_restore_receipt( return matches[-1] if matches else None +def owner_repair_session_paths(paths): + """Paths belonging to the open owner-repair session, all quarantines included. + + A multi-record repair quarantines several records before restoring them + one at a time. Each sibling stub, its preserved evidence, and its restore + receipts are part of the same owner-approved surgery, so the strict + baseline comparison must not read them as unrelated drift. + """ + allowed = set() + for directory in ( + paths.proposals, paths.decisions, paths.conflicts, paths.resolutions + ): + try: + children = list(directory.glob("*.json")) + except OSError: + continue + for path in children: + if path.is_symlink() or not path.is_file(): + continue + try: + record = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, ValueError): + continue + if is_quarantine_stub(record): + allowed.add(path.relative_to(paths.root).as_posix()) + try: + children = list(paths.recovery.iterdir()) + except OSError: + children = [] + for path in children: + if path.is_symlink() or not path.is_file(): + continue + if path.name.startswith("authority-"): + allowed.add(path.relative_to(paths.root).as_posix()) + return allowed + + def cmd_restore_authority( paths, kind, record_id, replacement, human_approved=False ): @@ -8898,6 +12054,11 @@ def cmd_restore_authority( replacement_hash = sha256_bytes(replacement_bytes) with engine_lock(paths): state = load_or_initialize(paths) + repair_prior_witness = ( + load_audit_blocked_prior_witness(paths) + or load_sealed_audit_witness(paths) + ) + repair_prior_manifest = load_audit_blocked_prior_manifest(paths) path = authority_record_path(paths, kind, record_id) target_path = path.relative_to(paths.root).as_posix() current = read_json_strict(path, path.name) @@ -8913,7 +12074,30 @@ def cmd_restore_authority( validate_authority_replacement_graph( paths, state, kind, record_id, replacement_data ) - _, remaining_issues = governance_snapshot(paths, state) + receipt_file = restore_receipt_path( + paths, + kind, + record_id, + receipt["original_sha256"], + replacement_hash, + ) + allowed_repair_paths = { + target_path, + receipt["recovery_file"], + receipt_file.relative_to(paths.root).as_posix(), + } | owner_repair_session_paths(paths) + _, remaining_issues = governance_snapshot( + paths, + state, + use_witness=False, + strict_prior_comparison=True, + audit_blocked=True, + recover_blocked=True, + prior_witness_override=repair_prior_witness, + prior_manifest_override=repair_prior_manifest, + allowed_manifest_paths=allowed_repair_paths, + allow_audit_epoch_advance=True, + ) if remaining_issues: print( f"BIMRI: {kind} {record_id} already restored from an " @@ -8973,9 +12157,37 @@ def cmd_restore_authority( target_path, ) else: + state["_audit_epoch"] = state_audit_epoch(state) + 1 + save_state(paths, state) + paths.validated_audit_witness = None exclusive_write_bytes(receipt_path, canonical_json_bytes(receipt)) + if receipt_path.exists() and state_audit_epoch(state) == ( + repair_prior_witness.get("audit_epoch", -1) + if repair_prior_witness is not None else -1 + ): + # A pre-existing receipt can still precede the first replacement + # byte (for example, a caught failure after receipt publication). + state["_audit_epoch"] = state_audit_epoch(state) + 1 + save_state(paths, state) + paths.validated_audit_witness = None atomic_write_json(path, replacement_data) - _, remaining_issues = governance_snapshot(paths, state) + allowed_repair_paths = { + target_path, + stub["recovery_file"], + receipt_path.relative_to(paths.root).as_posix(), + } | owner_repair_session_paths(paths) + _, remaining_issues = governance_snapshot( + paths, + state, + use_witness=False, + strict_prior_comparison=True, + audit_blocked=True, + recover_blocked=True, + prior_witness_override=repair_prior_witness, + prior_manifest_override=repair_prior_manifest, + allowed_manifest_paths=allowed_repair_paths, + allow_audit_epoch_advance=True, + ) unexpected_issues = [ issue for issue in remaining_issues @@ -9038,6 +12250,85 @@ def retention_order(entry, state): ) +def exact_current_recall_records(paths, state, key, hot_entries=None): + """Read one exact subject from current hot/cold storage only.""" + if hot_entries is None: + content = revision_path( + paths, state["head_revision"] + ).read_text(encoding="utf-8") + _, hot_entries, errors = parse_hot(content) + if errors: + raise BimriError( + "cannot recall malformed hot memory: " + "; ".join(errors) + ) + records = [] + for entry in hot_entries: + if entry.get("key") != key: + continue + records.append({ + "location": "HOT", + "key": key, + "id": entry["id"], + "detail": entry.get("text", ""), + "reason": "current", + "trust": entry.get("trust", ""), + "source": entry.get("source", ""), + }) + cold = state.get("cold_current", {}).get(key) + if cold is not None: + entry = parse_entry_line(cold["raw_line"]) + records.append({ + "location": "COLD", + "key": key, + "id": entry["id"], + "detail": entry.get("text", ""), + "reason": "current", + "trust": entry.get("trust", ""), + "source": entry.get("source", ""), + }) + return records + + +def held_recall_records(paths, state, key=None): + """Build held candidates live from authority, optionally for one key.""" + content = revision_path( + paths, state["head_revision"] + ).read_text(encoding="utf-8") + _, hot_entries, errors = parse_hot(content) + if errors: + raise BimriError("cannot recall malformed hot memory: " + "; ".join(errors)) + current_by_key = { + entry["key"]: entry for entry in hot_entries if entry.get("key") + } + for current_key, cold in state.get("cold_current", {}).items(): + current_by_key[current_key] = parse_entry_line(cold["raw_line"]) + records = [] + for path in sorted(paths.decisions.glob("R*-Q*.json")): + if path.is_symlink() or not path.is_file(): + continue + decision_data = read_json_strict(path, path.name) + if decision_data.get("outcome") != "held": + continue + decision = validate_decision(decision_data, path.stem) + proposal = authority_proposal(paths, state, path.stem) + if key is not None and proposal["key"] != key: + continue + if proposal_equivalent( + proposal, current_by_key.get(proposal["key"]) + ): + continue + records.append({ + "location": "HELD", + "key": proposal["key"], + "id": proposal["proposal_id"], + "detail": proposal["text"], + "reason": decision["reason"], + "trust": proposal.get("trust", ""), + "source": proposal.get("source", ""), + }) + return records + + def recall_records(paths, state): content = revision_path( paths, state["head_revision"] @@ -9143,14 +12434,37 @@ def cmd_recall(paths, key=None, query=None, history=True, limit=20): if search_text is not None and (not search_tokens or len(search_tokens) > 32): raise BimriError("recall query must contain 1 to 32 lexical tokens.") with existing_store_lock(paths): - state = load_current_state_read_only(paths) - _conflicts, authority_issues = governance_snapshot(paths, state) + authority_issues = [] + checkpoint = None + if exact_key is not None and not history: + checkpoint = load_exact_checkpoint_state(paths, exact_key) + if checkpoint is None and reconcile_engine_checkpoint_for_exact_read( + paths + ): + checkpoint = load_exact_checkpoint_state(paths, exact_key) + if checkpoint is not None: + state, hot_entries, _witness = checkpoint + records = exact_current_recall_records( + paths, state, exact_key, hot_entries=hot_entries + ) + else: + state = load_current_state_read_only(paths) + _conflicts, authority_issues = governance_snapshot( + paths, + state, + use_witness=False, + ) + if exact_key is not None and not history: + records = exact_current_recall_records( + paths, state, exact_key + ) + else: + records = recall_records(paths, state) if authority_issues: raise BimriError( "recall stopped because authority recovery is required: " + "; ".join(authority_issues[:3]) ) - records = recall_records(paths, state) matches = [] for record in records: if exact_key is not None: @@ -9205,7 +12519,7 @@ def cmd_maintain(paths): with engine_lock(paths): state = load_or_initialize(paths) sync_generated_view(paths, state) - require_governance_healthy(paths, state) + require_full_governance_healthy(paths, state) content = revision_path(paths, state["head_revision"]).read_text(encoding="utf-8") _, entries, errors = parse_hot(content) if errors: @@ -9238,6 +12552,8 @@ def cmd_maintain(paths): ) else: print("Residency is healthy; no pressure action is needed.") + index_count = build_index(paths, state) + print(f"BIMRI: index rebuilt with {index_count} rows.") def cmd_review(paths, conflict_id=None, show_all=False, offset=0, limit=20): @@ -9253,7 +12569,11 @@ def cmd_review(paths, conflict_id=None, show_all=False, offset=0, limit=20): ) with existing_store_lock(paths): state = load_current_state_read_only(paths) - conflicts, authority_issues = governance_snapshot(paths, state) + conflicts, authority_issues = governance_snapshot( + paths, + state, + use_witness=False, + ) if authority_issues: print_authority_recovery(authority_issues) return 1 @@ -9328,7 +12648,11 @@ def cmd_status(paths): with engine_lock(paths): state = load_or_initialize(paths) sync_generated_view(paths, state) - conflicts, authority_issues = governance_snapshot(paths, state) + conflicts, authority_issues = governance_snapshot( + paths, + state, + use_witness=False, + ) content = revision_path( paths, state["head_revision"] ).read_text(encoding="utf-8") @@ -9510,6 +12834,7 @@ def load_current_state_read_only(paths, accepted_versions=None): state = validate_state( merged, accepted_versions=accepted_versions ) + require_read_only_legacy_lineage(paths, state) head = revision_path(paths, state["head_revision"]) if path_is_redirected(head) or not head.is_file(): raise BimriError( @@ -9578,6 +12903,33 @@ def doctor_errors( "recovery evidence is damaged: " + issue for issue in restore_receipt_issues(paths) ) + drift_total, drift_records, drift_damaged = audit_drift_summary( + paths, limit=1 + ) + if drift_records: + newest = drift_records[0] + newest_line = "; ".join(newest.get("reasons", [])[:2]) + warnings.append( + f"{drift_total} unexplained-drift receipt(s) on record; newest " + f"valid ({newest.get('created_at', 'unknown time')}): " + f"{newest_line}" + ) + if newest.get("truncated"): + warnings.append( + "the newest drift receipt is truncated; its complete delta " + "is pinned in a validated attachment beside it" + ) + if drift_damaged: + warnings.append( + f"{len(drift_damaged)} drift receipt(s) failed validation and " + "are untrusted damaged evidence: " + + ", ".join(sorted(drift_damaged)[:3]) + ) + if paths.last_audit_drift: + warnings.append( + "this audit observed checkpoint drift: " + + "; ".join(paths.last_audit_drift[:3]) + ) if governance_issues: errors.extend( "authority recovery needed: " + issue @@ -9826,13 +13178,39 @@ def read_only_store_audit(paths, accepted_versions=None): state = load_current_state_read_only( paths, accepted_versions=accepted_versions ) - _, governance_issues = governance_snapshot(paths, state) + _, governance_issues = governance_snapshot( + paths, + state, + use_witness=False, + write_witness=False, + audit_blocked=True, + ) + transition_note = None + if paths.audit_transition.exists() or paths.audit_transition.is_symlink(): + try: + marker = load_audit_transition(paths) + detail = ( + f"{marker['operation']} {marker.get('run_id') or ''}" + ).strip() + # A loadable in-flight marker is recoverable state, not damage: + # the next writable command reconciles it. Read-only audit + # reports it without failing the store. + transition_note = ( + "audit transition is incomplete; the next writable command " + f"reconciles it: {detail}" + ) + except (BimriError, OSError, UnicodeError, ValueError, TypeError) as exc: + governance_issues.insert( + 0, f"audit transition marker is invalid: {exc}" + ) errors, warnings = doctor_errors( paths, state, governance_issues=governance_issues, repair_generated_view=False, ) + if transition_note: + warnings.insert(0, transition_note) return state, errors, warnings, governance_issues @@ -9847,12 +13225,20 @@ def cmd_doctor(paths, read_only=False): else: with engine_lock(paths): state = load_or_initialize(paths) + # Doctor proves the store from authority even when a matching + # derived witness exists. sync_error = None try: sync_generated_view(paths, state) except (BimriError, OSError, UnicodeError) as exc: sync_error = str(exc) - _, governance_issues = governance_snapshot(paths, state) + _, governance_issues = governance_snapshot( + paths, + state, + use_witness=False, + audit_blocked=True, + recover_blocked=True, + ) if sync_error: governance_issues.insert( 0, "generated view recovery failed: " + sync_error @@ -10055,7 +13441,7 @@ def rendered_hooks_snippet(template, python_executable): subcommand, ]: raise KeyError(event) - if command.get("timeout") != 15: + if command.get("timeout") != HOOK_TIMEOUT_SECONDS: raise KeyError(event) command["command"] = python_executable rendered += 1 @@ -11091,6 +14477,7 @@ def code_update_receipt_identity(manifest): ) if identity not in { ("5.0.3", V5_0_2_VERSION), + (V5_1_0_ENGINE_VERSION, MEMORY_FORMAT_VERSION), (ENGINE_VERSION, MEMORY_FORMAT_VERSION), }: raise BimriError( @@ -11408,8 +14795,15 @@ def _recover_prepared_code_updates(paths, policy, backup_root): if status == "prepared-for-authority-activation": prepared_projection = dict(manifest) prepared_projection["status"] = "prepared" + receipt_engine, receipt_memory = code_update_receipt_identity( + manifest + ) _validate_prepared_code_update_manifest( - paths, manifest_path.parent, prepared_projection + paths, + manifest_path.parent, + prepared_projection, + expected_engine=receipt_engine, + expected_memory=receipt_memory, ) backup = _safe_code_update_backup_file( manifest_path.parent, manifest.get("state_backup") @@ -12267,7 +15661,28 @@ def build_parser(): return parser +def reconfigure_console_streams(): + """Emit UTF-8 on every host. + + Memory text is UTF-8 on disk. On Windows a piped or redirected stdout + defaults to the ANSI code page, so printing an entry that carried a + character outside it raised UnicodeEncodeError from print() and the + command died with exit 2 (reproduced 2026-09-02). Hook and tool + consumers read UTF-8; a stream that cannot be reconfigured is left + alone. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is None: + continue + try: + reconfigure(encoding="utf-8", errors="backslashreplace") + except (ValueError, OSError, AttributeError, TypeError): + pass + + def main(argv=None): + reconfigure_console_streams() argv = list(sys.argv[1:] if argv is None else argv) if ( len(argv) == 2 @@ -12368,7 +15783,7 @@ def main(argv=None): with engine_lock(paths): state = load_or_initialize(paths) sync_generated_view(paths, state) - require_governance_healthy(paths, state) + require_full_governance_healthy(paths, state) count = build_index(paths, state) print(f"BIMRI: index rebuilt with {count} rows.") elif command == "maintain": @@ -12377,7 +15792,7 @@ def main(argv=None): with engine_lock(paths): state = load_or_initialize(paths) sync_generated_view(paths, state) - require_governance_healthy(paths, state) + require_full_governance_healthy(paths, state) build_index(paths, state) migration_errors, migration_warnings = doctor_errors( paths, state diff --git a/hooks-example.json b/hooks-example.json index 349fe60..fcbeaad 100644 --- a/hooks-example.json +++ b/hooks-example.json @@ -11,7 +11,7 @@ "${CLAUDE_PROJECT_DIR}/bimri-engine.py", "hook-start" ], - "timeout": 15 + "timeout": 90 } ] } @@ -27,7 +27,7 @@ "${CLAUDE_PROJECT_DIR}/bimri-engine.py", "hook-close" ], - "timeout": 15 + "timeout": 90 } ] } diff --git a/tests/benchmark_v511.py b/tests/benchmark_v511.py new file mode 100644 index 0000000..aa03fc6 --- /dev/null +++ b/tests/benchmark_v511.py @@ -0,0 +1,260 @@ +"""Reproducible latency benchmark for the v5.1.1 read/write paths. + +Not collected by unittest discovery: run it directly. + + tests/benchmark_v511.py --store + tests/benchmark_v511.py --store --grow 200 + +The store at --store is never touched: the benchmark copies it into a +disposable directory and measures there. --grow N first enlarges the copy by +driving the engine itself (N extra runs, each with a journal line and one +accepted proposal), which keeps every hash chain valid — synthesizing files +by hand would only benchmark a broken store. Growing is slow by design; +expect minutes for hundreds of runs. + +Reported per operation: p50 and p95 wall-clock over --samples CLI +invocations (interpreter startup included, matching what an agent pays). +The cold first operation that seeds the checkpoint is reported separately. +""" + +import argparse +import json +import math +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +REPOSITORY = Path(__file__).resolve().parents[1] +ENGINE = REPOSITORY / "bimri-engine.py" + +# Canonical release ceilings (--gate default). Deliberately generous: they +# catch an order-of-magnitude regression (the class this release fixed) on +# ordinary shared hardware, not millisecond noise. +CANONICAL_GATES = { + "cold_first_read_ms": 90000.0, + "warm_get_p50_ms": 2000.0, + "start_p50_ms": 2500.0, + "journal_p50_ms": 2500.0, + "propose_p50_ms": 6000.0, + "sync_p50_ms": 6000.0, + "close_p50_ms": 6000.0, +} + + +def cli(root, *arguments, check=True): + result = subprocess.run( + [sys.executable, str(ENGINE), "--root", str(root), *map(str, arguments)], + text=True, + capture_output=True, + timeout=600, + ) + if check and result.returncode != 0: + raise SystemExit( + f"benchmark command failed: {arguments}\n" + f"{result.stdout}\n{result.stderr}" + ) + return result + + +def timed(root, *arguments, check=True): + began = time.perf_counter() + cli(root, *arguments, check=check) + return (time.perf_counter() - began) * 1000.0 + + +RESULTS = {} + + +def summarize(label, samples, metric=None): + ordered = sorted(samples) + p50 = statistics.median(ordered) + p95 = ordered[max(0, int(round(0.95 * len(ordered))) - 1)] + print(f"{label:<28} p50 {p50:8.1f} ms p95 {p95:8.1f} ms n={len(ordered)}") + if metric: + RESULTS[f"{metric}_p50_ms"] = p50 + RESULTS[f"{metric}_p95_ms"] = p95 + + +def apply_gates(specification): + """Fail (exit 1) when any metric breaches its stated ceiling. + + Specification: "default" for the committed CANONICAL_GATES, or + comma-separated metric=max_ms pairs, for example + --gate "warm_get_p50_ms=1000,start_p50_ms=2000,propose_p50_ms=3000". + Known metrics: cold_first_read_ms plus _p50_ms/_p95_ms for + warm_get, start, journal, propose, sync, close. A non-finite measured + value or ceiling is a breach, never a bypass. + """ + if specification.strip() == "default": + gates = dict(CANONICAL_GATES) + else: + gates = {} + breaches = [] + for pair in filter( + None, (part.strip() for part in specification.split(",")) + ): + metric, _, ceiling = pair.partition("=") + try: + gates[metric.strip()] = float(ceiling) + except ValueError: + breaches.append(f"invalid gate ceiling: {pair}") + if breaches: + for breach in breaches: + print(f"GATE FAILED: {breach}") + raise SystemExit(1) + breaches = [] + for metric, maximum in gates.items(): + if metric not in RESULTS: + breaches.append(f"unknown gate metric: {metric}") + continue + value = RESULTS[metric] + if not math.isfinite(maximum) or not math.isfinite(value): + breaches.append(f"{metric} is not a finite measurement") + continue + if not value <= maximum: + breaches.append( + f"{metric} {value:.1f} ms exceeds {maximum:.1f} ms" + ) + if breaches: + for breach in breaches: + print(f"GATE FAILED: {breach}") + raise SystemExit(1) + print("all gates passed") + + +def pick_hot_key(root): + state = json.loads( + (Path(root) / ".bimri" / "state.json").read_text("utf-8") + ) + hot = (Path(root) / "bimri.md").read_text("utf-8") + for line in hot.splitlines(): + if "[K:" in line: + return line.split("[K:", 1)[1].split("]", 1)[0] + cold = state.get("cold_current") or {} + if cold: + return sorted(cold)[0] + raise SystemExit("store has no current keys to benchmark") + + +def start_run(root, actor): + result = cli(root, "start", "--actor", actor) + for token in result.stdout.split(): + if token.startswith("R") and token[1:7].isdigit(): + return token[:7] + raise SystemExit(f"could not parse run handle from: {result.stdout!r}") + + +def grow(root, runs): + for index in range(runs): + run_id = start_run(root, f"bench-grow-{index:05d}") + cli( + root, "journal", "--run", run_id, "--importance", "3", + "--text", f"Benchmark growth journal {index}.", + ) + cli( + root, "propose", "--run", run_id, "--operation", "set", + "--tier", "2", "--key", f"bench.grow.{index:05d}", + "--text", f"Benchmark growth subject {index}.", + "--source", "agent", "--trust", "working", "--new-subject", + ) + cli(root, "sync", "--run", run_id) + cli( + root, "close", "--run", run_id, "--outcome", "success", + "--summary", "benchmark growth run", + ) + if index and index % 25 == 0: + print(f" grew {index}/{runs} runs", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--store", default="", + help="store root to copy (omit with --init for a fresh store)", + ) + parser.add_argument( + "--init", action="store_true", + help="build a small fresh store instead of copying --store", + ) + parser.add_argument("--samples", type=int, default=9) + parser.add_argument( + "--grow", type=int, default=0, + help="extra engine-driven runs to add to the copy before measuring", + ) + parser.add_argument( + "--gate", default="", + help="comma-separated metric=max_ms ceilings; breach exits 1", + ) + arguments = parser.parse_args() + + workspace = Path(tempfile.mkdtemp(prefix="bimri-bench-")) + root = workspace / "store" + root.mkdir() + if arguments.init: + print("building a fresh benchmark store...") + grow(root, 3) + else: + if not arguments.store: + raise SystemExit("provide --store, or --init for a fresh store") + source = Path(arguments.store).resolve() + if not (source / ".bimri").is_dir(): + raise SystemExit(f"{source} does not contain a .bimri store") + if (source / "bimri.md").exists(): + shutil.copy2(source / "bimri.md", root / "bimri.md") + shutil.copytree(source / ".bimri", root / ".bimri") + print(f"benchmark copy: {root}") + + if arguments.grow: + print(f"growing the copy by {arguments.grow} engine-driven runs...") + grow(root, arguments.grow) + + (root / ".bimri" / "audit-witness.json").unlink(missing_ok=True) + key = pick_hot_key(root) + print(f"benchmark key: {key}") + + cold = timed(root, "get", "--key", key) + print(f"{'cold first read (seeds)':<28} once {cold:8.1f} ms") + RESULTS["cold_first_read_ms"] = cold + + reads = [ + timed(root, "get", "--key", key) + for _ in range(arguments.samples) + ] + summarize("warm get --key", reads, metric="warm_get") + + starts, journals, proposes, syncs, closes = [], [], [], [], [] + for index in range(arguments.samples): + began = time.perf_counter() + run_id = start_run(root, f"bench-{index:03d}") + starts.append((time.perf_counter() - began) * 1000.0) + journals.append(timed( + root, "journal", "--run", run_id, "--importance", "3", + "--text", f"benchmark journal {index}", + )) + proposes.append(timed( + root, "propose", "--run", run_id, "--operation", "set", + "--tier", "2", "--key", f"bench.probe.{index:03d}", + "--text", f"benchmark subject {index}", + "--source", "agent", "--trust", "working", "--new-subject", + )) + syncs.append(timed(root, "sync", "--run", run_id)) + closes.append(timed( + root, "close", "--run", run_id, "--outcome", "success", + "--summary", "benchmark run", + )) + summarize("warm start", starts, metric="start") + summarize("journal", journals, metric="journal") + summarize("propose", proposes, metric="propose") + summarize("sync", syncs, metric="sync") + summarize("close (authority)", closes, metric="close") + print(f"disposable copy retained at {workspace}") + if arguments.gate: + apply_gates(arguments.gate) + + +if __name__ == "__main__": + main() diff --git a/tests/crash_worker.py b/tests/crash_worker.py index f14155f..c6d05e2 100644 --- a/tests/crash_worker.py +++ b/tests/crash_worker.py @@ -35,6 +35,7 @@ def main(): root = Path(sys.argv[3]).resolve() command = sys.argv[4:] engine = load_engine(engine_path) + graph_replay_observed = None no_memory_temp_modes = { "code_update_forbid_bdir_temp", @@ -263,6 +264,15 @@ def crash_after_resolution_effect(*args, **kwargs): return result engine.apply_proposal = crash_after_resolution_effect + elif mode == "resolution_fail_during_force_apply": + original = engine.apply_proposal + + def fail_during_resolution_effect(*args, **kwargs): + if kwargs.get("force") and kwargs.get("human_confirmed"): + raise engine.BimriError("forced resolution effect failure") + return original(*args, **kwargs) + + engine.apply_proposal = fail_during_resolution_effect elif mode == "resolution_crash_after_applying_record": original = engine.atomic_write_json resolution_dir = root / ".bimri" / "resolutions" @@ -353,6 +363,100 @@ def fail_hot_write(path, content): return original(path, content) engine.atomic_write_text = fail_hot_write + elif mode == "witness_crash_before_replace": + original = engine.os.replace + witness = (root / ".bimri" / "audit-witness.json").resolve() + + def crash_before_witness_replace(source, destination, *args, **kwargs): + if Path(destination).resolve() == witness: + os._exit(110) + return original(source, destination, *args, **kwargs) + + engine.os.replace = crash_before_witness_replace + elif mode == "blocked_restore_crash_between_manifest_and_witness": + original_replace = engine.os.replace + manifest = (root / ".bimri" / "audit-manifest.json").resolve() + witness = (root / ".bimri" / "audit-witness.json").resolve() + publication_marker = root / ".test-restore-manifest-published" + + def crash_after_manifest_before_witness( + source, destination, *args, **kwargs + ): + resolved = Path(destination).resolve() + if resolved == witness and publication_marker.exists(): + os._exit(113) + result = original_replace(source, destination, *args, **kwargs) + if resolved == manifest: + publication_marker.write_text( + "manifest published\n", encoding="utf-8" + ) + return result + + engine.os.replace = crash_after_manifest_before_witness + elif mode == "exact_recall_forbid_global_collection": + + def reject_global_collection(*_args, **_kwargs): + raise AssertionError( + "exact-current recall attempted global history collection" + ) + + engine.recall_records = reject_global_collection + elif mode == "valid_witness_forbid_full_audit": + + def reject_full_authority_audit(*_args, **_kwargs): + raise AssertionError( + "valid audit witness fell back to a full governance audit" + ) + + engine.authority_storage_issues = reject_full_authority_audit + engine.audit_witness_manifest = reject_full_authority_audit + engine.audit_witness_run_facts = reject_full_authority_audit + engine.load_audit_manifest_evidence = reject_full_authority_audit + engine.scan_open_conflicts = reject_full_authority_audit + engine.held_recall_records = reject_full_authority_audit + elif mode == "valid_checkpoint_forbid_graph_replay": + + def reject_graph_replay(*_args, **_kwargs): + raise AssertionError( + "valid authority checkpoint triggered global graph replay" + ) + + engine.authority_storage_issues = reject_graph_replay + elif mode == "missing_checkpoint_require_graph_replay": + original = engine.authority_storage_issues + graph_replay_observed = {"called": False} + + def observe_graph_replay(*args, **kwargs): + graph_replay_observed["called"] = True + return original(*args, **kwargs) + + engine.authority_storage_issues = observe_graph_replay + elif mode == "journal_hold_lock": + original = engine.append_line + signal = root / ".test-writer-lock-held" + release = root / ".test-release-writer-lock" + held = False + + def hold_journal_write(path, line): + nonlocal held + candidate = Path(path) + if ( + not held + and candidate.parent == root / ".bimri" / "log" + and str(line).startswith("[ID:") + ): + held = True + signal.write_text("writer lock held\n", encoding="utf-8") + deadline = time.monotonic() + 20 + while not release.exists(): + if time.monotonic() >= deadline: + raise RuntimeError( + "timed out waiting to release journal writer lock" + ) + time.sleep(0.005) + return original(path, line) + + engine.append_line = hold_journal_write elif mode == "index_failure": def fail_index(_paths, _state): @@ -735,7 +839,14 @@ def fake_python_check(arguments, **_kwargs): else: raise SystemExit("unknown fault mode: " + mode) - return engine.main(["--root", str(root)] + command) + result = engine.main(["--root", str(root)] + command) + if graph_replay_observed is not None and not graph_replay_observed["called"]: + print( + "missing or invalid checkpoint bypassed global graph replay", + file=sys.stderr, + ) + return 119 + return result if __name__ == "__main__": diff --git a/tests/test_bimri_engine.py b/tests/test_bimri_engine.py index 14632cd..fd85bf2 100644 --- a/tests/test_bimri_engine.py +++ b/tests/test_bimri_engine.py @@ -351,7 +351,7 @@ def assert_installed_runtime_binding(self, target, agents_text=None): "engine_path": engine_path, "host_bound": True, "python_executable": python_executable, - "version": "5.1.0", + "version": "5.1.1", }) template = json.loads( @@ -376,7 +376,7 @@ def assert_installed_runtime_binding(self, target, agents_text=None): "${CLAUDE_PROJECT_DIR}/bimri-engine.py", subcommand, ]) - self.assertEqual(command["timeout"], 15) + self.assertEqual(command["timeout"], 90) self.assertNotIn("__BIMRI_VERIFIED_PYTHON__", json.dumps(hooks)) self.assertFalse((target / ".claude").exists()) @@ -772,7 +772,11 @@ def snapshot(): return { path.relative_to(self.root).as_posix(): path.read_bytes() for path in self.root.rglob("*") - if path.is_file() and not path.is_symlink() + if ( + path.is_file() + and not path.is_symlink() + and path != self.root / ".bimri" / "audit-witness.json" + ) } before_retry = snapshot() @@ -893,7 +897,7 @@ def test_v503_preflight_receipt_binds_current_head_after_unrelated_change(self): receipt = proposal["preflight_receipt"] self.assertEqual(proposal["bimri_version"], "5.1.0") self.assertEqual(proposal["base_revision"], state["head_revision"]) - self.assertEqual(receipt["engine_release"], "5.1.0") + self.assertEqual(receipt["engine_release"], "5.1.1") self.assertEqual(receipt["observed_head_revision"], state["head_revision"]) self.assertEqual(receipt["observed_head_hash"], state["head_hash"]) self.assertEqual(receipt["observed_key_hash"], "absent") @@ -1082,9 +1086,14 @@ def test_pending_proposal_does_not_reserve_a_soft_tier_target(self): "working", ) second_id = PROPOSAL_RE.search(second.stdout).group(0) + # Directory order is filesystem-dependent (unsorted on ext4, sorted on + # NTFS); compare the set of proposal files in a fixed order. self.assertEqual( - [path.stem for path in (self.root / ".bimri" / "proposals").glob("*.json")], - [first, second_id], + sorted( + path.stem + for path in (self.root / ".bimri" / "proposals").glob("*.json") + ), + sorted([first, second_id]), ) self.assertNotEqual(before, { path.relative_to(self.root).as_posix(): path.read_bytes() @@ -2118,8 +2127,9 @@ def test_resolution_and_decision_finalization_are_crash_idempotent(self): crash_candidate, "--human-approved", ) - self.assertIn( - f"already resolved as {crash_candidate}", recovered.stdout + self.assertRegex( + recovered.stdout, + f"(already resolved as|resolved with) {crash_candidate}", ) self.assertEqual(self.state()["head_revision"], revision_with_effect) recovered_resolution = json.loads( @@ -2579,8 +2589,10 @@ def test_applying_decision_replay_recovers_close_and_touch_effects(self): ) close_replay = self.cli("sync", "--run", closer) + # Finalizing this run's own interrupted apply counts as applied; the + # asserts below prove no second effect landed. self.assertIn( - "applied 0, held candidates 0, already satisfied/no change 0", + "applied 1, held candidates 0, already satisfied/no change 0", close_replay.stdout, ) self.assertEqual(self.state()["head_revision"], close_revision) @@ -2653,7 +2665,7 @@ def test_applying_decision_replay_recovers_close_and_touch_effects(self): touch_replay = self.cli("sync", "--run", toucher) self.assertIn( - "applied 0, held candidates 0, already satisfied/no change 0", + "applied 1, held candidates 0, already satisfied/no change 0", touch_replay.stdout, ) self.assertEqual(self.state()["head_revision"], touch_revision) @@ -2801,7 +2813,7 @@ def test_agent_change_to_confirmed_memory_requires_human(self): ) self.assertIn("Launch on Monday.", self.hot()) self.assertNotIn("Launch on Tuesday.", self.hot()) - recall = self.cli("recall", "--key", "launch.date") + recall = self.cli("recall", "--key", "launch.date", "--history") self.assertIn("Launch on Monday.", recall.stdout) self.assertIn("HELD", recall.stdout) self.assertIn("Launch on Tuesday.", recall.stdout) @@ -3301,8 +3313,21 @@ def test_semantic_or_orphan_authority_corruption_is_recoverable(self): orphan_status = self.cli("status", root=orphan_root, check=False) self.assertEqual(orphan_status.returncode, 1) self.assertIn(f"resolution {orphan_id}", orphan_status.stdout) - degraded = self.cli("start", "--actor", "recovery", root=orphan_root) - self.assertIn("AUTHORITY RECOVERY NEEDED", degraded.stdout) + # Owner-ruled 2026-09-02: once a full-audit boundary (status above) + # has condemned the store, the checkpoint no longer shields the warm + # path. Start re-proves the store and prints the recovery banner, + # exactly as v5.1.0 did and as BIMRI-PROTOCOL 9.5 requires. + warm_start = self.cli("start", "--actor", "recovery", root=orphan_root) + self.assertIn("AUTHORITY RECOVERY NEEDED", warm_start.stdout) + self.assertIn(f"resolution {orphan_id}", warm_start.stdout) + boundary = self.cli( + "doctor", "--read-only", root=orphan_root, check=False + ) + self.assertEqual(boundary.returncode, 1) + self.assertIn( + f"resolution {orphan_id}", + boundary.stdout + boundary.stderr, + ) self.cli( "quarantine-authority", "--kind", @@ -3930,10 +3955,52 @@ def test_started_resolution_freezes_conflict_candidate_snapshot(self): + "\n" ).encode("utf-8") ) - self.assertIn( - "Open conflicts: 1", - self.cli("status", root=root).stdout, - ) + if resolution_status == "failed": + status = self.cli("status", root=root, check=False) + self.assertEqual(status.returncode, 1) + self.assertIn("Open conflicts: 1", status.stdout) + self.assertIn("AUTHORITY RECOVERY NEEDED", status.stdout) + self.assertIn("explicit retry", status.stdout) + blocked = self.cli( + "sync", + "--run", + second_run, + root=root, + check=False, + ) + self.assertEqual(blocked.returncode, 2) + self.assertIn( + "authority recovery is required", blocked.stderr + ) + self.assertFalse( + root.joinpath( + ".bimri", + "decisions", + f"{second_candidate}.json", + ).exists() + ) + self.assertEqual( + first_conflict_path.read_bytes(), first_conflict_bytes + ) + self.assertEqual( + json.loads(resolution_path.read_text("utf-8"))[ + "proposal_ids" + ], + [first_candidate], + ) + self.cli( + "resolve", + first_conflict_id, + "--choose", + "current", + "--human-approved", + root=root, + ) + else: + self.assertIn( + "Open conflicts: 1", + self.cli("status", root=root).stdout, + ) self.cli("sync", "--run", second_run, root=root) second_decision = self.decision( @@ -3966,8 +4033,9 @@ def test_started_resolution_freezes_conflict_candidate_snapshot(self): second_conflict["current_hash"], first_conflict["current_hash"], ) + expected_open = 1 if resolution_status == "failed" else 2 self.assertIn( - "Open conflicts: 2", + f"Open conflicts: {expected_open}", self.cli("status", root=root).stdout, ) @@ -5550,7 +5618,7 @@ def test_generic_post_commit_view_failure_warns_and_self_heals(self): self.assertEqual(healed.returncode, 0) self.assertIn("[K:view.generic]", self.hot()) - def test_post_commit_index_failures_warn_without_undoing_operations(self): + def test_hot_operations_do_not_call_retired_automatic_index_rebuild(self): start_root = self.root / "index-start" started = self.worker( "index_failure", @@ -5565,74 +5633,102 @@ def test_post_commit_index_failures_warn_without_undoing_operations(self): ) self.assertIsNotNone(start_match, started.stdout) start_run = start_match.group(1) - self.assertIn("durable operation succeeded", started.stderr) - self.assertIn("forced index failure", started.stderr) + self.assertNotIn("forced index failure", started.stderr) self.assertIn( start_run, self.state(root=start_root)["active_runs"], ) + index_path = start_root / ".bimri" / "index.tsv" + index_path.write_bytes(b"owner-controlled stale index sentinel\n") - sync_root = self.root / "index-sync" - sync_run = self.start("index-sync", root=sync_root) - sync_proposal = self.propose( - sync_run, + journaled = self.worker( + "index_failure", + "journal", + "--run", + start_run, + "--text", + "Journal must not rebuild the unused derived index.", + root=start_root, + ) + self.assertNotIn("forced index failure", journaled.stderr) + self.assertEqual( + index_path.read_bytes(), b"owner-controlled stale index sentinel\n" + ) + + proposed = self.worker( + "index_failure", + "propose", + "--run", + start_run, + "--tier", + "2", + "--key", "index.sync", - "Sync remains committed when indexing fails.", - root=sync_root, + "--text", + "Sync commits without touching the retired hot-path index.", + "--source", + "agent", + "--trust", + "working", + "--new-subject", + root=start_root, ) + sync_proposal = PROPOSAL_RE.search(proposed.stdout).group(0) + self.assertNotIn("forced index failure", proposed.stderr) synced = self.worker( "index_failure", "sync", "--run", - sync_run, - root=sync_root, + start_run, + root=start_root, ) - self.assertIn("durable operation succeeded", synced.stderr) + self.assertNotIn("forced index failure", synced.stderr) self.assertEqual( - self.decision(sync_proposal, root=sync_root)["outcome"], + self.decision(sync_proposal, root=start_root)["outcome"], "accepted", ) - self.assertIn("[K:index.sync]", self.hot(root=sync_root)) + self.assertIn("[K:index.sync]", self.hot(root=start_root)) + self.assertEqual( + index_path.read_bytes(), b"owner-controlled stale index sentinel\n" + ) - close_root = self.root / "index-close" - close_run = self.start("index-close", root=close_root) closed = self.worker( "index_failure", "close", "--run", - close_run, + start_run, "--outcome", "success", "--summary", - "Close remains committed when indexing fails.", - root=close_root, + "Close also leaves the unused index untouched.", + root=start_root, ) - self.assertIn("durable operation succeeded", closed.stderr) + self.assertNotIn("forced index failure", closed.stderr) self.assertNotIn( - close_run, - self.state(root=close_root)["active_runs"], + start_run, + self.state(root=start_root)["active_runs"], ) close_log = ( - close_root / ".bimri" / "log" / f"{close_run}.md" + start_root / ".bimri" / "log" / f"{start_run}.md" ).read_text("utf-8") - self.assertIn(f"[CLOSED:{close_run} ", close_log) + self.assertIn(f"[CLOSED:{start_run} ", close_log) + self.assertEqual( + index_path.read_bytes(), b"owner-controlled stale index sentinel\n" + ) resolve_root = self.root / "index-resolve" resolve_run, candidate, _, _ = self.stage_concurrent_candidate( "index.resolve", - "Human approval should survive an index failure.", + "Human-approved resolution without automatic indexing.", root=resolve_root, candidate_actor="index-resolve", - writer_text="The competing live index value.", - ) - self.cli( - "sync", - "--run", - resolve_run, - root=resolve_root, + writer_text="The competing current value.", ) + self.cli("sync", "--run", resolve_run, root=resolve_root) contested = self.decision(candidate, root=resolve_root) self.assertEqual(contested["outcome"], "contested") + resolve_index = resolve_root / ".bimri" / "index.tsv" + resolve_index.write_bytes(b"resolution index sentinel\n") resolved = self.worker( "index_failure", "resolve", @@ -5642,12 +5738,14 @@ def test_post_commit_index_failures_warn_without_undoing_operations(self): "--human-approved", root=resolve_root, ) - self.assertIn("durable operation succeeded", resolved.stderr) + self.assertNotIn("forced index failure", resolved.stderr) self.assertEqual( self.decision(candidate, root=resolve_root)["outcome"], "accepted", ) - self.assertIn("[K:index.resolve]", self.hot(root=resolve_root)) + self.assertEqual( + resolve_index.read_bytes(), b"resolution index sentinel\n" + ) def test_index_and_doctor_are_deterministic(self): run_id = self.start("codex") @@ -7530,7 +7628,11 @@ def snapshot(): return { path.relative_to(self.root).as_posix(): path.read_bytes() for path in self.root.rglob("*") - if path.is_file() and not path.is_symlink() + if ( + path.is_file() + and not path.is_symlink() + and path != self.root / ".bimri" / "audit-witness.json" + ) } before = snapshot() @@ -7880,7 +7982,7 @@ def test_engine_release_is_separate_from_memory_format(self): self.assertIn("