From 227f587b0d9a02ef903f78a70cd8352fa0250b10 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Tue, 29 Sep 2026 16:29:42 +0900 Subject: [PATCH] refactor: simplify seal checks and remove stale prune script --- README.md | 2 +- crates/cmem-eval-runner/src/seal.rs | 65 +++++++---------------------- scripts/README.md | 11 ----- scripts/qdrant_prune_collections.sh | 44 ------------------- 4 files changed, 15 insertions(+), 107 deletions(-) delete mode 100755 scripts/qdrant_prune_collections.sh diff --git a/README.md b/README.md index 9485fc5d..7729a9fe 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ cargo run -p cmem-eval-runner -- seal ./runs/continuity/candidate cargo run -p cmem-eval-runner -- verify ./evidence/ ``` -The run ID must be one filename component. An existing evidence destination or source seal is never overwritten; every file is created atomically. If sealing fails after creating the destination, the command reports the written files and the failed operation and leaves all files in place for inspection. Inspect a partial destination before deliberately removing it; `seal` never removes files. Commit completed evidence and cite the SHA-256 of its `seal.json` in the findings register. Never edit sealed evidence; changed measurements require a new run and seal. `verify` checks the listed file hashes without parsing the artifacts and exits non-zero on missing or changed bytes. Compare the seal's own hash with the register citation as well. +An existing evidence destination or source seal is never overwritten; every file is created atomically. If sealing fails after creating the destination, the command reports the written files and the failed operation and leaves all files in place for inspection. Inspect a partial destination before deliberately removing it; `seal` never removes files. Commit completed evidence and cite the SHA-256 of its `seal.json` in the findings register. Never edit sealed evidence; changed measurements require a new run and seal. `verify` checks the listed file hashes without parsing the artifacts and exits non-zero on missing or changed bytes. Compare the seal's own hash with the register citation as well. The historical round-9 reference pair is preserved in `evidence/pr13r9ba/` and `evidence/pr13r9bb/`. Its promotion seals describe the existing files without inventing a header or changing their old shape. `pwsh -NoProfile -File evidence/derive-round9.ps1` (PowerShell 7.2+) reproduces the cited normalized-row and report-content hashes and the short-gap recall@5 mean from those tracked bytes alone. diff --git a/crates/cmem-eval-runner/src/seal.rs b/crates/cmem-eval-runner/src/seal.rs index 3581edf5..4ccab13d 100644 --- a/crates/cmem-eval-runner/src/seal.rs +++ b/crates/cmem-eval-runner/src/seal.rs @@ -4,30 +4,11 @@ use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::fs; use std::io::{self, BufReader, Read}; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; const STREAM_BUFFER_SIZE: usize = 64 * 1024; -fn file_name(name: &str) -> Result<()> { - ensure!( - !name.contains(['/', '\\', ':']) - && matches!( - Path::new(name).components().next(), - Some(Component::Normal(_)) - ), - "expected one filename component: {name:?}" - ); - Ok(()) -} - fn open_file(path: &Path) -> Result> { - let metadata = - fs::symlink_metadata(path).with_context(|| format!("inspect {}", path.display()))?; - ensure!( - metadata.is_file(), - "expected a regular file: {}", - path.display() - ); let file = fs::File::open(path).with_context(|| format!("open {}", path.display()))?; Ok(BufReader::with_capacity(STREAM_BUFFER_SIZE, file)) } @@ -64,7 +45,6 @@ pub(crate) fn seal(run_dir: &Path, evidence_root: &Path) -> Result { .file_name() .into_string() .map_err(|_| anyhow::anyhow!("artifact filename must be UTF-8"))?; - file_name(&artifact_name)?; let mut artifacts = BTreeMap::new(); for name in [&*artifact_name, "header.json", "report.json"] { artifacts.insert(name, run_dir.join(name)); @@ -79,14 +59,7 @@ pub(crate) fn seal(run_dir: &Path, evidence_root: &Path) -> Result { .get("run_id") .and_then(serde_json::Value::as_str) .context("header.json must be an object with a string run_id")?; - file_name(run_id)?; fs::create_dir_all(evidence_root)?; - ensure!( - !fs::symlink_metadata(evidence_root)? - .file_type() - .is_symlink(), - "evidence root must not be a link" - ); let destination = evidence_root.join(run_id); fs::create_dir(&destination).with_context(|| { format!( @@ -136,7 +109,6 @@ pub(crate) fn verify(evidence_dir: &Path) -> Result<()> { ensure!(!manifest.files.is_empty(), "seal contains no file hashes"); let mut failures = Vec::new(); for (name, expected) in manifest.files { - file_name(&name)?; ensure!(name != "seal.json", "seal must not hash itself"); match hash_file(&evidence_dir.join(&name)) { Ok(actual) => { @@ -259,7 +231,7 @@ mod tests { fs::remove_file(evidence.join("report.json")).unwrap(); let error = verify(&evidence).unwrap_err().to_string(); assert!(error.contains("traces.jsonl: expected")); - assert!(error.contains("report.json: inspect")); + assert!(error.contains("report.json: open")); } #[test] @@ -308,26 +280,17 @@ mod tests { ) .unwrap(); verify(directory.path()).unwrap(); - for name in [ - "", - "..", - "../outside", - "nested/file", - "nested\\file", - "C:outside", - "seal.json", - ] { - manifest["files"] = serde_json::json!({name: sha256(&[])}); - fs::write( - directory.path().join("seal.json"), - serde_json::to_vec(&manifest).unwrap(), - ) - .unwrap(); - let error = verify(directory.path()).unwrap_err().to_string(); - assert!( - error.contains("filename component") || error.contains("hash itself"), - "{name}: {error}" - ); - } + manifest["files"] = serde_json::json!({"seal.json": sha256(&[])}); + fs::write( + directory.path().join("seal.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + assert!( + verify(directory.path()) + .unwrap_err() + .to_string() + .contains("hash itself") + ); } } diff --git a/scripts/README.md b/scripts/README.md index 9a2fe52f..232ab784 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -15,14 +15,3 @@ cargo run -p cmem-eval-runner -- run locomo ... - Set `LOCOMO_OFFICIAL_DATASET` to the local official LoCoMo JSON file, then run `bash scripts/locomo_census.sh` to print annotation and derived-memory counts (`cargo run -p cmem-eval-locomo --example official_derived_content_census`). - Set `LONGMEMEVAL_DATASET` to the local official LongMemEval JSON file and `LONGMEMEVAL_IDENTITY_DUMP` to an output JSON path, then run `bash scripts/longmemeval_census.sh` to print repeated-session counts and write the identity mapping (`cargo run -p cmem-eval-longmemeval --example official_repeated_session_census`). The dump path must not exist; the census refuses to overwrite a file, so it can never truncate the dataset it reads. - -## Prune orphaned Qdrant collections - -`qdrant_prune_collections.sh` lists Qdrant collections whose names begin with an exact prefix and deletes them only when passed `--delete`. It uses `QDRANT_REST_URL`, defaulting to `http://127.0.0.1:6333`. - -Run it only while holding the orchestrator-managed live-run mutex. Known test collection families include `cmem_eval_*` and `test_collection_*`; always provide the narrowest run-specific prefix rather than either broad family prefix. - -```bash -./scripts/qdrant_prune_collections.sh cmem_eval_continuity_continuity_v1_ -./scripts/qdrant_prune_collections.sh cmem_eval_prune_selftest_12345 --delete -``` diff --git a/scripts/qdrant_prune_collections.sh b/scripts/qdrant_prune_collections.sh deleted file mode 100755 index 173d0f51..00000000 --- a/scripts/qdrant_prune_collections.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - echo "usage: qdrant_prune_collections.sh [--delete]" >&2 -} - -if (( $# < 1 || $# > 2 )) || { (( $# == 2 )) && [[ $2 != --delete ]]; }; then - usage - exit 2 -fi - -prefix=$1 -if (( ${#prefix} < 12 )); then - echo "error: collection prefix must contain at least 12 characters" >&2 - exit 2 -fi - -delete=false -[[ ${2-} == --delete ]] && delete=true -endpoint=${QDRANT_REST_URL:-http://127.0.0.1:6333} -endpoint=${endpoint%/} -response=$(curl -fsS "$endpoint/collections") -remaining=$response -name_pattern='"name"[[:space:]]*:[[:space:]]*"([^"]+)"' -matches=() - -while [[ $remaining =~ $name_pattern ]]; do - match=${BASH_REMATCH[0]} - name=${BASH_REMATCH[1]} - [[ $name == "$prefix"* ]] && matches+=("$name") - remaining=${remaining#*"$match"} -done - -for name in "${matches[@]}"; do - if [[ $delete == true ]]; then - curl -fsS -X DELETE "$endpoint/collections/$name" >/dev/null - echo "deleted: $name" - else - echo "dry-run: $name" - fi -done - -echo "matched collections: ${#matches[@]}"