Feat/benchmark artifact manifests

.github/workflows/test-level2.yml

@@ -11,6 +11,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.11", "3.12"]
 
@@ -26,7 +27,8 @@ jobs:
       - name: Install package and test dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e .
+          # Existing contract tests import pytest during unittest discovery.
+          python -m pip install -e . pytest
 
       - name: Run Level 2 tests
         run: python -m unittest discover -s tests -v

.github/workflows/test.yml

@@ -31,3 +31,6 @@ jobs:
 
       - name: Run benchmark contract tests
         run: python -m pytest -q tests/test_benchmark_utils_contract.py
+
+      - name: Run artifact manifest tests
+        run: python -m pytest -q tests/test_artifact_manifest.py

docs/benchmark-artifact-manifests.md

@@ -0,0 +1,30 @@
+# Benchmark artifact manifests
+
+PowerAgentBench can emit per-case CSVs, aggregate summaries, tool logs, and other experiment artifacts. A checksum manifest makes it easier to confirm that those files have not changed between evaluation, review, and publication.
+
+The manifest utility uses only the Python standard library and does not alter benchmark results.
+
+## Create a manifest
+
+Run from the repository root after a benchmark has finished:
+
+```bash
+python -m poweragentbench.artifact_manifest create results/steady_n2 \
+  --metadata-json '{"case_source":"case39","seed_start":1000,"cases":8,"budget":80}'
+```
+
+This writes `run_manifest.json` inside the selected results directory. It records each regular file's relative path, byte size, and SHA-256 digest, plus a UTC creation timestamp and the metadata supplied explicitly on the command line.
+
+Metadata is optional. Supply only non-sensitive details that help reproduce the run, such as scenario source, seed range, budget, model identifier, or code revision. The utility does not collect environment variables or credentials.
+
+## Verify a manifest
+
+Run verification before sharing or analyzing the artifacts:
+
+```bash
+python -m poweragentbench.artifact_manifest verify results/steady_n2
+```
+
+A successful verification exits with status 0. A missing or modified file, malformed manifest, unsafe path, or invalid checksum exits non-zero and reports the problem. The verifier checks the files listed in the manifest; it does not assert that the experiment itself is scientifically valid or that every unlisted file is absent.
+
+Use a separate manifest per result directory or campaign to keep artifact inventories scoped and understandable.

poweragentbench/artifact_manifest.py

@@ -0,0 +1,194 @@
+"""Create and verify checksummed manifests for benchmark result artifacts.
+
+The manifest deliberately records files and explicitly supplied metadata only.
+It does not capture environment variables, credentials, or machine-specific paths.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path, PurePosixPath
+from typing import Any, Mapping
+
+SCHEMA_VERSION = 1
+DEFAULT_MANIFEST = "run_manifest.json"
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _safe_relative_path(value: object) -> str | None:
+    if not isinstance(value, str) or not value or "\\" in value:
+        return None
+    candidate = PurePosixPath(value)
+    if candidate.is_absolute() or any(part in ("", ".", "..") for part in candidate.parts):
+        return None
+    return candidate.as_posix()
+
+
+def build_manifest(
+    root: str | Path,
+    metadata: Mapping[str, Any] | None = None,
+    manifest_path: str = DEFAULT_MANIFEST,
+) -> dict[str, Any]:
+    """Build a manifest for regular files beneath *root*, excluding the manifest."""
+    root_path = Path(root).resolve()
+    if not root_path.is_dir():
+        raise NotADirectoryError(f"artifact root is not a directory: {root}")
+    safe_manifest = _safe_relative_path(manifest_path)
+    if safe_manifest is None:
+        raise ValueError("manifest_path must be a safe relative path")
+    try:
+        json.dumps(metadata if metadata is not None else {})
+    except (TypeError, ValueError) as exc:
+        raise ValueError("metadata must be JSON-serializable") from exc
+
+    files: list[dict[str, Any]] = []
+    for path in sorted(root_path.rglob("*")):
+        if path.is_symlink() or not path.is_file():
+            continue
+        resolved = path.resolve()
+        if not resolved.is_relative_to(root_path):
+            continue
+        relative = path.relative_to(root_path).as_posix()
+        if relative == safe_manifest:
+            continue
+        files.append({
+            "path": relative,
+            "size_bytes": path.stat().st_size,
+            "sha256": _sha256(path),
+        })
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"),
+        "metadata": dict(metadata or {}),
+        "files": files,
+    }
+
+
+def write_manifest(
+    root: str | Path,
+    metadata: Mapping[str, Any] | None = None,
+    manifest_path: str = DEFAULT_MANIFEST,
+) -> Path:
+    """Create a JSON manifest under *root* and return its path."""
+    root_path = Path(root).resolve()
+    manifest = build_manifest(root_path, metadata=metadata, manifest_path=manifest_path)
+    target = root_path / manifest_path
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return target
+
+
+def verify_manifest(root: str | Path, manifest_path: str = DEFAULT_MANIFEST) -> list[str]:
+    """Return integrity errors for a manifest; an empty list means it verified."""
+    root_path = Path(root).resolve()
+    if not root_path.is_dir():
+        return [f"artifact root is not a directory: {root}"]
+    safe_manifest = _safe_relative_path(manifest_path)
+    if safe_manifest is None:
+        return ["manifest path must be a safe relative path"]
+    manifest_file = root_path / safe_manifest
+    if manifest_file.is_symlink() or not manifest_file.is_file():
+        return [f"manifest not found: {safe_manifest}"]
+    try:
+        data = json.loads(manifest_file.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        return [f"cannot read manifest: {exc}"]
+    if not isinstance(data, dict):
+        return ["manifest root must be a JSON object"]
+    if data.get("schema_version") != SCHEMA_VERSION:
+        return [f"schema_version must equal {SCHEMA_VERSION}"]
+    entries = data.get("files")
+    if not isinstance(entries, list):
+        return ["files must be a list"]
+
+    errors: list[str] = []
+    seen: set[str] = set()
+    for index, entry in enumerate(entries):
+        prefix = f"files[{index}]"
+        if not isinstance(entry, dict):
+            errors.append(f"{prefix} must be an object")
+            continue
+        relative = _safe_relative_path(entry.get("path"))
+        if relative is None:
+            errors.append(f"{prefix}.path must be a safe relative path")
+            continue
+        if relative == safe_manifest:
+            errors.append(f"{prefix}.path must not reference the manifest itself")
+            continue
+        if relative in seen:
+            errors.append(f"{prefix}.path duplicates {relative!r}")
+            continue
+        seen.add(relative)
+        target = root_path / relative
+        if target.is_symlink() or not target.is_file():
+            errors.append(f"missing artifact: {relative}")
+            continue
+        try:
+            if not target.resolve().is_relative_to(root_path):
+                errors.append(f"artifact escapes root: {relative}")
+                continue
+        except OSError:
+            errors.append(f"cannot resolve artifact: {relative}")
+            continue
+        expected_size = entry.get("size_bytes")
+        if not isinstance(expected_size, int) or isinstance(expected_size, bool) or expected_size < 0:
+            errors.append(f"{prefix}.size_bytes must be a non-negative integer")
+        elif target.stat().st_size != expected_size:
+            errors.append(f"size mismatch: {relative}")
+        expected_hash = entry.get("sha256")
+        if not isinstance(expected_hash, str) or len(expected_hash) != 64 or any(
+            char not in "0123456789abcdef" for char in expected_hash
+        ):
+            errors.append(f"{prefix}.sha256 must be a lowercase SHA-256 hex digest")
+        elif _sha256(target) != expected_hash:
+            errors.append(f"SHA-256 mismatch: {relative}")
+    return errors
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(dest="action", required=True)
+    create_parser = subparsers.add_parser("create", help="create a checksummed artifact manifest")
+    create_parser.add_argument("root", type=Path, help="directory containing benchmark artifacts")
+    create_parser.add_argument("--output", default=DEFAULT_MANIFEST, help="relative manifest path")
+    create_parser.add_argument(
+        "--metadata-json", default="{}", help="explicit JSON metadata, e.g. '{\"seed\": 42}'"
+    )
+    verify_parser = subparsers.add_parser("verify", help="verify artifacts against a manifest")
+    verify_parser.add_argument("root", type=Path, help="directory containing benchmark artifacts")
+    verify_parser.add_argument("--manifest", default=DEFAULT_MANIFEST, help="relative manifest path")
+    args = parser.parse_args(argv)
+
+    if args.action == "create":
+        try:
+            metadata = json.loads(args.metadata_json)
+            if not isinstance(metadata, dict):
+                raise ValueError("metadata JSON must be an object")
+            path = write_manifest(args.root, metadata=metadata, manifest_path=args.output)
+        except (OSError, ValueError, json.JSONDecodeError) as exc:
+            print(f"error: {exc}", file=sys.stderr)
+            return 2
+        print(f"Manifest written: {path}")
+        return 0
+
+    errors = verify_manifest(args.root, manifest_path=args.manifest)
+    if errors:
+        print("\n".join(f"ERROR: {error}" for error in errors), file=sys.stderr)
+        return 1
+    print(f"Manifest verified: {args.root / args.manifest}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_artifact_manifest.py

@@ -0,0 +1,92 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from poweragentbench.artifact_manifest import (
+    DEFAULT_MANIFEST,
+    build_manifest,
+    verify_manifest,
+    write_manifest,
+)
+
+
+class ArtifactManifestTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp.name)
+        (self.root / "baseline_per_case.csv").write_text("agent,score\nA,0.5\n", encoding="utf-8")
+        (self.root / "nested").mkdir()
+        (self.root / "nested" / "summary.json").write_text('{"score": 0.5}\n', encoding="utf-8")
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def test_manifest_lists_sorted_relative_files_with_hashes(self):
+        manifest = build_manifest(self.root, metadata={"seed": 42})
+        paths = [entry["path"] for entry in manifest["files"]]
+        self.assertEqual(paths, ["baseline_per_case.csv", "nested/summary.json"])
+        self.assertEqual(manifest["metadata"], {"seed": 42})
+        self.assertEqual(len(manifest["files"][0]["sha256"]), 64)
+        self.assertNotIn(DEFAULT_MANIFEST, paths)
+
+    def test_written_manifest_verifies(self):
+        write_manifest(self.root, metadata={"case_source": "case39", "seed": 7})
+        self.assertEqual(verify_manifest(self.root), [])
+
+    def test_modified_file_is_detected(self):
+        write_manifest(self.root)
+        (self.root / "baseline_per_case.csv").write_text("agent,score\nA,0.9\n", encoding="utf-8")
+        errors = verify_manifest(self.root)
+        self.assertIn("SHA-256 mismatch: baseline_per_case.csv", errors)
+
+    def test_missing_file_is_detected(self):
+        write_manifest(self.root)
+        (self.root / "nested" / "summary.json").unlink()
+        self.assertIn("missing artifact: nested/summary.json", verify_manifest(self.root))
+
+    def test_manifest_path_rejects_traversal(self):
+        with self.assertRaises(ValueError):
+            build_manifest(self.root, manifest_path="../outside.json")
+        self.assertIn("manifest path must be a safe relative path",
+                      verify_manifest(self.root, manifest_path="../outside.json"))
+
+    def test_manifest_rejects_non_json_metadata(self):
+        with self.assertRaises(ValueError):
+            build_manifest(self.root, metadata={"bad": object()})
+
+    def test_invalid_manifest_json_is_reported(self):
+        (self.root / DEFAULT_MANIFEST).write_text("{bad", encoding="utf-8")
+        self.assertTrue(verify_manifest(self.root)[0].startswith("cannot read manifest:"))
+
+    def test_manifest_root_must_be_object(self):
+        (self.root / DEFAULT_MANIFEST).write_text("[]", encoding="utf-8")
+        self.assertEqual(verify_manifest(self.root), ["manifest root must be a JSON object"])
+
+    def test_unsafe_artifact_path_is_rejected(self):
+        write_manifest(self.root)
+        manifest_path = self.root / DEFAULT_MANIFEST
+        data = json.loads(manifest_path.read_text(encoding="utf-8"))
+        data["files"][0]["path"] = "../outside.csv"
+        manifest_path.write_text(json.dumps(data), encoding="utf-8")
+        self.assertIn("files[0].path must be a safe relative path", verify_manifest(self.root))
+
+    def test_duplicate_artifact_paths_are_rejected(self):
+        write_manifest(self.root)
+        manifest_path = self.root / DEFAULT_MANIFEST
+        data = json.loads(manifest_path.read_text(encoding="utf-8"))
+        data["files"].append(dict(data["files"][0]))
+        manifest_path.write_text(json.dumps(data), encoding="utf-8")
+        self.assertTrue(any("duplicates" in error for error in verify_manifest(self.root)))
+
+    def test_wrong_schema_version_is_rejected(self):
+        write_manifest(self.root)
+        manifest_path = self.root / DEFAULT_MANIFEST
+        data = json.loads(manifest_path.read_text(encoding="utf-8"))
+        data["schema_version"] = 99
+        manifest_path.write_text(json.dumps(data), encoding="utf-8")
+        self.assertEqual(verify_manifest(self.root), ["schema_version must equal 1"])
+
+
+if __name__ == "__main__":
+    unittest.main()