diff --git a/.github/workflows/test-level2.yml b/.github/workflows/test-level2.yml index da83c90..d8adf33 100644 --- a/.github/workflows/test-level2.yml +++ b/.github/workflows/test-level2.yml @@ -11,6 +11,7 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.11", "3.12"] @@ -26,7 +27,8 @@ jobs: - name: Install package and test dependencies run: | python -m pip install --upgrade pip - python -m pip install -e . + # Existing contract tests import pytest during unittest discovery. + python -m pip install -e . pytest - name: Run Level 2 tests run: python -m unittest discover -s tests -v diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 807fa1a..d8987ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,3 +31,6 @@ jobs: - name: Run benchmark contract tests run: python -m pytest -q tests/test_benchmark_utils_contract.py + + - name: Run artifact manifest tests + run: python -m pytest -q tests/test_artifact_manifest.py diff --git a/docs/benchmark-artifact-manifests.md b/docs/benchmark-artifact-manifests.md new file mode 100644 index 0000000..e51e193 --- /dev/null +++ b/docs/benchmark-artifact-manifests.md @@ -0,0 +1,30 @@ +# Benchmark artifact manifests + +PowerAgentBench can emit per-case CSVs, aggregate summaries, tool logs, and other experiment artifacts. A checksum manifest makes it easier to confirm that those files have not changed between evaluation, review, and publication. + +The manifest utility uses only the Python standard library and does not alter benchmark results. + +## Create a manifest + +Run from the repository root after a benchmark has finished: + +```bash +python -m poweragentbench.artifact_manifest create results/steady_n2 \ + --metadata-json '{"case_source":"case39","seed_start":1000,"cases":8,"budget":80}' +``` + +This writes `run_manifest.json` inside the selected results directory. It records each regular file's relative path, byte size, and SHA-256 digest, plus a UTC creation timestamp and the metadata supplied explicitly on the command line. + +Metadata is optional. Supply only non-sensitive details that help reproduce the run, such as scenario source, seed range, budget, model identifier, or code revision. The utility does not collect environment variables or credentials. + +## Verify a manifest + +Run verification before sharing or analyzing the artifacts: + +```bash +python -m poweragentbench.artifact_manifest verify results/steady_n2 +``` + +A successful verification exits with status 0. A missing or modified file, malformed manifest, unsafe path, or invalid checksum exits non-zero and reports the problem. The verifier checks the files listed in the manifest; it does not assert that the experiment itself is scientifically valid or that every unlisted file is absent. + +Use a separate manifest per result directory or campaign to keep artifact inventories scoped and understandable. diff --git a/poweragentbench/artifact_manifest.py b/poweragentbench/artifact_manifest.py new file mode 100644 index 0000000..bd5668e --- /dev/null +++ b/poweragentbench/artifact_manifest.py @@ -0,0 +1,194 @@ +"""Create and verify checksummed manifests for benchmark result artifacts. + +The manifest deliberately records files and explicitly supplied metadata only. +It does not capture environment variables, credentials, or machine-specific paths. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Mapping + +SCHEMA_VERSION = 1 +DEFAULT_MANIFEST = "run_manifest.json" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_relative_path(value: object) -> str | None: + if not isinstance(value, str) or not value or "\\" in value: + return None + candidate = PurePosixPath(value) + if candidate.is_absolute() or any(part in ("", ".", "..") for part in candidate.parts): + return None + return candidate.as_posix() + + +def build_manifest( + root: str | Path, + metadata: Mapping[str, Any] | None = None, + manifest_path: str = DEFAULT_MANIFEST, +) -> dict[str, Any]: + """Build a manifest for regular files beneath *root*, excluding the manifest.""" + root_path = Path(root).resolve() + if not root_path.is_dir(): + raise NotADirectoryError(f"artifact root is not a directory: {root}") + safe_manifest = _safe_relative_path(manifest_path) + if safe_manifest is None: + raise ValueError("manifest_path must be a safe relative path") + try: + json.dumps(metadata if metadata is not None else {}) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON-serializable") from exc + + files: list[dict[str, Any]] = [] + for path in sorted(root_path.rglob("*")): + if path.is_symlink() or not path.is_file(): + continue + resolved = path.resolve() + if not resolved.is_relative_to(root_path): + continue + relative = path.relative_to(root_path).as_posix() + if relative == safe_manifest: + continue + files.append({ + "path": relative, + "size_bytes": path.stat().st_size, + "sha256": _sha256(path), + }) + + return { + "schema_version": SCHEMA_VERSION, + "created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), + "metadata": dict(metadata or {}), + "files": files, + } + + +def write_manifest( + root: str | Path, + metadata: Mapping[str, Any] | None = None, + manifest_path: str = DEFAULT_MANIFEST, +) -> Path: + """Create a JSON manifest under *root* and return its path.""" + root_path = Path(root).resolve() + manifest = build_manifest(root_path, metadata=metadata, manifest_path=manifest_path) + target = root_path / manifest_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return target + + +def verify_manifest(root: str | Path, manifest_path: str = DEFAULT_MANIFEST) -> list[str]: + """Return integrity errors for a manifest; an empty list means it verified.""" + root_path = Path(root).resolve() + if not root_path.is_dir(): + return [f"artifact root is not a directory: {root}"] + safe_manifest = _safe_relative_path(manifest_path) + if safe_manifest is None: + return ["manifest path must be a safe relative path"] + manifest_file = root_path / safe_manifest + if manifest_file.is_symlink() or not manifest_file.is_file(): + return [f"manifest not found: {safe_manifest}"] + try: + data = json.loads(manifest_file.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + return [f"cannot read manifest: {exc}"] + if not isinstance(data, dict): + return ["manifest root must be a JSON object"] + if data.get("schema_version") != SCHEMA_VERSION: + return [f"schema_version must equal {SCHEMA_VERSION}"] + entries = data.get("files") + if not isinstance(entries, list): + return ["files must be a list"] + + errors: list[str] = [] + seen: set[str] = set() + for index, entry in enumerate(entries): + prefix = f"files[{index}]" + if not isinstance(entry, dict): + errors.append(f"{prefix} must be an object") + continue + relative = _safe_relative_path(entry.get("path")) + if relative is None: + errors.append(f"{prefix}.path must be a safe relative path") + continue + if relative == safe_manifest: + errors.append(f"{prefix}.path must not reference the manifest itself") + continue + if relative in seen: + errors.append(f"{prefix}.path duplicates {relative!r}") + continue + seen.add(relative) + target = root_path / relative + if target.is_symlink() or not target.is_file(): + errors.append(f"missing artifact: {relative}") + continue + try: + if not target.resolve().is_relative_to(root_path): + errors.append(f"artifact escapes root: {relative}") + continue + except OSError: + errors.append(f"cannot resolve artifact: {relative}") + continue + expected_size = entry.get("size_bytes") + if not isinstance(expected_size, int) or isinstance(expected_size, bool) or expected_size < 0: + errors.append(f"{prefix}.size_bytes must be a non-negative integer") + elif target.stat().st_size != expected_size: + errors.append(f"size mismatch: {relative}") + expected_hash = entry.get("sha256") + if not isinstance(expected_hash, str) or len(expected_hash) != 64 or any( + char not in "0123456789abcdef" for char in expected_hash + ): + errors.append(f"{prefix}.sha256 must be a lowercase SHA-256 hex digest") + elif _sha256(target) != expected_hash: + errors.append(f"SHA-256 mismatch: {relative}") + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="action", required=True) + create_parser = subparsers.add_parser("create", help="create a checksummed artifact manifest") + create_parser.add_argument("root", type=Path, help="directory containing benchmark artifacts") + create_parser.add_argument("--output", default=DEFAULT_MANIFEST, help="relative manifest path") + create_parser.add_argument( + "--metadata-json", default="{}", help="explicit JSON metadata, e.g. '{\"seed\": 42}'" + ) + verify_parser = subparsers.add_parser("verify", help="verify artifacts against a manifest") + verify_parser.add_argument("root", type=Path, help="directory containing benchmark artifacts") + verify_parser.add_argument("--manifest", default=DEFAULT_MANIFEST, help="relative manifest path") + args = parser.parse_args(argv) + + if args.action == "create": + try: + metadata = json.loads(args.metadata_json) + if not isinstance(metadata, dict): + raise ValueError("metadata JSON must be an object") + path = write_manifest(args.root, metadata=metadata, manifest_path=args.output) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + print(f"Manifest written: {path}") + return 0 + + errors = verify_manifest(args.root, manifest_path=args.manifest) + if errors: + print("\n".join(f"ERROR: {error}" for error in errors), file=sys.stderr) + return 1 + print(f"Manifest verified: {args.root / args.manifest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_artifact_manifest.py b/tests/test_artifact_manifest.py new file mode 100644 index 0000000..beb7a2c --- /dev/null +++ b/tests/test_artifact_manifest.py @@ -0,0 +1,92 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from poweragentbench.artifact_manifest import ( + DEFAULT_MANIFEST, + build_manifest, + verify_manifest, + write_manifest, +) + + +class ArtifactManifestTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + (self.root / "baseline_per_case.csv").write_text("agent,score\nA,0.5\n", encoding="utf-8") + (self.root / "nested").mkdir() + (self.root / "nested" / "summary.json").write_text('{"score": 0.5}\n', encoding="utf-8") + + def tearDown(self): + self.temp.cleanup() + + def test_manifest_lists_sorted_relative_files_with_hashes(self): + manifest = build_manifest(self.root, metadata={"seed": 42}) + paths = [entry["path"] for entry in manifest["files"]] + self.assertEqual(paths, ["baseline_per_case.csv", "nested/summary.json"]) + self.assertEqual(manifest["metadata"], {"seed": 42}) + self.assertEqual(len(manifest["files"][0]["sha256"]), 64) + self.assertNotIn(DEFAULT_MANIFEST, paths) + + def test_written_manifest_verifies(self): + write_manifest(self.root, metadata={"case_source": "case39", "seed": 7}) + self.assertEqual(verify_manifest(self.root), []) + + def test_modified_file_is_detected(self): + write_manifest(self.root) + (self.root / "baseline_per_case.csv").write_text("agent,score\nA,0.9\n", encoding="utf-8") + errors = verify_manifest(self.root) + self.assertIn("SHA-256 mismatch: baseline_per_case.csv", errors) + + def test_missing_file_is_detected(self): + write_manifest(self.root) + (self.root / "nested" / "summary.json").unlink() + self.assertIn("missing artifact: nested/summary.json", verify_manifest(self.root)) + + def test_manifest_path_rejects_traversal(self): + with self.assertRaises(ValueError): + build_manifest(self.root, manifest_path="../outside.json") + self.assertIn("manifest path must be a safe relative path", + verify_manifest(self.root, manifest_path="../outside.json")) + + def test_manifest_rejects_non_json_metadata(self): + with self.assertRaises(ValueError): + build_manifest(self.root, metadata={"bad": object()}) + + def test_invalid_manifest_json_is_reported(self): + (self.root / DEFAULT_MANIFEST).write_text("{bad", encoding="utf-8") + self.assertTrue(verify_manifest(self.root)[0].startswith("cannot read manifest:")) + + def test_manifest_root_must_be_object(self): + (self.root / DEFAULT_MANIFEST).write_text("[]", encoding="utf-8") + self.assertEqual(verify_manifest(self.root), ["manifest root must be a JSON object"]) + + def test_unsafe_artifact_path_is_rejected(self): + write_manifest(self.root) + manifest_path = self.root / DEFAULT_MANIFEST + data = json.loads(manifest_path.read_text(encoding="utf-8")) + data["files"][0]["path"] = "../outside.csv" + manifest_path.write_text(json.dumps(data), encoding="utf-8") + self.assertIn("files[0].path must be a safe relative path", verify_manifest(self.root)) + + def test_duplicate_artifact_paths_are_rejected(self): + write_manifest(self.root) + manifest_path = self.root / DEFAULT_MANIFEST + data = json.loads(manifest_path.read_text(encoding="utf-8")) + data["files"].append(dict(data["files"][0])) + manifest_path.write_text(json.dumps(data), encoding="utf-8") + self.assertTrue(any("duplicates" in error for error in verify_manifest(self.root))) + + def test_wrong_schema_version_is_rejected(self): + write_manifest(self.root) + manifest_path = self.root / DEFAULT_MANIFEST + data = json.loads(manifest_path.read_text(encoding="utf-8")) + data["schema_version"] = 99 + manifest_path.write_text(json.dumps(data), encoding="utf-8") + self.assertEqual(verify_manifest(self.root), ["schema_version must equal 1"]) + + +if __name__ == "__main__": + unittest.main()