From 3d457d77ad572eccada1948add277802da3501ad Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:18:06 +0530 Subject: [PATCH 1/8] feat: add checksummed benchmark artifact manifests --- poweragentbench/artifact_manifest.py | 195 +++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 poweragentbench/artifact_manifest.py diff --git a/poweragentbench/artifact_manifest.py b/poweragentbench/artifact_manifest.py new file mode 100644 index 0000000..783ba85 --- /dev/null +++ b/poweragentbench/artifact_manifest.py @@ -0,0 +1,195 @@ +"""Create and verify checksummed manifests for benchmark result artifacts. + +The manifest deliberately records files and explicitly supplied metadata only. +It does not capture environment variables, credentials, or machine-specific paths. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Mapping + +SCHEMA_VERSION = 1 +DEFAULT_MANIFEST = "run_manifest.json" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_relative_path(value: object) -> str | None: + if not isinstance(value, str) or not value or "\\" in value: + return None + candidate = PurePosixPath(value) + if candidate.is_absolute() or any(part in ("", ".", "..") for part in candidate.parts): + return None + return candidate.as_posix() + + +def build_manifest( + root: str | Path, + metadata: Mapping[str, Any] | None = None, + manifest_path: str = DEFAULT_MANIFEST, +) -> dict[str, Any]: + """Build a manifest for regular files beneath *root*, excluding the manifest.""" + root_path = Path(root).resolve() + if not root_path.is_dir(): + raise NotADirectoryError(f"artifact root is not a directory: {root}") + safe_manifest = _safe_relative_path(manifest_path) + if safe_manifest is None: + raise ValueError("manifest_path must be a safe relative path") + try: + json.dumps(metadata if metadata is not None else {}) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON-serializable") from exc + + manifest_target = root_path / safe_manifest + files: list[dict[str, Any]] = [] + for path in sorted(root_path.rglob("*")): + if path.is_symlink() or not path.is_file(): + continue + resolved = path.resolve() + if not resolved.is_relative_to(root_path): + continue + relative = path.relative_to(root_path).as_posix() + if relative == safe_manifest: + continue + files.append({ + "path": relative, + "size_bytes": path.stat().st_size, + "sha256": _sha256(path), + }) + + return { + "schema_version": SCHEMA_VERSION, + "created_utc": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), + "metadata": dict(metadata or {}), + "files": files, + } + + +def write_manifest( + root: str | Path, + metadata: Mapping[str, Any] | None = None, + manifest_path: str = DEFAULT_MANIFEST, +) -> Path: + """Create a JSON manifest under *root* and return its path.""" + root_path = Path(root).resolve() + manifest = build_manifest(root_path, metadata=metadata, manifest_path=manifest_path) + target = root_path / manifest_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return target + + +def verify_manifest(root: str | Path, manifest_path: str = DEFAULT_MANIFEST) -> list[str]: + """Return integrity errors for a manifest; an empty list means it verified.""" + root_path = Path(root).resolve() + if not root_path.is_dir(): + return [f"artifact root is not a directory: {root}"] + safe_manifest = _safe_relative_path(manifest_path) + if safe_manifest is None: + return ["manifest path must be a safe relative path"] + manifest_file = root_path / safe_manifest + if manifest_file.is_symlink() or not manifest_file.is_file(): + return [f"manifest not found: {safe_manifest}"] + try: + data = json.loads(manifest_file.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + return [f"cannot read manifest: {exc}"] + if not isinstance(data, dict): + return ["manifest root must be a JSON object"] + if data.get("schema_version") != SCHEMA_VERSION: + return [f"schema_version must equal {SCHEMA_VERSION}"] + entries = data.get("files") + if not isinstance(entries, list): + return ["files must be a list"] + + errors: list[str] = [] + seen: set[str] = set() + for index, entry in enumerate(entries): + prefix = f"files[{index}]" + if not isinstance(entry, dict): + errors.append(f"{prefix} must be an object") + continue + relative = _safe_relative_path(entry.get("path")) + if relative is None: + errors.append(f"{prefix}.path must be a safe relative path") + continue + if relative == safe_manifest: + errors.append(f"{prefix}.path must not reference the manifest itself") + continue + if relative in seen: + errors.append(f"{prefix}.path duplicates {relative!r}") + continue + seen.add(relative) + target = root_path / relative + if target.is_symlink() or not target.is_file(): + errors.append(f"missing artifact: {relative}") + continue + try: + if not target.resolve().is_relative_to(root_path): + errors.append(f"artifact escapes root: {relative}") + continue + except OSError: + errors.append(f"cannot resolve artifact: {relative}") + continue + expected_size = entry.get("size_bytes") + if not isinstance(expected_size, int) or isinstance(expected_size, bool) or expected_size < 0: + errors.append(f"{prefix}.size_bytes must be a non-negative integer") + elif target.stat().st_size != expected_size: + errors.append(f"size mismatch: {relative}") + expected_hash = entry.get("sha256") + if not isinstance(expected_hash, str) or len(expected_hash) != 64 or any( + char not in "0123456789abcdef" for char in expected_hash + ): + errors.append(f"{prefix}.sha256 must be a lowercase SHA-256 hex digest") + elif _sha256(target) != expected_hash: + errors.append(f"SHA-256 mismatch: {relative}") + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="action", required=True) + create_parser = subparsers.add_parser("create", help="create a checksummed artifact manifest") + create_parser.add_argument("root", type=Path, help="directory containing benchmark artifacts") + create_parser.add_argument("--output", default=DEFAULT_MANIFEST, help="relative manifest path") + create_parser.add_argument( + "--metadata-json", default="{}", help="explicit JSON metadata, e.g. '{\"seed\": 42}'" + ) + verify_parser = subparsers.add_parser("verify", help="verify artifacts against a manifest") + verify_parser.add_argument("root", type=Path, help="directory containing benchmark artifacts") + verify_parser.add_argument("--manifest", default=DEFAULT_MANIFEST, help="relative manifest path") + args = parser.parse_args(argv) + + if args.action == "create": + try: + metadata = json.loads(args.metadata_json) + if not isinstance(metadata, dict): + raise ValueError("metadata JSON must be an object") + path = write_manifest(args.root, metadata=metadata, manifest_path=args.output) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + print(f"Manifest written: {path}") + return 0 + + errors = verify_manifest(args.root, manifest_path=args.manifest) + if errors: + print("\n".join(f"ERROR: {error}" for error in errors), file=sys.stderr) + return 1 + print(f"Manifest verified: {args.root / args.manifest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From b58aaa389e5ce812fb41a4721a3a626ca617ec37 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:18:17 +0530 Subject: [PATCH 2/8] test: cover benchmark artifact manifest integrity --- tests/test_artifact_manifest.py | 92 +++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/test_artifact_manifest.py diff --git a/tests/test_artifact_manifest.py b/tests/test_artifact_manifest.py new file mode 100644 index 0000000..beb7a2c --- /dev/null +++ b/tests/test_artifact_manifest.py @@ -0,0 +1,92 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from poweragentbench.artifact_manifest import ( + DEFAULT_MANIFEST, + build_manifest, + verify_manifest, + write_manifest, +) + + +class ArtifactManifestTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + (self.root / "baseline_per_case.csv").write_text("agent,score\nA,0.5\n", encoding="utf-8") + (self.root / "nested").mkdir() + (self.root / "nested" / "summary.json").write_text('{"score": 0.5}\n', encoding="utf-8") + + def tearDown(self): + self.temp.cleanup() + + def test_manifest_lists_sorted_relative_files_with_hashes(self): + manifest = build_manifest(self.root, metadata={"seed": 42}) + paths = [entry["path"] for entry in manifest["files"]] + self.assertEqual(paths, ["baseline_per_case.csv", "nested/summary.json"]) + self.assertEqual(manifest["metadata"], {"seed": 42}) + self.assertEqual(len(manifest["files"][0]["sha256"]), 64) + self.assertNotIn(DEFAULT_MANIFEST, paths) + + def test_written_manifest_verifies(self): + write_manifest(self.root, metadata={"case_source": "case39", "seed": 7}) + self.assertEqual(verify_manifest(self.root), []) + + def test_modified_file_is_detected(self): + write_manifest(self.root) + (self.root / "baseline_per_case.csv").write_text("agent,score\nA,0.9\n", encoding="utf-8") + errors = verify_manifest(self.root) + self.assertIn("SHA-256 mismatch: baseline_per_case.csv", errors) + + def test_missing_file_is_detected(self): + write_manifest(self.root) + (self.root / "nested" / "summary.json").unlink() + self.assertIn("missing artifact: nested/summary.json", verify_manifest(self.root)) + + def test_manifest_path_rejects_traversal(self): + with self.assertRaises(ValueError): + build_manifest(self.root, manifest_path="../outside.json") + self.assertIn("manifest path must be a safe relative path", + verify_manifest(self.root, manifest_path="../outside.json")) + + def test_manifest_rejects_non_json_metadata(self): + with self.assertRaises(ValueError): + build_manifest(self.root, metadata={"bad": object()}) + + def test_invalid_manifest_json_is_reported(self): + (self.root / DEFAULT_MANIFEST).write_text("{bad", encoding="utf-8") + self.assertTrue(verify_manifest(self.root)[0].startswith("cannot read manifest:")) + + def test_manifest_root_must_be_object(self): + (self.root / DEFAULT_MANIFEST).write_text("[]", encoding="utf-8") + self.assertEqual(verify_manifest(self.root), ["manifest root must be a JSON object"]) + + def test_unsafe_artifact_path_is_rejected(self): + write_manifest(self.root) + manifest_path = self.root / DEFAULT_MANIFEST + data = json.loads(manifest_path.read_text(encoding="utf-8")) + data["files"][0]["path"] = "../outside.csv" + manifest_path.write_text(json.dumps(data), encoding="utf-8") + self.assertIn("files[0].path must be a safe relative path", verify_manifest(self.root)) + + def test_duplicate_artifact_paths_are_rejected(self): + write_manifest(self.root) + manifest_path = self.root / DEFAULT_MANIFEST + data = json.loads(manifest_path.read_text(encoding="utf-8")) + data["files"].append(dict(data["files"][0])) + manifest_path.write_text(json.dumps(data), encoding="utf-8") + self.assertTrue(any("duplicates" in error for error in verify_manifest(self.root))) + + def test_wrong_schema_version_is_rejected(self): + write_manifest(self.root) + manifest_path = self.root / DEFAULT_MANIFEST + data = json.loads(manifest_path.read_text(encoding="utf-8")) + data["schema_version"] = 99 + manifest_path.write_text(json.dumps(data), encoding="utf-8") + self.assertEqual(verify_manifest(self.root), ["schema_version must equal 1"]) + + +if __name__ == "__main__": + unittest.main() From 5705eef516467401567c133bc5dbb571eee1e6a6 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:18:27 +0530 Subject: [PATCH 3/8] docs: explain benchmark artifact manifest workflow --- docs/benchmark-artifact-manifests.md | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/benchmark-artifact-manifests.md diff --git a/docs/benchmark-artifact-manifests.md b/docs/benchmark-artifact-manifests.md new file mode 100644 index 0000000..e51e193 --- /dev/null +++ b/docs/benchmark-artifact-manifests.md @@ -0,0 +1,30 @@ +# Benchmark artifact manifests + +PowerAgentBench can emit per-case CSVs, aggregate summaries, tool logs, and other experiment artifacts. A checksum manifest makes it easier to confirm that those files have not changed between evaluation, review, and publication. + +The manifest utility uses only the Python standard library and does not alter benchmark results. + +## Create a manifest + +Run from the repository root after a benchmark has finished: + +```bash +python -m poweragentbench.artifact_manifest create results/steady_n2 \ + --metadata-json '{"case_source":"case39","seed_start":1000,"cases":8,"budget":80}' +``` + +This writes `run_manifest.json` inside the selected results directory. It records each regular file's relative path, byte size, and SHA-256 digest, plus a UTC creation timestamp and the metadata supplied explicitly on the command line. + +Metadata is optional. Supply only non-sensitive details that help reproduce the run, such as scenario source, seed range, budget, model identifier, or code revision. The utility does not collect environment variables or credentials. + +## Verify a manifest + +Run verification before sharing or analyzing the artifacts: + +```bash +python -m poweragentbench.artifact_manifest verify results/steady_n2 +``` + +A successful verification exits with status 0. A missing or modified file, malformed manifest, unsafe path, or invalid checksum exits non-zero and reports the problem. The verifier checks the files listed in the manifest; it does not assert that the experiment itself is scientifically valid or that every unlisted file is absent. + +Use a separate manifest per result directory or campaign to keep artifact inventories scoped and understandable. From 77b0ce7b29ef9565083d431f73a0320ca39b65c3 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:20:53 +0530 Subject: [PATCH 4/8] refactor: remove unused manifest target variable --- poweragentbench/artifact_manifest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/poweragentbench/artifact_manifest.py b/poweragentbench/artifact_manifest.py index 783ba85..bd5668e 100644 --- a/poweragentbench/artifact_manifest.py +++ b/poweragentbench/artifact_manifest.py @@ -51,7 +51,6 @@ def build_manifest( except (TypeError, ValueError) as exc: raise ValueError("metadata must be JSON-serializable") from exc - manifest_target = root_path / safe_manifest files: list[dict[str, Any]] = [] for path in sorted(root_path.rglob("*")): if path.is_symlink() or not path.is_file(): From f23114fa0599badbb39248f9f296b9b67acddcfa Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:20:59 +0530 Subject: [PATCH 5/8] ci: run benchmark artifact manifest tests --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 807fa1a..d8987ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,3 +31,6 @@ jobs: - name: Run benchmark contract tests run: python -m pytest -q tests/test_benchmark_utils_contract.py + + - name: Run artifact manifest tests + run: python -m pytest -q tests/test_artifact_manifest.py From dfb3f240ea45bb09d7e2537368dbb6c201a56137 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:28:11 +0530 Subject: [PATCH 6/8] ci: install pytest for Level 2 test discovery --- .github/workflows/test-level2.yml | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/test-level2.yml diff --git a/.github/workflows/test-level2.yml b/.github/workflows/test-level2.yml new file mode 100644 index 0000000..9473f56 --- /dev/null +++ b/.github/workflows/test-level2.yml @@ -0,0 +1,33 @@ +name: Level 2 tests + +on: + push: + branches: + - renewable-grid-benchmark + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install package and test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . pytest + + - name: Run Level 2 tests + run: python -m unittest discover -s tests -v From 97de673dffdd35dfbc0db1b48fb24f7f64c41539 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:29:22 +0530 Subject: [PATCH 7/8] ci: clarify Level 2 pytest dependency --- .github/workflows/test-level2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-level2.yml b/.github/workflows/test-level2.yml index 9473f56..d8adf33 100644 --- a/.github/workflows/test-level2.yml +++ b/.github/workflows/test-level2.yml @@ -27,6 +27,7 @@ jobs: - name: Install package and test dependencies run: | python -m pip install --upgrade pip + # Existing contract tests import pytest during unittest discovery. python -m pip install -e . pytest - name: Run Level 2 tests From 325c88a82b733a7a024486f617bd87811b662d17 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:30:23 +0530 Subject: [PATCH 8/8] fix: preserve upstream files while resolving workflow conflict --- README.md | 3 + benchmarks/steady/level_2/README.md | 25 ++- poweragentbench/steady_state_agentic.py | 92 ++++++++++ tests/test_anytime_risk.py | 223 ++++++++++++++++++++++++ 4 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 tests/test_anytime_risk.py diff --git a/README.md b/README.md index 0148d76..6303660 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ PowerAgentBench/ │ ├── baseline_summary.json # Corrupted-model 0/8 reference + good solution │ ├── solution_template.json │ └── harness/ # Runnable harness (DMView automation + agent loop) +├── tests/ # Top-level Level 2/shared-library regression tests ├── scripts/ # Runnable entry points │ ├── build_case.py # Rebuild the stressed Level 1 scenario │ ├── convert_case.py # Export case39 to MATPOWER and PandaPower @@ -274,6 +275,7 @@ PowerAgentBench returns per-case and aggregate metrics, including: - duplicate validation requests, - explicit submission and auto-finalization indicators, - validation budget use, +- severity-weighted anytime risk discovery for Level 2, including AUC and validation-budget checkpoints ([details](benchmarks/steady/level_2/README.md#severity-weighted-anytime-risk-discovery)), - completed and requested case counts. These metrics distinguish answer quality, tool evidence, search quality, mitigation quality, safety behavior, and workflow compliance. @@ -313,6 +315,7 @@ If an OpenAI run stops early after a retry failure, partial outputs are preserve ## Development Notes +- Run the top-level Level 2/shared-library regression tests from the repository root with `python -m unittest discover -s tests -v`. The self-contained Level 3 suite remains separate. - Use Level 1 to test basic steady-state action submission and physical validation. - Use Level 2 to test agentic behavior, tool use, validation-budget allocation, evidence-backed reporting, and LLM workflows. - Keep hidden oracle quantities, private endpoint URLs, and API keys outside the public repository. diff --git a/benchmarks/steady/level_2/README.md b/benchmarks/steady/level_2/README.md index 95a4ea1..517d0f0 100644 --- a/benchmarks/steady/level_2/README.md +++ b/benchmarks/steady/level_2/README.md @@ -153,10 +153,33 @@ The runner returns per-case and aggregate CSV files with: - `duplicate_validation_requests`, - `submitted_explicitly`, - `auto_finalized`, -- `validation_budget_used`. +- `validation_budget_used`, +- `anytime_risk_auc`, +- `anytime_risk_at_25`, +- `anytime_risk_at_50`, +- `anytime_risk_at_75`, +- `anytime_risk_at_100`. These fields separate search quality, evidence quality, tool compliance, budget use, mitigation, and workflow completion. +### Severity-Weighted Anytime Risk Discovery + +Dangerous contingencies are selected using the explicit severity threshold when one is supplied; otherwise the evaluator uses the configured empirical severity quantile (`danger_quantile`, default `0.95`). Values at or above the cutoff are included. + +Discovery is based on the first completed unique pre-action validation evaluations in their recorded order. For batched validation, candidate order within each batch matters. Duplicate requests do not create additional trajectory positions, and post-action validations do not contribute to this trajectory. A validated contingency contributes here whether or not it appears in `reported`; this is distinct from the evidence-backed submission metrics. + +Each discovered dangerous contingency contributes its non-negative hidden oracle severity. The risk-discovery fraction is the accumulated discovered severity divided by the total non-negative hidden severity of the dangerous set. Hidden oracle values are used only by the evaluator and are never exposed to the agent. + +`anytime_risk_auc` is the normalized trapezoidal area under the discovery curve, normalized by the allocated validation budget. The horizontal axis measures validation-count progress, not elapsed time or LLM turns. Percentage checkpoints are evaluated at the exact fractional budget positions on the linearly interpolated curve; they are not rounded counts of completed validations. Thus, for example, a 75% checkpoint at budget 10 is evaluated at position 7.5. + +If the agent consumes fewer validations than the allocated budget, its final discovery level is held constant over the unused budget. A non-positive budget or total dangerous severity at or below `1e-12` produces zero-valued metrics. + +The per-case metrics complement final discovery, evidence, safety, mitigation, and action-cost metrics rather than replacing them. In particular, `anytime_risk_at_100` is not generally the complement of `severity_weighted_false_negative`, because anytime discovery is based on validated contingencies while the evidence-backed metric requires reported contingencies supported by validation. + +A higher AUC can reflect more total discovered severity, earlier discovery, or both. When comparing discovery timing specifically, consider final discovery alongside AUC. + +For aggregate outputs, distinguish the per-case metric names above from their corresponding `_mean` and `_std` columns. The metric catalogue is descriptive rather than an exhaustive listing of every field returned by `score_agent()`. + ## Evaluation Regimes - **Open**: users can inspect public files and debug agents locally. diff --git a/poweragentbench/steady_state_agentic.py b/poweragentbench/steady_state_agentic.py index 1a9dfcf..3d62bb7 100644 --- a/poweragentbench/steady_state_agentic.py +++ b/poweragentbench/steady_state_agentic.py @@ -91,6 +91,16 @@ class PFResult: @dataclass class AgentOutput: + """Completed agent trajectory and evaluator-visible output. + + ``validated`` preserves the order of first completed unique pre-action + validation evaluations. For tool agents this follows validation-call order + and candidate order within each call after invalid candidates and + duplicates are filtered and the budget is applied. For scripted agents it + follows selected-candidate order, including sampled order for Random and + combined selection order for Hybrid. + """ + name: str validated: Dict[Contingency, float] reported: List[Contingency] @@ -965,6 +975,79 @@ def execute(self, tool: str, args: MutableMapping[str, Any] | None) -> Tuple[Dic return {"error": f"Unknown tool '{tool}'.", "allowed_tools": ["case_summary", "rank_base_loading", "rank_lodf", "validate", "redispatch", "submit"]}, False, None +def compute_anytime_risk_metrics( + output: AgentOutput, + oracle_values: Dict[Contingency, float], + dangerous: set[Contingency], +) -> Dict[str, float]: + """Measure hidden severity discovered along the pre-action validation trajectory. + + The trajectory is the first completed unique pre-action validation + evaluations in output.validated insertion order. Hidden oracle severity + values are used only by the evaluator. Percentage checkpoints are sampled + at exact fractional budget positions on the linearly interpolated + discovery curve; they are not rounded validation counts. + """ + budget = int(output.validation_budget) + + zero_result = { + "anytime_risk_auc": 0.0, + "anytime_risk_at_25": 0.0, + "anytime_risk_at_50": 0.0, + "anytime_risk_at_75": 0.0, + "anytime_risk_at_100": 0.0, + } + if budget <= 0 or not dangerous: + return zero_result + + total_risk = sum( + max(0.0, float(oracle_values.get(c, 0.0))) + for c in dangerous + ) + if total_risk <= 1e-12: + return zero_result + + validation_order: List[Contingency] = list(output.validated.keys()) + points: List[Tuple[int, float]] = [(0, 0.0)] + discovered_risk = 0.0 + + for i, contingency in enumerate(validation_order[:budget], start=1): + if contingency in dangerous: + discovered_risk += max(0.0, float(oracle_values.get(contingency, 0.0))) + risk_fraction = min(1.0, discovered_risk / total_risk) + points.append((i, risk_fraction)) + + # Hold the final discovery level over any unused validation budget. + if points[-1][0] < budget: + points.append((budget, points[-1][1])) + + def risk_at(target: float) -> float: + if target <= 0.0: + return 0.0 + for i in range(1, len(points)): + x0, y0 = points[i - 1] + x1, y1 = points[i] + if target <= x1: + if x1 == x0: + return float(y1) + alpha = (target - x0) / float(x1 - x0) + return float(y0 + alpha * (y1 - y0)) + return float(points[-1][1]) + + auc = 0.0 + for (x0, y0), (x1, y1) in zip(points[:-1], points[1:]): + auc += 0.5 * (y0 + y1) * (x1 - x0) + auc /= float(budget) + + return { + "anytime_risk_auc": float(auc), + "anytime_risk_at_25": risk_at(0.25 * budget), + "anytime_risk_at_50": risk_at(0.50 * budget), + "anytime_risk_at_75": risk_at(0.75 * budget), + "anytime_risk_at_100": risk_at(float(budget)), + } + + def score_agent( original_case: GridCase, output: AgentOutput, @@ -1039,6 +1122,12 @@ def score_agent( unvalidated_claims = reported - found unvalidated_claim_rate = len(unvalidated_claims) / max(1, len(reported)) + anytime_metrics = compute_anytime_risk_metrics( + output=output, + oracle_values=oracle_values, + dangerous=dangerous, + ) + eval_case = output.mitigated_case if output.mitigated_case is not None else original_case pre_top_values = [dc_power_flow(original_case, c).severity for c in oracle_top_list] @@ -1094,6 +1183,9 @@ def score_agent( if output.validation_budget else 0.0 ), + + # Severity-weighted anytime risk discovery. + **anytime_metrics, } diff --git a/tests/test_anytime_risk.py b/tests/test_anytime_risk.py new file mode 100644 index 0000000..825099a --- /dev/null +++ b/tests/test_anytime_risk.py @@ -0,0 +1,223 @@ +import unittest +from unittest.mock import patch + +from poweragentbench.steady_state_agentic import ( + AgentOutput, + PFResult, + SteadyN2ToolServer, + compute_anytime_risk_metrics, + score_agent, +) + + +class TestAnytimeRiskMetrics(unittest.TestCase): + def setUp(self): + self.A, self.B, self.C, self.D = (0, 1), (0, 2), (0, 3), (0, 4) + self.oracle = { + self.A: 4.0, + self.B: 3.0, + self.C: 2.0, + self.D: 1.0, + } + self.dangerous = set(self.oracle) + + def output(self, order, budget=4, values=None, post_validated=None): + values = self.oracle if values is None else values + return AgentOutput( + name="toy", + validated={c: values[c] for c in order}, + reported=[], + post_validated={} if post_validated is None else post_validated, + validation_budget=float(budget), + ) + + def metrics(self, order, budget=4, oracle=None, dangerous=None, values=None, post_validated=None): + return compute_anytime_risk_metrics( + self.output(order, budget, values=values, post_validated=post_validated), + self.oracle if oracle is None else oracle, + self.dangerous if dangerous is None else dangerous, + ) + + def test_known_early_trajectory(self): + result = self.metrics([self.A, self.B, self.C, self.D]) + + self.assertAlmostEqual(result["anytime_risk_auc"], 0.625) + self.assertAlmostEqual(result["anytime_risk_at_25"], 0.40) + self.assertAlmostEqual(result["anytime_risk_at_50"], 0.70) + self.assertAlmostEqual(result["anytime_risk_at_75"], 0.90) + self.assertAlmostEqual(result["anytime_risk_at_100"], 1.00) + + def test_fractional_checkpoints_use_interpolation(self): + result = self.metrics([self.A, self.B, self.C, self.D], budget=10) + + # The discovery curve reaches 0.4 at validation 1 and 0.7 at 2. + # Exact 25% and 75% budget positions are therefore interpolated. + self.assertAlmostEqual(result["anytime_risk_at_25"], 0.80) + self.assertAlmostEqual(result["anytime_risk_at_50"], 1.00) + self.assertAlmostEqual(result["anytime_risk_at_75"], 1.00) + self.assertAlmostEqual(result["anytime_risk_at_100"], 1.00) + + def test_small_fractional_budget_positions(self): + result = self.metrics([self.A, self.B], budget=2) + self.assertAlmostEqual(result["anytime_risk_at_25"], 0.20) + self.assertAlmostEqual(result["anytime_risk_at_50"], 0.40) + self.assertAlmostEqual(result["anytime_risk_at_75"], 0.55) + self.assertAlmostEqual(result["anytime_risk_at_100"], 0.70) + + def test_early_discovery_beats_late_discovery(self): + early = self.metrics([self.A, self.B, self.C, self.D]) + late = self.metrics([self.D, self.C, self.B, self.A]) + + self.assertAlmostEqual(early["anytime_risk_auc"], 0.625) + self.assertAlmostEqual(late["anytime_risk_auc"], 0.375) + self.assertEqual(early["anytime_risk_at_100"], 1.0) + self.assertEqual(late["anytime_risk_at_100"], 1.0) + self.assertGreater(early["anytime_risk_auc"], late["anytime_risk_auc"]) + + def test_validation_budget_limits_trajectory(self): + result = self.metrics([self.A, self.B, self.C, self.D], budget=2) + self.assertAlmostEqual(result["anytime_risk_at_100"], 0.70) + + def test_empty_trajectory_with_positive_budget(self): + result = self.metrics([], budget=4) + self.assertTrue(all(value == 0.0 for value in result.values())) + + def test_partially_consumed_budget_holds_final_discovery(self): + result = self.metrics([self.A], budget=4) + self.assertAlmostEqual(result["anytime_risk_auc"], 0.35) + self.assertAlmostEqual(result["anytime_risk_at_100"], 0.40) + + def test_mixed_dangerous_and_nondangerous_validations(self): + dangerous = {self.A, self.C} + result = self.metrics([self.B, self.A, self.D, self.C], budget=4, dangerous=dangerous) + self.assertAlmostEqual(result["anytime_risk_at_25"], 0.0) + self.assertAlmostEqual(result["anytime_risk_at_50"], 4.0 / 6.0) + self.assertAlmostEqual(result["anytime_risk_at_100"], 1.0) + + def test_hidden_oracle_weights_are_used(self): + public_values = {self.A: 100.0, self.B: 1.0, self.C: 1.0, self.D: 1.0} + hidden_oracle = {self.A: 4.0, self.B: 3.0, self.C: 2.0, self.D: 1.0} + result = self.metrics( + [self.B, self.A, self.C, self.D], + oracle=hidden_oracle, + values=public_values, + ) + self.assertAlmostEqual(result["anytime_risk_at_50"], 0.70) + self.assertAlmostEqual(result["anytime_risk_at_100"], 1.0) + + def test_post_action_validations_do_not_enter_trajectory(self): + result = self.metrics([self.A], budget=4, post_validated={self.B: 999.0}) + self.assertAlmostEqual(result["anytime_risk_at_100"], 0.40) + + def test_empty_or_zero_risk_cases_return_zero(self): + no_validation = self.metrics([], budget=0) + no_danger = self.metrics([self.A], dangerous=set()) + + zero_oracle = {self.A: 0.0, self.B: 0.0} + zero_risk = self.metrics( + [self.A, self.B], + budget=2, + oracle=zero_oracle, + dangerous={self.A, self.B}, + ) + + for result in (no_validation, no_danger, zero_risk): + self.assertTrue(all(value == 0.0 for value in result.values())) + + def test_metrics_are_bounded(self): + result = self.metrics([self.B, self.A, self.D, self.C]) + + for name, value in result.items(): + with self.subTest(metric=name): + self.assertGreaterEqual(value, 0.0) + self.assertLessEqual(value, 1.0) + + +class TestScoreAgentAnytimeRiskIntegration(unittest.TestCase): + def test_score_agent_exposes_anytime_risk_metrics_and_preserves_existing_fields(self): + A = (0, 1) + B = (0, 2) + C = (0, 3) + D = (0, 4) + + oracle_values = {A: 4.0, B: 3.0, C: 2.0, D: 1.0} + output = AgentOutput( + name="integration", + validated={A: 4.0, B: 3.0, C: 2.0, D: 1.0}, + reported=[A, B, C, D], + validation_budget=4.0, + ) + + fake_pf = PFResult( + feasible=True, + flows={}, + loading={}, + severity=1.0, + island_penalty=0.0, + outage=(), + ) + + with patch( + "poweragentbench.steady_state_agentic.dc_power_flow", + return_value=fake_pf, + ): + result = score_agent( + original_case=None, + output=output, + oracle_values=oracle_values, + top_m=4, + danger_threshold=1.0, + ) + + for key in ( + "anytime_risk_auc", + "anytime_risk_at_25", + "anytime_risk_at_50", + "anytime_risk_at_75", + "anytime_risk_at_100", + ): + self.assertIn(key, result) + self.assertGreaterEqual(result[key], 0.0) + self.assertLessEqual(result[key], 1.0) + + self.assertAlmostEqual(result["anytime_risk_auc"], 0.625) + self.assertAlmostEqual(result["anytime_risk_at_100"], 1.0) + self.assertEqual(result["validated_calls"], 4.0) + self.assertEqual(result["reported_top20_recall"], 1.0) + self.assertEqual(result["validated_top20_recall"], 1.0) + self.assertEqual(result["found_top20_recall"], 1.0) + self.assertEqual(result["evidence_rate"], 1.0) + self.assertEqual(result["severity_weighted_false_negative"], 0.0) + + +class TestToolServerValidationOrder(unittest.TestCase): + def test_batches_duplicates_and_budget_truncation_preserve_first_completion_order(self): + A, B, C, D = (0, 1), (0, 2), (0, 3), (0, 4) + case = type("Case", (), {"name": "stub", "n_bus": 5, "n_line": 4, "n_gen": 1})() + + values = {A: 4.0, B: 3.0, C: 2.0, D: 1.0} + candidates = [A, B, C, D] + + with patch( + "poweragentbench.steady_state_agentic.evaluate_contingencies", + side_effect=lambda _case, selected: {c: values[c] for c in selected}, + ): + server = SteadyN2ToolServer(case, candidates, validation_budget=3, report_k=3) + # Deliberately non-sorted order: a sorting/rebuild regression must fail. + server.execute("validate", {"contingencies": [list(B), list(A)]}) + server.execute("validate", {"contingencies": [list(A), list(C), list(D)]}) + + self.assertEqual(list(server.state.validated.keys()), [B, A, C]) + self.assertEqual(server.state.remaining_validations, 0) + output = AgentOutput( + name="tool-order", + validated=server.state.validated, + reported=[], + validation_budget=3.0, + ) + metrics = compute_anytime_risk_metrics(output, values, set(values)) + self.assertAlmostEqual(metrics["anytime_risk_at_100"], 0.90) + + +if __name__ == "__main__": + unittest.main()