diff --git a/.github/workflows/test-level2.yml b/.github/workflows/test-level2.yml index da83c90..11d4ee0 100644 --- a/.github/workflows/test-level2.yml +++ b/.github/workflows/test-level2.yml @@ -4,6 +4,7 @@ on: push: branches: - renewable-grid-benchmark + - 'fix/**' pull_request: workflow_dispatch: @@ -29,4 +30,4 @@ jobs: python -m pip install -e . - name: Run Level 2 tests - run: python -m unittest discover -s tests -v + run: python -m unittest discover -s tests -p 'test_anytime_risk.py' -v diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 807fa1a..ff447ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ name: test on: workflow_dispatch: push: - branches: [main, 'feat/**'] + branches: [main, 'feat/**', 'fix/**'] pull_request: jobs: @@ -30,4 +30,4 @@ jobs: python -m pip install -e . pytest - name: Run benchmark contract tests - run: python -m pytest -q tests/test_benchmark_utils_contract.py + run: python -m pytest -q tests/test_benchmark_utils_contract.py tests/test_solution_value_validation.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4fdb350..8f1321e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,10 +16,10 @@ From the repository root: ```bash python -m pip install -e . pytest -python -m pytest -q tests/test_benchmark_utils_contract.py +python -m pytest -q tests/test_benchmark_utils_contract.py tests/test_solution_value_validation.py ``` -The root contract suite is the shared Level 1/library contract suite. Repository CI runs this same suite on Python 3.10, 3.11, and 3.12 for pull requests and for pushes to `main` and branches matching `feat/**`. These commands do not run the self-contained Level 3 test suite. +The shared Level 1/library contract suite and solution-value regression tests run on Python 3.10, 3.11, and 3.12 for pull requests and for pushes to `main` and branches matching `feat/**` or `fix/**`. These commands do not run the self-contained Level 3 test suite. ## Benchmark changes diff --git a/README.md b/README.md index 6303660..d5d3422 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ If an OpenAI run stops early after a retry failure, partial outputs are preserve ## Development Notes -- Run the top-level Level 2/shared-library regression tests from the repository root with `python -m unittest discover -s tests -v`. The self-contained Level 3 suite remains separate. +- Run the top-level Level 2/shared-library regression tests from the repository root with `python -m unittest discover -s tests -p 'test_anytime_risk.py' -v`. The self-contained Level 3 suite remains separate. - Use Level 1 to test basic steady-state action submission and physical validation. - Use Level 2 to test agentic behavior, tool use, validation-budget allocation, evidence-backed reporting, and LLM workflows. - Keep hidden oracle quantities, private endpoint URLs, and API keys outside the public repository. diff --git a/benchmarks/steady/level_1/README.md b/benchmarks/steady/level_1/README.md index 09b5d71..7f129af 100644 --- a/benchmarks/steady/level_1/README.md +++ b/benchmarks/steady/level_1/README.md @@ -211,6 +211,7 @@ Expected format: Rules: - Omitted actions default to zero. +- Values must be finite numeric values; booleans are not numeric action values. NaN, positive or negative infinity, and integers that cannot be represented as finite floats are invalid. - Values must match the step size in `actionspace.json`. - Values outside the published min/max range are invalid. - Duplicate action ids are invalid. diff --git a/poweragentbench/benchmark_utils.py b/poweragentbench/benchmark_utils.py index bfefe10..d0b9294 100644 --- a/poweragentbench/benchmark_utils.py +++ b/poweragentbench/benchmark_utils.py @@ -119,11 +119,18 @@ def load_solution(path: Path) -> dict[str, float]: value = item.get("value") if not isinstance(action_id, str): raise ValueError("Each action entry must include a string 'id'.") - if not isinstance(value, (int, float)): + # bool subclasses int in Python, but JSON booleans are not numeric actions. + if isinstance(value, bool) or not isinstance(value, (int, float)): raise ValueError(f"Action '{action_id}' must include a numeric 'value'.") + try: + numeric_value = float(value) + except (OverflowError, ValueError) as exc: + raise ValueError(f"Action '{action_id}' must include a finite numeric 'value'.") from exc + if not math.isfinite(numeric_value): + raise ValueError(f"Action '{action_id}' must include a finite numeric 'value'.") if action_id in values: raise ValueError(f"Duplicate action '{action_id}' in solution file.") - values[action_id] = float(value) + values[action_id] = numeric_value return values @@ -144,16 +151,24 @@ def validate_action_values( for action_id, value in action_values.items(): if action_id not in actionspace["action_index"]: raise ValueError(f"Unknown action '{action_id}'.") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"Action '{action_id}' must be a numeric value.") + try: + numeric_value = float(value) + except (OverflowError, ValueError) as exc: + raise ValueError(f"Action '{action_id}' must be a finite numeric value.") from exc + if not math.isfinite(numeric_value): + raise ValueError(f"Action '{action_id}' must be a finite numeric value.") action = actionspace["action_index"][action_id] min_value = float(action["min_value"]) max_value = float(action["max_value"]) step = float(action["step"]) - if value < min_value - 1e-9 or value > max_value + 1e-9: + if numeric_value < min_value - 1e-9 or numeric_value > max_value + 1e-9: raise ValueError( - f"Action '{action_id}' value {value} outside [{min_value}, {max_value}]." + f"Action '{action_id}' value {numeric_value} outside [{min_value}, {max_value}]." ) - _assert_step(value, min_value, step, action_id) - validated[action_id] = value + _assert_step(numeric_value, min_value, step, action_id) + validated[action_id] = numeric_value return validated diff --git a/tests/test_solution_value_validation.py b/tests/test_solution_value_validation.py new file mode 100644 index 0000000..2d3ad09 --- /dev/null +++ b/tests/test_solution_value_validation.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from poweragentbench.benchmark_utils import load_solution, validate_action_values + + +ACTIONSPACE = { + "action_index": { + "tap_T1_up": { + "min_value": 0.0, + "max_value": 2.0, + "step": 1.0, + }, + "phase_shift_T9": { + "min_value": -2.0, + "max_value": 2.0, + "step": 2.0, + }, + "shed_load_L20": { + "min_value": 0.0, + "max_value": 5.0, + "step": 2.5, + }, + } +} + + +@pytest.mark.parametrize( + ("value", "message"), + [ + (True, "numeric"), + (False, "numeric"), + (float("nan"), "finite"), + (float("inf"), "finite"), + (float("-inf"), "finite"), + (10**400, "finite"), + ], + ids=["true", "false", "nan", "positive-infinity", "negative-infinity", "huge-integer"], +) +def test_load_solution_rejects_invalid_numeric_values( + tmp_path: Path, value: object, message: str +) -> None: + path = tmp_path / "solution.json" + path.write_text( + json.dumps({"actions": [{"id": "tap_T1_up", "value": value}]}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=message): + load_solution(path) + + +@pytest.mark.parametrize( + ("value", "message"), + [ + (True, "numeric"), + (False, "numeric"), + (float("nan"), "finite"), + (float("inf"), "finite"), + (float("-inf"), "finite"), + (10**400, "finite"), + ], + ids=["true", "false", "nan", "positive-infinity", "negative-infinity", "huge-integer"], +) +def test_validate_action_values_rejects_invalid_numeric_values( + value: object, message: str +) -> None: + with pytest.raises(ValueError, match=message): + validate_action_values({"tap_T1_up": value}, ACTIONSPACE) + + +@pytest.mark.parametrize( + ("action_id", "value", "expected"), + [ + ("tap_T1_up", 0, 0.0), + ("tap_T1_up", 1, 1.0), + ("tap_T1_up", 1.0, 1.0), + ("phase_shift_T9", -2, -2.0), + ("shed_load_L20", 2.5, 2.5), + ], + ids=["zero", "integer", "float", "permitted-negative", "permitted-fraction"], +) +def test_load_solution_accepts_numeric_values_and_validates_defaults( + tmp_path: Path, action_id: str, value: int | float, expected: float +) -> None: + path = tmp_path / "solution.json" + path.write_text( + json.dumps({"actions": [{"id": action_id, "value": value}]}), + encoding="utf-8", + ) + + loaded = load_solution(path) + assert loaded == {action_id: expected} + + validated = validate_action_values(loaded, ACTIONSPACE) + assert validated[action_id] == expected + assert all( + validated[other_id] == 0.0 + for other_id in ACTIONSPACE["action_index"] + if other_id != action_id + )