From b0a3a82d2d1030ba1b073831ef0f2c526686d555 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:16:00 +0530 Subject: [PATCH 01/10] test: include solution value validation in Level 1 CI --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 807fa1a..702a125 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,4 +30,4 @@ jobs: python -m pip install -e . pytest - name: Run benchmark contract tests - run: python -m pytest -q tests/test_benchmark_utils_contract.py + run: python -m pytest -q tests/test_benchmark_utils_contract.py tests/test_solution_value_validation.py From 2f3c12ee8526ada9a4bcc1b61eb028f5f28a6bbc Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:16:03 +0530 Subject: [PATCH 02/10] docs: align Level 1 test command --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4fdb350..bea1fbd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,10 +16,10 @@ From the repository root: ```bash python -m pip install -e . pytest -python -m pytest -q tests/test_benchmark_utils_contract.py +python -m pytest -q tests/test_benchmark_utils_contract.py tests/test_solution_value_validation.py ``` -The root contract suite is the shared Level 1/library contract suite. Repository CI runs this same suite on Python 3.10, 3.11, and 3.12 for pull requests and for pushes to `main` and branches matching `feat/**`. These commands do not run the self-contained Level 3 test suite. +The shared Level 1/library contract suite and solution-value regression tests run on Python 3.10, 3.11, and 3.12 for pull requests and for pushes to `main` and branches matching `feat/**`. These commands do not run the self-contained Level 3 test suite. ## Benchmark changes From fe114fef381a16872b23085bf8f82e3e99cd6f26 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:16:07 +0530 Subject: [PATCH 03/10] docs: clarify finite solution values --- benchmarks/steady/level_1/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/steady/level_1/README.md b/benchmarks/steady/level_1/README.md index 09b5d71..7f129af 100644 --- a/benchmarks/steady/level_1/README.md +++ b/benchmarks/steady/level_1/README.md @@ -211,6 +211,7 @@ Expected format: Rules: - Omitted actions default to zero. +- Values must be finite numeric values; booleans are not numeric action values. NaN, positive or negative infinity, and integers that cannot be represented as finite floats are invalid. - Values must match the step size in `actionspace.json`. - Values outside the published min/max range are invalid. - Duplicate action ids are invalid. From e5313568ffef22f75d88d28b5c5efae7433e766f Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:16:10 +0530 Subject: [PATCH 04/10] test: scope Level 2 unittest discovery --- .github/workflows/test-level2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-level2.yml b/.github/workflows/test-level2.yml index da83c90..e6a3fce 100644 --- a/.github/workflows/test-level2.yml +++ b/.github/workflows/test-level2.yml @@ -29,4 +29,4 @@ jobs: python -m pip install -e . - name: Run Level 2 tests - run: python -m unittest discover -s tests -v + run: python -m unittest discover -s tests -p 'test_anytime_risk.py' -v From 0d6f8f3d7c5b3ba14c1bb7e985b3da7888d097b1 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:16:14 +0530 Subject: [PATCH 05/10] fix: reject non-finite and boolean solution values --- poweragentbench/benchmark_utils.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/poweragentbench/benchmark_utils.py b/poweragentbench/benchmark_utils.py index bfefe10..d0b9294 100644 --- a/poweragentbench/benchmark_utils.py +++ b/poweragentbench/benchmark_utils.py @@ -119,11 +119,18 @@ def load_solution(path: Path) -> dict[str, float]: value = item.get("value") if not isinstance(action_id, str): raise ValueError("Each action entry must include a string 'id'.") - if not isinstance(value, (int, float)): + # bool subclasses int in Python, but JSON booleans are not numeric actions. + if isinstance(value, bool) or not isinstance(value, (int, float)): raise ValueError(f"Action '{action_id}' must include a numeric 'value'.") + try: + numeric_value = float(value) + except (OverflowError, ValueError) as exc: + raise ValueError(f"Action '{action_id}' must include a finite numeric 'value'.") from exc + if not math.isfinite(numeric_value): + raise ValueError(f"Action '{action_id}' must include a finite numeric 'value'.") if action_id in values: raise ValueError(f"Duplicate action '{action_id}' in solution file.") - values[action_id] = float(value) + values[action_id] = numeric_value return values @@ -144,16 +151,24 @@ def validate_action_values( for action_id, value in action_values.items(): if action_id not in actionspace["action_index"]: raise ValueError(f"Unknown action '{action_id}'.") + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"Action '{action_id}' must be a numeric value.") + try: + numeric_value = float(value) + except (OverflowError, ValueError) as exc: + raise ValueError(f"Action '{action_id}' must be a finite numeric value.") from exc + if not math.isfinite(numeric_value): + raise ValueError(f"Action '{action_id}' must be a finite numeric value.") action = actionspace["action_index"][action_id] min_value = float(action["min_value"]) max_value = float(action["max_value"]) step = float(action["step"]) - if value < min_value - 1e-9 or value > max_value + 1e-9: + if numeric_value < min_value - 1e-9 or numeric_value > max_value + 1e-9: raise ValueError( - f"Action '{action_id}' value {value} outside [{min_value}, {max_value}]." + f"Action '{action_id}' value {numeric_value} outside [{min_value}, {max_value}]." ) - _assert_step(value, min_value, step, action_id) - validated[action_id] = value + _assert_step(numeric_value, min_value, step, action_id) + validated[action_id] = numeric_value return validated From 28eb73f75f44faebad5354a255576caeb243ccfd Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:16:17 +0530 Subject: [PATCH 06/10] test: cover invalid and valid solution numeric values --- tests/test_solution_value_validation.py | 105 ++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tests/test_solution_value_validation.py diff --git a/tests/test_solution_value_validation.py b/tests/test_solution_value_validation.py new file mode 100644 index 0000000..2d3ad09 --- /dev/null +++ b/tests/test_solution_value_validation.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from poweragentbench.benchmark_utils import load_solution, validate_action_values + + +ACTIONSPACE = { + "action_index": { + "tap_T1_up": { + "min_value": 0.0, + "max_value": 2.0, + "step": 1.0, + }, + "phase_shift_T9": { + "min_value": -2.0, + "max_value": 2.0, + "step": 2.0, + }, + "shed_load_L20": { + "min_value": 0.0, + "max_value": 5.0, + "step": 2.5, + }, + } +} + + +@pytest.mark.parametrize( + ("value", "message"), + [ + (True, "numeric"), + (False, "numeric"), + (float("nan"), "finite"), + (float("inf"), "finite"), + (float("-inf"), "finite"), + (10**400, "finite"), + ], + ids=["true", "false", "nan", "positive-infinity", "negative-infinity", "huge-integer"], +) +def test_load_solution_rejects_invalid_numeric_values( + tmp_path: Path, value: object, message: str +) -> None: + path = tmp_path / "solution.json" + path.write_text( + json.dumps({"actions": [{"id": "tap_T1_up", "value": value}]}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=message): + load_solution(path) + + +@pytest.mark.parametrize( + ("value", "message"), + [ + (True, "numeric"), + (False, "numeric"), + (float("nan"), "finite"), + (float("inf"), "finite"), + (float("-inf"), "finite"), + (10**400, "finite"), + ], + ids=["true", "false", "nan", "positive-infinity", "negative-infinity", "huge-integer"], +) +def test_validate_action_values_rejects_invalid_numeric_values( + value: object, message: str +) -> None: + with pytest.raises(ValueError, match=message): + validate_action_values({"tap_T1_up": value}, ACTIONSPACE) + + +@pytest.mark.parametrize( + ("action_id", "value", "expected"), + [ + ("tap_T1_up", 0, 0.0), + ("tap_T1_up", 1, 1.0), + ("tap_T1_up", 1.0, 1.0), + ("phase_shift_T9", -2, -2.0), + ("shed_load_L20", 2.5, 2.5), + ], + ids=["zero", "integer", "float", "permitted-negative", "permitted-fraction"], +) +def test_load_solution_accepts_numeric_values_and_validates_defaults( + tmp_path: Path, action_id: str, value: int | float, expected: float +) -> None: + path = tmp_path / "solution.json" + path.write_text( + json.dumps({"actions": [{"id": action_id, "value": value}]}), + encoding="utf-8", + ) + + loaded = load_solution(path) + assert loaded == {action_id: expected} + + validated = validate_action_values(loaded, ACTIONSPACE) + assert validated[action_id] == expected + assert all( + validated[other_id] == 0.0 + for other_id in ACTIONSPACE["action_index"] + if other_id != action_id + ) From 3314c0f32ad127a5f8f40a38b13fe78542c7dde1 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:17:06 +0530 Subject: [PATCH 07/10] ci: run scoped Level 2 checks on fix branches --- .github/workflows/test-level2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-level2.yml b/.github/workflows/test-level2.yml index e6a3fce..11d4ee0 100644 --- a/.github/workflows/test-level2.yml +++ b/.github/workflows/test-level2.yml @@ -4,6 +4,7 @@ on: push: branches: - renewable-grid-benchmark + - 'fix/**' pull_request: workflow_dispatch: From c01cd1e5828abf031ac2cb0368571ee03aa2e824 Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:17:10 +0530 Subject: [PATCH 08/10] ci: run contract tests on fix branches --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 702a125..ff447ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ name: test on: workflow_dispatch: push: - branches: [main, 'feat/**'] + branches: [main, 'feat/**', 'fix/**'] pull_request: jobs: From aa6f439b9d5f13b02d2b36abceda0c3272d50d1b Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:37:02 +0530 Subject: [PATCH 09/10] docs: document fix branch CI triggers --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bea1fbd..8f1321e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,7 +19,7 @@ python -m pip install -e . pytest python -m pytest -q tests/test_benchmark_utils_contract.py tests/test_solution_value_validation.py ``` -The shared Level 1/library contract suite and solution-value regression tests run on Python 3.10, 3.11, and 3.12 for pull requests and for pushes to `main` and branches matching `feat/**`. These commands do not run the self-contained Level 3 test suite. +The shared Level 1/library contract suite and solution-value regression tests run on Python 3.10, 3.11, and 3.12 for pull requests and for pushes to `main` and branches matching `feat/**` or `fix/**`. These commands do not run the self-contained Level 3 test suite. ## Benchmark changes From c4cc6aab494465ef886ee4310a34b240571be2ab Mon Sep 17 00:00:00 2001 From: Burhan Abdullah <131237388+BurhanAbdullah@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:37:05 +0530 Subject: [PATCH 10/10] docs: scope Level 2 test command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6303660..d5d3422 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ If an OpenAI run stops early after a retry failure, partial outputs are preserve ## Development Notes -- Run the top-level Level 2/shared-library regression tests from the repository root with `python -m unittest discover -s tests -v`. The self-contained Level 3 suite remains separate. +- Run the top-level Level 2/shared-library regression tests from the repository root with `python -m unittest discover -s tests -p 'test_anytime_risk.py' -v`. The self-contained Level 3 suite remains separate. - Use Level 1 to test basic steady-state action submission and physical validation. - Use Level 2 to test agentic behavior, tool use, validation-budget allocation, evidence-backed reporting, and LLM workflows. - Keep hidden oracle quantities, private endpoint URLs, and API keys outside the public repository.