diff --git a/benchmarks/steady/level_3/README.md b/benchmarks/steady/level_3/README.md index 1b5b1a9..3752f65 100644 --- a/benchmarks/steady/level_3/README.md +++ b/benchmarks/steady/level_3/README.md @@ -153,11 +153,16 @@ write it to JSON, and the scorer replays it against the frozen snapshot: ```bash uv run restorebench-score attempt.json uv run restorebench-score --batch attempts/ --out results/my_run +uv run restorebench-score --batch attempts/ --data-dir dataset/pegase89 --out results/pegase89_run ``` +`--data-dir` accepts either a dataset root such as `dataset/pegase89` or its `full/` directory; +batch scoring derives the matching `evaluation_manifest.json` from the same corpus. + The report gives the status (`SUCCESS` / `BUDGET_EXHAUSTED`), the per-step outcome of every -maneuver, the number of invalid actions, and the quality of the terminal state. A batch report -additionally aggregates success rate and maneuver counts, broken down by dataset split. +maneuver, the number of invalid actions, and the quality of the terminal state. Batch mode writes +one `attempt_NNNN_score.json` per input plus `benchmark_report.json`, which aggregates success rate +and maneuver counts broken down by dataset split. To sanity-check the setup, score a scenario's own witness from `private/witnesses.json` — it must come back `SUCCESS`. @@ -188,7 +193,7 @@ is the only thing you have to supply: ```bash uv run restorebench-sweep --campaign ieee118-anthropic --dry-run # print the queue, spend nothing -uv run restorebench-sweep --campaign ieee118-anthropic # 46 scenarios × 9 cells +uv run restorebench-sweep --campaign ieee118-anthropic # 46 scenarios × 27 cells uv run restorebench-sweep --campaign ieee118-bedrock uv run restorebench-sweep --campaign ieee118-openai uv run restorebench-sweep --campaign pegase89-anthropic @@ -196,14 +201,16 @@ uv run restorebench-sweep --campaign pegase89-bedrock uv run restorebench-sweep --campaign pegase89-openai ``` +Every campaign runs three repetitions (`rep0`, `rep1`, and `rep2`). + | Campaign | Corpus | Models | Cells | |---|---|---|---:| -| `ieee118-anthropic` | IEEE 118, all 46 | Haiku 4.5, Sonnet 5, Opus 5 | 414 | -| `ieee118-bedrock` | the same 46 scenarios | DeepSeek V3.2, Kimi K2.5, GLM-5 | 414 | -| `ieee118-openai` | the same 46 scenarios | GPT-5.6 Sol | 138 | -| `pegase89-anthropic` | PEGASE 89, all 46 | Haiku 4.5, Sonnet 5, Opus 5 | 414 | -| `pegase89-bedrock` | the same 46 scenarios | DeepSeek V3.2, Kimi K2.5, GLM-5 | 414 | -| `pegase89-openai` | the same 46 scenarios | GPT-5.6 Sol | 138 | +| `ieee118-anthropic` | IEEE 118, all 46 | Haiku 4.5, Sonnet 5, Opus 5 | 1,242 | +| `ieee118-bedrock` | the same 46 scenarios | DeepSeek V3.2, Kimi K2.5, GLM-5 | 1,242 | +| `ieee118-openai` | the same 46 scenarios | GPT-5.6 Sol | 414 | +| `pegase89-anthropic` | PEGASE 89, all 46 | Haiku 4.5, Sonnet 5, Opus 5 | 1,242 | +| `pegase89-bedrock` | the same 46 scenarios | DeepSeek V3.2, Kimi K2.5, GLM-5 | 1,242 | +| `pegase89-openai` | the same 46 scenarios | GPT-5.6 Sol | 414 | The sweep is resumable and writes one scenario at a time, so an interrupted run leaves whole cases rather than half-measured ones. It waits out provider outages instead of recording them as model diff --git a/benchmarks/steady/level_3/dataset/ieee118/private/witnesses.json b/benchmarks/steady/level_3/dataset/ieee118/private/witnesses.json index 9b31bfa..db037d8 100644 --- a/benchmarks/steady/level_3/dataset/ieee118/private/witnesses.json +++ b/benchmarks/steady/level_3/dataset/ieee118/private/witnesses.json @@ -26853,8 +26853,8 @@ { "maneuvers": [ { - "gen_id": 1, - "new_vm_pu": 0.988, + "gen_id": 27, + "new_vm_pu": 1.015, "type": "GEN_V_SETPOINT" } ], @@ -26862,7 +26862,7 @@ "search_policy_version": "deterministic-bfs-beam-qv-v3", "state_hashes": [ "66f2285f65b6399739a74fb51fb09b6f1b919a4bbf322f1302c3e72edf7adc95", - "0a7063a799019b260a4b95fa1b7574d2100cf8e53f4b33b19401d841ea13ad5b" + "bbe5c8b5cae723543d526a93f0cc05e507157c206426dc9acf0d69a8e31fcb99" ], "terminal_pf": { "converged": true, @@ -26874,7 +26874,7 @@ "failure_reasons": [ { "code": "HARD_VOLTAGE_ENVELOPE", - "detail": "bus voltage range [0.736715747134, 1.05] outside [0.9, 1.1] pu", + "detail": "bus voltage range [0.737490976544, 1.05] outside [0.9, 1.1] pu", "element_id": null } ], @@ -26883,91 +26883,91 @@ "generator_q_status": [ { "gen_id": 0, - "lower_headroom_mvar": 5.360567871233329, + "lower_headroom_mvar": 1.8513829050157682, "max_q_mvar": 15.0, "min_q_mvar": -5.0, - "near_limit": false, + "near_limit": true, "policy_version": "directional-q-limit-v1", - "q_mvar": 0.3605678712333287, + "q_mvar": -3.148617094984232, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 14.639432128766671 + "upper_headroom_mvar": 18.148617094984232 }, { "gen_id": 1, - "lower_headroom_mvar": 229.6034452339075, + "lower_headroom_mvar": 284.1100238017639, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -70.39655476609249, + "q_mvar": -15.889976198236127, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 370.3965547660925 + "upper_headroom_mvar": 315.8899761982361 }, { "gen_id": 2, - "lower_headroom_mvar": 40.8057911838221, + "lower_headroom_mvar": 28.94825248862871, "max_q_mvar": 50.0, "min_q_mvar": -13.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 27.8057911838221, + "q_mvar": 15.948252488628711, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 22.1942088161779 + "upper_headroom_mvar": 34.05174751137129 }, { "gen_id": 3, - "lower_headroom_mvar": 429.2937782485922, + "lower_headroom_mvar": 404.6155792491274, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 129.29377824859222, + "q_mvar": 104.61557924912739, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 170.70622175140778 + "upper_headroom_mvar": 195.3844207508726 }, { "gen_id": 4, - "lower_headroom_mvar": 117.28965638261954, + "lower_headroom_mvar": 117.28965638261994, "max_q_mvar": 200.0, "min_q_mvar": -147.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -29.71034361738046, + "q_mvar": -29.71034361738006, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 229.71034361738046 + "upper_headroom_mvar": 229.71034361738006 }, { "gen_id": 5, - "lower_headroom_mvar": 137.50391218454178, + "lower_headroom_mvar": 121.84093525215442, "max_q_mvar": 120.0, "min_q_mvar": -35.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 102.50391218454178, + "q_mvar": 86.84093525215442, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 17.496087815458225 + "upper_headroom_mvar": 33.15906474784558 }, { "gen_id": 6, - "lower_headroom_mvar": 17.847201166309272, + "lower_headroom_mvar": 16.977882516538347, "max_q_mvar": 30.0, "min_q_mvar": -10.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 7.847201166309272, + "q_mvar": 6.977882516538347, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 22.152798833690728 + "upper_headroom_mvar": 23.022117483461653 }, { "gen_id": 7, - "lower_headroom_mvar": 43.62672651009109, + "lower_headroom_mvar": 43.63465005744729, "max_q_mvar": 50.0, "min_q_mvar": -16.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 27.62672651009109, + "q_mvar": 27.63465005744729, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 22.37327348990891 + "upper_headroom_mvar": 22.36534994255271 }, { "gen_id": 8, @@ -26982,58 +26982,58 @@ }, { "gen_id": 9, - "lower_headroom_mvar": 307.22657465919946, + "lower_headroom_mvar": 307.9527950780775, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 7.226574659199457, + "q_mvar": 7.952795078077486, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 292.77342534080054 + "upper_headroom_mvar": 292.0472049219225 }, { "gen_id": 10, - "lower_headroom_mvar": 97.13206764405216, + "lower_headroom_mvar": 97.11613055320339, "max_q_mvar": 140.0, "min_q_mvar": -47.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 50.13206764405216, + "q_mvar": 50.11613055320339, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 89.86793235594784 + "upper_headroom_mvar": 89.88386944679661 }, { "gen_id": 11, - "lower_headroom_mvar": 1020.487788663409, + "lower_headroom_mvar": 1020.466182409278, "max_q_mvar": 1000.0, "min_q_mvar": -1000.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 20.487788663408992, + "q_mvar": 20.466182409277963, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 979.512211336591 + "upper_headroom_mvar": 979.533817590722 }, { "gen_id": 12, - "lower_headroom_mvar": 305.61709029911657, + "lower_headroom_mvar": 305.6034284757834, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 5.617090299116569, + "q_mvar": 5.603428475783403, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 294.38290970088343 + "upper_headroom_mvar": 294.3965715242166 }, { "gen_id": 13, - "lower_headroom_mvar": 331.5820607678841, + "lower_headroom_mvar": 331.58398454607976, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 31.582060767884116, + "q_mvar": 31.583984546079762, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 268.4179392321159 + "upper_headroom_mvar": 268.41601545392024 }, { "gen_id": 14, @@ -27059,124 +27059,124 @@ }, { "gen_id": 16, - "lower_headroom_mvar": 7.49979392062458, + "lower_headroom_mvar": 7.503976381687043, "max_q_mvar": 24.0, "min_q_mvar": -8.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -0.5002060793754204, + "q_mvar": -0.4960236183129574, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 24.50020607937542 + "upper_headroom_mvar": 24.496023618312957 }, { "gen_id": 17, - "lower_headroom_mvar": 329.3096972646947, + "lower_headroom_mvar": 329.31734774078893, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 29.309697264694705, + "q_mvar": 29.317347740788932, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 270.6903027353053 + "upper_headroom_mvar": 270.68265225921107 }, { "gen_id": 18, - "lower_headroom_mvar": 337.9091780971322, + "lower_headroom_mvar": 337.9024858467397, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 37.90917809713221, + "q_mvar": 37.902485846739694, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 262.0908219028678 + "upper_headroom_mvar": 262.0975141532603 }, { "gen_id": 19, - "lower_headroom_mvar": 93.1692528999001, + "lower_headroom_mvar": 93.1639861057424, "max_q_mvar": 100.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -6.830747100099899, + "q_mvar": -6.836013894257604, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 106.8307471000999 + "upper_headroom_mvar": 106.8360138942576 }, { "gen_id": 20, - "lower_headroom_mvar": 183.30852965137157, + "lower_headroom_mvar": 183.28877572587396, "max_q_mvar": 210.0, "min_q_mvar": -85.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 98.30852965137157, + "q_mvar": 98.28877572587396, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 111.69147034862843 + "upper_headroom_mvar": 111.71122427412604 }, { "gen_id": 21, - "lower_headroom_mvar": 300.2798529478665, + "lower_headroom_mvar": 300.28029779969705, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 0.27985294786651593, + "q_mvar": 0.2802977996970526, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 299.7201470521335 + "upper_headroom_mvar": 299.71970220030295 }, { "gen_id": 22, - "lower_headroom_mvar": 12.63772110450467, + "lower_headroom_mvar": 12.63773943349172, "max_q_mvar": 23.0, "min_q_mvar": -8.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 4.637721104504671, + "q_mvar": 4.637739433491721, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 18.36227889549533 + "upper_headroom_mvar": 18.36226056650828 }, { "gen_id": 23, - "lower_headroom_mvar": 4.998934460115802, + "lower_headroom_mvar": 4.998996898805277, "max_q_mvar": 15.0, "min_q_mvar": -8.0, "near_limit": true, "policy_version": "directional-q-limit-v1", - "q_mvar": -3.0010655398841983, + "q_mvar": -3.0010031011947227, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 18.0010655398842 + "upper_headroom_mvar": 18.001003101194723 }, { "gen_id": 24, - "lower_headroom_mvar": 127.89585071606496, + "lower_headroom_mvar": 127.89476001325905, "max_q_mvar": 180.0, "min_q_mvar": -60.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 67.89585071606496, + "q_mvar": 67.89476001325905, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 112.10414928393504 + "upper_headroom_mvar": 112.10523998674095 }, { "gen_id": 25, - "lower_headroom_mvar": 53.49944796273613, + "lower_headroom_mvar": 53.497118850753225, "max_q_mvar": 300.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -46.50055203726387, + "q_mvar": -46.502881149246775, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 346.5005520372639 + "upper_headroom_mvar": 346.5028811492468 }, { "gen_id": 26, - "lower_headroom_mvar": 21.186417505052162, + "lower_headroom_mvar": 21.186544151786933, "max_q_mvar": 20.0, "min_q_mvar": -20.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 1.1864175050521624, + "q_mvar": 1.1865441517869328, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 18.813582494947838 + "upper_headroom_mvar": 18.813455848213067 }, { "gen_id": 27, @@ -27191,47 +27191,47 @@ }, { "gen_id": 28, - "lower_headroom_mvar": 65.94553902901708, + "lower_headroom_mvar": 65.94300344091698, "max_q_mvar": 200.0, "min_q_mvar": -67.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -1.054460970982916, + "q_mvar": -1.056996559083018, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 201.05446097098292 + "upper_headroom_mvar": 201.05699655908302 }, { "gen_id": 29, - "lower_headroom_mvar": 42.0, + "lower_headroom_mvar": 31.94985196449798, "max_q_mvar": 32.0, "min_q_mvar": -10.0, - "near_limit": true, + "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 32.0, - "status": "Q_LIMITED_UPPER", - "upper_headroom_mvar": 0.0 + "q_mvar": 21.94985196449798, + "status": "PV_CONTROLLABLE", + "upper_headroom_mvar": 10.05014803550202 }, { "gen_id": 30, - "lower_headroom_mvar": 91.13255331713401, + "lower_headroom_mvar": 92.0446071400052, "max_q_mvar": 100.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -8.867446682865989, + "q_mvar": -7.955392859994802, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 108.86744668286599 + "upper_headroom_mvar": 107.9553928599948 }, { "gen_id": 31, - "lower_headroom_mvar": 106.40258779053875, + "lower_headroom_mvar": 110.04315827538964, "max_q_mvar": 100.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 6.402587790538746, + "q_mvar": 10.043158275389644, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 93.59741220946125 + "upper_headroom_mvar": 89.95684172461036 }, { "gen_id": 32, @@ -27290,14 +27290,14 @@ }, { "gen_id": 37, - "lower_headroom_mvar": 195.00551475631613, + "lower_headroom_mvar": 194.7400928728215, "max_q_mvar": 1000.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 95.00551475631613, + "q_mvar": 94.74009287282149, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 904.9944852436838 + "upper_headroom_mvar": 905.2599071271785 }, { "gen_id": 38, @@ -27400,14 +27400,14 @@ }, { "gen_id": 47, - "lower_headroom_mvar": 221.96276869678172, + "lower_headroom_mvar": 221.78578904061376, "max_q_mvar": 200.0, "min_q_mvar": -200.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 21.962768696781723, + "q_mvar": 21.785789040613764, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 178.03723130321828 + "upper_headroom_mvar": 178.21421095938624 }, { "gen_id": 48, @@ -27422,47 +27422,47 @@ }, { "gen_id": 49, - "lower_headroom_mvar": 97.14621010586505, + "lower_headroom_mvar": 97.06983540642908, "max_q_mvar": 1000.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -2.853789894134948, + "q_mvar": -2.9301645935709217, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 1002.8537898941349 + "upper_headroom_mvar": 1002.9301645935709 }, { "gen_id": 50, - "lower_headroom_mvar": 143.11885794282355, + "lower_headroom_mvar": 143.0262458137593, "max_q_mvar": 1000.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 43.11885794282355, + "q_mvar": 43.02624581375929, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 956.8811420571765 + "upper_headroom_mvar": 956.9737541862407 }, { "gen_id": 51, - "lower_headroom_mvar": 108.56798996156715, + "lower_headroom_mvar": 108.5835544387685, "max_q_mvar": 200.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 8.56798996156715, + "q_mvar": 8.583554438768502, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 191.43201003843285 + "upper_headroom_mvar": 191.4164455612315 }, { "gen_id": 52, - "lower_headroom_mvar": 1320.7102460457527, + "lower_headroom_mvar": 1320.6799570307146, "max_q_mvar": 1000.0, "min_q_mvar": -1000.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 320.7102460457527, + "q_mvar": 320.6799570307146, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 679.2897539542473 + "upper_headroom_mvar": 679.3200429692854 } ], "generator_q_within_limits": true, @@ -27473,7 +27473,6 @@ 14, 15, 27, - 29, 32, 33, 34, @@ -27493,20 +27492,19 @@ "slack_results": [ { "ext_grid_id": 0, - "p_mw": 780.8818585446998, + "p_mw": 780.3534273527885, "p_within_limits": true, - "q_mvar": -6.894049956363409, + "q_mvar": -2.874678523876897, "q_within_limits": true } ], "voltage": { "hard_envelope_ok": false, - "high_bus_ids": [ - 24 - ], + "high_bus_ids": [], "low_bus_ids": [ 52, 73, + 74, 75, 76, 77, @@ -27528,99 +27526,99 @@ 105, 117 ], - "max_vm_pu": 1.0500000000000003, - "min_vm_pu": 0.7367157471338502, + "max_vm_pu": 1.05, + "min_vm_pu": 0.7374909765437455, "runtime_quality_ok": false } }, "generator_q_status": [ { "gen_id": 0, - "lower_headroom_mvar": 5.360567871233329, + "lower_headroom_mvar": 1.8513829050157682, "max_q_mvar": 15.0, "min_q_mvar": -5.0, - "near_limit": false, + "near_limit": true, "policy_version": "directional-q-limit-v1", - "q_mvar": 0.3605678712333287, + "q_mvar": -3.148617094984232, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 14.639432128766671 + "upper_headroom_mvar": 18.148617094984232 }, { "gen_id": 1, - "lower_headroom_mvar": 229.6034452339075, + "lower_headroom_mvar": 284.1100238017639, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -70.39655476609249, + "q_mvar": -15.889976198236127, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 370.3965547660925 + "upper_headroom_mvar": 315.8899761982361 }, { "gen_id": 2, - "lower_headroom_mvar": 40.8057911838221, + "lower_headroom_mvar": 28.94825248862871, "max_q_mvar": 50.0, "min_q_mvar": -13.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 27.8057911838221, + "q_mvar": 15.948252488628711, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 22.1942088161779 + "upper_headroom_mvar": 34.05174751137129 }, { "gen_id": 3, - "lower_headroom_mvar": 429.2937782485922, + "lower_headroom_mvar": 404.6155792491274, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 129.29377824859222, + "q_mvar": 104.61557924912739, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 170.70622175140778 + "upper_headroom_mvar": 195.3844207508726 }, { "gen_id": 4, - "lower_headroom_mvar": 117.28965638261954, + "lower_headroom_mvar": 117.28965638261994, "max_q_mvar": 200.0, "min_q_mvar": -147.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -29.71034361738046, + "q_mvar": -29.71034361738006, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 229.71034361738046 + "upper_headroom_mvar": 229.71034361738006 }, { "gen_id": 5, - "lower_headroom_mvar": 137.50391218454178, + "lower_headroom_mvar": 121.84093525215442, "max_q_mvar": 120.0, "min_q_mvar": -35.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 102.50391218454178, + "q_mvar": 86.84093525215442, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 17.496087815458225 + "upper_headroom_mvar": 33.15906474784558 }, { "gen_id": 6, - "lower_headroom_mvar": 17.847201166309272, + "lower_headroom_mvar": 16.977882516538347, "max_q_mvar": 30.0, "min_q_mvar": -10.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 7.847201166309272, + "q_mvar": 6.977882516538347, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 22.152798833690728 + "upper_headroom_mvar": 23.022117483461653 }, { "gen_id": 7, - "lower_headroom_mvar": 43.62672651009109, + "lower_headroom_mvar": 43.63465005744729, "max_q_mvar": 50.0, "min_q_mvar": -16.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 27.62672651009109, + "q_mvar": 27.63465005744729, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 22.37327348990891 + "upper_headroom_mvar": 22.36534994255271 }, { "gen_id": 8, @@ -27635,58 +27633,58 @@ }, { "gen_id": 9, - "lower_headroom_mvar": 307.22657465919946, + "lower_headroom_mvar": 307.9527950780775, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 7.226574659199457, + "q_mvar": 7.952795078077486, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 292.77342534080054 + "upper_headroom_mvar": 292.0472049219225 }, { "gen_id": 10, - "lower_headroom_mvar": 97.13206764405216, + "lower_headroom_mvar": 97.11613055320339, "max_q_mvar": 140.0, "min_q_mvar": -47.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 50.13206764405216, + "q_mvar": 50.11613055320339, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 89.86793235594784 + "upper_headroom_mvar": 89.88386944679661 }, { "gen_id": 11, - "lower_headroom_mvar": 1020.487788663409, + "lower_headroom_mvar": 1020.466182409278, "max_q_mvar": 1000.0, "min_q_mvar": -1000.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 20.487788663408992, + "q_mvar": 20.466182409277963, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 979.512211336591 + "upper_headroom_mvar": 979.533817590722 }, { "gen_id": 12, - "lower_headroom_mvar": 305.61709029911657, + "lower_headroom_mvar": 305.6034284757834, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 5.617090299116569, + "q_mvar": 5.603428475783403, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 294.38290970088343 + "upper_headroom_mvar": 294.3965715242166 }, { "gen_id": 13, - "lower_headroom_mvar": 331.5820607678841, + "lower_headroom_mvar": 331.58398454607976, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 31.582060767884116, + "q_mvar": 31.583984546079762, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 268.4179392321159 + "upper_headroom_mvar": 268.41601545392024 }, { "gen_id": 14, @@ -27712,124 +27710,124 @@ }, { "gen_id": 16, - "lower_headroom_mvar": 7.49979392062458, + "lower_headroom_mvar": 7.503976381687043, "max_q_mvar": 24.0, "min_q_mvar": -8.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -0.5002060793754204, + "q_mvar": -0.4960236183129574, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 24.50020607937542 + "upper_headroom_mvar": 24.496023618312957 }, { "gen_id": 17, - "lower_headroom_mvar": 329.3096972646947, + "lower_headroom_mvar": 329.31734774078893, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 29.309697264694705, + "q_mvar": 29.317347740788932, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 270.6903027353053 + "upper_headroom_mvar": 270.68265225921107 }, { "gen_id": 18, - "lower_headroom_mvar": 337.9091780971322, + "lower_headroom_mvar": 337.9024858467397, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 37.90917809713221, + "q_mvar": 37.902485846739694, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 262.0908219028678 + "upper_headroom_mvar": 262.0975141532603 }, { "gen_id": 19, - "lower_headroom_mvar": 93.1692528999001, + "lower_headroom_mvar": 93.1639861057424, "max_q_mvar": 100.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -6.830747100099899, + "q_mvar": -6.836013894257604, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 106.8307471000999 + "upper_headroom_mvar": 106.8360138942576 }, { "gen_id": 20, - "lower_headroom_mvar": 183.30852965137157, + "lower_headroom_mvar": 183.28877572587396, "max_q_mvar": 210.0, "min_q_mvar": -85.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 98.30852965137157, + "q_mvar": 98.28877572587396, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 111.69147034862843 + "upper_headroom_mvar": 111.71122427412604 }, { "gen_id": 21, - "lower_headroom_mvar": 300.2798529478665, + "lower_headroom_mvar": 300.28029779969705, "max_q_mvar": 300.0, "min_q_mvar": -300.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 0.27985294786651593, + "q_mvar": 0.2802977996970526, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 299.7201470521335 + "upper_headroom_mvar": 299.71970220030295 }, { "gen_id": 22, - "lower_headroom_mvar": 12.63772110450467, + "lower_headroom_mvar": 12.63773943349172, "max_q_mvar": 23.0, "min_q_mvar": -8.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 4.637721104504671, + "q_mvar": 4.637739433491721, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 18.36227889549533 + "upper_headroom_mvar": 18.36226056650828 }, { "gen_id": 23, - "lower_headroom_mvar": 4.998934460115802, + "lower_headroom_mvar": 4.998996898805277, "max_q_mvar": 15.0, "min_q_mvar": -8.0, "near_limit": true, "policy_version": "directional-q-limit-v1", - "q_mvar": -3.0010655398841983, + "q_mvar": -3.0010031011947227, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 18.0010655398842 + "upper_headroom_mvar": 18.001003101194723 }, { "gen_id": 24, - "lower_headroom_mvar": 127.89585071606496, + "lower_headroom_mvar": 127.89476001325905, "max_q_mvar": 180.0, "min_q_mvar": -60.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 67.89585071606496, + "q_mvar": 67.89476001325905, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 112.10414928393504 + "upper_headroom_mvar": 112.10523998674095 }, { "gen_id": 25, - "lower_headroom_mvar": 53.49944796273613, + "lower_headroom_mvar": 53.497118850753225, "max_q_mvar": 300.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -46.50055203726387, + "q_mvar": -46.502881149246775, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 346.5005520372639 + "upper_headroom_mvar": 346.5028811492468 }, { "gen_id": 26, - "lower_headroom_mvar": 21.186417505052162, + "lower_headroom_mvar": 21.186544151786933, "max_q_mvar": 20.0, "min_q_mvar": -20.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 1.1864175050521624, + "q_mvar": 1.1865441517869328, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 18.813582494947838 + "upper_headroom_mvar": 18.813455848213067 }, { "gen_id": 27, @@ -27844,47 +27842,47 @@ }, { "gen_id": 28, - "lower_headroom_mvar": 65.94553902901708, + "lower_headroom_mvar": 65.94300344091698, "max_q_mvar": 200.0, "min_q_mvar": -67.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -1.054460970982916, + "q_mvar": -1.056996559083018, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 201.05446097098292 + "upper_headroom_mvar": 201.05699655908302 }, { "gen_id": 29, - "lower_headroom_mvar": 42.0, + "lower_headroom_mvar": 31.94985196449798, "max_q_mvar": 32.0, "min_q_mvar": -10.0, - "near_limit": true, + "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 32.0, - "status": "Q_LIMITED_UPPER", - "upper_headroom_mvar": 0.0 + "q_mvar": 21.94985196449798, + "status": "PV_CONTROLLABLE", + "upper_headroom_mvar": 10.05014803550202 }, { "gen_id": 30, - "lower_headroom_mvar": 91.13255331713401, + "lower_headroom_mvar": 92.0446071400052, "max_q_mvar": 100.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -8.867446682865989, + "q_mvar": -7.955392859994802, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 108.86744668286599 + "upper_headroom_mvar": 107.9553928599948 }, { "gen_id": 31, - "lower_headroom_mvar": 106.40258779053875, + "lower_headroom_mvar": 110.04315827538964, "max_q_mvar": 100.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 6.402587790538746, + "q_mvar": 10.043158275389644, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 93.59741220946125 + "upper_headroom_mvar": 89.95684172461036 }, { "gen_id": 32, @@ -27943,14 +27941,14 @@ }, { "gen_id": 37, - "lower_headroom_mvar": 195.00551475631613, + "lower_headroom_mvar": 194.7400928728215, "max_q_mvar": 1000.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 95.00551475631613, + "q_mvar": 94.74009287282149, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 904.9944852436838 + "upper_headroom_mvar": 905.2599071271785 }, { "gen_id": 38, @@ -28053,14 +28051,14 @@ }, { "gen_id": 47, - "lower_headroom_mvar": 221.96276869678172, + "lower_headroom_mvar": 221.78578904061376, "max_q_mvar": 200.0, "min_q_mvar": -200.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 21.962768696781723, + "q_mvar": 21.785789040613764, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 178.03723130321828 + "upper_headroom_mvar": 178.21421095938624 }, { "gen_id": 48, @@ -28075,62 +28073,62 @@ }, { "gen_id": 49, - "lower_headroom_mvar": 97.14621010586505, + "lower_headroom_mvar": 97.06983540642908, "max_q_mvar": 1000.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": -2.853789894134948, + "q_mvar": -2.9301645935709217, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 1002.8537898941349 + "upper_headroom_mvar": 1002.9301645935709 }, { "gen_id": 50, - "lower_headroom_mvar": 143.11885794282355, + "lower_headroom_mvar": 143.0262458137593, "max_q_mvar": 1000.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 43.11885794282355, + "q_mvar": 43.02624581375929, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 956.8811420571765 + "upper_headroom_mvar": 956.9737541862407 }, { "gen_id": 51, - "lower_headroom_mvar": 108.56798996156715, + "lower_headroom_mvar": 108.5835544387685, "max_q_mvar": 200.0, "min_q_mvar": -100.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 8.56798996156715, + "q_mvar": 8.583554438768502, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 191.43201003843285 + "upper_headroom_mvar": 191.4164455612315 }, { "gen_id": 52, - "lower_headroom_mvar": 1320.7102460457527, + "lower_headroom_mvar": 1320.6799570307146, "max_q_mvar": 1000.0, "min_q_mvar": -1000.0, "near_limit": false, "policy_version": "directional-q-limit-v1", - "q_mvar": 320.7102460457527, + "q_mvar": 320.6799570307146, "status": "PV_CONTROLLABLE", - "upper_headroom_mvar": 679.2897539542473 + "upper_headroom_mvar": 679.3200429692854 } ], - "iterations": 4, + "iterations": 5, "quality": null, - "recovery_used": true, + "recovery_used": false, "runtime_ms": 0.0, "slack": { "ext_grid_id": 0, - "p_mw": 780.8818585446998, + "p_mw": 780.3534273527885, "p_within_limits": true, - "q_mvar": -6.894049956363409, + "q_mvar": -2.874678523876897, "q_within_limits": true }, - "solver_attempt_count": 2, - "tolerance_used": 1e-06, + "solver_attempt_count": 1, + "tolerance_used": 1e-08, "warnings": [] } }, diff --git a/benchmarks/steady/level_3/restorebench/scoring/score_maneuvers.py b/benchmarks/steady/level_3/restorebench/scoring/score_maneuvers.py index 2bb00d9..e40a236 100644 --- a/benchmarks/steady/level_3/restorebench/scoring/score_maneuvers.py +++ b/benchmarks/steady/level_3/restorebench/scoring/score_maneuvers.py @@ -29,8 +29,6 @@ MANEUVER_BUDGET = 10 DEFAULT_DATA_DIR = ROOT / "dataset/ieee118/full" DEFAULT_DATASET_ROOT = ROOT / "dataset/ieee118" -DEFAULT_EVALUATION_MANIFEST_PATH = DEFAULT_DATASET_ROOT / "evaluation_manifest.json" -DEFAULT_SPLIT_PATH = DEFAULT_EVALUATION_MANIFEST_PATH DEFAULT_BATCH_OUT = ROOT / "results/manual_scoring" ACTION_ADAPTER: TypeAdapter[Any] = TypeAdapter(Action) SOLVER_SETTINGS = {"init": INIT, "max_iteration": MAX_ITERATION} @@ -105,28 +103,37 @@ def score_batch( *, out_dir: Path = DEFAULT_BATCH_OUT, data_dir: Path = DEFAULT_DATA_DIR, - index_path: Path = DEFAULT_SPLIT_PATH, + index_path: Path | None = None, ) -> dict[str, Any]: attempts = _load_batch_attempts(batch_path) reports = [score_attempt(attempt, data_dir=data_dir) for attempt in attempts] - scenario_index = _load_scenario_index(index_path) + resolved_index_path = index_path or _dataset_paths(data_dir)[1] + scenario_index = _load_scenario_index(resolved_index_path) report = _aggregate_batch(reports, scenario_index) out_dir.mkdir(parents=True, exist_ok=True) + for index, attempt_report in enumerate(reports, start=1): + _write_json(out_dir / f"attempt_{index:04d}_score.json", attempt_report) _write_json(out_dir / "benchmark_report.json", report) return report def main(argv: Sequence[str] | None = None) -> None: args = _parse_args(argv) + data_dir, index_path = _dataset_paths(args.data_dir) if args.batch is not None: out_dir = args.out or DEFAULT_BATCH_OUT - report = score_batch(args.batch, out_dir=out_dir) + report = score_batch( + args.batch, + out_dir=out_dir, + data_dir=data_dir, + index_path=index_path, + ) print(_json_dumps(report)) return if args.input is None: raise SystemExit("one maneuver JSON file is required unless --batch is used") - report = score_attempt_file(args.input) + report = score_attempt_file(args.input, data_dir=data_dir) _write_json(_single_report_path(args.input, args.out), report) print(_json_dumps(report)) @@ -135,10 +142,23 @@ def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Score maneuver proposals against frozen benchmark scenarios.") parser.add_argument("input", nargs="?", type=Path, help="single maneuver JSON attempt") parser.add_argument("--batch", type=Path, help="directory, JSON file, or JSONL file of maneuver attempts") + parser.add_argument( + "--data-dir", + type=Path, + default=DEFAULT_DATASET_ROOT, + help="dataset root or its full/ scenario directory (default: dataset/ieee118)", + ) parser.add_argument("--out", type=Path, help="output directory, or output JSON file for a single attempt") return parser.parse_args(argv) +def _dataset_paths(data_dir: Path) -> tuple[Path, Path]: + """Resolve one corpus selection for both scenario loading and batch stratification.""" + dataset_root = data_dir.parent if data_dir.name == "full" else data_dir + full_dir = data_dir if data_dir.name == "full" else data_dir / "full" + return full_dir, dataset_root / "evaluation_manifest.json" + + def _load_scenario(scenario_id: str, data_dir: Path) -> Any: return pp.from_json(str(data_dir / f"{scenario_id}.json")) diff --git a/benchmarks/steady/level_3/restorebench/sweeps/campaigns.json b/benchmarks/steady/level_3/restorebench/sweeps/campaigns.json index 431dfba..9b4c8fe 100644 --- a/benchmarks/steady/level_3/restorebench/sweeps/campaigns.json +++ b/benchmarks/steady/level_3/restorebench/sweeps/campaigns.json @@ -61,7 +61,7 @@ 2, 3 ], - "repetitions": 1, + "repetitions": 3, "max_runtime_seconds": 2400, "results_dir": "results/ieee118-anthropic" }, @@ -126,7 +126,7 @@ 2, 3 ], - "repetitions": 1, + "repetitions": 3, "max_runtime_seconds": 2400, "results_dir": "results/ieee118-bedrock" }, @@ -189,7 +189,7 @@ 2, 3 ], - "repetitions": 1, + "repetitions": 3, "max_runtime_seconds": 2400, "results_dir": "results/ieee118-openai" }, @@ -207,7 +207,7 @@ 2, 3 ], - "repetitions": 1, + "repetitions": 3, "max_runtime_seconds": 2400, "results_dir": "results/pegase89-bedrock" }, @@ -225,7 +225,7 @@ 2, 3 ], - "repetitions": 1, + "repetitions": 3, "max_runtime_seconds": 2400, "results_dir": "results/pegase89-anthropic" }, @@ -241,7 +241,7 @@ 2, 3 ], - "repetitions": 1, + "repetitions": 3, "max_runtime_seconds": 2400, "results_dir": "results/pegase89-openai" } diff --git a/benchmarks/steady/level_3/restorebench/sweeps/extract_three_run_data.py b/benchmarks/steady/level_3/restorebench/sweeps/extract_three_run_data.py new file mode 100644 index 0000000..134d59f --- /dev/null +++ b/benchmarks/steady/level_3/restorebench/sweeps/extract_three_run_data.py @@ -0,0 +1,927 @@ +# ABOUTME: Extracts run-preserving episode and event CSVs from RestoreBench result stores. +# ABOUTME: Validates three-run coverage without fabricating absent cells or missing measurements. +from __future__ import annotations + +import argparse +import csv +import math +import statistics +from collections import Counter, defaultdict +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from restorebench.eval.store import load_response, model_id_from_response +from restorebench.llm.models import model_slug +from restorebench.schemas.actions import Maneuver +from restorebench.schemas.response import ResolutionResponse +from restorebench.sweeps.build_paper_figures import ( + ARCH_LABELS, + BUDGET_EVENT_LABELS, + MANEUVER_BUDGET, + MODEL_DISPLAY, + OVERSTRESS_EPS, + committed_overstress_chain, +) +from restorebench.sweeps.run_sweep import Campaign, campaign_scenarios, load_campaigns + + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_OUT_DIR = ROOT / "reports/three_run_extraction" +NETWORK_CAMPAIGNS = { + "ieee118": ("ieee118-anthropic", "ieee118-bedrock", "ieee118-openai"), + "pegase89": ("pegase89-anthropic", "pegase89-bedrock", "pegase89-openai"), +} +EXPECTED_REPETITIONS = (0, 1, 2) + +EPISODE_COLUMNS = ( + "network", + "model", + "architecture", + "run_id", + "case_id", + "success", + "final_status", + "termination_reason", + "cost_usd", + "runtime_s", + "in_band_buses_pct", + "maneuvers_used", + "maneuver_budget", + "llm_calls", + "budget_events_consumed", + "maneuvers_remaining", + "llm_calls_per_committed_maneuver", +) + +EVENT_COLUMNS = ( + "network", + "model", + "architecture", + "run_id", + "case_id", + "event_index", + "event_category", + "episode_final_status", + "progress_verdict", + "llm_calls_before_event", + "llm_calls_per_committed_maneuver", + "action_type", + "action_target", + "action_value", + "solver_status_after_action", + "converged_after_action", + "electrically_feasible_after_action", + "native_event_kind", + "native_iteration_index", + "native_event_detail", + "maneuvers_consumed", + "maneuvers_remaining", + "maximum_maneuver_budget", + "total_llm_calls", +) + +SUMMARY_COLUMNS = ( + "network", + "model", + "architecture", + "run_id", + "number_of_cases", + "number_successful", + "success_rate", + "mean_cost_per_case", + "median_cost_per_case", + "mean_runtime", + "median_runtime", + "number_budget_exhausted", + "number_timeout", + "number_other_terminal_statuses", + "complete_46_cases", + "missing_case_count", +) + + +@dataclass(frozen=True) +class SourceCell: + network: str + campaign: str + store: Path + path: Path + response: ResolutionResponse + + +@dataclass +class Validation: + unreadable_files: list[str] + unexpected_repetitions: list[str] + filename_mismatches: list[str] + duplicate_episodes: dict[tuple[str, str, str, int, str], list[str]] + missing_cost: list[tuple[str, str, str, int, str]] + missing_runtime: list[tuple[str, str, str, int, str]] + missing_final_status: list[tuple[str, str, str, int, str]] + impossible_success_status: list[tuple[str, str, str, int, str]] + progress_chain_mismatches: list[str] + over_budget_events: list[str] + version_stamp_errors: list[str] + + +def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR) + parser.add_argument( + "--store", + action="append", + default=[], + metavar="CAMPAIGN=PATH", + help="Override one campaign result root. Repeat for multiple campaigns.", + ) + return parser.parse_args(argv) + + +def _store_overrides(values: Sequence[str]) -> dict[str, Path]: + overrides: dict[str, Path] = {} + for value in values: + name, separator, raw_path = value.partition("=") + if not separator or not name or not raw_path: + raise SystemExit(f"invalid --store {value!r}; expected CAMPAIGN=PATH") + overrides[name] = Path(raw_path) + return overrides + + +def _model_display(model_id: str) -> str: + return MODEL_DISPLAY.get(model_id, model_id) + + +def _cells_dir(store: Path) -> Path: + # Published campaign paths name the result root; CLI overrides may name cells/ directly. + return store if store.name == "cells" else store / "cells" + + +def _campaign_networks(campaigns: dict[str, Campaign]) -> dict[str, str]: + return { + campaign_name: network + for network, campaign_names in NETWORK_CAMPAIGNS.items() + for campaign_name in campaign_names + if campaign_name in campaigns + } + + +def load_source_cells( + campaigns: dict[str, Campaign], overrides: dict[str, Path], validation: Validation +) -> tuple[list[SourceCell], dict[str, Path]]: + campaign_network = _campaign_networks(campaigns) + stores: dict[str, Path] = {} + cells: list[SourceCell] = [] + for campaign_name, network in sorted(campaign_network.items()): + campaign = campaigns[campaign_name] + store = overrides.get(campaign_name, ROOT / campaign.results_dir) + stores[campaign_name] = store + cells_dir = _cells_dir(store) + if not cells_dir.is_dir(): + continue + for path in sorted(cells_dir.glob("*.json")): + if "manifest" in path.name: + continue + try: + response = load_response(path) + except Exception as exc: + validation.unreadable_files.append(f"{path}: {type(exc).__name__}: {exc}") + continue + if response.repetition_index not in EXPECTED_REPETITIONS: + validation.unexpected_repetitions.append( + f"{path}: repetition_index={response.repetition_index}" + ) + continue + expected_name = ( + f"{response.scenario_id}__config{response.configuration}__" + f"{model_slug(model_id_from_response(response))}__rep{response.repetition_index}.json" + ) + if path.name != expected_name: + validation.filename_mismatches.append(f"{path}: expected filename {expected_name}") + cells.append( + SourceCell( + network=network, + campaign=campaign_name, + store=store, + path=path, + response=response, + ) + ) + return cells, stores + + +def _empty_validation() -> Validation: + return Validation([], [], [], {}, [], [], [], [], [], [], []) + + +def _episode_key(cell: SourceCell) -> tuple[str, str, str, int, str]: + response = cell.response + return ( + cell.network, + _model_display(model_id_from_response(response)), + ARCH_LABELS[response.configuration], + response.repetition_index + 1, + response.scenario_id, + ) + + +def _budget_events(response: ResolutionResponse) -> int: + return sum(1 for feedback in response.failure_feedback if feedback.kind in BUDGET_EVENT_LABELS) + + +def _cost(response: ResolutionResponse) -> float | None: + from restorebench.analysis.cost import run_cost_usd + + try: + value = run_cost_usd(response) + except (KeyError, TypeError, ValueError): + return None + return value if math.isfinite(value) else None + + +def episode_rows(cells: Sequence[SourceCell], validation: Validation) -> list[dict[str, Any]]: + by_key: dict[tuple[str, str, str, int, str], list[SourceCell]] = defaultdict(list) + for cell in cells: + by_key[_episode_key(cell)].append(cell) + validation.duplicate_episodes = { + key: [str(cell.path) for cell in matches] + for key, matches in by_key.items() + if len(matches) > 1 + } + + rows: list[dict[str, Any]] = [] + for key, matches in sorted(by_key.items()): + cell = matches[0] + response = cell.response + cost = _cost(response) + runtime = response.total_runtime_seconds + consumed = _budget_events(response) + ratio = ( + response.trace.n_llm_calls / response.n_maneuvers + if response.n_maneuvers > 0 + else None + ) + quality = response.quality + bus_count = 118 if cell.network == "ieee118" else 89 + in_band = ( + 100.0 * (bus_count - quality.n_buses_out_of_band) / bus_count + if response.status == "SUCCESS" and quality is not None + else None + ) + if cost is None: + validation.missing_cost.append(key) + if not math.isfinite(runtime): + validation.missing_runtime.append(key) + runtime_value: float | None = None + else: + runtime_value = runtime + if not response.status: + validation.missing_final_status.append(key) + success = int(response.status == "SUCCESS") + if bool(success) != response.converged: + validation.impossible_success_status.append(key) + if consumed > MANEUVER_BUDGET: + validation.over_budget_events.append( + f"{cell.path}: {consumed} budget events exceed {MANEUVER_BUDGET}" + ) + rows.append( + { + "network": cell.network, + "model": key[1], + "architecture": key[2], + "run_id": key[3], + "case_id": response.scenario_id, + "success": success, + "final_status": response.status, + # ResolutionResponse has one terminal-status field and no second reason field. + "termination_reason": response.status, + "cost_usd": cost, + "runtime_s": runtime_value, + "in_band_buses_pct": in_band, + "maneuvers_used": response.n_maneuvers, + "maneuver_budget": MANEUVER_BUDGET, + "llm_calls": response.trace.n_llm_calls, + "budget_events_consumed": consumed, + "maneuvers_remaining": MANEUVER_BUDGET - consumed if response.status == "TIMEOUT" else None, + "llm_calls_per_committed_maneuver": ratio, + } + ) + return rows + + +def _accepted_iterations(response: ResolutionResponse) -> list[int]: + iterations = [] + for event in response.trace.events: + if event.phase == "solve" and event.event_name == "run_ac_pf": + iteration = event.payload.get("iteration") + if isinstance(iteration, int): + iterations.append(iteration) + return iterations + + +def _llm_calls_by_iteration(response: ResolutionResponse) -> dict[int, int | None]: + if response.configuration == 1: + iterations = {feedback.iteration for feedback in response.failure_feedback} + iterations.update(_accepted_iterations(response)) + return {iteration: response.trace.n_llm_calls for iteration in iterations} + + calls: dict[int, int | None] = {} + cumulative = 0 + exact = True + for event in response.trace.events: + if event.phase != "diagnosis" or event.event_name != "agent_step": + continue + iteration = event.payload.get("iteration") + if not isinstance(iteration, int): + continue + count = event.payload.get("llm_calls") + if exact and isinstance(count, int): + cumulative += count + calls[iteration] = cumulative + else: + exact = False + calls[iteration] = None + return calls + + +def _progress_verdict(pair: tuple[float, float] | None) -> str | None: + if pair is None: + return None + before, after = pair + if after < before - OVERSTRESS_EPS: + return "Improved" + if after > before + OVERSTRESS_EPS: + return "Worsened" + return "Unchanged" + + +def _action_fields(maneuver: Maneuver | None) -> tuple[Any, Any, Any]: + if maneuver is None: + return None, None, None + action = maneuver.action.model_dump(mode="json") + action_type = action["type"] + if action_type == "GEN_V_SETPOINT": + return action_type, action["gen_id"], action["new_vm_pu"] + if action_type == "SHUNT_STEP": + return action_type, action["shunt_id"], action["new_step"] + if action_type == "TAP_ADJUSTMENT": + return action_type, action["trafo_id"], action["new_tap_pos"] + return action_type, None, None + + +def _solver_fields(kind: str) -> tuple[str | None, bool | None, bool | None]: + if kind == "STILL_DIVERGED": + return "DIVERGED", False, None + if kind == "SOLVED_INFEASIBLE": + return "CONVERGED_INFEASIBLE", True, False + if kind == "SUCCESS": + return "CONVERGED_FEASIBLE", True, True + return None, None, None + + +def _event_base(cell: SourceCell) -> dict[str, Any]: + response = cell.response + consumed = _budget_events(response) + return { + "network": cell.network, + "model": _model_display(model_id_from_response(response)), + "architecture": ARCH_LABELS[response.configuration], + "run_id": response.repetition_index + 1, + "case_id": response.scenario_id, + "episode_final_status": response.status, + "llm_calls_per_committed_maneuver": ( + response.trace.n_llm_calls / response.n_maneuvers + if response.n_maneuvers > 0 + else None + ), + "maneuvers_consumed": consumed if response.status == "TIMEOUT" else None, + "maneuvers_remaining": MANEUVER_BUDGET - consumed if response.status == "TIMEOUT" else None, + "maximum_maneuver_budget": MANEUVER_BUDGET, + "total_llm_calls": response.trace.n_llm_calls, + } + + +def _event_row( + base: dict[str, Any], + *, + event_index: int, + event_category: str | None, + progress_verdict: str | None, + llm_calls_before_event: int | None, + maneuver: Maneuver | None, + native_kind: str, + native_iteration: int, + native_detail: str | None, +) -> dict[str, Any]: + action_type, action_target, action_value = _action_fields(maneuver) + solver_status, converged, feasible = _solver_fields(native_kind) + return { + **base, + "event_index": event_index, + "event_category": event_category, + "progress_verdict": progress_verdict, + "llm_calls_before_event": llm_calls_before_event, + "action_type": action_type, + "action_target": action_target, + "action_value": action_value, + "solver_status_after_action": solver_status, + "converged_after_action": converged, + "electrically_feasible_after_action": feasible, + "native_event_kind": native_kind, + "native_iteration_index": native_iteration, + "native_event_detail": native_detail, + } + + +def event_rows(cells: Sequence[SourceCell], validation: Validation) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + unique_cells: dict[tuple[str, str, str, int, str], SourceCell] = {} + for cell in cells: + unique_cells.setdefault(_episode_key(cell), cell) + + for _, cell in sorted(unique_cells.items()): + response = cell.response + base = _event_base(cell) + progress = committed_overstress_chain(response) + if len(progress) != response.n_maneuvers or any(pair is None for pair in progress): + if response.n_maneuvers: + validation.progress_chain_mismatches.append(str(cell.path)) + accepted_iterations = _accepted_iterations(response) + llm_calls = _llm_calls_by_iteration(response) + committed_index = 0 + + for feedback in response.failure_feedback: + if feedback.kind not in BUDGET_EVENT_LABELS: + continue + pair = None + maneuver = feedback.maneuver + if feedback.kind in {"STILL_DIVERGED", "SOLVED_INFEASIBLE"}: + if committed_index < len(progress): + pair = progress[committed_index] + if committed_index < len(response.maneuvers): + maneuver = response.maneuvers[committed_index] + committed_index += 1 + rows.append( + _event_row( + base, + event_index=feedback.iteration + 1, + event_category=BUDGET_EVENT_LABELS[feedback.kind], + progress_verdict=_progress_verdict(pair), + llm_calls_before_event=llm_calls.get(feedback.iteration), + maneuver=maneuver, + native_kind=feedback.kind, + native_iteration=feedback.iteration, + native_detail=feedback.detail, + ) + ) + + # A successful terminal maneuver has no FailureFeedback entry, but it is part of the + # paper's committed-overstress chain and therefore needs one event-level observation. + if response.status == "SUCCESS" and committed_index < response.n_maneuvers: + terminal_index = response.n_maneuvers - 1 + native_iteration = ( + accepted_iterations[terminal_index] + if terminal_index < len(accepted_iterations) + else max((feedback.iteration for feedback in response.failure_feedback), default=-1) + 1 + ) + pair = progress[terminal_index] if terminal_index < len(progress) else None + rows.append( + _event_row( + base, + event_index=native_iteration + 1, + event_category=None, + progress_verdict=_progress_verdict(pair), + llm_calls_before_event=llm_calls.get(native_iteration), + maneuver=response.maneuvers[terminal_index], + native_kind="SUCCESS", + native_iteration=native_iteration, + native_detail=None, + ) + ) + return sorted( + rows, + key=lambda row: ( + row["network"], + row["model"], + row["architecture"], + row["run_id"], + row["case_id"], + row["event_index"], + row["native_event_kind"], + ), + ) + + +def _expected_groups( + campaigns: dict[str, Campaign], +) -> tuple[list[tuple[str, str, str, int, str]], dict[str, set[str]]]: + groups = [] + expected_cases: dict[str, set[str]] = {} + for network, campaign_names in NETWORK_CAMPAIGNS.items(): + available = [campaigns[name] for name in campaign_names if name in campaigns] + if not available: + continue + expected_cases[network] = set(campaign_scenarios(available[0])) + for campaign in available: + for model_id in campaign.models: + for configuration in campaign.configurations: + for repetition in EXPECTED_REPETITIONS: + groups.append( + ( + network, + _model_display(model_id), + ARCH_LABELS[configuration], + repetition + 1, + campaign.name, + ) + ) + return sorted(groups), expected_cases + + +def _finite_values(rows: Sequence[dict[str, Any]], field: str) -> list[float]: + return [ + float(row[field]) + for row in rows + if row[field] is not None and math.isfinite(float(row[field])) + ] + + +def summary_rows( + episodes: Sequence[dict[str, Any]], + campaigns: dict[str, Campaign], +) -> tuple[list[dict[str, Any]], dict[tuple[str, str, str, int], set[str]], dict[str, set[str]]]: + groups, expected_cases = _expected_groups(campaigns) + indexed: dict[tuple[str, str, str, int], list[dict[str, Any]]] = defaultdict(list) + for row in episodes: + indexed[(row["network"], row["model"], row["architecture"], int(row["run_id"]))].append(row) + + summaries: list[dict[str, Any]] = [] + missing: dict[tuple[str, str, str, int], set[str]] = {} + for network, model, architecture, run_id, _campaign in groups: + key = (network, model, architecture, run_id) + rows = indexed.get(key, []) + unique_cases = {row["case_id"] for row in rows} + missing[key] = expected_cases[network] - unique_cases + costs = _finite_values(rows, "cost_usd") + runtimes = _finite_values(rows, "runtime_s") + successes = sum(int(row["success"]) for row in rows) + statuses = Counter(row["final_status"] for row in rows) + summaries.append( + { + "network": network, + "model": model, + "architecture": architecture, + "run_id": run_id, + "number_of_cases": len(unique_cases), + "number_successful": successes, + "success_rate": successes / len(unique_cases) if unique_cases else None, + "mean_cost_per_case": statistics.mean(costs) if costs else None, + "median_cost_per_case": statistics.median(costs) if costs else None, + "mean_runtime": statistics.mean(runtimes) if runtimes else None, + "median_runtime": statistics.median(runtimes) if runtimes else None, + "number_budget_exhausted": statuses["BUDGET_EXHAUSTED"], + "number_timeout": statuses["TIMEOUT"], + "number_other_terminal_statuses": sum( + count + for status, count in statuses.items() + if status not in {"SUCCESS", "BUDGET_EXHAUSTED", "TIMEOUT"} + ), + "complete_46_cases": len(unique_cases) == 46 and not missing[key], + "missing_case_count": len(missing[key]), + } + ) + return summaries, missing, expected_cases + + +def _write_csv(path: Path, columns: Sequence[str], rows: Iterable[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + +def _markdown_value(value: Any) -> str: + if value is None: + return "NA" + if isinstance(value, float): + return f"{value:.6g}" + if isinstance(value, bool): + return "yes" if value else "no" + return str(value).replace("|", "\\|") + + +def _markdown_table(columns: Sequence[str], rows: Sequence[dict[str, Any]]) -> str: + lines = [ + "| " + " | ".join(columns) + " |", + "|" + "---|" * len(columns), + ] + lines.extend( + "| " + " | ".join(_markdown_value(row.get(column)) for column in columns) + " |" + for row in rows + ) + return "\n".join(lines) + + +def _source_mapping_rows( + campaigns: dict[str, Campaign], stores: dict[str, Path], summaries: Sequence[dict[str, Any]] +) -> list[dict[str, Any]]: + campaign_for_model = { + (network, _model_display(model_id)): campaign_name + for network, campaign_names in NETWORK_CAMPAIGNS.items() + for campaign_name in campaign_names + if campaign_name in campaigns + for model_id in campaigns[campaign_name].models + } + rows = [] + for summary in summaries: + key = (summary["network"], summary["model"]) + campaign_name = campaign_for_model[key] + store = stores.get(campaign_name, ROOT / campaigns[campaign_name].results_dir) + configuration = next( + number for number, label in ARCH_LABELS.items() if label == summary["architecture"] + ) + model_id = next( + model_id + for model_id in campaigns[campaign_name].models + if _model_display(model_id) == summary["model"] + ) + pattern = ( + f"*__config{configuration}__{model_slug(model_id)}__" + f"rep{int(summary['run_id']) - 1}.json" + ) + rows.append( + { + "network": summary["network"], + "model": summary["model"], + "architecture": summary["architecture"], + "run_id": summary["run_id"], + "source path/store": f"{_cells_dir(store)}/{pattern}", + "source state": "present" if summary["number_of_cases"] else "absent", + } + ) + return rows + + +def _issue_lines(title: str, values: Sequence[Any]) -> list[str]: + if not values: + return [f"- {title}: none detected among readable source cells."] + lines = [f"- {title}: {len(values)}."] + lines.extend(f" - `{value}`" for value in values[:50]) + if len(values) > 50: + lines.append(f" - ... {len(values) - 50} more") + return lines + + +def write_report( + path: Path, + *, + campaigns: dict[str, Campaign], + stores: dict[str, Path], + episodes: Sequence[dict[str, Any]], + events: Sequence[dict[str, Any]], + summaries: Sequence[dict[str, Any]], + missing: dict[tuple[str, str, str, int], set[str]], + validation: Validation, +) -> None: + mapping = _source_mapping_rows(campaigns, stores, summaries) + expected_total = sum(len(case_ids) for case_ids in missing.values()) + len(episodes) + incomplete = [row for row in summaries if not row["complete_46_cases"]] + campaign_repetitions = { + name: campaigns[name].repetitions + for names in NETWORK_CAMPAIGNS.values() + for name in names + if name in campaigns + } + duplicate_details = [ + f"{key}: {', '.join(paths)}" + for key, paths in sorted(validation.duplicate_episodes.items()) + ] + missing_sets: dict[tuple[str, tuple[str, ...]], list[tuple[str, str, str, int]]] = defaultdict(list) + for key, case_ids in missing.items(): + missing_sets[(key[0], tuple(sorted(case_ids)))].append(key) + + lines = [ + "# RestoreBench three-run extraction report", + "", + "## Outcome", + "", + f"Readable LLM episode cells found: **{len(episodes)}** of **{expected_total}** expected " + "(2 networks × 7 models × 3 architectures × 3 runs × 46 cases).", + "", + "No values were reconstructed from the published PNG/PDF figures, hard-coded narrative " + "numbers, aggregate statistics, or another repository. Missing observations remain absent.", + "", + "`restorebench_baselines.csv` was not created: this checkout contains no deterministic " + "greedy result store, random-baseline repetitions, or baseline-result schema. The LLM " + "chatbot architecture is not treated as a greedy/random baseline.", + "", + "## Source paths inspected", + "", + ] + for campaign_name in sorted(stores): + store = stores[campaign_name] + cells_dir = _cells_dir(store) + state = "present" if cells_dir.is_dir() else "missing" + lines.append(f"- `{cells_dir}` — {state}") + lines.extend( + [ + "- `results/` and `reports/` — absent before this extraction; both are ignored run outputs.", + "- Repository-wide search for `*.db`, `*.sqlite*`, `*.jsonl`, `*.csv`, logs, result " + "files, and `cells/` directories — no experimental stores found.", + "- All local and remote Git refs/history — no tracked historical `results/`, " + "`reports/`, CSV, JSONL, or SQLite result artifact found.", + "- `docs/figs/results/` — static PNG/PDF figures only; excluded as non-raw sources.", + "- `restorebench/sweeps/build_figure_tables.py` — contains aggregate/narrative values; " + "excluded as non-raw sources.", + "", + "The repository README explicitly states: **No result cells ship with this repository**.", + "", + "## Run identification", + "", + "The native run identity is explicit, not inferred from directory order: " + "`ResolutionResponse.repetition_index` and filename suffix `__repN.json`. The export " + "maps native `rep0`, `rep1`, and `rep2` to `run_id` 1, 2, and 3 respectively. " + "Because no cells are present, every mapping row below is an expected filename pattern, " + "not a claim that the source exists.", + "", + _markdown_table( + ("network", "model", "architecture", "run_id", "source path/store", "source state"), + mapping, + ), + "", + "## Campaign repetitions", + "", + "All six published campaign definitions specify the three-run design:", + "", + ] + ) + lines.extend(f"- `{name}`: `repetitions = {count}`" for name, count in sorted(campaign_repetitions.items())) + lines.extend( + [ + "", + "The evaluation harness encodes the three repetitions as `rep0`, `rep1`, and `rep2`.", + "", + "## Field mappings", + "", + "- `network`: campaign/corpus identity (`ieee118` or `pegase89`).", + "- `model`: model id from `llm_assignment`, normalized through the paper display map.", + "- `architecture`: configuration 1/2/3 → chatbot/single agent/multi-agent.", + "- `run_id`: `repetition_index + 1`; the native index is encoded in every expected filename.", + "- `case_id`: `scenario_id`.", + "- `success`: `status == SUCCESS`.", + "- `final_status`: `status`.", + "- `termination_reason`: `status`; `ResolutionResponse` has no separate terminal-reason field.", + "- `cost_usd`: derived per episode from stored input/output token counts via " + "`restorebench.analysis.cost.run_cost_usd` and the model price registry. No aggregate cost is used.", + "- `runtime_s`: `total_runtime_seconds`.", + "- `in_band_buses_pct`: `(network_bus_count - quality.n_buses_out_of_band) / " + "network_bus_count × 100`.", + "- `maneuvers_used`: `n_maneuvers` (committed maneuvers only).", + f"- `maneuver_budget`: `{MANEUVER_BUDGET}`, reused from the current paper-analysis constant; " + "the response schema does not store the budget per episode.", + "- `llm_calls`: `trace.n_llm_calls`.", + "- Extra timeout columns preserve the paper logic: budget events consumed, remaining slots, " + "and total calls divided by committed maneuvers.", + "", + "## Current filters", + "", + "### Runtime", + "", + "The current runtime analysis includes every readable tested episode, regardless of terminal " + "status. It uses `total_runtime_seconds`; SUCCESS, BUDGET_EXHAUSTED, TIMEOUT, LLM_FAILURE, " + "and TOOL_FAILURE are not filtered out. Missing/non-finite runtime is left blank and flagged.", + "", + "### In-band buses", + "", + "The current voltage-quality analysis includes only episodes with `status == SUCCESS` and a " + "non-null `quality` object. Other statuses remain blank, never zero. The denominator is 118 " + "for IEEE 118 and 89 for PEGASE 89.", + "", + "## Existing event classification reused", + "", + "The export imports the mapping directly from `restorebench/sweeps/build_paper_figures.py`:", + "", + "- `STILL_DIVERGED` → Still diverged", + "- `MALFORMED_OUTPUT` → Structured-output failure", + "- `INVALID_ACTION` → Invalid action", + "- `SOLVED_INFEASIBLE` → Solved infeasible", + "", + "Only those four `FailureFeedback.kind` values consume slots in the current failure and " + "timeout analyses. Preview failures and LLM_FAILURE do not consume slots.", + "", + "Maneuver progress reuses `committed_overstress_chain` and the paper figure's exact " + f"tolerance (`OVERSTRESS_EPS = {OVERSTRESS_EPS}`). A maneuver is Improved when " + "`after < before - eps`, Worsened when `after > before + eps`, and Unchanged otherwise. " + "STILL_DIVERGED uses the next diagnostics' overstress; SOLVED_INFEASIBLE and the terminal " + "SUCCESS maneuver use zero. Unclassifiable pairs remain blank.", + "", + "For TIMEOUT episodes the current paper logic counts the four slot-consuming feedback " + f"kinds, computes `{MANEUVER_BUDGET} - consumed`, and computes " + "`trace.n_llm_calls / n_maneuvers` only when at least one maneuver was committed.", + "", + "## Validation results", + "", + f"- Duplicate episode keys: {len(validation.duplicate_episodes)}.", + f"- Configurations/runs with 46 unique cases: {len(summaries) - len(incomplete)} of {len(summaries)}.", + f"- Incomplete configuration/runs: {len(incomplete)}.", + f"- Missing episode observations: {sum(len(case_ids) for case_ids in missing.values())}.", + ] + ) + lines.extend(_issue_lines("Duplicate case IDs / duplicated episodes", duplicate_details)) + lines.extend(_issue_lines("Unreadable result files", validation.unreadable_files)) + lines.extend(_issue_lines("Unexpected repetition indices", validation.unexpected_repetitions)) + lines.extend(_issue_lines("Filename/payload run-id mismatches", validation.filename_mismatches)) + lines.extend(_issue_lines("Missing episode cost", validation.missing_cost)) + lines.extend(_issue_lines("Missing/non-finite runtime", validation.missing_runtime)) + lines.extend(_issue_lines("Missing final status", validation.missing_final_status)) + lines.extend(_issue_lines("Impossible success/status combinations", validation.impossible_success_status)) + lines.extend(_issue_lines("Unclassifiable or mismatched maneuver-progress chains", validation.progress_chain_mismatches)) + lines.extend(_issue_lines("Episodes exceeding the event budget", validation.over_budget_events)) + lines.extend(_issue_lines("Version-stamp errors", validation.version_stamp_errors)) + lines.extend(["", "### Exact missing case IDs", ""]) + for (network, case_ids), affected_groups in sorted(missing_sets.items()): + if not case_ids: + continue + lines.append( + f"- `{network}` — {len(affected_groups)} configuration/run groups: " + + ", ".join(case_ids) + ) + lines.extend( + [ + "", + "No validation problem was silently repaired. With zero readable source cells, duplicate " + "and field-level checks are necessarily vacuous; the material failure is complete source absence.", + "", + "## Per-run summary", + "", + _markdown_table(SUMMARY_COLUMNS, list(summaries)), + "", + "## Generated files", + "", + f"- `restorebench_episode_level_3runs.csv`: {len(episodes)} data rows.", + f"- `restorebench_event_level_3runs.csv`: {len(events)} data rows.", + "- `restorebench_baselines.csv`: not generated because no baseline raw results are available.", + "", + ] + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines), encoding="utf-8") + + +def _validate_version_stamps(cells: Sequence[SourceCell], validation: Validation) -> None: + fields = ( + "dataset_version", + "solver_version", + "action_policy_version", + "ranking_policy_version", + "result_schema_version", + ) + by_network: dict[str, list[SourceCell]] = defaultdict(list) + for cell in cells: + by_network[cell.network].append(cell) + for network, network_cells in by_network.items(): + for field in fields: + values = {getattr(cell.response, field) for cell in network_cells} + if None in values: + validation.version_stamp_errors.append(f"{network}: {field} is missing") + if len(values) > 1: + validation.version_stamp_errors.append( + f"{network}: incompatible {field} values {sorted(str(value) for value in values)}" + ) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + campaigns = load_campaigns() + validation = _empty_validation() + overrides = _store_overrides(args.store) + cells, stores = load_source_cells(campaigns, overrides, validation) + _validate_version_stamps(cells, validation) + episodes = episode_rows(cells, validation) + events = event_rows(cells, validation) + summaries, missing, _expected_cases = summary_rows(episodes, campaigns) + + args.out_dir.mkdir(parents=True, exist_ok=True) + _write_csv(args.out_dir / "restorebench_episode_level_3runs.csv", EPISODE_COLUMNS, episodes) + _write_csv(args.out_dir / "restorebench_event_level_3runs.csv", EVENT_COLUMNS, events) + write_report( + args.out_dir / "extraction_report.md", + campaigns=campaigns, + stores=stores, + episodes=episodes, + events=events, + summaries=summaries, + missing=missing, + validation=validation, + ) + print( + f"{len(episodes)} episodes and {len(events)} events -> {args.out_dir}; " + f"{sum(len(case_ids) for case_ids in missing.values())} expected episodes missing" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/steady/level_3/tests/scoring/test_score_maneuvers.py b/benchmarks/steady/level_3/tests/scoring/test_score_maneuvers.py index a2ec59a..eb76216 100644 --- a/benchmarks/steady/level_3/tests/scoring/test_score_maneuvers.py +++ b/benchmarks/steady/level_3/tests/scoring/test_score_maneuvers.py @@ -256,6 +256,9 @@ def test_batch_report_aggregates_real_scenarios_and_is_deterministic(tmp_path: P second = score_batch(batch_dir, out_dir=second_out, data_dir=DATASET_FULL, index_path=DATASET_INDEX) assert (first_out / "benchmark_report.json").exists() + attempt_reports = sorted(first_out.glob("attempt_*_score.json")) + assert len(attempt_reports) == len(SUCCESS_FIXTURES) + assert json.loads(attempt_reports[0].read_text(encoding="utf-8"))["scenario_id"] == "S0008" assert first["n_attempts"] == 5 assert first["success_rate"] == 1.0 assert first["n_maneuvers_success"]["mean"] == 1.0 @@ -269,6 +272,90 @@ def test_batch_report_aggregates_real_scenarios_and_is_deterministic(tmp_path: P assert _without_runtime(first) == _without_runtime(second) +def test_cli_data_dir_selects_corpus_for_single_attempt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from restorebench.scoring import score_maneuvers + + attempt = tmp_path / "attempt.json" + output = tmp_path / "score.json" + _write_attempt(attempt, SUCCESS_FIXTURES[0]) + calls: list[tuple[Path, Path]] = [] + + def fake_score_attempt_file(path: Path, *, data_dir: Path) -> dict[str, Any]: + calls.append((path, data_dir)) + return {"scenario_id": "S0001", "status": "SUCCESS"} + + monkeypatch.setattr(score_maneuvers, "score_attempt_file", fake_score_attempt_file) + + score_maneuvers.main([ + str(attempt), + "--data-dir", + "dataset/pegase89", + "--out", + str(output), + ]) + + assert calls == [(attempt, Path("dataset/pegase89/full"))] + assert output.is_file() + + +def test_cli_data_dir_selects_matching_corpus_and_index_for_batch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from restorebench.scoring import score_maneuvers + + batch = tmp_path / "attempts.jsonl" + batch.write_text("", encoding="utf-8") + output = tmp_path / "scores" + calls: list[tuple[Path, Path, Path, Path]] = [] + + def fake_score_batch( + batch_path: Path, + *, + out_dir: Path, + data_dir: Path, + index_path: Path, + ) -> dict[str, Any]: + calls.append((batch_path, out_dir, data_dir, index_path)) + return {"n_attempts": 0} + + monkeypatch.setattr(score_maneuvers, "score_batch", fake_score_batch) + + score_maneuvers.main([ + "--batch", + str(batch), + "--data-dir", + "dataset/pegase89/full", + "--out", + str(output), + ]) + + assert calls == [( + batch, + output, + Path("dataset/pegase89/full"), + Path("dataset/pegase89/evaluation_manifest.json"), + )] + + +def test_published_s0081_witness_converges_at_primary_tolerance() -> None: + from restorebench.scoring.score_maneuvers import score_attempt + + witnesses = json.loads(Path("dataset/ieee118/private/witnesses.json").read_text(encoding="utf-8")) + witness = next(row for row in witnesses if row["scenario_id"] == "S0081") + + report = score_attempt( + {"scenario_id": "S0081", "maneuvers": witness["maneuvers"]}, + data_dir=DATASET_FULL, + ) + + assert report["status"] == "SUCCESS" + assert report["steps"][-1]["tolerance_used"] == 1e-8 + + def test_aggregate_reports_tolerates_success_with_null_quality() -> None: # A converged attempt can carry quality=None (the key is present, so a plain # .get('quality', {}) returns None and None.get('clean') raises AttributeError). diff --git a/benchmarks/steady/level_3/tests/sweeps/test_run_sweep.py b/benchmarks/steady/level_3/tests/sweeps/test_run_sweep.py index 4cf257c..821be50 100644 --- a/benchmarks/steady/level_3/tests/sweeps/test_run_sweep.py +++ b/benchmarks/steady/level_3/tests/sweeps/test_run_sweep.py @@ -28,7 +28,7 @@ def test_published_campaigns_are_runnable_without_any_result_store(): for name, campaign in campaigns.items(): assert campaign.models, f"{name} names no models" assert campaign.configurations, f"{name} names no configurations" - assert campaign.repetitions >= 1 + assert campaign.repetitions == 3 assert campaign.max_runtime_seconds > 0 assert campaign.data_dir.is_dir(), f"{name} points at a corpus that is not shipped" @@ -41,7 +41,7 @@ def test_the_published_ieee118_campaigns_pin_the_same_46_scenarios(): # Both providers ran the identical subset; a comparison table is only meaningful if they did. assert anthropic.scenarios == bedrock.scenarios assert len(anthropic.scenarios) == 46 - assert anthropic.cells_per_scenario == 9 + assert anthropic.cells_per_scenario == 27 def test_frozen_scenarios_must_belong_to_the_held_out_split():