From beed234eb4e53294385a4c45348fa6d61a6671e5 Mon Sep 17 00:00:00 2001 From: primorLee Date: Sat, 22 Aug 2026 06:48:16 +0800 Subject: [PATCH 1/3] Fix incomplete UniGenBench evaluation coverage --- .../run_with_cosmos_framework.ipynb | 12 +- .../generator/unigenbench/test_ugb_scorer.py | 81 +++++++++++++ .../generator/unigenbench/ugb_scorer.py | 110 +++++++++++++----- 3 files changed, 172 insertions(+), 31 deletions(-) create mode 100644 evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py diff --git a/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb b/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb index d54ddeea..a0206841 100644 --- a/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb +++ b/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb @@ -437,7 +437,8 @@ "\n", "id_list = list(PROMPTS.keys())\n", "\n", - "# If full run, please comment out the debug_limit_n\n", + "# If full run, please comment out the debug_limit_n. Partial scoring also\n", + "# requires --allow_missing_images in the scorer command below.\n", "# debug_limit_n = 10\n", "# if debug_limit_n > 0:\n", "# id_list = id_list[:debug_limit_n]\n", @@ -502,12 +503,19 @@ "all_images_dir.mkdir(exist_ok=True)\n", "\n", "for subdir in sorted(Path(UNIGENBENCH_IMAGE).iterdir()):\n", - " if subdir.is_dir() and re.fullmatch(r\"orig\\d+\", subdir.name):\n", + " if subdir.is_dir() and re.fullmatch(r\"(?:orig|phi)\\d+\", subdir.name):\n", " src = subdir / \"vision.jpg\"\n", " dst = all_images_dir / f\"{subdir.name}_0.jpg\"\n", " if src.exists() and not dst.exists():\n", " shutil.copy2(src, dst)\n", "copied = list(all_images_dir.glob(\"*.jpg\"))\n", + "copied_ids = {path.stem.removesuffix(\"_0\") for path in copied}\n", + "missing_ids = sorted(set(PROMPTS) - copied_ids)\n", + "if missing_ids:\n", + " preview = \", \".join(missing_ids[:5])\n", + " raise FileNotFoundError(\n", + " f\"Missing {len(missing_ids)} of {len(PROMPTS)} generated images: {preview}\"\n", + " )\n", "print(f\"Done. {len(copied)} images in {all_images_dir}\")" ] }, diff --git a/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py b/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py new file mode 100644 index 00000000..eb01a8d2 --- /dev/null +++ b/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py @@ -0,0 +1,81 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from ugb_scorer import build_benchmark_samples, validate_cached_results + + +class BuildBenchmarkSamplesTest(unittest.TestCase): + def setUp(self): + self.rows = [ + {"id": "orig0", "prompt": "original", "sub_dims": {}}, + {"id": "phi0", "prompt": "physical", "sub_dims": {}}, + ] + + def test_rejects_incomplete_benchmark_generation(self): + with tempfile.TemporaryDirectory() as directory: + image_folder = Path(directory) + (image_folder / "orig0_0.png").touch() + + with self.assertRaisesRegex( + FileNotFoundError, r"Missing 1 of 2.*phi0_0\.png" + ): + build_benchmark_samples(self.rows, image_folder, "png") + + def test_builds_samples_for_both_benchmark_splits(self): + with tempfile.TemporaryDirectory() as directory: + image_folder = Path(directory) + for row in self.rows: + (image_folder / f"{row['id']}_0.png").touch() + + samples = build_benchmark_samples(self.rows, image_folder, "png") + + self.assertEqual( + [sample["image_path"].name for sample in samples], + ["orig0_0.png", "phi0_0.png"], + ) + + def test_allows_explicit_partial_debug_generation(self): + with tempfile.TemporaryDirectory() as directory: + image_folder = Path(directory) + (image_folder / "orig0_0.png").touch() + + samples = build_benchmark_samples( + self.rows, image_folder, "png", allow_missing_images=True + ) + + self.assertEqual( + [sample["image_path"].name for sample in samples], ["orig0_0.png"] + ) + + def test_bundled_benchmark_discovers_all_orig_and_phi_samples(self): + prompt_file = Path(__file__).parent / "assets" / "unigenbench_prompt.json" + rows = json.loads(prompt_file.read_text(encoding="utf-8"))["benchmark"] + with tempfile.TemporaryDirectory() as directory: + image_folder = Path(directory) + for row in rows: + (image_folder / f"{row['id']}_0.jpg").touch() + + samples = build_benchmark_samples(rows, image_folder, "jpg") + + names = [sample["image_path"].name for sample in samples] + self.assertEqual(len(names), 1170) + self.assertEqual(sum(name.startswith("orig") for name in names), 600) + self.assertEqual(sum(name.startswith("phi") for name in names), 570) + + +class ValidateCachedResultsTest(unittest.TestCase): + def test_rejects_stale_partial_result(self): + samples = [ + {"image_path": Path("orig0_0.png")}, + {"image_path": Path("phi0_0.png")}, + ] + score_final = {"breakdown": {"orig0_0.png": {}}} + + with self.assertRaisesRegex(ValueError, r"cover 1 of 2"): + validate_cached_results(score_final, samples, Path("scores.json")) + + +if __name__ == "__main__": + unittest.main() diff --git a/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py b/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py index 3550adc6..d71f5997 100644 --- a/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py +++ b/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py @@ -301,6 +301,62 @@ def split_score_breakdown_by_prefix(score_breakdown: dict) -> tuple[dict, dict]: return orig, phi +def build_benchmark_samples( + benchmark_rows: list[dict], + image_folder: Path, + extension: str, + allow_missing_images: bool = False, +) -> list[dict[str, Any]]: + """Build scorer inputs and reject incomplete benchmark generations.""" + samples = [] + missing_paths = [] + for row in benchmark_rows: + index = row.get("index") if row.get("index") is not None else row.get("id") + if index is None: + raise ValueError("Each benchmark row must define either 'index' or 'id'.") + + prompt = row.get("prompt_en") or row.get("prompt") + sub_dims = row.get("sub_dims_en") or row.get("sub_dims") + metadata = ast.literal_eval(sub_dims) if isinstance(sub_dims, str) else sub_dims + image_path = image_folder / f"{index}_0.{extension}" + if not image_path.exists(): + missing_paths.append(image_path) + continue + + samples.append( + { + "image_path": image_path, + "prompt": prompt, + "metadata": metadata, + "extension": extension, + "try_num": 0, + } + ) + + if missing_paths and not allow_missing_images: + preview = ", ".join(path.name for path in missing_paths[:5]) + if len(missing_paths) > 5: + preview += ", ..." + raise FileNotFoundError( + f"Missing {len(missing_paths)} of {len(benchmark_rows)} expected image(s) " + f"under {image_folder}: {preview}" + ) + + return samples + + +def validate_cached_results(score_final: dict, samples: list[dict], result_path: Path) -> int: + """Reject a cached score whose coverage differs from the current image set.""" + expected_results = {sample["image_path"].name for sample in samples} + cached_results = set(score_final.get("breakdown", {})) + if cached_results != expected_results: + raise ValueError( + f"Existing results at {result_path} cover {len(cached_results)} of " + f"{len(expected_results)} currently available image(s); remove the stale result file to rescore." + ) + return len(cached_results) + + def print_stats_from_json(score_final: dict, input_folder: str, result_path: str) -> None: stats = score_final["stats"] success_count = score_final.get("success_count", "N/A") @@ -349,43 +405,34 @@ def main() -> None: ) print(f"\nParams:\n{params_disp}") - if result_path.exists(): - print(f"Found existing results at {result_path}, skipping scoring") - score_final = json.loads(result_path.read_text()) - print_stats_from_json(score_final, str(image_folder), str(result_path)) - return - benchmark_data = json.loads(prompt_file.read_text()) if isinstance(benchmark_data, dict) and "benchmark" in benchmark_data: benchmark_rows = benchmark_data["benchmark"] else: benchmark_rows = benchmark_data - samples_todo = [] - missing_images = 0 - for row in benchmark_rows: - index = row.get("index") or row.get("id") - prompt = row.get("prompt_en") or row.get("prompt") - sub_dims = row.get("sub_dims_en") or row.get("sub_dims") - metadata = ast.literal_eval(sub_dims) if isinstance(sub_dims, str) else sub_dims - image_path = image_folder / f"{index}_0.{args.extension}" - if not image_path.exists(): - missing_images += 1 - continue - samples_todo.append( - { - "image_path": image_path, - "prompt": prompt, - "metadata": metadata, - "extension": args.extension, - "try_num": 0, - } + samples_todo = build_benchmark_samples( + benchmark_rows, + image_folder, + args.extension, + allow_missing_images=args.allow_missing_images, + ) + samples_total = len(benchmark_rows) + missing_images = samples_total - len(samples_todo) + if missing_images: + print( + f"Warning: scoring {len(samples_todo)} of {samples_total} benchmark images " + "because --allow_missing_images was set" ) + print(f"Total samples: {len(samples_todo)}/{samples_total}") - samples_total = len(samples_todo) - if missing_images: - print(f"Missing {missing_images} expected image(s) under {image_folder}") - print(f"Total samples: {samples_total}") + if result_path.exists(): + score_final = json.loads(result_path.read_text()) + cached_count = validate_cached_results(score_final, samples_todo, result_path) + score_final["success_count"] = f"{cached_count}/{samples_total}" + print(f"Found complete existing results at {result_path}, skipping scoring") + print_stats_from_json(score_final, str(image_folder), str(result_path)) + return vlm = GatewayVLM( gateway_url=args.gateway_url, @@ -473,6 +520,11 @@ def parse_arguments() -> argparse.Namespace: parser.add_argument("--num_concurrency", type=int, default=128) parser.add_argument("--batch_size", type=int, default=1170) parser.add_argument("--max_retry", type=int, default=3) + parser.add_argument( + "--allow_missing_images", + action="store_true", + help="Score an explicitly incomplete generation for debugging; coverage still uses the full benchmark total.", + ) return parser.parse_args() From 21a3e8581886cce7b7b5d1efa46b5bd26d7191c3 Mon Sep 17 00:00:00 2001 From: primorLee Date: Sat, 5 Sep 2026 00:01:56 +0800 Subject: [PATCH 2/3] Fix stale UniGenBench run outputs --- .../run_with_cosmos_framework.ipynb | 32 +++++++++++-------- .../generator/unigenbench/test_ugb_scorer.py | 31 ++++++++++++++++++ 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb b/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb index a0206841..bdecabd4 100644 --- a/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb +++ b/evaluation/cosmos3/generator/unigenbench/run_with_cosmos_framework.ipynb @@ -105,6 +105,7 @@ "import os.path as osp\n", "import socket\n", "import sys\n", + "from datetime import datetime, timezone\n", "from pathlib import Path\n", "\n", "\n", @@ -432,8 +433,9 @@ "metadata": {}, "outputs": [], "source": [ - "output_dir = UNIGENBENCH_IMAGE\n", - "os.makedirs(output_dir, exist_ok=True)\n", + "run_id = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%S.%fZ\")\n", + "output_dir = osp.join(UNIGENBENCH_IMAGE, run_id)\n", + "os.makedirs(output_dir, exist_ok=False)\n", "\n", "id_list = list(PROMPTS.keys())\n", "\n", @@ -499,22 +501,23 @@ "import re\n", "import shutil\n", "\n", - "all_images_dir = Path(UNIGENBENCH_IMAGE) / \"all_images\"\n", - "all_images_dir.mkdir(exist_ok=True)\n", + "all_images_dir = Path(output_dir) / \"all_images\"\n", + "all_images_dir.mkdir(exist_ok=False)\n", "\n", - "for subdir in sorted(Path(UNIGENBENCH_IMAGE).iterdir()):\n", - " if subdir.is_dir() and re.fullmatch(r\"(?:orig|phi)\\d+\", subdir.name):\n", - " src = subdir / \"vision.jpg\"\n", - " dst = all_images_dir / f\"{subdir.name}_0.jpg\"\n", - " if src.exists() and not dst.exists():\n", - " shutil.copy2(src, dst)\n", + "for sample_id in id_list:\n", + " if not re.fullmatch(r\"(?:orig|phi)\\d+\", sample_id):\n", + " raise ValueError(f\"Unexpected benchmark sample ID: {sample_id}\")\n", + " src = Path(output_dir) / sample_id / \"vision.jpg\"\n", + " dst = all_images_dir / f\"{sample_id}_0.jpg\"\n", + " if src.exists():\n", + " shutil.copy2(src, dst)\n", "copied = list(all_images_dir.glob(\"*.jpg\"))\n", "copied_ids = {path.stem.removesuffix(\"_0\") for path in copied}\n", - "missing_ids = sorted(set(PROMPTS) - copied_ids)\n", + "missing_ids = sorted(set(id_list) - copied_ids)\n", "if missing_ids:\n", " preview = \", \".join(missing_ids[:5])\n", " raise FileNotFoundError(\n", - " f\"Missing {len(missing_ids)} of {len(PROMPTS)} generated images: {preview}\"\n", + " f\"Missing {len(missing_ids)} of {len(id_list)} generated images: {preview}\"\n", " )\n", "print(f\"Done. {len(copied)} images in {all_images_dir}\")" ] @@ -542,17 +545,18 @@ " COSMOS3_ROOT, \"evaluation\", \"cosmos3\", \"generator\", \"unigenbench\", \"ugb_scorer.py\"\n", ")\n", "python_bin = osp.join(COSMOS3_UV_ENV, \"bin\", \"python\")\n", + "run_score_path = osp.join(output_dir, \"unigenbench_result.json\")\n", "\n", "subprocess.run(\n", " [\n", " python_bin,\n", " scorer_path,\n", " \"--image_folder\",\n", - " osp.join(UNIGENBENCH_IMAGE, \"all_images\"),\n", + " str(all_images_dir),\n", " \"--benchmark_prompt_json\",\n", " UNIGENBENCH_PROMPT,\n", " \"--output_score_json\",\n", - " UNIGENBENCH_SCORE,\n", + " run_score_path,\n", " \"--extension\",\n", " \"jpg\",\n", " \"--gateway_url\",\n", diff --git a/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py b/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py index eb01a8d2..bf3f5096 100644 --- a/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py +++ b/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py @@ -6,6 +6,37 @@ from ugb_scorer import build_benchmark_samples, validate_cached_results +class NotebookImageCollectionTest(unittest.TestCase): + def test_does_not_reuse_images_from_an_earlier_run(self): + notebook_path = Path(__file__).with_name("run_with_cosmos_framework.ipynb") + notebook = json.loads(notebook_path.read_text(encoding="utf-8")) + collection_cell = next( + cell + for cell in notebook["cells"] + if cell.get("id") == "5add5da2-05a4-43c7-a5d5-f9b2b47b1595" + ) + collection_source = "".join(collection_cell["source"]) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + previous_run = root / "previous" + current_run = root / "current" + (previous_run / "orig0").mkdir(parents=True) + (previous_run / "orig0" / "vision.jpg").write_bytes(b"stale") + (current_run / "phi0").mkdir(parents=True) + (current_run / "phi0" / "vision.jpg").write_bytes(b"current") + + with self.assertRaisesRegex(FileNotFoundError, r"Missing 1 of 2.*orig0"): + exec( + compile(collection_source, str(notebook_path), "exec"), + { + "Path": Path, + "id_list": ["orig0", "phi0"], + "output_dir": str(current_run), + }, + ) + + class BuildBenchmarkSamplesTest(unittest.TestCase): def setUp(self): self.rows = [ From 8753a29826e8f48a916a0b5b26e267f63e9aff4d Mon Sep 17 00:00:00 2001 From: primorLee Date: Thu, 1 Oct 2026 12:14:35 +0800 Subject: [PATCH 3/3] fix: validate UniGenBench cache input and judge identity --- .../generator/unigenbench/test_ugb_scorer.py | 104 +++++++++++++++++- .../generator/unigenbench/ugb_scorer.py | 46 +++++++- 2 files changed, 142 insertions(+), 8 deletions(-) diff --git a/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py b/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py index bf3f5096..4bcc393a 100644 --- a/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py +++ b/evaluation/cosmos3/generator/unigenbench/test_ugb_scorer.py @@ -1,9 +1,14 @@ +import argparse +import copy +import io import json import tempfile import unittest +from contextlib import redirect_stdout from pathlib import Path +from unittest.mock import patch -from ugb_scorer import build_benchmark_samples, validate_cached_results +from ugb_scorer import SYSTEM_PROMPT, build_benchmark_samples, main, validate_cached_results class NotebookImageCollectionTest(unittest.TestCase): @@ -105,7 +110,102 @@ def test_rejects_stale_partial_result(self): score_final = {"breakdown": {"orig0_0.png": {}}} with self.assertRaisesRegex(ValueError, r"cover 1 of 2"): - validate_cached_results(score_final, samples, Path("scores.json")) + validate_cached_results(score_final, samples, Path("scores.json"), {}) + + +class ScorerCacheIdentityTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + root = Path(self.directory.name) + self.rows = [ + { + "id": index, + "prompt": "a red ball", + "sub_dims": { + "Testpoints": ["Attribute - Color"], + "Testpoint Description": ["The ball is red."], + }, + } + for index in ("orig0", "phi0") + ] + self.args = argparse.Namespace( + image_folder=str(root), + benchmark_prompt_json=str(root / "prompts.json"), + output_score_json=str(root / "scores.json"), + modelstr="judge-A", + gateway_url="https://judge.example/v1", + gateway_api="private-test-credential", + extension="png", + allow_missing_images=False, + num_concurrency=1, + batch_size=2, + max_retry=1, + ) + self.write_inputs(self.rows) + with patch("ugb_scorer.parse_arguments", return_value=self.args), patch("ugb_scorer.GatewayVLM") as factory: + judge = factory.return_value + judge.build_request.side_effect = lambda **kwargs: kwargs + judge.query.side_effect = lambda requests, **kwargs: [ + '- testpoint: "Attribute - Color"\n analysis: "red ball"\n verdict: true' + for _ in requests + ] + with redirect_stdout(io.StringIO()): + main() + factory.assert_called_once() + judge.close.assert_called_once() + self.cached_result = json.loads(Path(self.args.output_score_json).read_text(encoding="utf-8")) + + def write_inputs(self, rows): + Path(self.args.benchmark_prompt_json).write_text(json.dumps(rows), encoding="utf-8") + for row in rows: + (Path(self.args.image_folder) / f"{row['id']}_0.png").write_bytes(b"image") + + def test_persists_identity_and_reuses_matching_cache_without_calling_judge(self): + self.assertEqual(self.cached_result["success_count"], "2/2") + self.assertEqual(self.cached_result["run_manifest"]["judge_model"], "judge-A") + self.assertNotIn(self.args.gateway_api, json.dumps(self.cached_result)) + with patch("ugb_scorer.parse_arguments", return_value=self.args), patch("ugb_scorer.GatewayVLM") as factory: + with redirect_stdout(io.StringIO()): + main() + factory.assert_not_called() + + def test_rejects_changed_identity_even_when_image_names_match(self): + original_args = copy.deepcopy(self.args) + for change in ("model", "endpoint", "prompt", "testpoint", "image", "system_prompt", "scorer", "legacy"): + with self.subTest(change=change): + self.args = copy.deepcopy(original_args) + rows = copy.deepcopy(self.rows) + self.write_inputs(rows) + cached = copy.deepcopy(self.cached_result) + system_prompt = SYSTEM_PROMPT + if change == "model": + self.args.modelstr = "judge-B" + elif change == "endpoint": + self.args.gateway_url = "https://different-judge.example/v1" + elif change == "prompt": + rows[0]["prompt"] = "a blue ball" + elif change == "testpoint": + rows[0]["sub_dims"]["Testpoint Description"] = ["The ball is blue."] + elif change == "image": + (Path(self.args.image_folder) / "orig0_0.png").write_bytes(b"different image") + elif change == "system_prompt": + system_prompt += "\nUpdated judge instructions." + elif change == "scorer": + cached["run_manifest"]["scorer_sha256"]["ugb_scorer.py"] = "older-scorer-version" + elif change == "legacy": + del cached["run_manifest"] + Path(self.args.benchmark_prompt_json).write_text(json.dumps(rows), encoding="utf-8") + Path(self.args.output_score_json).write_text(json.dumps(cached), encoding="utf-8") + with ( + patch("ugb_scorer.parse_arguments", return_value=self.args), + patch("ugb_scorer.SYSTEM_PROMPT", system_prompt), + patch("ugb_scorer.GatewayVLM") as factory, + redirect_stdout(io.StringIO()), + ): + with self.assertRaisesRegex(ValueError, "missing or mismatched run manifest"): + main() + factory.assert_not_called() if __name__ == "__main__": diff --git a/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py b/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py index d71f5997..102361f3 100644 --- a/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py +++ b/evaluation/cosmos3/generator/unigenbench/ugb_scorer.py @@ -15,6 +15,7 @@ import argparse import ast import difflib +import hashlib import json import os.path as osp import re @@ -345,8 +346,34 @@ def build_benchmark_samples( return samples -def validate_cached_results(score_final: dict, samples: list[dict], result_path: Path) -> int: - """Reject a cached score whose coverage differs from the current image set.""" +def build_run_manifest( + benchmark_rows: list[dict], samples: list[dict], judge_model: str, gateway_url: str +) -> dict[str, Any]: + """Identify the inputs and scoring implementation without storing API credentials.""" + scorer_dir = Path(__file__).resolve().parent + return { + "schema_version": 1, + "judge_model": judge_model, + "gateway_url": gateway_url.rstrip("/"), + "benchmark_sha256": hashlib.sha256( + json.dumps(benchmark_rows, sort_keys=True, ensure_ascii=False).encode("utf-8") + ).hexdigest(), + "images_sha256": { + sample["image_path"].name: hashlib.sha256(sample["image_path"].read_bytes()).hexdigest() + for sample in samples + }, + "scorer_sha256": { + filename: hashlib.sha256((scorer_dir / filename).read_text(encoding="utf-8").encode("utf-8")).hexdigest() + for filename in ("ugb_scorer.py", "utils.py") + }, + "system_prompt_sha256": hashlib.sha256(SYSTEM_PROMPT.encode("utf-8")).hexdigest(), + } + + +def validate_cached_results( + score_final: dict, samples: list[dict], result_path: Path, run_manifest: dict +) -> int: + """Reject cached scores from different inputs or a different scoring run.""" expected_results = {sample["image_path"].name for sample in samples} cached_results = set(score_final.get("breakdown", {})) if cached_results != expected_results: @@ -354,6 +381,11 @@ def validate_cached_results(score_final: dict, samples: list[dict], result_path: f"Existing results at {result_path} cover {len(cached_results)} of " f"{len(expected_results)} currently available image(s); remove the stale result file to rescore." ) + if score_final.get("run_manifest") != run_manifest: + raise ValueError( + f"Existing results at {result_path} have a missing or mismatched run manifest; " + "remove the stale result file to rescore with the requested inputs and judge." + ) return len(cached_results) @@ -405,7 +437,7 @@ def main() -> None: ) print(f"\nParams:\n{params_disp}") - benchmark_data = json.loads(prompt_file.read_text()) + benchmark_data = json.loads(prompt_file.read_text(encoding="utf-8")) if isinstance(benchmark_data, dict) and "benchmark" in benchmark_data: benchmark_rows = benchmark_data["benchmark"] else: @@ -425,10 +457,11 @@ def main() -> None: "because --allow_missing_images was set" ) print(f"Total samples: {len(samples_todo)}/{samples_total}") + run_manifest = build_run_manifest(benchmark_rows, samples_todo, args.modelstr, args.gateway_url) if result_path.exists(): - score_final = json.loads(result_path.read_text()) - cached_count = validate_cached_results(score_final, samples_todo, result_path) + score_final = json.loads(result_path.read_text(encoding="utf-8")) + cached_count = validate_cached_results(score_final, samples_todo, result_path, run_manifest) score_final["success_count"] = f"{cached_count}/{samples_total}" print(f"Found complete existing results at {result_path}, skipping scoring") print_stats_from_json(score_final, str(image_folder), str(result_path)) @@ -500,11 +533,12 @@ def main() -> None: "all": compute_final_metrics(score_breakdown, verbose=False), }, "judge_model": args.modelstr, + "run_manifest": run_manifest, "success_count": f"{len(score_breakdown)}/{samples_total}", "breakdown": score_breakdown, } - result_path.write_text(json.dumps(score_final, indent=4)) + result_path.write_text(json.dumps(score_final, indent=4), encoding="utf-8") print_stats_from_json(score_final, str(image_folder), str(result_path))