diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 102951b..df56d7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,9 @@ jobs: - name: V6 Continuation and Permission Safety Tests run: python -m unittest discover -s tools -p "test_servy_v6_continuation.py" -v + - name: Frozen-Candidate Accuracy Diagnostics Regression Tests + run: python -m unittest discover -s tools -p "test_servy_accuracy_diagnostics.py" -v + - name: V6 Holdout Independence and Leakage Safety Tests run: python -m unittest discover -s tools -p "test_servy_v6_holdout_audit.py" -v diff --git a/SERVY_ACCURACY_EVIDENCE_GATES.md b/SERVY_ACCURACY_EVIDENCE_GATES.md new file mode 100644 index 0000000..3a6af11 --- /dev/null +++ b/SERVY_ACCURACY_EVIDENCE_GATES.md @@ -0,0 +1,85 @@ +# SERVY — Accuracy evidence and fine-tuning gates + +## What this PR actually fixes + +* V6 now writes V5 and V6 per-query evaluation records, a paired movement ledger, + and a candidate-recall/head-pool ceiling report into its **private experiment + directory**. Equal aggregate Hit@1 cannot conceal offsetting query regressions. +* Frozen candidate identifiers, training subset, original V5 weights, corpus + and development split are SHA-256 checked both before and after training. + An in-place mutation aborts reporting a valid comparison. +* Research-only counterfactuals reuse **already computed logits** to estimate + Hit@1/5/10 and MRR at pool sizes 5, 8 and 10. They never change model + weights, production retrieval, tenant permissions or serving pool settings. +* Synthetic tests cover upper bounds, top-1 exchanges with unchanged aggregate + scores, candidate drift, label mismatch, missing examples and invalid logits. + +## Findings from the existing V5/V6 result + +On the reported 42-query frozen development candidates: + +* Hit@1: 35/42 = 83.33% for both models +* Hit@5: 37/42 = 88.10% +* Candidate Recall@10: 38/42 = 90.48% +* Four queries have no gold evidence in the frozen top ten +* The first five candidates contain gold evidence for at most 37/42 queries: + **88.10% is the head-five top-one oracle ceiling** on this frozen set. +* The earlier reported 37/42 V5 Hit@1 is from a different/unreconciled + evaluation. Do **not** overwrite one metric with the other. + +These are DEVELOPMENT diagnostics on provisional labels, **not independent +holdout or production metrics**. Improving reranker training cannot retrieve +chunks entirely absent from the frozen candidate set. Any production candidate +retrieval change needs separate authorization, source integrity, latency and +quality tests. + +## How to diagnose the EXISTING V5/V6 experiment without retraining + +The original V6 experiment has already completed and does not contain +the new per-query outputs. The existing read-only checkpoint audit can replay +the two *known* local checkpoints against the same frozen candidates: + +```bash +python tools/servy_checkpoint_audit.py \ + --cases /content/servy_v6_inputs/recovery_v4/family_resplit_20261018_234238/dev_unapproved_candidate.jsonl \ + --corpus /content/servy_v6_inputs/recovery_v4/servy_corpus_recovered_20261018T150933Z.jsonl \ + --candidates /content/servy_v6_inputs/recovery_v4/research_artifacts_resplit_20261019/frozen_candidates.jsonl \ + --model v5=/content/servy_v6_inputs/best_model \ + --model v6=/content/drive/MyDrive/Servy_V6_Authorized/outputs/v6_research_first_run/best_model \ + --pool-size 5 \ + --top-k 10 \ + --out-dir /content/drive/MyDrive/Servy_V6_Authorized/outputs/v5_v6_independent_replay_01 +``` + +Use a **NEW** destination directory; the existing audit tool can write +to an existing directory, so choose a unique path and inspect it first. +This replays development evaluation only; it does not certify independence, +answer grounding, production serving, or the company permission scope. + +Output is private and includes source/query IDs and text. Do not commit +private per-query output, training files or model weights to GitHub. + +## Further training decision gate + +**Retain the existing verified V5 model. Do not automatically fine-tune V7.** + +1. Identify whether each top-one miss is due to absent candidate evidence, + a gold passage outside the eligible head pool, or a reranker ordering miss. +2. Compare per-query V5 and V6 results on the same checkpoint hashes, candidate + SHA, pool size, split SHA and model inference settings. +3. If retraining is proposed, select a new label-reviewed and genuinely + independent evaluation population. Do not reuse known dev failures as + training examples and then claim gains on the same dev set. +4. Keep independent holdout and human label approval false unless supported by + independent records. Never bypass the existing permission gate. +5. Measure runtime CPU/GPU latency and memory, safety, citations and tenant + isolation before considering any production rollout. + +No code change or increased pool limit is an accuracy improvement until a real +replay demonstrates it. No company-private artifact is available in Git CI. + +## New regression test + +```bash +python -m unittest discover -s tools -p "test_servy_accuracy_diagnostics.py" -v +``` diff --git a/tools/servy_accuracy_diagnostics.py b/tools/servy_accuracy_diagnostics.py new file mode 100644 index 0000000..647ad85 --- /dev/null +++ b/tools/servy_accuracy_diagnostics.py @@ -0,0 +1,212 @@ +"""Read-only paired ranking diagnostics for research checkpoints. + +Provides auditable, honest candidate-pool ceilings and paired query movements. +Neither development-set improvement nor this analysis authorizes promotion. +Per-query output can include proprietary IDs; keep it in approved private storage. +""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + + +PROTOCOL = "servy-frozen-pool-paired-v1" + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _validate(results, frozen, model_name): + rows = results.get("queries") + if not isinstance(rows, list) or not rows: + raise ValueError(f"{model_name}: missing per-query evaluation") + indexed = {} + for row in rows: + qid = str(row.get("query_id", "")) + if not qid or qid in indexed or qid not in frozen: + raise ValueError(f"{model_name}: duplicate/unknown query ID: {qid}") + expected = [str(x) for x in frozen[qid]] + candidate_ids = [str(x) for x in row["candidate_chunk_ids"]] + ranked_ids = [str(x) for x in row["ranked_chunk_ids"]] + gold_ids = {str(x) for x in row["gold_chunk_ids"]} + if not expected or len(expected) != len(set(expected)): + raise ValueError(f"{model_name}: invalid frozen candidates: {qid}") + if candidate_ids != expected or len(ranked_ids) != len(expected) or set(ranked_ids) != set(expected): + raise ValueError(f"{model_name}: candidate scope or membership changed: {qid}") + actual_original_rank = next((i for i, cid in enumerate(expected, 1) if cid in gold_ids), None) + actual_rank = next((i for i, cid in enumerate(ranked_ids, 1) if cid in gold_ids), None) + if row["original_gold_rank"] != actual_original_rank or row["gold_rank"] != actual_rank: + raise ValueError(f"{model_name}: reported gold rank contradicts candidate ordering: {qid}") + indexed[qid] = row + if set(indexed) != set(frozen): + raise ValueError(f"{model_name}: missing query rows") + return indexed + + +def compare_frozen_pool(before, after, frozen, *, pool_size=5): + """Compare two full evaluate_one results on identical frozen candidates. + + The top-1 ceiling is bounded by original recall within the head pool. + A gold chunk outside the pool cannot move to rank 1 when the tail is fixed. + """ + if not isinstance(pool_size, int) or pool_size < 1: + raise ValueError("pool_size must be a positive integer") + old = _validate(before, frozen, "before") + new = _validate(after, frozen, "after") + movement = { + "up": 0, "down": 0, "unchanged_rank": 0, + "found": 0, "lost": 0, "both_missing": 0, + } + rows = [] + head_reachable = 0 + candidate_reachable = 0 + changed_top_one = 0 + regression_from_top_one = 0 + improvement_to_top_one = 0 + improved_gold = 0 + worsened_gold = 0 + + for qid in frozen: + first, second = old[qid], new[qid] + if first["gold_chunk_ids"] != second["gold_chunk_ids"]: + raise ValueError(f"Gold labels changed during evaluation: {qid}") + original_rank = first["original_gold_rank"] + if original_rank != second["original_gold_rank"]: + raise ValueError(f"Frozen candidate rank changed: {qid}") + a, b = first["gold_rank"], second["gold_rank"] + # A gold rank cannot disappear when both models only reorder the same candidates. + if (a is None) != (b is None): + raise ValueError(f"Inconsistent candidate membership: {qid}") + if a is None: + label = "both_missing" + elif b < a: + label = "up" + improved_gold += 1 + elif b > a: + label = "down" + worsened_gold += 1 + else: + label = "unchanged_rank" + movement[label] += 1 + + if original_rank is not None: + candidate_reachable += 1 + if original_rank <= pool_size: + head_reachable += 1 + if first["ranked_chunk_ids"][0] != second["ranked_chunk_ids"][0]: + changed_top_one += 1 + if a == 1 and b != 1: + regression_from_top_one += 1 + if a != 1 and b == 1: + improvement_to_top_one += 1 + + rows.append({ + "query_id": qid, + "original_gold_rank": original_rank, + "before_gold_rank": a, + "after_gold_rank": b, + "gold_rank_movement": label, + "before_top_chunk_id": first["ranked_chunk_ids"][0], + "after_top_chunk_id": second["ranked_chunk_ids"][0], + "changed_top_one": first["ranked_chunk_ids"][0] != second["ranked_chunk_ids"][0], + "head_pool_reachable": original_rank is not None and original_rank <= pool_size, + "candidate_top10_reachable": original_rank is not None and original_rank <= 10, + }) + n = len(frozen) + pct = lambda v: round(100 * v / n, 2) + summary = { + "protocol": PROTOCOL, + "queries": n, + "pool_size": pool_size, + "candidate_recall_at_10_count": sum( + row["candidate_top10_reachable"] for row in rows + ), + "candidate_recall_at_10_pct": pct(sum(row["candidate_top10_reachable"] for row in rows)), + "head_pool_oracle_hit_at_1_ceiling_count": head_reachable, + "head_pool_oracle_hit_at_1_ceiling_pct": pct(head_reachable), + "queries_with_gold_outside_head_but_inside_candidates": candidate_reachable - head_reachable, + "queries_missing_gold_from_candidates": n - candidate_reachable, + "before_top1_count": sum(row["before_gold_rank"] == 1 for row in rows), + "after_top1_count": sum(row["after_gold_rank"] == 1 for row in rows), + "before_missed_head_reachable": sum( + row["head_pool_reachable"] and row["before_gold_rank"] != 1 for row in rows + ), + "after_missed_head_reachable": sum( + row["head_pool_reachable"] and row["after_gold_rank"] != 1 for row in rows + ), + "query_gold_rank_movements": movement, + "queries_with_different_top_chunk": changed_top_one, + "queries_improved_to_top1": improvement_to_top_one, + "queries_regressed_from_top1": regression_from_top_one, + "queries_with_improved_gold_rank": improved_gold, + "queries_with_worsened_gold_rank": worsened_gold, + "evaluation_status": "REUSED_PROVISIONAL_DEVELOPMENT_ONLY", + "independent_holdout_verified": False, + "production_promotion_allowed": False, + "model_decision": "KEEP_V5_UNTIL_INDEPENDENT_EVIDENCE", + } + return {"summary": summary, "per_query": rows} + + +def simulate_pool_sweep(result, frozen, *, pool_sizes=(5, 8, 10)): + """Offline counterfactual, using ALREADY-SCORED logits, not another model run. + + It does not modify the frozen candidate set. Unlike serving, an experimental + pool of 10 is permitted here to diagnose the candidate-head bottleneck. + Changing a development pool is NOT evidence for production improvement. + """ + rows = _validate(result, frozen, "pool_sweep") + observations = [] + for size in pool_sizes: + if not isinstance(size, int) or size < 1 or size > 10: + raise ValueError("Pool sweep size must be 1..10") + hit1 = hit5 = hit10 = 0 + reciprocal_sum = 0.0 + for qid, original in frozen.items(): + row = rows[qid] + scores = row.get("reranker_scores") + if not isinstance(scores, dict) or set(scores) != set(original): + raise ValueError(f"Missing complete candidate scores for {qid}") + values = {} + import math + for cid in original: + value = float(scores[cid]) + if not math.isfinite(value): + raise ValueError(f"Non-finite candidate score for {qid}") + values[cid] = value + head = list(original[:size]) + # Python stable sort preserves original tie order, just like rerank_fixed. + reranked = sorted(head, key=lambda cid: -values[cid]) + list(original[size:]) + gold = set(row["gold_chunk_ids"]) + rank = next((index for index, cid in enumerate(reranked, 1) if cid in gold), None) + if rank is not None: + hit1 += rank == 1 + hit5 += rank <= 5 + hit10 += rank <= 10 + if rank <= 10: + reciprocal_sum += 1 / rank + n = len(frozen) + observations.append({ + "pool_size": size, + "queries": n, + "hit_at_1": round(hit1 * 100 / n, 2), + "hit_at_5": round(hit5 * 100 / n, 2), + "hit_at_10": round(hit10 * 100 / n, 2), + "mrr_at_10": round(reciprocal_sum / n, 4), + "research_counterfactual_only": True, + "production_change_approved": False, + }) + return observations + + +def write_jsonl_exclusive(path, records): + """Never silently overwrite a previous private experiment output.""" + with Path(path).open("x", encoding="utf-8") as stream: + for row in records: + stream.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") diff --git a/tools/servy_v6_continuation.py b/tools/servy_v6_continuation.py index 7dde39d..512f385 100644 --- a/tools/servy_v6_continuation.py +++ b/tools/servy_v6_continuation.py @@ -24,6 +24,9 @@ from servy_v5_training_preflight import admit, read_jsonl, sha256 from servy_checkpoint_audit import evaluate_one from servy_checkpoint_backup import backup_checkpoint +from servy_accuracy_diagnostics import ( + compare_frozen_pool, sha256_file, simulate_pool_sweep, write_jsonl_exclusive, +) V5_WEIGHTS_SHA256 = "b1cf23ba0806f0057d73a5ab93fde88d634fc278f71b27f3a2a915afd8f8bcda" @@ -158,8 +161,34 @@ def train(args, plan, permission): out = Path(args.out_dir).expanduser().resolve() check_storage(out) out.mkdir(mode=0o700) + input_hashes = { + "v5_weights": sha256_file(Path(args.v5_model) / "model.safetensors"), + "train_subset": sha256_file(args.candidate_train), + "frozen_candidates": sha256_file(args.candidates), + "corpus": sha256_file(args.corpus), + "dev_split": sha256_file(Path(args.split_dir) / "dev_unapproved_candidate.jsonl"), + } + if input_hashes["v5_weights"] != V5_WEIGHTS_SHA256: + raise ValueError("V5_WEIGHTS_CHANGED_BEFORE_TRAINING") + if input_hashes["train_subset"] != plan["selected_train_sha256"]: + raise ValueError("V6_TRAIN_SUBSET_CHANGED_AFTER_PREFLIGHT") summary = { **plan, + "input_sha256": input_hashes, + "evaluation_protocol": { + "candidate_source": "frozen_input", + "pool_size": 5, + "top_k": 10, + "checkpoint_selection": "minimum_development_bce_loss", + "no_independent_test": True, + }, + "training_config": { + "epochs": args.epochs, + "batch_size": args.batch_size, + "learning_rate": args.learning_rate, + "seed": 42, + "objective": "BinaryCrossEntropyLoss", + }, "status": "V6_EXPLORATORY_IN_PROGRESS", "permission": permission, "data_status": "PROVISIONAL_TECHNICALLY_REVIEWED_NO_HUMAN_LABEL_SIGNOFF", @@ -190,6 +219,9 @@ def scorer(pairs_to_score): (out / "v5_dev_metrics.json").write_text( json.dumps(before["metrics"], indent=2) + "\n", encoding="utf-8" ) + # Per-query records remain in the approved private experiment directory. + # Without these records, identical aggregate metrics conceal regressions. + write_jsonl_exclusive(out / "v5_dev_per_query.jsonl", before["queries"]) use_bf16 = bool(torch.cuda.is_bf16_supported()) training_args = CrossEncoderTrainingArguments( output_dir=str(out / "trainer"), @@ -231,6 +263,36 @@ def scorer(pairs_to_score): data["cases"], data["chunks"], data["frozen"], score(trained), pool_size=5, ) + final_hashes = { + "v5_weights": sha256_file(Path(args.v5_model) / "model.safetensors"), + "train_subset": sha256_file(args.candidate_train), + "frozen_candidates": sha256_file(args.candidates), + "corpus": sha256_file(args.corpus), + "dev_split": sha256_file(Path(args.split_dir) / "dev_unapproved_candidate.jsonl"), + } + if final_hashes != input_hashes: + raise ValueError("V6_INPUT_DRIFT_DETECTED_NO_VALID_COMPARISON") + audit = compare_frozen_pool(before, after, data["frozen"], pool_size=5) + write_jsonl_exclusive(out / "v6_dev_per_query.jsonl", after["queries"]) + write_jsonl_exclusive(out / "v5_v6_paired_per_query.jsonl", audit["per_query"]) + (out / "v5_v6_accuracy_diagnostics.json").write_text( + json.dumps(audit["summary"], indent=2) + "\n", encoding="utf-8" + ) + # Counterfactual analysis reuses logits already scored on frozen candidates. + # It does not change the serving pool, select checkpoints or promote a model. + pool_audit = { + "v5": simulate_pool_sweep(before, data["frozen"]), + "v6": simulate_pool_sweep(after, data["frozen"]), + "status": "REUSED_DEV_COUNTERFACTUAL_NOT_PRODUCTION_CONFIG", + } + for checkpoint, report in (("v5", before), ("v6", after)): + pool_five = pool_audit[checkpoint][0] + for metric in ("hit_at_1", "hit_at_5", "hit_at_10", "mrr_at_10"): + if pool_five[metric] != report["metrics"][metric]: + raise ValueError("V6_POOL_SWEEP_METRIC_REPLAY_MISMATCH:" + checkpoint + ":" + metric) + (out / "v5_v6_pool_sweep.json").write_text( + json.dumps(pool_audit, indent=2) + "\n", encoding="utf-8" + ) summary.update({ "status": "V6_EXPLORATORY_COMPLETED_NOT_PROMOTED", "training_performed": True, @@ -238,6 +300,7 @@ def scorer(pairs_to_score): "v6_weights_sha256": sha256(out / "best_model" / "model.safetensors"), "v5_dev_metrics": before["metrics"], "v6_dev_metrics": after["metrics"], + "paired_diagnostics": audit["summary"], "dev_comparison_is_reused_diagnostic_only": True, "independent_holdout_verified": False, "production_promotion_allowed": False, diff --git a/tools/test_servy_accuracy_diagnostics.py b/tools/test_servy_accuracy_diagnostics.py new file mode 100644 index 0000000..c3e4e31 --- /dev/null +++ b/tools/test_servy_accuracy_diagnostics.py @@ -0,0 +1,139 @@ +"""Pure CPU-only diagnostics tests: no private data or model weights.""" +import json +from pathlib import Path +import tempfile +import unittest + +from servy_accuracy_diagnostics import compare_frozen_pool, simulate_pool_sweep, write_jsonl_exclusive + + +def case(qid, frozen, ordered, gold): + original = next((i for i, cid in enumerate(frozen, 1) if cid in gold), None) + rank = next((i for i, cid in enumerate(ordered, 1) if cid in gold), None) + return { + "query_id": qid, + "candidate_chunk_ids": list(frozen), + "ranked_chunk_ids": list(ordered), + "gold_chunk_ids": list(gold), + "original_gold_rank": original, + "gold_rank": rank, + } + + +class PairedAccuracyDiagnosticsTests(unittest.TestCase): + def setUp(self): + self.frozen = { + "q1": ["a", "b", "c"], + "q2": ["d", "e", "f"], + "q3": ["g", "h", "i"], + "q4": ["j", "k"], + } + self.before = {"queries": [ + case("q1", self.frozen["q1"], ["a", "b", "c"], {"b"}), + case("q2", self.frozen["q2"], ["e", "d", "f"], {"e"}), + case("q3", self.frozen["q3"], ["g", "h", "i"], {"i"}), + case("q4", self.frozen["q4"], ["j", "k"], {"not-in-candidates"}), + ]} + self.after = {"queries": [ + case("q1", self.frozen["q1"], ["b", "a", "c"], {"b"}), + case("q2", self.frozen["q2"], ["d", "e", "f"], {"e"}), + case("q3", self.frozen["q3"], ["g", "h", "i"], {"i"}), + case("q4", self.frozen["q4"], ["j", "k"], {"not-in-candidates"}), + ]} + + def test_ceiling_separates_retrieval_failures_from_reranking(self): + result = compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + s = result["summary"] + self.assertEqual(s["queries"], 4) + self.assertEqual(s["head_pool_oracle_hit_at_1_ceiling_count"], 2) + self.assertEqual(s["candidate_recall_at_10_count"], 3) + self.assertEqual(s["queries_with_gold_outside_head_but_inside_candidates"], 1) + self.assertEqual(s["queries_missing_gold_from_candidates"], 1) + self.assertEqual(s["queries_improved_to_top1"], 1) + self.assertEqual(s["queries_regressed_from_top1"], 1) + self.assertEqual(s["before_top1_count"], s["after_top1_count"]) + self.assertEqual(s["query_gold_rank_movements"]["up"], 1) + self.assertEqual(s["query_gold_rank_movements"]["down"], 1) + self.assertEqual(s["query_gold_rank_movements"]["both_missing"], 1) + self.assertFalse(s["production_promotion_allowed"]) + + def test_top1_same_does_not_prove_same_rankings(self): + result = compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + self.assertEqual(result["summary"]["before_top1_count"], 1) + self.assertEqual(result["summary"]["after_top1_count"], 1) + self.assertEqual(result["summary"]["queries_with_different_top_chunk"], 2) + + def test_pool_change_changes_oracle_not_metrics(self): + a = compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + b = compare_frozen_pool(self.before, self.after, self.frozen, pool_size=3) + self.assertEqual(a["summary"]["head_pool_oracle_hit_at_1_ceiling_count"], 2) + self.assertEqual(b["summary"]["head_pool_oracle_hit_at_1_ceiling_count"], 3) + self.assertEqual(a["summary"]["before_top1_count"], b["summary"]["before_top1_count"]) + + def test_frozen_candidates_cannot_change_between_runs(self): + self.after["queries"][0]["candidate_chunk_ids"] = ["b", "a", "c"] + with self.assertRaisesRegex(ValueError, "candidate scope"): + compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + + def test_reported_rank_must_match_actual_order(self): + self.after["queries"][0]["gold_rank"] = 2 + with self.assertRaisesRegex(ValueError, "contradicts"): + compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + + def test_gold_cannot_change(self): + self.after["queries"][0]["gold_chunk_ids"] = ["a"] + self.after["queries"][0]["original_gold_rank"] = 1 + self.after["queries"][0]["gold_rank"] = 2 + with self.assertRaisesRegex(ValueError, "Gold labels changed"): + compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + + def test_missing_query_is_fatal_not_dropped(self): + self.after["queries"].pop() + with self.assertRaisesRegex(ValueError, "missing query"): + compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + + def test_same_candidates_cannot_lose_gold(self): + self.after["queries"][0]["ranked_chunk_ids"] = ["a", "b", "bad"] + with self.assertRaisesRegex(ValueError, "candidate scope"): + compare_frozen_pool(self.before, self.after, self.frozen, pool_size=2) + + def test_negative_or_zero_pool_rejected(self): + with self.assertRaises(ValueError): + compare_frozen_pool(self.before, self.after, self.frozen, pool_size=0) + + def test_pool_sweep_reuses_existing_logits_to_surface_head_bottleneck(self): + frozen = {"query": ["a", "b", "c"]} + result = {"queries": [{ + **case("query", frozen["query"], ["a", "b", "c"], {"c"}), + "reranker_scores": {"a": 0.0, "b": 1.0, "c": 20.0}, + }]} + sweep = simulate_pool_sweep(result, frozen, pool_sizes=(2, 3)) + self.assertEqual(sweep[0]["hit_at_1"], 0.0) + self.assertEqual(sweep[1]["hit_at_1"], 100.0) + self.assertTrue(all(row["research_counterfactual_only"] for row in sweep)) + self.assertFalse(any(row["production_change_approved"] for row in sweep)) + + def test_pool_sweep_rejects_missing_or_nonfinite_logits(self): + frozen = {"query": ["a", "b"]} + row = { + **case("query", frozen["query"], ["a", "b"], {"b"}), + "reranker_scores": {"a": 1.0}, + } + with self.assertRaisesRegex(ValueError, "Missing complete"): + simulate_pool_sweep({"queries": [row]}, frozen) + row["reranker_scores"]["b"] = float("nan") + with self.assertRaisesRegex(ValueError, "Non-finite"): + simulate_pool_sweep({"queries": [row]}, frozen) + + def test_private_report_refuses_overwrite(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "paired.jsonl" + write_jsonl_exclusive(path, [{"query_id": "private", "rank": 1}]) + self.assertEqual(json.loads(path.read_text().strip())["rank"], 1) + with self.assertRaises(FileExistsError): + write_jsonl_exclusive(path, [{"query_id": "private", "rank": 9}]) + self.assertEqual(json.loads(path.read_text().strip())["rank"], 1) + + +if __name__ == "__main__": + unittest.main()