diff --git a/evals/README.md b/evals/README.md index 9ed2720..2a52f82 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,6 +1,6 @@ # Skill evaluation scenarios -One JSON file per skill, recording what a correct activation and a correct run look like. This directory is a pilot (three skills). There is no runner yet and nothing here calls a model; `tests/test_eval_scenarios.py` validates the files structurally. +One JSON file per skill, recording what a correct activation and a correct run look like. This directory is a pilot (three skills). There is no runner and nothing here calls a model; `trial.py` below records and reports a reviewer's observations; `tests/test_eval_scenarios.py` validates the files structurally. ## File format (`schema_version` 1) @@ -63,3 +63,39 @@ All other prompts remain activation-only; they have no procedure fixture yet. Th ### Fixture validation is not agent success `tests/test_eval_fixtures.py` checks deterministic facts only: each prompt and rubric text still exists in its scenario, starting-file hashes are unchanged, the materializer is repeatable and refuses unsafe requests, and the `ci-gate-mismatch` reproduction behaves as documented (local command exits 0, the workflow command exits 1, aligned configuration plus the code fix exit 0, using pinned `ruff==0.9.10` through `uvx`; skipped when `uv` is absent). It never runs an agent. A passing suite says the inputs are intact, not that any skill works; no success rate is measured or claimed here. Changing a fixture file means regenerating its hashes in `fixture.json` and re-reading the affected trials. + +## Trial worksheets and reports + +`evals/trial.py` is a stdlib-only helper with two operations. It makes no network calls, launches no agent and executes nothing from a scenario. + +```bash +# 1. Write an unscored worksheet to a NEW file outside evals/ (existing files are never overwritten) +uv run --no-project --python 3.12 python evals/trial.py init /tmp/trial-1.json \ + --trial-id trial-1 --client "" --model "" --reviewer "" + +# 2. Edit the worksheet by hand while reviewing a trial (see below) + +# 3. Validate and report (text by default; --check-current notes scenarios edited since the trial) +uv run --no-project --python 3.12 python evals/trial.py report /tmp/trial-1.json +uv run --no-project --python 3.12 python evals/trial.py report /tmp/trial-1.json --format json +``` + +`init` fills `trial_date` (today) and `catalog_commit` (`git rev-parse HEAD`) unless given as `--trial-date` / `--catalog-commit`. Provenance fields still empty are listed; `report` refuses a worksheet that lacks any of `trial_id`, `trial_date`, `catalog_commit`, `client`, `model`, `reviewer`. Keep worksheets out of `evals/` (it refuses that location) so they are never read as scenario files, and do not commit real transcripts. + +The worksheet snapshots each scenario (content plus hashes) and assigns case ids `/p` (positive prompt N) and `/n` (negative prompt N). Reports read only that snapshot, so an old worksheet stays reportable after scenarios change. + +Per case, set `activation`: + +- `status`: `pending` (not yet recorded), `observed`, or `unobserved` (could not tell which skill activated). +- `activated_skills`: every skill you saw activate (list all of them; extra skills are kept and do not fail a case). Only with `observed`, which also needs `evidence` (a transcript excerpt or log reference). + +For positive cases, set each rubric record's `status` to `pass`, `fail`, `not-applicable` or `unscored`. `pass`/`fail` need `evidence`; `not-applicable` needs `na_reason`. Rubrics are scenario-wide, so mark items a given prompt does not call for (for example CI artifacts when only a scaffold was requested) `not-applicable`. + +Reading the report: + +- Activation passes when a positive case shows the expected skill, or a negative case shows its `expected_skill` and not the skill under evaluation. `unobserved` and `pending` cases are never passes and are excluded from the observed denominator, with their counts shown. +- Procedure is summarised separately: scored items passed, passed out of all applicable items (unscored counts as not passed), failed, unscored, and not-applicable (visibly excluded). A case with unscored items and no failure is `incomplete`; a case with only not-applicable items is `undefined`. +- A zero denominator prints `undefined`, never 100%. +- Everything reported is what the reviewer recorded. It is not automatic proof of agent behavior, and one trial is not a success rate. + +`tests/test_eval_trials.py` runs the helper on a mixed-result worksheet with exact expected counts. No model or network access is needed. diff --git a/evals/trial.py b/evals/trial.py new file mode 100644 index 0000000..6a0ba89 --- /dev/null +++ b/evals/trial.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +"""Local worksheet and report helper for skill evaluation trials. + +Stdlib only; no network, no model calls, nothing is executed for the trial. + trial.py init DESTINATION [--trial-id ...] write an unscored worksheet + trial.py report WORKSHEET [--format text|json] validate and summarise it + +A report describes what a reviewer recorded, not verified agent behaviour. +""" +import argparse +import datetime +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +EVALS = Path(__file__).resolve().parent +WORKSHEET_VERSION = 1 +PROVENANCE = ("trial_id", "trial_date", "catalog_commit", "client", "model", "reviewer") +ACTIVATION_STATUSES = ("pending", "observed", "unobserved") +RUBRIC_STATUSES = ("pass", "fail", "not-applicable", "unscored") +UNDEFINED = "undefined" + + +class WorksheetError(Exception): + pass + + +def canonical_sha256(data) -> str: + text = json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(text.encode()).hexdigest() + + +def blank_text(value) -> bool: + return not isinstance(value, str) or not value.strip() + + +def case_skeletons(skill: str, content: dict) -> list[dict]: + cases = [] + for n, prompt in enumerate(content["positive_prompts"], 1): + cases.append( + { + "case_id": f"{skill}/p{n}", + "skill": skill, + "kind": "positive", + "prompt": prompt, + "expected_skill": skill, + "activation": {"status": "pending", "activated_skills": [], "evidence": ""}, + "rubric": [ + {"rubric_id": f"r{i}", "item": item, "status": "unscored", "evidence": "", "na_reason": ""} + for i, item in enumerate(content["rubric"], 1) + ], + } + ) + for n, entry in enumerate(content["negative_prompts"], 1): + cases.append( + { + "case_id": f"{skill}/n{n}", + "skill": skill, + "kind": "negative", + "prompt": entry["prompt"], + "expected_skill": entry["expected_skill"], + "activation": {"status": "pending", "activated_skills": [], "evidence": ""}, + } + ) + return cases + + +def build_worksheet(scenarios_dir: Path, provenance: dict) -> dict: + files = sorted(scenarios_dir.glob("*.json")) + if not files: + raise WorksheetError(f"no scenario files in {scenarios_dir}") + scenarios, cases = {}, [] + for path in files: + content = json.loads(path.read_text()) + skill = content["skill"] + scenarios[skill] = { + "file": path.name, + "file_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "content_sha256": canonical_sha256(content), + "content": content, + } + cases.extend(case_skeletons(skill, content)) + worksheet = {"worksheet_version": WORKSHEET_VERSION} + worksheet.update({key: provenance.get(key) for key in PROVENANCE}) + worksheet["scenarios"] = scenarios + worksheet["cases"] = cases + return worksheet + + +def git_head() -> str | None: + try: + out = subprocess.run( + ["git", "-C", str(EVALS), "rev-parse", "HEAD"], capture_output=True, text=True, timeout=10, check=False + ) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 and out.stdout.strip() else None + + +def validate_worksheet(ws) -> list[str]: + """Check a worksheet against its own embedded snapshots (never the live scenarios).""" + if not isinstance(ws, dict): + return ["worksheet must be a JSON object"] + errors = [] + if ws.get("worksheet_version") != WORKSHEET_VERSION: + errors.append(f"worksheet_version must be {WORKSHEET_VERSION}") + for key in PROVENANCE: + if blank_text(ws.get(key)): + errors.append(f"missing provenance: {key} (fill it in before reporting)") + date = ws.get("trial_date") + if isinstance(date, str) and date.strip(): + try: + datetime.date.fromisoformat(date) + except ValueError: + errors.append(f"trial_date must be YYYY-MM-DD, got {date!r}") + expected = {} + scenarios = ws.get("scenarios") + if not isinstance(scenarios, dict) or not scenarios: + errors.append("scenarios snapshot is missing or empty") + scenarios = {} + for skill, snap in sorted(scenarios.items()): + content = snap.get("content") if isinstance(snap, dict) else None + try: + skeletons = case_skeletons(skill, content) + except (KeyError, TypeError, AttributeError): + errors.append(f"scenario snapshot for {skill!r} is malformed") + continue + if canonical_sha256(content) != snap.get("content_sha256"): + errors.append(f"scenario snapshot for {skill!r} does not match its recorded content_sha256") + for skeleton in skeletons: + expected[skeleton["case_id"]] = skeleton + cases = ws.get("cases") + if not isinstance(cases, list): + return errors + ["cases must be a list"] + seen = set() + for index, case in enumerate(cases): + if not isinstance(case, dict): + errors.append(f"cases[{index}] must be an object") + continue + cid = case.get("case_id") + if cid in seen: + errors.append(f"duplicate record for case {cid!r}") + continue + seen.add(cid) + if cid not in expected: + errors.append(f"unknown case id {cid!r} (not in the scenario snapshot)") + continue + errors.extend(case_errors(case, expected[cid])) + for cid in sorted(set(expected) - seen): + errors.append(f"case {cid!r} is missing from the worksheet") + return errors + + +def case_errors(case: dict, skeleton: dict) -> list[str]: + cid = skeleton["case_id"] + errors = [] + for field in ("skill", "kind", "prompt", "expected_skill"): + if case.get(field) != skeleton[field]: + errors.append(f"{cid}: {field} {case.get(field)!r} does not match the scenario snapshot {skeleton[field]!r}") + act = case.get("activation") + if not isinstance(act, dict): + errors.append(f"{cid}: activation record is missing") + else: + status = act.get("status") + if status not in ACTIVATION_STATUSES: + errors.append(f"{cid}: invalid activation status {status!r} (use one of {', '.join(ACTIVATION_STATUSES)})") + skills = act.get("activated_skills") + if not isinstance(skills, list) or any(blank_text(s) for s in skills): + errors.append(f"{cid}: activated_skills must be a list of skill names") + elif status != "observed" and skills: + errors.append(f"{cid}: activated_skills must be empty unless activation status is 'observed'") + if status == "observed" and blank_text(act.get("evidence")): + errors.append(f"{cid}: observed activation needs evidence") + if skeleton["kind"] == "negative": + if case.get("rubric"): + errors.append(f"{cid}: negative cases carry no rubric records") + return errors + records = case.get("rubric") + if not isinstance(records, list): + return errors + [f"{cid}: rubric must be a list"] + want = {r["rubric_id"]: r for r in skeleton["rubric"]} + seen = set() + for record in records: + rid = record.get("rubric_id") if isinstance(record, dict) else None + if rid in seen: + errors.append(f"{cid}: duplicate rubric record {rid!r}") + continue + seen.add(rid) + if rid not in want: + errors.append(f"{cid}: unknown rubric id {rid!r}") + continue + errors.extend(rubric_errors(cid, record, want[rid])) + for rid in sorted(set(want) - seen): + errors.append(f"{cid}: rubric record {rid!r} is missing") + return errors + + +def rubric_errors(cid: str, record: dict, skeleton: dict) -> list[str]: + rid = skeleton["rubric_id"] + errors = [] + if record.get("item") != skeleton["item"]: + errors.append(f"{cid}/{rid}: item text does not match the scenario snapshot") + status = record.get("status") + if status not in RUBRIC_STATUSES: + errors.append(f"{cid}/{rid}: invalid status {status!r} (use one of {', '.join(RUBRIC_STATUSES)})") + elif status in ("pass", "fail") and blank_text(record.get("evidence")): + errors.append(f"{cid}/{rid}: {status} needs evidence") + elif status == "not-applicable" and blank_text(record.get("na_reason")): + errors.append(f"{cid}/{rid}: not-applicable needs na_reason") + return errors + + +def activation_outcome(case: dict) -> str: + act = case["activation"] + if act["status"] == "pending": + return "unrecorded" + if act["status"] == "unobserved": + return "unobserved" + activated = set(act["activated_skills"]) + if case["kind"] == "positive": + return "pass" if case["skill"] in activated else "fail" + ok = case["expected_skill"] in activated and case["skill"] not in activated + return "pass" if ok else "fail" + + +def procedure_outcome(case: dict) -> tuple[str, dict]: + counts = {status: 0 for status in RUBRIC_STATUSES} + for record in case["rubric"]: + counts[record["status"]] += 1 + if counts["fail"]: + outcome = "fail" + elif counts["unscored"]: + outcome = "incomplete" + elif counts["pass"] == 0: + outcome = UNDEFINED + else: + outcome = "pass" + return outcome, counts + + +def rate(numerator: int, denominator: int): + return UNDEFINED if denominator == 0 else numerator / denominator + + +def summarize_worksheet(ws: dict) -> dict: + cases_out = [] + activation = {"pass": 0, "fail": 0, "unobserved": 0, "unrecorded": 0} + by_kind = {k: {"pass": 0, "fail": 0, "unobserved": 0, "unrecorded": 0} for k in ("positive", "negative")} + proc_cases = {"pass": 0, "fail": 0, "incomplete": 0, UNDEFINED: 0} + items = {status: 0 for status in RUBRIC_STATUSES} + for case in ws["cases"]: + entry = { + "case_id": case["case_id"], + "kind": case["kind"], + "activation": activation_outcome(case), + "activated_skills": list(case["activation"]["activated_skills"]), + } + activation[entry["activation"]] += 1 + by_kind[case["kind"]][entry["activation"]] += 1 + if case["kind"] == "positive": + outcome, counts = procedure_outcome(case) + entry["procedure"] = outcome + entry["rubric_counts"] = counts + proc_cases[outcome] += 1 + for status, n in counts.items(): + items[status] += n + cases_out.append(entry) + observed = activation["pass"] + activation["fail"] + scored = items["pass"] + items["fail"] + applicable = scored + items["unscored"] + return { + "trial": {key: ws[key] for key in PROVENANCE}, + "activation": { + **activation, + "total": observed + activation["unobserved"] + activation["unrecorded"], + "observed": observed, + "rate_observed": rate(activation["pass"], observed), + "by_kind": by_kind, + "complete": activation["unobserved"] + activation["unrecorded"] == 0, + }, + "procedure": { + "cases": proc_cases, + "items": { + "pass": items["pass"], + "fail": items["fail"], + "unscored": items["unscored"], + "not_applicable_excluded": items["not-applicable"], + "scored": scored, + "applicable": applicable, + "rate_scored": rate(items["pass"], scored), + "rate_applicable": rate(items["pass"], applicable), + }, + "complete": items["unscored"] == 0, + }, + "cases": cases_out, + "note": "Reviewer-recorded observations; not automatic proof of agent behavior.", + } + + +def fmt_rate(value, num: int, den: int) -> str: + return f"{num}/{den} = undefined" if value == UNDEFINED else f"{num}/{den} = {value:.0%}" + + +def render_text(summary: dict, drift: list[str] | None = None) -> str: + t, a, p = summary["trial"], summary["activation"], summary["procedure"] + lines = [ + f"Trial {t['trial_id']} on {t['trial_date']} (catalog {t['catalog_commit']})", + f"Client {t['client']}, model {t['model']}, reviewer {t['reviewer']}", + summary["note"], + ] + if drift: + lines.append("Scenarios changed since this trial (snapshot still used): " + ", ".join(drift)) + lines += [ + "", + "ACTIVATION (expected skill present; pilot skill absent on negative cases)", + f" passed {fmt_rate(a['rate_observed'], a['pass'], a['observed'])} observed cases", + f" failed {a['fail']}, unobserved {a['unobserved']}, unrecorded {a['unrecorded']} " + f"(excluded from the denominator, never counted as pass)", + ] + for kind, c in a["by_kind"].items(): + lines.append(f" {kind}: pass {c['pass']}, fail {c['fail']}, unobserved {c['unobserved']}, unrecorded {c['unrecorded']}") + lines.append(f" status: {'complete' if a['complete'] else 'INCOMPLETE'}") + i = p["items"] + lines += [ + "", + "PROCEDURE (rubric items on positive cases)", + f" scored items passed {fmt_rate(i['rate_scored'], i['pass'], i['scored'])}", + f" passed of all applicable {fmt_rate(i['rate_applicable'], i['pass'], i['applicable'])} " + f"(unscored counted as not passed)", + f" failed {i['fail']}, unscored {i['unscored']}, not-applicable {i['not_applicable_excluded']} (excluded)", + " cases: " + ", ".join(f"{k} {v}" for k, v in p["cases"].items()), + f" status: {'complete' if p['complete'] else 'INCOMPLETE'}", + "", + "CASES", + ] + for c in summary["cases"]: + extra = f", procedure {c['procedure']}" if "procedure" in c else "" + seen = ",".join(c["activated_skills"]) or "-" + lines.append(f" {c['case_id']}: activation {c['activation']} (activated: {seen}){extra}") + return "\n".join(lines) + "\n" + + +def load_worksheet(path: Path) -> dict: + try: + ws = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise WorksheetError(f"cannot read worksheet {path}: {exc}") from exc + errors = validate_worksheet(ws) + if errors: + raise WorksheetError("worksheet is invalid:\n" + "\n".join(f" - {e}" for e in errors)) + return ws + + +def drifted_skills(ws: dict, scenarios_dir: Path) -> list[str]: + changed = [] + for skill, snap in sorted(ws["scenarios"].items()): + path = scenarios_dir / snap["file"] + current = hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + if current != snap["file_sha256"]: + changed.append(skill) + return changed + + +def write_new_worksheet(destination: Path, worksheet: dict) -> None: + destination = destination.resolve() + if destination.parent == EVALS: + raise WorksheetError("destination must be outside evals/ so it is not picked up as a scenario file") + if not destination.parent.is_dir(): + raise WorksheetError(f"destination directory does not exist: {destination.parent}") + try: + with open(destination, "x", encoding="utf-8") as handle: + handle.write(json.dumps(worksheet, indent=2, ensure_ascii=False) + "\n") + except FileExistsError: + raise WorksheetError(f"refusing to overwrite existing file: {destination}") from None + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + init = sub.add_parser("init", help="write an unscored worksheet to a new file") + init.add_argument("destination", type=Path) + for key in PROVENANCE: + init.add_argument("--" + key.replace("_", "-"), dest=key) + report = sub.add_parser("report", help="validate a completed worksheet and report") + report.add_argument("worksheet", type=Path) + report.add_argument("--format", choices=("text", "json"), default="text") + report.add_argument("--check-current", action="store_true", help="note scenarios changed since the trial") + args = parser.parse_args(argv) + try: + if args.command == "init": + provenance = {key: getattr(args, key) for key in PROVENANCE} + provenance["trial_date"] = provenance["trial_date"] or datetime.date.today().isoformat() + provenance["catalog_commit"] = provenance["catalog_commit"] or git_head() + write_new_worksheet(args.destination, build_worksheet(EVALS, provenance)) + print(f"Wrote unscored worksheet: {args.destination}") + missing = [k for k in PROVENANCE if blank_text(provenance[k])] + if missing: + print("Fill in before reporting: " + ", ".join(missing)) + return 0 + ws = load_worksheet(args.worksheet) + summary = summarize_worksheet(ws) + drift = drifted_skills(ws, EVALS) if args.check_current else None + if args.format == "json": + if drift is not None: + summary["scenarios_changed_since_trial"] = drift + print(json.dumps(summary, indent=2)) + else: + print(render_text(summary, drift), end="") + return 0 + except WorksheetError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_eval_trials.py b/tests/test_eval_trials.py new file mode 100644 index 0000000..f2d302d --- /dev/null +++ b/tests/test_eval_trials.py @@ -0,0 +1,228 @@ +import contextlib +import copy +import importlib.util +import io +import json +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "evals" / "trial.py" +spec = importlib.util.spec_from_file_location("eval_trial", SCRIPT) +TRIAL = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = TRIAL +spec.loader.exec_module(TRIAL) + +PROVENANCE = { + "trial_id": "trial-a", + "trial_date": "2026-01-02", + "catalog_commit": "abc1234", + "client": "example-client 1.0", + "model": "example-model 1", + "reviewer": "reviewer-1", +} +SETUP = "setting-up-python-libraries" +SECURITY = "auditing-python-security" + + +def case(ws, case_id): + return next(c for c in ws["cases"] if c["case_id"] == case_id) + + +def observe(ws, case_id, skills, evidence="transcript line"): + case(ws, case_id)["activation"] = {"status": "observed", "activated_skills": skills, "evidence": evidence} + + +def rubric(ws, case_id, **statuses): + for rid, status in statuses.items(): + record = next(r for r in case(ws, case_id)["rubric"] if r["rubric_id"] == rid) + record["status"] = status + if status in ("pass", "fail"): + record["evidence"] = "file listing" + if status == "not-applicable": + record["na_reason"] = "prompt only asks for a scaffold" + + +def blank_worksheet(scenarios_dir=TRIAL.EVALS): + return TRIAL.build_worksheet(scenarios_dir, PROVENANCE) + + +def mixed_worksheet(): + ws = blank_worksheet() + observe(ws, f"{SETUP}/p1", [SETUP]) + rubric(ws, f"{SETUP}/p1", r1="pass", r2="pass", r3="pass", r4="pass", r5="pass", r6="not-applicable") + observe(ws, f"{SETUP}/p2", [SETUP, "testing-python-libraries"]) # extra skill is preserved + rubric(ws, f"{SETUP}/p2", r1="pass", r2="fail") + case(ws, f"{SETUP}/p3")["activation"] = {"status": "unobserved", "activated_skills": [], "evidence": "no log"} + observe(ws, f"{SETUP}/n1", ["testing-python-libraries"]) + observe(ws, f"{SETUP}/n2", [SETUP]) # pilot wrongly activated + observe(ws, f"{SECURITY}/p1", [SECURITY]) # all rubric items unscored + observe(ws, f"{SECURITY}/p2", ["keeping-git-repos-clean"]) + rubric(ws, f"{SECURITY}/p2", **{f"r{i}": "not-applicable" for i in range(1, 6)}) + observe(ws, f"{SECURITY}/n1", ["rendering-untrusted-content", SECURITY]) # expected skill and pilot both + observe(ws, f"{SECURITY}/n2", ["keeping-git-repos-clean"]) + case(ws, f"{SECURITY}/n3")["activation"] = {"status": "unobserved", "activated_skills": [], "evidence": ""} + return ws + + +class SummaryTests(unittest.TestCase): + def setUp(self): + self.ws = mixed_worksheet() + self.assertEqual(TRIAL.validate_worksheet(self.ws), []) + self.summary = TRIAL.summarize_worksheet(self.ws) + + def test_exact_per_case_outcomes(self): + got = {c["case_id"]: (c["activation"], c.get("procedure")) for c in self.summary["cases"]} + self.assertEqual(got[f"{SETUP}/p1"], ("pass", "pass")) + self.assertEqual(got[f"{SETUP}/p2"], ("pass", "fail")) + self.assertEqual(got[f"{SETUP}/p3"], ("unobserved", "incomplete")) + self.assertEqual(got[f"{SETUP}/n1"], ("pass", None)) + self.assertEqual(got[f"{SETUP}/n2"], ("fail", None)) + self.assertEqual(got[f"{SETUP}/n3"], ("unrecorded", None)) + self.assertEqual(got[f"{SECURITY}/p1"], ("pass", "incomplete")) + self.assertEqual(got[f"{SECURITY}/p2"], ("fail", "undefined")) + self.assertEqual(got[f"{SECURITY}/n1"], ("fail", None)) + self.assertEqual(got[f"{SECURITY}/n2"], ("pass", None)) + self.assertEqual(got[f"{SECURITY}/n3"], ("unobserved", None)) + extra = next(c for c in self.summary["cases"] if c["case_id"] == f"{SETUP}/p2") + self.assertEqual(extra["activated_skills"], [SETUP, "testing-python-libraries"]) + + def test_activation_summary_is_separate_with_exact_counts(self): + a = self.summary["activation"] + self.assertEqual((a["pass"], a["fail"], a["unobserved"], a["unrecorded"]), (5, 3, 2, 8)) + self.assertEqual((a["observed"], a["total"]), (8, 18)) + self.assertEqual(a["rate_observed"], 5 / 8) + self.assertFalse(a["complete"]) + self.assertEqual(a["by_kind"]["positive"], {"pass": 3, "fail": 1, "unobserved": 1, "unrecorded": 4}) + self.assertEqual(a["by_kind"]["negative"], {"pass": 2, "fail": 2, "unobserved": 1, "unrecorded": 4}) + + def test_procedure_summary_is_separate_with_exact_counts(self): + p = self.summary["procedure"] + self.assertEqual(p["cases"], {"pass": 1, "fail": 1, "incomplete": 6, "undefined": 1}) + items = p["items"] + self.assertEqual((items["pass"], items["fail"], items["unscored"]), (6, 1, 35)) + self.assertEqual(items["not_applicable_excluded"], 6) + self.assertEqual((items["scored"], items["applicable"]), (7, 42)) + self.assertEqual(items["rate_scored"], 6 / 7) + self.assertEqual(items["rate_applicable"], 6 / 42) + self.assertFalse(p["complete"]) + + def test_zero_denominators_are_undefined_never_full(self): + summary = TRIAL.summarize_worksheet(blank_worksheet()) + self.assertEqual(summary["activation"]["rate_observed"], "undefined") + self.assertEqual(summary["procedure"]["items"]["rate_scored"], "undefined") + text = TRIAL.render_text(summary) + self.assertIn("0/0 = undefined", text) + self.assertNotIn("100%", text) + + def test_text_report_shows_exclusions_and_disclaimer(self): + text = TRIAL.render_text(self.summary) + self.assertIn("5/8 = 62%", text) + self.assertIn("6/7 = 86%", text) + self.assertIn("not-applicable 6 (excluded)", text) + self.assertIn("INCOMPLETE", text) + self.assertIn("not automatic proof", text) + + +class ValidationTests(unittest.TestCase): + def assert_error(self, mutate, fragment): + ws = mixed_worksheet() + mutate(ws) + errors = TRIAL.validate_worksheet(ws) + self.assertTrue(any(fragment in e for e in errors), errors) + + def test_unknown_case_id(self): + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1").update(case_id="nope/p9"), "unknown case id 'nope/p9'") + + def test_duplicate_record(self): + self.assert_error(lambda ws: ws["cases"].append(copy.deepcopy(ws["cases"][0])), "duplicate record") + + def test_missing_case(self): + self.assert_error(lambda ws: ws["cases"].pop(), "is missing from the worksheet") + + def test_invalid_statuses(self): + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1")["activation"].update(status="maybe"), "invalid activation status") + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1")["rubric"][0].update(status="passed"), "invalid status 'passed'") + + def test_missing_provenance(self): + for key in TRIAL.PROVENANCE: + with self.subTest(key=key): + self.assert_error(lambda ws: ws.update({key: None}), f"missing provenance: {key}") + + def test_mismatched_snapshot(self): + self.assert_error(lambda ws: ws["scenarios"][SETUP]["content"]["rubric"].append("extra"), "content_sha256") + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1").update(prompt="edited"), "prompt") + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1")["rubric"][0].update(item="edited"), "item text") + + def test_evidence_and_reason_requirements(self): + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1")["rubric"][5].update(na_reason=""), "needs na_reason") + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1")["rubric"][0].update(evidence=""), "pass needs evidence") + self.assert_error(lambda ws: case(ws, f"{SETUP}/p1")["activation"].update(evidence=""), "needs evidence") + + def test_unscored_is_valid_but_incomplete(self): + self.assertEqual(TRIAL.validate_worksheet(blank_worksheet()), []) + + +class HistoricalAndCliTests(unittest.TestCase): + def run_main(self, *argv): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = TRIAL.main(list(argv)) + return code, out.getvalue(), err.getvalue() + + def test_report_reads_from_snapshot_after_scenarios_change(self): + with tempfile.TemporaryDirectory() as tmp: + scenarios = Path(tmp) + for path in (ROOT / "evals").glob("*.json"): + shutil.copy(path, scenarios) + ws = TRIAL.build_worksheet(scenarios, PROVENANCE) + target = scenarios / f"{SETUP}.json" + data = json.loads(target.read_text()) + data["rubric"] = ["a different rubric"] + target.write_text(json.dumps(data)) + self.assertEqual(TRIAL.validate_worksheet(ws), []) + self.assertEqual(len(case(ws, f"{SETUP}/p1")["rubric"]), 6) + self.assertEqual(TRIAL.drifted_skills(ws, scenarios), [SETUP]) + + def test_init_then_report_round_trip(self): + with tempfile.TemporaryDirectory() as tmp: + dest = Path(tmp) / "ws.json" + args = ["init", str(dest)] + for key, value in PROVENANCE.items(): + args += ["--" + key.replace("_", "-"), value] + self.assertEqual(self.run_main(*args)[0], 0) + ws = json.loads(dest.read_text()) + self.assertEqual(len(ws["cases"]), 18) + code, _, err = self.run_main("init", str(dest)) + self.assertEqual(code, 1) + self.assertIn("refusing to overwrite", err) + case(ws, f"{SETUP}/p1")["activation"]["status"] = "bogus" + dest.write_text(json.dumps(ws)) + code, _, err = self.run_main("report", str(dest)) + self.assertEqual(code, 1) + self.assertIn("invalid activation status", err) + observe(ws, f"{SETUP}/p1", [SETUP]) + dest.write_text(json.dumps(ws)) + code, out, _ = self.run_main("report", str(dest), "--format", "json") + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)["activation"]["pass"], 1) + + def test_init_refuses_destination_inside_evals(self): + code, _, err = self.run_main("init", str(ROOT / "evals" / "worksheet.json")) + self.assertEqual(code, 1) + self.assertIn("outside evals/", err) + self.assertFalse((ROOT / "evals" / "worksheet.json").exists()) + + def test_missing_provenance_is_reported_at_init(self): + with tempfile.TemporaryDirectory() as tmp: + code, out, _ = self.run_main("init", str(Path(tmp) / "w.json"), "--catalog-commit", "abc") + self.assertEqual(code, 0) + self.assertIn("Fill in before reporting", out) + self.assertIn("client", out) + + +if __name__ == "__main__": + unittest.main()