diff --git a/evals/README.md b/evals/README.md index 3d536f9..9ed2720 100644 --- a/evals/README.md +++ b/evals/README.md @@ -34,3 +34,32 @@ Bump `schema_version` for any change that alters how a file must be read. 2. For positive prompts, run the agent in a scratch project and check the `rubric` items against the resulting files and command transcript. This scores the procedure. The two scores should be reported separately and versioned with the model and catalog commit. Live-model runs are opt-in and must stay out of default CI. + +## Scratch fixtures for procedure trials + +`evals/fixtures//` holds a small, versioned set of starting projects so reviewers run a procedure trial on the same inputs. Each has `fixture.json` (metadata) and `files/` (the starting tree). Scenario files above are unchanged (schema version 1); a fixture refers to its scenario by file name, the exact prompt text and the exact rubric item text. + +| Fixture | Skill | Prompt (exact) | +|---|---|---| +| `setup-clean-parent` | `setting-up-python-libraries` | Create a new Python library called acme-utils with pytest, ruff and CI ready to go. | +| `security-inert-examples` | `auditing-python-security` | Audit my Python library for security vulnerabilities before we release. | +| `ci-gate-mismatch` | `reproducing-ci-locally` | The lint job fails in CI but passes on my machine; help me reproduce it locally. | + +All other prompts remain activation-only; they have no procedure fixture yet. The insecure code in `security-inert-examples` is inert: never called, no payloads, no credentials. Do not import or run it. + +### Manual trial workflow + +1. Materialize into a NEW directory outside this checkout (existing destinations and unknown or unsafe fixture ids are refused before anything is written; nothing is executed or installed): + + ```bash + uv run --no-project --python 3.12 python evals/fixtures/materialize.py ci-gate-mismatch /tmp/trial-ci-1 + ``` + + It prints the fixture id and `tree_sha256`. +2. Follow the fixture's `setup` list, give the agent the `prompt` verbatim with the skill installed, and let it work in that directory. +3. Compare the result with the fixture's `expected_observations` and score the listed `rubric_items` from the resulting files and transcript. For `setup-clean-parent`, assess the generated child directory against those scaffold rubric items. +4. Record with the trial: fixture id, `tree_sha256`, the catalog commit, the agent client and model version, and which rubric items were observed, not observed, or not assessable. Worksheets and reports (proposed separately) can adopt these fields unchanged. + +### Fixture validation is not agent success + +`tests/test_eval_fixtures.py` checks deterministic facts only: each prompt and rubric text still exists in its scenario, starting-file hashes are unchanged, the materializer is repeatable and refuses unsafe requests, and the `ci-gate-mismatch` reproduction behaves as documented (local command exits 0, the workflow command exits 1, aligned configuration plus the code fix exit 0, using pinned `ruff==0.9.10` through `uvx`; skipped when `uv` is absent). It never runs an agent. A passing suite says the inputs are intact, not that any skill works; no success rate is measured or claimed here. Changing a fixture file means regenerating its hashes in `fixture.json` and re-reading the affected trials. diff --git a/evals/fixtures/ci-gate-mismatch/files/.github/workflows/ci.yml b/evals/fixtures/ci-gate-mismatch/files/.github/workflows/ci.yml new file mode 100644 index 0000000..a19e6fe --- /dev/null +++ b/evals/fixtures/ci-gate-mismatch/files/.github/workflows/ci.yml @@ -0,0 +1,9 @@ +name: ci +on: [pull_request] +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uvx --from ruff==0.9.10 ruff check --isolated --select E4,E7,E9,F . diff --git a/evals/fixtures/ci-gate-mismatch/files/Makefile b/evals/fixtures/ci-gate-mismatch/files/Makefile new file mode 100644 index 0000000..36799d5 --- /dev/null +++ b/evals/fixtures/ci-gate-mismatch/files/Makefile @@ -0,0 +1,3 @@ +.PHONY: lint +lint: + uvx --from ruff==0.9.10 ruff check . diff --git a/evals/fixtures/ci-gate-mismatch/files/pyproject.toml b/evals/fixtures/ci-gate-mismatch/files/pyproject.toml new file mode 100644 index 0000000..327a099 --- /dev/null +++ b/evals/fixtures/ci-gate-mismatch/files/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "sample-pkg" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" diff --git a/evals/fixtures/ci-gate-mismatch/files/ruff.toml b/evals/fixtures/ci-gate-mismatch/files/ruff.toml new file mode 100644 index 0000000..88f4b0b --- /dev/null +++ b/evals/fixtures/ci-gate-mismatch/files/ruff.toml @@ -0,0 +1,4 @@ +extend-exclude = ["tests"] + +[lint] +select = ["E4", "E7", "E9", "F"] diff --git a/evals/fixtures/ci-gate-mismatch/files/src/sample_pkg/__init__.py b/evals/fixtures/ci-gate-mismatch/files/src/sample_pkg/__init__.py new file mode 100644 index 0000000..e1829c3 --- /dev/null +++ b/evals/fixtures/ci-gate-mismatch/files/src/sample_pkg/__init__.py @@ -0,0 +1,2 @@ +def add(a: int, b: int) -> int: + return a + b diff --git a/evals/fixtures/ci-gate-mismatch/files/tests/test_add.py b/evals/fixtures/ci-gate-mismatch/files/tests/test_add.py new file mode 100644 index 0000000..fcdea5a --- /dev/null +++ b/evals/fixtures/ci-gate-mismatch/files/tests/test_add.py @@ -0,0 +1,7 @@ +import os + +from sample_pkg import add + + +def test_add(): + assert add(1, 2) == 3 diff --git a/evals/fixtures/ci-gate-mismatch/fixture.json b/evals/fixtures/ci-gate-mismatch/fixture.json new file mode 100644 index 0000000..5982676 --- /dev/null +++ b/evals/fixtures/ci-gate-mismatch/fixture.json @@ -0,0 +1,80 @@ +{ + "fixture_version": 1, + "id": "ci-gate-mismatch", + "skill": "reproducing-ci-locally", + "scenario": "reproducing-ci-locally.json", + "scenario_schema_version": 1, + "prompt": "The lint job fails in CI but passes on my machine; help me reproduce it locally.", + "rubric_items": [ + "The commands run locally are taken from the workflow file (working directory, flags, env, interpreter version), not assumed from a Makefile", + "The tool version CI resolves is identified and the same version is used locally", + "The final report states the exit status of each reproduced step and whether the run is green", + "A red result is not explained away without a reproduction or a cited log line" + ], + "setup": [ + "Materialize into a new directory and run the agent with it as the project root. Network access to fetch ruff 0.9.10 through uvx is needed.", + "Local command (what the Makefile runs): uvx --from ruff==0.9.10 ruff check . -> exit 0.", + "Workflow command (the run: line in .github/workflows/ci.yml): uvx --from ruff==0.9.10 ruff check --isolated --select E4,E7,E9,F . -> exit 1 (F401 in tests/test_add.py).", + "Cause: ruff.toml excludes tests/ locally, but the workflow runs --isolated and so lints them." + ], + "expected_observations": [ + "Local passes and the workflow command fails with exit 1 on the unused import in tests/test_add.py.", + "Config alignment (remove the extend-exclude line from ruff.toml) makes the local command fail the same way, exit 1.", + "After also removing the unused import, both commands exit 0.", + "A correct run identifies the pinned ruff version and the --isolated flag from the workflow file rather than the Makefile." + ], + "files": { + ".github/workflows/ci.yml": "8a9ceb0228ccd42d8429fed7d5abe4f376bc16b28f1d77ef2af02a88d2c55c51", + "Makefile": "a4a3a8ebfdeae1d8a823bd2a9d5e3e1e6f19e884027b28d6f7fc6aadc1551819", + "pyproject.toml": "2e0dd90ae65d2564442d9da7c21619e9e7b41225205efd4167c83977a77b2d82", + "ruff.toml": "8e95761db53ad1afa902fd7d23c63e0e5204b1a56e0efd3d2c942487eeaa4f91", + "src/sample_pkg/__init__.py": "b414e3e8a1cc84d091471727544c353f60db59fc2bea10ee562ece77410915be", + "tests/test_add.py": "a4d9f900d266670b34d84e1453ecba46d1f29ad16dfc036a8ef6aa675df94e76" + }, + "tree_sha256": "af714c7e6469f90c776d7349fbcf5531e287d2390fd36c6d0ecde5bac8e255fb", + "reproduction": { + "tool": "ruff==0.9.10", + "local": { + "argv": [ + "uvx", + "--from", + "ruff==0.9.10", + "ruff", + "check", + "." + ], + "source": "Makefile", + "expect_exit": 0 + }, + "workflow": { + "argv": [ + "uvx", + "--from", + "ruff==0.9.10", + "ruff", + "check", + "--isolated", + "--select", + "E4,E7,E9,F", + "." + ], + "source": ".github/workflows/ci.yml", + "expect_exit": 1 + }, + "align": [ + { + "file": "ruff.toml", + "old": "extend-exclude = [\"tests\"]\n\n", + "new": "", + "then_local_exit": 1 + }, + { + "file": "tests/test_add.py", + "old": "import os\n\n", + "new": "", + "then_local_exit": 0, + "then_workflow_exit": 0 + } + ] + } +} diff --git a/evals/fixtures/materialize.py b/evals/fixtures/materialize.py new file mode 100644 index 0000000..5d868d1 --- /dev/null +++ b/evals/fixtures/materialize.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Copy a versioned evaluation fixture into a new directory. + +Only copies files; nothing from the fixture is executed or installed. +Usage: materialize.py FIXTURE_ID DESTINATION +""" +import argparse +import hashlib +import json +import re +import shutil +import sys +from pathlib import Path + +FIXTURES = Path(__file__).resolve().parent +EVALS = FIXTURES.parent +ID_PATTERN = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +MANIFEST_VERSION = 1 + + +class FixtureError(Exception): + pass + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def hash_tree(files_dir: Path) -> dict[str, str]: + """Map relative POSIX path -> sha256 for every file; reject symlinks.""" + hashes = {} + for path in sorted(files_dir.rglob("*")): + if path.is_symlink(): + raise FixtureError(f"symlink in fixture: {path.relative_to(files_dir)}") + if path.is_file(): + hashes[path.relative_to(files_dir).as_posix()] = sha256_bytes(path.read_bytes()) + return hashes + + +def tree_digest(hashes: dict[str, str]) -> str: + text = "".join(f"{name}\0{digest}\n" for name, digest in sorted(hashes.items())) + return sha256_bytes(text.encode()) + + +def load_manifest(fixture_id: str, root: Path = FIXTURES) -> dict: + if not isinstance(fixture_id, str) or not ID_PATTERN.fullmatch(fixture_id): + raise FixtureError(f"unsafe fixture id: {fixture_id!r}") + manifest_path = root / fixture_id / "fixture.json" + if not manifest_path.is_file(): + raise FixtureError(f"unknown fixture: {fixture_id}") + manifest = json.loads(manifest_path.read_text()) + if manifest.get("fixture_version") != MANIFEST_VERSION or manifest.get("id") != fixture_id: + raise FixtureError(f"bad manifest header for {fixture_id}") + return manifest + + +def input_errors(manifest: dict, root: Path = FIXTURES) -> list[str]: + """Compare the starting files on disk with the recorded hashes.""" + files_dir = root / manifest["id"] / "files" + try: + actual = hash_tree(files_dir) + except FixtureError as exc: + return [str(exc)] + recorded = manifest["files"] + errors = [f"unexpected file: {n}" for n in sorted(set(actual) - set(recorded))] + errors += [f"missing file: {n}" for n in sorted(set(recorded) - set(actual))] + errors += [f"hash changed: {n}" for n in sorted(set(actual) & set(recorded)) if actual[n] != recorded[n]] + if not errors and tree_digest(actual) != manifest["tree_sha256"]: + errors.append("tree_sha256 does not match files") + return errors + + +def prompt_errors(manifest: dict, evals_dir: Path = EVALS) -> list[str]: + """Check the exact prompt and rubric text still exist in the scenario file.""" + scenario_path = evals_dir / manifest["scenario"] + if not scenario_path.is_file(): + return [f"scenario file missing: {manifest['scenario']}"] + scenario = json.loads(scenario_path.read_text()) + errors = [] + if scenario.get("schema_version") != manifest["scenario_schema_version"]: + errors.append("scenario schema_version differs from the fixture's") + if scenario.get("skill") != manifest["skill"]: + errors.append("scenario skill differs from the fixture's") + if manifest["prompt"] not in scenario.get("positive_prompts", []): + errors.append(f"prompt not found in positive_prompts: {manifest['prompt']!r}") + rubric = scenario.get("rubric", []) + errors += [f"rubric item not found: {item!r}" for item in manifest["rubric_items"] if item not in rubric] + return errors + + +def materialize(fixture_id: str, destination: Path, root: Path = FIXTURES) -> dict: + """Validate everything first, then create destination and copy files.""" + manifest = load_manifest(fixture_id, root) + errors = input_errors(manifest, root) + if errors: + raise FixtureError("fixture inputs invalid: " + "; ".join(errors)) + destination = Path(destination) + if destination.exists() or destination.is_symlink(): + raise FixtureError(f"destination already exists: {destination}") + if not destination.parent.is_dir(): + raise FixtureError(f"destination parent is not a directory: {destination.parent}") + source = root / fixture_id / "files" + destination.mkdir() + for name in manifest["files"]: + target = destination / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source / name, target) + return manifest + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description="Copy an evaluation fixture into a new directory.") + parser.add_argument("fixture_id") + parser.add_argument("destination", type=Path) + args = parser.parse_args(argv) + try: + manifest = materialize(args.fixture_id, args.destination) + except FixtureError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + print(f"fixture: {manifest['id']}") + print(f"tree_sha256: {manifest['tree_sha256']}") + print(f"destination: {args.destination}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/fixtures/security-inert-examples/files/README.md b/evals/fixtures/security-inert-examples/files/README.md new file mode 100644 index 0000000..3b38506 --- /dev/null +++ b/evals/fixtures/security-inert-examples/files/README.md @@ -0,0 +1,5 @@ +# sample-service (evaluation fixture) + +Anonymous project for auditing practice. `src/sample_service/` contains +deliberately unsafe patterns that are INERT: nothing here is called, no payload +is included, and there are no credentials. Do not import or execute it. diff --git a/evals/fixtures/security-inert-examples/files/pyproject.toml b/evals/fixtures/security-inert-examples/files/pyproject.toml new file mode 100644 index 0000000..5e2f491 --- /dev/null +++ b/evals/fixtures/security-inert-examples/files/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "sample-service" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" diff --git a/evals/fixtures/security-inert-examples/files/src/sample_service/__init__.py b/evals/fixtures/security-inert-examples/files/src/sample_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evals/fixtures/security-inert-examples/files/src/sample_service/commands.py b/evals/fixtures/security-inert-examples/files/src/sample_service/commands.py new file mode 100644 index 0000000..56bf4ee --- /dev/null +++ b/evals/fixtures/security-inert-examples/files/src/sample_service/commands.py @@ -0,0 +1,7 @@ +"""INERT EVALUATION EXAMPLE: shell command built from a parameter.""" +import subprocess + + +def archive(directory: str): + # Unsafe on purpose: shell=True with an interpolated argument. + return subprocess.run(f"tar -czf backup.tgz {directory}", shell=True) diff --git a/evals/fixtures/security-inert-examples/files/src/sample_service/loaders.py b/evals/fixtures/security-inert-examples/files/src/sample_service/loaders.py new file mode 100644 index 0000000..df4bd8d --- /dev/null +++ b/evals/fixtures/security-inert-examples/files/src/sample_service/loaders.py @@ -0,0 +1,7 @@ +"""INERT EVALUATION EXAMPLE: deserializing bytes from an unspecified source.""" +import pickle + + +def load_state(blob: bytes): + # Unsafe on purpose: pickle can run code when loading untrusted bytes. + return pickle.loads(blob) diff --git a/evals/fixtures/security-inert-examples/files/src/sample_service/queries.py b/evals/fixtures/security-inert-examples/files/src/sample_service/queries.py new file mode 100644 index 0000000..b4adde4 --- /dev/null +++ b/evals/fixtures/security-inert-examples/files/src/sample_service/queries.py @@ -0,0 +1,11 @@ +"""INERT EVALUATION EXAMPLE: SQL built by string formatting.""" +import sqlite3 + + +def find_user(conn: sqlite3.Connection, name: str): + # Unsafe on purpose: caller-supplied text is interpolated into SQL. + return conn.execute(f"SELECT * FROM users WHERE name = '{name}'").fetchall() + + +def find_user_safely(conn: sqlite3.Connection, name: str): + return conn.execute("SELECT * FROM users WHERE name = ?", (name,)).fetchall() diff --git a/evals/fixtures/security-inert-examples/fixture.json b/evals/fixtures/security-inert-examples/fixture.json new file mode 100644 index 0000000..081a577 --- /dev/null +++ b/evals/fixtures/security-inert-examples/fixture.json @@ -0,0 +1,35 @@ +{ + "fixture_version": 1, + "id": "security-inert-examples", + "skill": "auditing-python-security", + "scenario": "auditing-python-security.json", + "scenario_schema_version": 1, + "prompt": "Audit my Python library for security vulnerabilities before we release.", + "rubric_items": [ + "Runs skills/python/security-audit/scripts/security_scan.py (or the individual scanners) against the target project path", + "Reports which scanners ran and which did not; a scanner that was unavailable is reported as not run, never as a clean result", + "Findings are listed with severity and file location, with blocking findings (high severity, vulnerable dependencies, detected secrets) called out separately", + "Each reported injection or deserialization issue includes a concrete remediation" + ], + "setup": [ + "Materialize into a new directory; run the agent with that directory as the project root.", + "Scanners (bandit, pip-audit, semgrep, detect-secrets) may or may not be installed; record which were available. Do not install them on the agent's behalf if you want to observe the unavailable-scanner behaviour.", + "The fixture has no dependencies, so pip-audit has nothing to find." + ], + "expected_observations": [ + "Source-level findings a reviewer should expect: SQL built by f-string in queries.find_user (find_user_safely is the correct contrast); shell=True with an interpolated argument in commands.archive; pickle.loads of caller-supplied bytes in loaders.load_state.", + "Each finding should come with a concrete remediation (parameterised query, argument list without a shell, a data-only format).", + "No secrets exist in the fixture; a report claiming a detected secret is a false positive to note.", + "Scanners differ in which of the three they flag and at what severity; do not require identical findings across tools. A scanner that did not run must be reported as not run.", + "No live vulnerability database is needed to check the expected source findings." + ], + "files": { + "README.md": "caaa84058996dd7c9b60706e83ed9c9097ac69fde3a867bff563a27905d639ce", + "pyproject.toml": "8d4c2993bac0c21b665a7ccdd4aadfc5df80195f658eb20d856bc3797009b6e2", + "src/sample_service/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "src/sample_service/commands.py": "9b860ef030e16ffdab6b0defa59431d6a8463ca681903e4dfe5bf3533c27e6c5", + "src/sample_service/loaders.py": "b27f6599d0e82301c5f5818187f9e1839b4f450d31fb15316a1da630dc67fbdf", + "src/sample_service/queries.py": "603255f7a247c7c984e72a68c95649b7f7e957699e47a34306b74be1eb1dbfd4" + }, + "tree_sha256": "ed64dfe7d2c10da2b8b277cf613c4bef5955832cf985509a7ee7307853b2dc3b" +} diff --git a/evals/fixtures/setup-clean-parent/files/PARENT.md b/evals/fixtures/setup-clean-parent/files/PARENT.md new file mode 100644 index 0000000..8dcfa23 --- /dev/null +++ b/evals/fixtures/setup-clean-parent/files/PARENT.md @@ -0,0 +1,5 @@ +# Scratch parent directory + +This directory is intentionally empty of Python projects. Create the new +library as a child directory here; do not run anything inside the skills +repository checkout. diff --git a/evals/fixtures/setup-clean-parent/fixture.json b/evals/fixtures/setup-clean-parent/fixture.json new file mode 100644 index 0000000..22dff66 --- /dev/null +++ b/evals/fixtures/setup-clean-parent/fixture.json @@ -0,0 +1,31 @@ +{ + "fixture_version": 1, + "id": "setup-clean-parent", + "skill": "setting-up-python-libraries", + "scenario": "setting-up-python-libraries.json", + "scenario_schema_version": 1, + "prompt": "Create a new Python library called acme-utils with pytest, ruff and CI ready to go.", + "rubric_items": [ + "Runs skills/python/project-setup/scripts/create_project.py (or equivalent steps) from a scratch parent directory, producing a src-layout package directory", + "pyproject.toml exists with [project] metadata, a build backend, and a src//py.typed marker", + "A tests/ directory exists and the generated test passes when pytest is run", + "Ruff lint and format checks pass on the generated project", + ".github/workflows contains a CI workflow that runs lint and tests", + "The helper is not run inside the skills repository checkout, leaving no stray generated directory" + ], + "setup": [ + "Materialize into a new directory outside the skills repository checkout; that directory is the scratch parent.", + "Start the agent with its working directory at the materialized directory, with the skill installed, and give it the prompt verbatim.", + "Do not pre-install anything. Python 3.12 and uv are assumed available." + ], + "expected_observations": [ + "Before the trial the parent contains only PARENT.md: no pyproject.toml, src/ or acme_utils/ directory.", + "After the trial a new child directory (for example acme-utils/) holds the generated package; PARENT.md is untouched.", + "Assess each listed rubric item against that child directory and the transcript; the generated project's own pytest and ruff commands give the exit statuses.", + "Nothing is generated inside the skills repository checkout." + ], + "files": { + "PARENT.md": "e5cc0a4345ad2f72ff023755638ac74a36479f6cad263b989d4b0918da6cdd0d" + }, + "tree_sha256": "58452dcdf2c4bc178b5695c0a3846993ab3d637ac5078427c9050d093bdc0586" +} diff --git a/tests/test_eval_fixtures.py b/tests/test_eval_fixtures.py new file mode 100644 index 0000000..bee3e65 --- /dev/null +++ b/tests/test_eval_fixtures.py @@ -0,0 +1,155 @@ +import copy +import importlib.util +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "evals" / "fixtures" / "materialize.py" +spec = importlib.util.spec_from_file_location("materialize_fixture", SCRIPT) +MATERIALIZE = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = MATERIALIZE +spec.loader.exec_module(MATERIALIZE) + +FIXTURE_IDS = ["setup-clean-parent", "security-inert-examples", "ci-gate-mismatch"] +UV = shutil.which("uv") + + +def snapshot(directory: Path) -> dict: + return {p.relative_to(directory).as_posix(): p.read_bytes() for p in sorted(directory.rglob("*")) if p.is_file()} + + +class FixtureContractTests(unittest.TestCase): + def test_each_fixture_maps_to_one_existing_prompt_and_unchanged_inputs(self): + prompts = set() + for fid in FIXTURE_IDS: + with self.subTest(fixture=fid): + manifest = MATERIALIZE.load_manifest(fid) + self.assertEqual(MATERIALIZE.prompt_errors(manifest), []) + self.assertEqual(MATERIALIZE.input_errors(manifest), []) + self.assertTrue(manifest["setup"] and manifest["expected_observations"]) + prompts.add((manifest["skill"], manifest["prompt"])) + self.assertEqual(len(prompts), len(FIXTURE_IDS)) + + def test_stale_prompt_mapping_is_detected(self): + manifest = MATERIALIZE.load_manifest("ci-gate-mismatch") + for label, change in { + "prompt reworded": {"prompt": manifest["prompt"] + " Please."}, + "rubric reworded": {"rubric_items": ["no longer in the scenario"]}, + "schema bumped": {"scenario_schema_version": 2}, + "wrong skill": {"skill": "auditing-python-security"}, + "scenario missing": {"scenario": "gone.json"}, + }.items(): + with self.subTest(case=label): + self.assertTrue(MATERIALIZE.prompt_errors(dict(manifest, **change))) + + def test_changed_fixture_input_is_detected_and_refused(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + root = tmp / "fixtures" + shutil.copytree(ROOT / "evals" / "fixtures", root, ignore=shutil.ignore_patterns("__pycache__")) + manifest = MATERIALIZE.load_manifest("security-inert-examples", root) + self.assertEqual(MATERIALIZE.input_errors(manifest, root), []) + target = root / "security-inert-examples" / "files" / "src" / "sample_service" / "loaders.py" + target.write_text(target.read_text() + "\n# edited\n") + self.assertEqual(MATERIALIZE.input_errors(manifest, root), ["hash changed: src/sample_service/loaders.py"]) + (root / "security-inert-examples" / "files" / "extra.txt").write_text("x") + self.assertIn("unexpected file: extra.txt", MATERIALIZE.input_errors(manifest, root)) + dest = tmp / "out" + with self.assertRaises(MATERIALIZE.FixtureError): + MATERIALIZE.materialize("security-inert-examples", dest, root) + self.assertFalse(dest.exists()) + + def test_security_fixture_is_inert_and_documents_findings(self): + manifest = MATERIALIZE.load_manifest("security-inert-examples") + source = {n: (SCRIPT.parent / "security-inert-examples" / "files" / n).read_text() for n in manifest["files"]} + code = "\n".join(t for n, t in source.items() if n.endswith(".py")) + for marker in ("INERT EVALUATION EXAMPLE",): + self.assertIn(marker, code) + for banned in ("password", "secret", "token", "api_key", "AKIA", "BEGIN PRIVATE", "os.system", "__reduce__"): + self.assertNotIn(banned.lower(), code.lower()) + expected = " ".join(manifest["expected_observations"]) + for name in ("queries.find_user", "commands.archive", "loaders.load_state"): + self.assertIn(name, expected) + + def test_setup_fixture_has_no_generated_package(self): + manifest = MATERIALIZE.load_manifest("setup-clean-parent") + self.assertEqual(list(manifest["files"]), ["PARENT.md"]) + + +class MaterializerTests(unittest.TestCase): + def test_two_materializations_are_identical(self): + for fid in FIXTURE_IDS: + with self.subTest(fixture=fid), tempfile.TemporaryDirectory() as tmp: + a, b = Path(tmp) / "a", Path(tmp) / "b" + MATERIALIZE.materialize(fid, a) + MATERIALIZE.materialize(fid, b) + manifest = MATERIALIZE.load_manifest(fid) + self.assertEqual(snapshot(a), snapshot(b)) + self.assertEqual(sorted(snapshot(a)), sorted(manifest["files"])) + self.assertEqual(MATERIALIZE.tree_digest(MATERIALIZE.hash_tree(a)), manifest["tree_sha256"]) + + def test_refusals_happen_before_any_write(self): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + sentinel = tmp / "existing" + sentinel.mkdir() + (sentinel / "keep.txt").write_text("keep") + before = snapshot(tmp) + for fid in ("../ci-gate-mismatch", "CI-gate", "ci/gate", "", "nonexistent", "-x"): + with self.subTest(fixture=fid), self.assertRaises(MATERIALIZE.FixtureError): + MATERIALIZE.materialize(fid, tmp / "new") + with self.assertRaises(MATERIALIZE.FixtureError): + MATERIALIZE.materialize("ci-gate-mismatch", sentinel) + link = tmp / "link" + link.symlink_to(sentinel) + with self.assertRaises(MATERIALIZE.FixtureError): + MATERIALIZE.materialize("ci-gate-mismatch", link) + with self.assertRaises(MATERIALIZE.FixtureError): + MATERIALIZE.materialize("ci-gate-mismatch", tmp / "missing-parent" / "new") + self.assertEqual(snapshot(tmp), before) + self.assertFalse((tmp / "new").exists()) + + def test_cli_reports_identity_and_exit_codes(self): + with tempfile.TemporaryDirectory() as tmp: + dest = Path(tmp) / "d" + run = lambda: subprocess.run( # noqa: E731 + [sys.executable, "-I", str(SCRIPT), "setup-clean-parent", str(dest)], capture_output=True, text=True + ) + first = run() + self.assertEqual(first.returncode, 0, first.stderr) + self.assertIn(MATERIALIZE.load_manifest("setup-clean-parent")["tree_sha256"], first.stdout) + self.assertEqual(run().returncode, 1) + + +@unittest.skipUnless(UV, "uv is required to run the pinned ruff reproduction") +class CiReproductionTests(unittest.TestCase): + def run_cmd(self, spec, cwd): + return subprocess.run(spec["argv"], cwd=cwd, capture_output=True, text=True).returncode + + def test_local_passes_workflow_fails_and_alignment_fixes_it(self): + repro = MATERIALIZE.load_manifest("ci-gate-mismatch")["reproduction"] + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp) / "p" + MATERIALIZE.materialize("ci-gate-mismatch", project) + self.assertIn(" ".join(repro["local"]["argv"][:-1]), (project / "Makefile").read_text()) + workflow = (project / ".github/workflows/ci.yml").read_text() + self.assertIn(" ".join(repro["workflow"]["argv"]), workflow) + self.assertEqual(self.run_cmd(repro["local"], project), repro["local"]["expect_exit"]) + self.assertEqual(self.run_cmd(repro["workflow"], project), repro["workflow"]["expect_exit"]) + for step in repro["align"]: + path = project / step["file"] + text = path.read_text() + self.assertEqual(text.count(step["old"]), 1) + path.write_text(text.replace(step["old"], step["new"])) + self.assertEqual(self.run_cmd(repro["local"], project), step["then_local_exit"]) + if "then_workflow_exit" in step: + self.assertEqual(self.run_cmd(repro["workflow"], project), step["then_workflow_exit"]) + + +if __name__ == "__main__": + unittest.main()