diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 01a4c1db..45c64904 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -96,6 +96,15 @@ jobs: - name: Type check with pyright run: .venv/bin/pyright + # The CE036 contract engine lives under tests/, which [tool.pyright] excludes + # -- and `exclude` beats both a CLI file arg and an `include` entry, so it can + # only be reached through a config of its own, derived from [tool.pyright] so + # the two passes cannot drift. Mirrors `make typecheck`. + - name: Type check the CE036 contract engine + run: | + .venv/bin/python -m tests.lint.pyright_config .pyright-tests.json + .venv/bin/pyright -p .pyright-tests.json + # PHASE 3: Security scanning - name: Security - Dependency vulnerabilities (pip-audit) run: .venv/bin/pip-audit --desc --skip-editable --ignore-vuln CVE-2026-4539 --ignore-vuln CVE-2026-3219 --ignore-vuln PYSEC-2025-183 # pygments 2.19.2 ReDoS + pip 26.0.1 tar/ZIP ambiguity + pyjwt 2.12.1 weak-encryption (disputed by supplier; key length is application-chosen); no fixes available on PyPI yet — revisit quarterly @@ -387,6 +396,11 @@ jobs: - name: Type check with pyright run: .venv/Scripts/pyright + - name: Type check the CE036 contract engine + run: | + .venv/Scripts/python -m tests.lint.pyright_config .pyright-tests.json + .venv/Scripts/pyright -p .pyright-tests.json + - name: Run test suite run: .venv/Scripts/pytest tests/ -v -m "not live and not lint" diff --git a/.gitignore b/.gitignore index 949f035d..ad83136b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,6 @@ refs/ # SkillsBench tasks for testing /resources/ + +# Derived pyright config for the CE036 contract engine (tests/lint/pyright_config.py) +.pyright-tests.json diff --git a/CLAUDE.md b/CLAUDE.md index 14865966..8013648c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -214,7 +214,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033, CE034, CE035 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) +When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) Adding a user-facing field to one of the models CE030 tracks (`TaskDefinition`, `RunLimits`, `Dataset`, `SimulationConfig` — see `tests/lint/doc_schema_parity.py`) means documenting it in its guide (mention the field name as inline code) or adding an `EXEMPT` entry with a reason it is not user-authored. `make lint` fails otherwise. diff --git a/Makefile b/Makefile index 510b5705..65d0a7c1 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,15 @@ plugin-reference: ## Regenerate the plugin's bundled criteria reference from th typecheck: ## Run type checking with pyright uv run pyright + # The CE036 contract engine executes checker code and feeds the early-stop + # design; it is the one tests/ surface worth type-checking. It needs its own + # config: pyproject.toml excludes "tests", and pyright's `exclude` beats BOTH + # an explicitly-passed CLI file arg AND an `include` entry naming the file -- + # either shortcut analyzes ZERO files and exits 0, a gate that checks nothing. + # The config below is DERIVED from [tool.pyright] (same rules, only + # include/exclude swapped), so the two passes cannot drift apart. + uv run python -m tests.lint.pyright_config .pyright-tests.json + uv run pyright -p .pyright-tests.json test: ## Run test suite (excludes live + lint tests; run `make lint` for those) uv run pytest -n auto -m "not live and not lint" tests/ diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 6c681bcd..5d7e0052 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -220,6 +220,21 @@ Notes: LiveSuccessCriterion)` directly, no separate checker-side flag. A lint rule (`tests/test_custom_lint.py::TestCE025LiveVerdictConsistency`) keeps the model subclassing and the checker's `live_verdict` override paired. +- Your `live_verdict` must be **deterministic** (a pure function of the + `turn_records` prefix — no wall-clock, randomness, or hidden instance state) + and **monotonic** (once it returns `"pass"`/`"fail"` for some prefix, every + longer prefix returns that same verdict) — `EarlyStopWatcher`'s verdict + latching and deferred stops silently depend on both. Lint rule CE036 + (`tests/lint/live_verdict_contract.py`) enforces this by replaying each live + criterion against every prefix of recorded trajectories, and **fails until + you add `ContractCase` fixtures** for the new type in the same change, + reaching every polarity its instances claim via + `live_decidable_polarities()`. An out-of-tree plugin criterion is invisible + to CE036's union walk — and the module lives under `tests/`, which is not + shipped in the PyPI wheel — so copy the replay pattern (a `ContractCase`-style + fixture plus the prefix-by-prefix determinism/monotonicity walk) into your + plugin's own test suite, using `tests/lint/live_verdict_contract.py` in this + repo as the reference implementation. > A duplicate `criterion_type` **overwrites** the earlier checker with a warning (not > a hard error, unlike agents) — keep type strings unique. diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 8306c492..2d6de9c0 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -468,6 +468,26 @@ Semantics: before weighting) — only the combination rule (weighted average vs strict AND) changes, which is what makes the `gate_threshold=1.0` default an exact equivalence with the strict `all(...)` rule. +- **Why the bounds stop at the armed subset (design decision).** The + ceiling/floor rule deliberately does **not** extend to unarmed or + non-observable criteria (`llm_judge`, `reference_comparison`, the file + checks). A mid-run score bound for those would require either scoring the + unfinished sandbox (the end-state peeking `live_verdict` forbids by + construction — it reads only `turn_records`) or re-running judges on every + tool call (expensive, and judge scores over a partial trajectory are not + monotonic — exactly the false-stop risk the bound design exists to rule + out). So a non-observable criterion's bound can never tighten past the + vacuous `[0, 1]` — and folding permanently-vacuous bounds into the gate + degenerates to "never stop": an undecided criterion holds the ceiling up + (suppressing every fail-stop) and the floor down (vetoing every pass-stop) + for the whole run. Scoping the gate to the armed subset is therefore not a + simplification but the design: **arming is the author's declaration of + which criteria the smoke verdict is allowed to hinge on**, and the + authoritative full-set score always comes from the kill-switched run. If a + run should end early on overall-score grounds, arm the observable criteria + with appropriate `weight`s and lower `stop_early_gate_threshold` — that is + the weighted-score break, expressed over the subset that can actually + decide mid-run. - **Decision-step timeout.** `stop_early: {decide_within: N}`. If the criterion is still **undecided** after N tool-call steps, the watcher latches an **effective fail** for it and diff --git a/src/coder_eval/criteria/base.py b/src/coder_eval/criteria/base.py index cbe7b9a8..df35053d 100644 --- a/src/coder_eval/criteria/base.py +++ b/src/coder_eval/criteria/base.py @@ -37,8 +37,16 @@ # (early_stop.py::_prev_verdicts) are correct only because both existing # implementations (skill_triggered, command_executed) honor this. A non-monotonic or # non-deterministic override compiles and passes CE025 (which only checks -# LiveSuccessCriterion subclassing / live_verdict pairing, not this) but silently corrupts the stop -# logic — there is currently no automated enforcement beyond this docstring. +# LiveSuccessCriterion subclassing / live_verdict pairing, not this) but silently corrupts +# the stop logic. +# +# ENFORCEMENT: lint rule CE036 (tests/lint/live_verdict_contract.py) replays every live +# criterion against every prefix of recorded trajectories and asserts both properties — +# monotonicity over arbitrary Python is undecidable, so replay is the only sound check. +# Adding a LiveSuccessCriterion REQUIRES adding ContractCase fixtures for it in the same +# change (CE036 fails on a live type with no cases, and on a polarity its instances claim +# decidable but no fixture reaches). Note the limit: CE036 proves the contract on the +# trajectories an author supplied, not in general — honoring it is still on the author. LiveVerdict = Literal["pass", "fail", "undecided"] @@ -438,8 +446,9 @@ def live_verdict( source of truth for "is this criterion type live-observable", checked by ``validate_early_stop`` / ``EarlyStopWatcher`` and enforced by lint rule CE025. An override MUST also satisfy the deterministic + monotonic - contract documented on the ``LiveVerdict`` type above (not enforced by - CE025 or any other automated check). + contract documented on the ``LiveVerdict`` type above, enforced by lint + rule CE036 — which requires this criterion type to supply replay fixtures + (``tests/lint/live_verdict_contract.py::CASES``) in the same change. """ return "undecided" diff --git a/tests/_fixtures/live_criteria.py b/tests/_fixtures/live_criteria.py new file mode 100644 index 00000000..b64de79e --- /dev/null +++ b/tests/_fixtures/live_criteria.py @@ -0,0 +1,47 @@ +"""Shared builders for live-criterion trajectories (early-stop tests + CE036). + +``tests/test_early_stop.py`` (the watcher's behavioral suite) and +``tests/lint/live_verdict_contract.py`` (the CE036 contract-replay fixtures) both +hand-build ``CommandTelemetry``/``TurnRecord`` trajectories for the same two live +checkers. The primitives live here so a telemetry field addition is threaded +through once; the *criterion* builders deliberately stay in each file — they +encode different defaults (armed with ``stop_early`` blocks vs unarmed contract +instances) and sharing them would just move the divergence into keyword soup. + +The timestamp is frozen: CE036's determinism replay requires fixtures that carry +no nondeterminism of their own, and the watcher tests never read wall-clock off +telemetry either. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal + +from coder_eval.models import CommandTelemetry, TurnRecord + + +FROZEN_TS = datetime(2026, 1, 1, 0, 0, 0) + + +def make_command( + tool_name: str, + parameters: dict[str, Any], + *, + tool_id: str | None = None, + sequence_number: int = 0, + result_status: Literal["success", "error", "unknown"] = "success", +) -> CommandTelemetry: + """One recorded tool call. ``tool_id`` defaults to ``tool-``.""" + return CommandTelemetry( + tool_name=tool_name, + tool_id=tool_id if tool_id is not None else f"tool-{sequence_number}", + timestamp=FROZEN_TS, + parameters=parameters, + result_status=result_status, + sequence_number=sequence_number, + ) + + +def make_turn(*commands: CommandTelemetry, iteration: int = 1) -> TurnRecord: + return TurnRecord(iteration=iteration, user_input="", agent_output="", commands=list(commands)) diff --git a/tests/lint/live_verdict_contract.py b/tests/lint/live_verdict_contract.py new file mode 100644 index 00000000..75ab8c5e --- /dev/null +++ b/tests/lint/live_verdict_contract.py @@ -0,0 +1,555 @@ +"""CE036 — every live-observable criterion must honor the ``live_verdict`` contract. + +``EarlyStopWatcher``'s deferred fail-stop, verdict latching, and ``_prev_verdicts`` +flip-attribution (``orchestration/early_stop.py``) are correct ONLY because every +armed criterion's ``live_verdict`` is: + +* **deterministic** — a pure function of the ``turn_records`` prefix handed in, with + no wall-clock, randomness, or hidden instance state; and +* **monotonic** — once it returns ``"pass"``/``"fail"`` for some trajectory prefix it + returns that SAME verdict for every longer prefix. ``"undecided"`` is the only + verdict allowed to change. + +That contract is documented on ``LiveVerdict`` / ``BaseCriterion.live_verdict`` +(``criteria/base.py``) but, until this rule, nothing enforced it: a third criterion +(in-tree or third-party plugin) implementing ``live_verdict`` non-monotonically would +type-check, pass CE025, and silently corrupt the stop logic — latching a verdict the +run then contradicts. See GitHub issue #61 item 2. + +Design choices, each load-bearing: + +* **Replay, not static analysis.** Monotonicity over arbitrary Python is undecidable, + so there is no sound *static* check to write. What IS mechanical is replaying a + criterion against every prefix of a recorded trajectory and asserting the property + directly. That is what ``contract_violations`` does. +* **Seeded permutations widen the walk.** ``permuted_violations`` re-runs the + determinism + monotonicity walk over seeded reorderings of each case's commands — + an order-sensitive bug (verdict read off the *latest* command instead of the + accumulated set) can look perfectly monotone on the one ordering the author wrote + and flip on a reordering. Each shuffle is RENUMBERED (``sequence_number`` reassigned + in the new order) so it stays a trajectory the watcher could actually hand over — it + sorts by that field before calling ``live_verdict`` — which also keeps the layer + effective for a checker that sorts by it too. The terminal-verdict and polarity + checks stay authored-ordering-only, where they are sound. +* **Fixtures are mandatory, and the registry says so.** A property test over random + trajectories would return ``"undecided"`` almost always and pass *vacuously*, + proving nothing. So each live criterion type must supply cases in ``CASES``, and + ``missing_case_types`` — driven by the ``SuccessCriterion`` union, exactly like + CE025 — fails when a newly added ``LiveSuccessCriterion`` has none. Adding a live + criterion now forces the author to demonstrate the contract in the same change. +* **Each case declares what it reaches.** ``ContractCase.reaches`` pins the verdict on + the FULL trajectory, so a fixture that quietly stops exercising its decision path + (a renamed tool, a changed regex) fails loudly instead of degrading into another + vacuous all-``undecided`` replay. +* **Polarity honesty is checked too.** ``live_decidable_polarities`` (on the model) is + documented as a subset of what the checker's ``live_verdict`` can emit for that + instance. A case that terminally decides a polarity the instance does NOT claim is a + real bug — the watcher would treat that trigger as inert while the checker decides + it — so ``contract_violations`` reports it. + +**Honest limits.** (1) This proves the contract holds *on the trajectories the author +supplied*, not in general. A careless implementation with an agreeable fixture still +passes. The rule raises the cost of the bug and puts the contract in front of the next +implementer; it does not close the hole. Nothing short of a proof would. (2) It covers +the in-tree ``SuccessCriterion`` union only — an out-of-tree plugin criterion never +appears in ``live_criterion_types``, and this module lives under ``tests/`` (not shipped +in the wheel), so a plugin shipping a live criterion should copy the replay pattern — +a ``ContractCase``-style fixture plus the prefix walk — into its own test suite, with +this module as the reference implementation (docs/EXTENDING.md says so where plugin +authors will read it). (3) The determinism probe is two +back-to-back calls on identical input: it catches RNG and per-call mutable state, but +two calls microseconds apart will rarely disagree on a *wall-clock* read, so a +slowly-varying ``datetime.now()`` dependency largely escapes it (the monotonicity +replay is the likelier tripwire for one, and only if the fixture happens to straddle +the flip). + +Like CE025/CE030, this is intentionally NOT a ``BaseRule`` registered in +``tests/lint/runner.py`` (that runner is AST-only, one ``.py`` file at a time); it +reasons over the criteria registry and executes checkers, and is wired as +``tests/test_custom_lint.py::TestCE036LiveVerdictContract``. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from typing import TYPE_CHECKING, Annotated, Any, Literal, get_args, get_origin + +from coder_eval.models import ( + CommandExecutedCriterion, + CommandTelemetry, + LiveSuccessCriterion, + SkillTriggeredCriterion, + SuccessCriterion, +) +from tests._fixtures.live_criteria import make_command as cmd +from tests._fixtures.live_criteria import make_turn # shared builders (frozen timestamp) + + +if TYPE_CHECKING: + from coder_eval.criteria.base import BaseCriterion, LiveVerdict + + +@dataclass(frozen=True) +class ContractCase: + """One replayable trajectory for one criterion instance. + + ``commands`` is replayed prefix by prefix (0 .. len), so a case is only as + strong as the decision path it actually walks: prefer trajectories where the + verdict flips partway through over ones that decide on the first command. + """ + + label: str + criterion: LiveSuccessCriterion + commands: tuple[CommandTelemetry, ...] + reaches: LiveVerdict + """Verdict on the FULL trajectory. ``"undecided"`` is a legitimate (and useful) + expectation — it pins a shape the criterion deliberately never decides live.""" + + +def _skill_crit(*, skill_name: str, expected_skill: str) -> SkillTriggeredCriterion: + return SkillTriggeredCriterion( + type="skill_triggered", + description=f"skill_triggered[{skill_name}]", + skill_name=skill_name, + expected_skill=expected_skill, + ) + + +def _cmd_crit( + *, + pattern: str | None = "curl", + min_count: int = 1, + max_count: int | None = None, + require_success: bool = False, +) -> CommandExecutedCriterion: + return CommandExecutedCriterion( + type="command_executed", + description=f"command_executed[{pattern}]", + tool_name="Bash", + command_pattern=pattern, + min_count=min_count, + max_count=max_count, + require_success=require_success, + ) + + +def _bash( + command: str, + *, + sequence_number: int, + result_status: Literal["success", "error", "unknown"] = "success", +) -> CommandTelemetry: + return cmd("Bash", {"command": command}, sequence_number=sequence_number, result_status=result_status) + + +def _skill(name: str, *, sequence_number: int) -> CommandTelemetry: + return cmd("Skill", {"skill": name}, sequence_number=sequence_number) + + +# --------------------------------------------------------------------------- # +# The fixture table. Every LiveSuccessCriterion type in the SuccessCriterion +# union MUST appear here (enforced by ``missing_case_types``), and every polarity +# its instances claim decidable must be reached by some case (``polarity_gaps``). +# --------------------------------------------------------------------------- # + +CASES: dict[str, tuple[ContractCase, ...]] = { + "skill_triggered": ( + ContractCase( + label="positive row: expected skill engages via the Skill tool", + criterion=_skill_crit(skill_name="uipath-agents", expected_skill="uipath-agents"), + commands=( + _bash("ls -la", sequence_number=0), + _skill("uipath-agents", sequence_number=1), + _bash("echo done", sequence_number=2), + ), + reaches="pass", + ), + ContractCase( + label="positive row: a distractor engages FIRST, expected skill still passes", + # The any-engagement recall path: an earlier wrong touch must not + # freeze this instance, and the late "pass" must survive the trailing + # commands unchanged. + criterion=_skill_crit(skill_name="uipath-agents", expected_skill="uipath-agents"), + commands=( + _skill("uipath-rpa", sequence_number=0), + _skill("uipath-agents", sequence_number=1), + _skill("uipath-maestro-flow", sequence_number=2), + ), + reaches="pass", + ), + ContractCase( + label="positive row: non-Claude engagement by reading the skill off disk", + criterion=_skill_crit(skill_name="uipath-agents", expected_skill="uipath-agents"), + commands=( + _bash("ls .agents/skills", sequence_number=0), + _bash("cat .agents/skills/uipath-agents/SKILL.md", sequence_number=1), + ), + reaches="pass", + ), + ContractCase( + label="positive row: expected skill never engages -> never decides", + criterion=_skill_crit(skill_name="uipath-agents", expected_skill="uipath-agents"), + commands=( + _bash("ls -la", sequence_number=0), + _bash("cat README.md", sequence_number=1), + ), + reaches="undecided", + ), + ContractCase( + label="positive row: foreign skill path read FIRST via shell, expected read later still passes", + # Pinned from a live counterfactual run (PR #126): a first-engagement + # regression (fail on any foreign engagement while the target is + # unengaged) is non-monotonic exactly on this walk — live, it fail-stopped + # at tool call 1, truncating the run before the expected engagement, and + # flipped SUCCESS to FAILURE. This case IS the enforced form of that + # evidence: the probe task it came from is deliberately not checked in, + # since the mutant half is not reproducible from the repo. + criterion=_skill_crit(skill_name="beta", expected_skill="beta"), + commands=( + _bash("ls skills/alpha/", sequence_number=0), + _bash("ls skills/beta/", sequence_number=1), + _bash("echo done", sequence_number=2), + ), + reaches="pass", + ), + ContractCase( + label="distractor row: a wrong skill engaging is a decidable miss", + criterion=_skill_crit(skill_name="uipath-rpa", expected_skill="uipath-agents"), + commands=( + _bash("ls -la", sequence_number=0), + _skill("uipath-rpa", sequence_number=1), + _skill("uipath-agents", sequence_number=2), + ), + reaches="fail", + ), + ), + "command_executed": ( + ContractCase( + label="no upper bound + positive floor: passes when the count reaches min_count", + criterion=_cmd_crit(min_count=2, max_count=None), + commands=( + _bash("echo hello", sequence_number=0), + _bash("curl https://example.com", sequence_number=1), + _bash("curl https://example.org", sequence_number=2), + _bash("echo bye", sequence_number=3), + ), + reaches="pass", + ), + ContractCase( + label="must-NOT-run form (min 0 / max 0): the first forbidden match fails", + criterion=_cmd_crit(pattern="rm -rf", min_count=0, max_count=0), + commands=( + _bash("ls -la", sequence_number=0), + _bash("rm -rf /tmp/scratch", sequence_number=1), + _bash("echo done", sequence_number=2), + ), + reaches="fail", + ), + ContractCase( + label="upper bound exceeded: fails only once the count passes max_count", + criterion=_cmd_crit(min_count=1, max_count=1), + commands=( + _bash("curl https://example.com", sequence_number=0), + _bash("curl https://example.org", sequence_number=1), + ), + reaches="fail", + ), + ContractCase( + label="bounded range: a pass is not final until end-of-run, so never decides live", + criterion=_cmd_crit(min_count=1, max_count=3), + commands=( + _bash("curl https://example.com", sequence_number=0), + _bash("echo done", sequence_number=1), + ), + reaches="undecided", + ), + ContractCase( + label="bounded window then overrun: undecided through [min, max], fail past max", + # Pinned from a live counterfactual run (PR #126): a two-sided mutant + # that latches a premature pass at min_count is non-monotonic exactly on + # this walk (pass at count 1, fail at count 3) — live, it froze a + # still-compliant count and flipped FAILURE to SUCCESS. As above, this + # case is the enforced form of that evidence; the probe task is not + # checked in. + criterion=_cmd_crit(pattern="echo ping", min_count=1, max_count=2), + commands=( + _bash("echo ping", sequence_number=0), + _bash("echo ping", sequence_number=1), + _bash("echo ping", sequence_number=2), + ), + reaches="fail", + ), + ContractCase( + label="no bounds at all (min 0 / max None): neither polarity is decidable", + criterion=_cmd_crit(min_count=0, max_count=None), + commands=( + _bash("curl https://example.com", sequence_number=0), + _bash("curl https://example.org", sequence_number=1), + ), + reaches="undecided", + ), + ContractCase( + label="malformed regex degrades to undecided rather than raising", + criterion=_cmd_crit(pattern="[unclosed", min_count=1, max_count=None), + commands=(_bash("curl https://example.com", sequence_number=0),), + reaches="undecided", + ), + ContractCase( + label="require_success: a crashed match never counts toward the live pass", + # The CE034-motivating hazard, pinned in the contract table: without + # require_success an errored invocation would live-PASS this criterion + # (and could fire on_pass: stop). WITH it, the shared matcher filters + # the error out of BOTH live_verdict and _check_impl, so the verdict + # stays undecided across the whole trajectory. + criterion=_cmd_crit(min_count=1, max_count=None, require_success=True), + commands=( + _bash("curl https://example.com", sequence_number=0, result_status="error"), + _bash("echo done", sequence_number=1), + ), + reaches="undecided", + ), + ContractCase( + label="require_success: the pass latches only on the successful match", + # An errored match first, a successful one later: the verdict must go + # undecided -> undecided -> pass and hold — replaying every prefix pins + # that the error can neither count nor un-count anything. + criterion=_cmd_crit(min_count=1, max_count=None, require_success=True), + commands=( + _bash("curl https://example.com", sequence_number=0, result_status="error"), + _bash("curl https://example.org", sequence_number=1), + _bash("echo done", sequence_number=2), + ), + reaches="pass", + ), + ), +} + + +# --------------------------------------------------------------------------- # +# The replay engine +# --------------------------------------------------------------------------- # + + +def verdict_at( + checker: BaseCriterion[Any], + criterion: LiveSuccessCriterion, + commands: tuple[CommandTelemetry, ...], + prefix_len: int, +) -> LiveVerdict: + """``live_verdict`` over the first ``prefix_len`` commands. + + Wraps the prefix in a SINGLE ``TurnRecord``, which is exactly how + ``EarlyStopWatcher._collect_verdicts`` calls it (``records = [record]``) — the + watcher rebuilds one record from its own collector on every round rather than + accumulating a list. + """ + record = make_turn(*commands[:prefix_len]) + return checker.live_verdict(criterion, [record]) + + +def _walk_prefixes( + checker: BaseCriterion[Any], + criterion: LiveSuccessCriterion, + commands: tuple[CommandTelemetry, ...], + label: str, +) -> tuple[list[str], LiveVerdict | None]: + """Prefix-by-prefix determinism + monotonicity walk over ONE command ordering. + + The shared core of both replay modes: ``contract_violations`` walks the + fixture's authored ordering (and layers the terminal-verdict/polarity checks + on top), ``permuted_violations`` walks seeded reorderings (where those extra + checks would be unsound — see its docstring). Returns the breach list and the + full-trajectory verdict — or ``None`` for that verdict when the TERMINAL prefix + raised, since there is then no verdict to compare against and the stale value + from the previous prefix would stack a bogus breach on the real one. + + 1. **Determinism** — ``live_verdict`` called twice on an identical prefix must + agree. Catches RNG and per-call mutable state; NOT a reliable wall-clock + tripwire — the two calls land microseconds apart (module docstring, honest + limit 3). + 2. **Monotonicity** — once a prefix decides, every longer prefix returns that + same verdict. + 3. **No raising** — an exception from ``live_verdict`` is reported as a labeled + violation (case + prefix length) rather than crashing the walk; the remaining + prefixes still replay so one bad prefix does not mask breaches elsewhere. The + watcher runs mid-turn where a raise would take down the stop logic, and the + shape ``command_executed`` pins for a malformed regex — degrade to + ``"undecided"``, never raise — is the contract for every implementation. + """ + violations: list[str] = [] + decided: LiveVerdict | None = None + decided_at = 0 + final: LiveVerdict | None = "undecided" + + for prefix_len in range(len(commands) + 1): + try: + first = verdict_at(checker, criterion, commands, prefix_len) + second = verdict_at(checker, criterion, commands, prefix_len) + except Exception as exc: # any raise, of any type, IS the violation being reported + violations.append( + f"{label}: live_verdict RAISED {exc!r} at prefix length {prefix_len} — it must " + + "degrade to 'undecided' on inputs it cannot judge, never raise." + ) + # No verdict for THIS prefix. Clear the running terminal value so a raise on + # the last prefix cannot leave the previous prefix's verdict standing in for + # it (which would stack a phantom `reaches` breach on top of the real one). + final = None + continue + if first != second: + violations.append( + f"{label}: live_verdict is NON-DETERMINISTIC at prefix length {prefix_len} " + + f"({first!r} then {second!r} for the same input) — it must be a pure function of turn_records." + ) + if decided is not None and first != decided: + violations.append( + f"{label}: live_verdict is NON-MONOTONIC — decided {decided!r} at prefix length " + + f"{decided_at}, then returned {first!r} at prefix length {prefix_len}. Once decided, " + + "a verdict must hold for every longer prefix." + ) + elif decided is None and first != "undecided": + decided = first + decided_at = prefix_len + final = first + + return violations, final + + +def contract_violations(checker: BaseCriterion[Any], case: ContractCase) -> list[str]: + """Replay every prefix of ``case``; return contract breaches (empty list = clean). + + Checks five things: determinism, monotonicity, and no-raising (via + ``_walk_prefixes``), then two checks specific to the authored ordering: + + 4. **Declared terminal verdict** — the full trajectory reaches ``case.reaches``, + so a fixture cannot rot into a vacuous all-``undecided`` replay. + 5. **Polarity honesty** — a terminal decision must be a polarity the instance's + own ``live_decidable_polarities()`` claims; deciding one it does not claim + leaves the watcher treating a live trigger as inert. + + Checks 4 and 5 are skipped when the terminal prefix RAISED (``final is None``): + there is no verdict to judge, and the raise reported by ``_walk_prefixes`` is + already the finding — adding a derived ``reaches`` breach would only bury it. + """ + violations, final = _walk_prefixes(checker, case.criterion, case.commands, repr(case.label)) + + if final is None: + return violations + + if final != case.reaches: + violations.append( + f"{case.label!r}: full trajectory reaches {final!r}, but the case declares {case.reaches!r}. " + + "Update ContractCase.reaches, or fix the fixture so it exercises the intended decision path." + ) + + if final != "undecided": + claimed = case.criterion.live_decidable_polarities() + if final not in claimed: + violations.append( + f"{case.label!r}: live_verdict decided {final!r}, but this instance's " + + f"live_decidable_polarities() claims only {set(claimed) or '{}'}. EarlyStopWatcher " + + "would treat that trigger as inert while the checker actually decides it." + ) + + return violations + + +# Fixed seed: every CI run replays the exact same shuffles (a flaky lint rule +# would erode trust in the gate faster than any coverage it adds). +_PERMUTATION_SEED = 20260816 + + +def permuted_violations( + checker: BaseCriterion[Any], + case: ContractCase, + *, + shuffles: int = 5, + seed: int = _PERMUTATION_SEED, +) -> list[str]: + """Determinism + monotonicity under seeded reorderings of the case's commands. + + ``contract_violations`` walks ONE ordering — the one the fixture author wrote. + But the contract quantifies over ANY trajectory, and the orderings an author + does not think of are exactly where an order-sensitive bug (e.g. a verdict + computed from the *latest* command instead of the accumulated set) hides: + such a checker can look perfectly monotone on the authored ordering and flip + on a reordering. Seeded shuffles probe those orderings essentially for free. + + Each shuffle is RENUMBERED (``sequence_number`` reassigned 0..N-1 in the new + order) so the permuted trajectory is one the runtime could actually produce: + ``EarlyStopWatcher._collect_verdicts`` keeps its partial trajectory sorted by + ``sequence_number``, so ``live_verdict`` never sees a list whose order + contradicts those numbers. Without the renumber this layer would (a) report + breaches on inputs the watcher cannot construct, and (b) degrade to a silent + no-op for any future checker that sorts by ``sequence_number`` itself — the + shuffle would just sort straight back to the authored ordering. + + Deliberately NOT checked here: ``case.reaches`` and polarity honesty. A + reordering may legitimately change the terminal verdict for a criterion whose + semantics are order-sensitive, so pinning either would make this layer + unsound for exactly the criteria it exists to probe. Both stay enforced on + the authored ordering by ``contract_violations``. + """ + rng = random.Random(seed) + violations: list[str] = [] + for round_no in range(shuffles): + shuffled = list(case.commands) + rng.shuffle(shuffled) + renumbered = tuple( + command.model_copy(update={"sequence_number": position}) for position, command in enumerate(shuffled) + ) + walk, _final = _walk_prefixes( + checker, + case.criterion, + renumbered, + f"{case.label!r} [shuffle {round_no + 1}/{shuffles}, seed {seed}]", + ) + violations.extend(walk) + return violations + + +# --------------------------------------------------------------------------- # +# Registry-derived coverage +# --------------------------------------------------------------------------- # + + +def live_criterion_types() -> dict[str, type[LiveSuccessCriterion]]: + """Every ``LiveSuccessCriterion`` member of the ``SuccessCriterion`` union, by discriminator. + + Walks the discriminated union rather than the checker registry (mirroring CE025's + ``_type_to_model``): ``LiveSuccessCriterion`` subclassing on the MODEL is the single + source of truth for "is this criterion type live-observable". In-tree types only — + plugin criteria are not in the union (see the module docstring's honest limits). + """ + assert get_origin(SuccessCriterion) is Annotated + inner, *_ = get_args(SuccessCriterion) + return { + model.model_fields["type"].default: model + for model in get_args(inner) + if issubclass(model, LiveSuccessCriterion) + } + + +def missing_case_types(cases: dict[str, tuple[ContractCase, ...]] | None = None) -> list[str]: + """Live criterion types with no contract cases — a vacuous, unenforced contract.""" + table = CASES if cases is None else cases + return sorted(ctype for ctype in live_criterion_types() if not table.get(ctype)) + + +def polarity_gaps(cases: dict[str, tuple[ContractCase, ...]] | None = None) -> list[str]: + """Polarities a type's fixtures claim decidable but never actually demonstrate. + + Without this, a type could satisfy ``missing_case_types`` with a single + always-``undecided`` case and enforce nothing about its decision paths. + """ + table = CASES if cases is None else cases + gaps: list[str] = [] + for ctype, type_cases in sorted(table.items()): + claimed = {p for case in type_cases for p in case.criterion.live_decidable_polarities()} + reached = {case.reaches for case in type_cases} + for polarity in sorted(claimed - reached): + gaps.append( + f"{ctype}: fixtures claim polarity {polarity!r} is live-decidable, but no ContractCase " + + "reaches it — that decision path is untested." + ) + return gaps diff --git a/tests/lint/pyright_config.py b/tests/lint/pyright_config.py new file mode 100644 index 00000000..f472c45e --- /dev/null +++ b/tests/lint/pyright_config.py @@ -0,0 +1,64 @@ +"""Emit a pyright config that type-checks the CE036 contract engine under `tests/`. + +`make typecheck`'s main pass cannot reach those modules. `pyproject.toml`'s +`[tool.pyright]` excludes `"tests"`, and pyright's `exclude` beats BOTH of the +obvious shortcuts (verified against a probe file carrying a deliberate error): + +* `pyright tests/lint/live_verdict_contract.py` — an explicitly-passed CLI file + arg is still excluded: `filesAnalyzed: 0`, exit 0. A gate that checks nothing. +* adding the path to `include` — likewise dropped; the probe never appears in + the analyzed set. + +So the second pass needs its own config. This script DERIVES it from +`[tool.pyright]` — every rule setting is copied verbatim, and only `include` +(the modules below) and `exclude` (minus `"tests"`) are swapped. That is the +point of generating it instead of checking in a hand-written twin: a rule tuned +in `pyproject.toml` applies to both passes, and the two can never drift. + +Usage: `python -m tests.lint.pyright_config ` +""" + +from __future__ import annotations + +import json +import sys +import tomllib +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# The `tests/` modules worth type-checking: CE036's contract engine executes real +# checker code and encodes the early-stop design, so a type error there is a bug in +# the gate itself. Add a path here only for a tests/ module with that character — +# this is deliberately not "all of tests/". +INCLUDE = [ + "tests/lint/live_verdict_contract.py", + "tests/_fixtures/live_criteria.py", +] + + +def build_config() -> dict[str, object]: + with (REPO_ROOT / "pyproject.toml").open("rb") as handle: + settings = dict(tomllib.load(handle)["tool"]["pyright"]) + + settings["include"] = list(INCLUDE) + settings["exclude"] = [pattern for pattern in settings.get("exclude", []) if pattern != "tests"] + return settings + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(f"usage: {Path(sys.argv[0]).name} ") + out = Path(sys.argv[1]).resolve() + if out.parent != REPO_ROOT: + # Every path in the config stays relative, exactly as authored in + # pyproject.toml. pyright resolves those (and the root for `tests.*` import + # resolution) against the CONFIG FILE's directory, so the file has to sit at + # the repo root to mean the same thing the main pass does. + raise SystemExit(f"output must be written to the repo root ({REPO_ROOT}), got {out.parent}") + out.write_text(json.dumps(build_config(), indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index daaa3e0c..0c4d79ea 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -2964,3 +2964,270 @@ def test_finding_reports_the_line_of_the_offending_reference(self, tmp_path: Pat assert len(findings) == 1 assert findings[0].line == 7, f"expected line 7, got {findings[0].line}" assert str(findings[0]).startswith(f"{wf}:7 — ") + + +@pytest.mark.lint +class TestCE036LiveVerdictContract: + """CE036 — every live-observable criterion's `live_verdict` must be deterministic + and monotonic (GitHub issue #61 item 2). + + `EarlyStopWatcher` latches verdicts, defers the fail-stop, and attributes pass-stop + flips against the previous round — all correct only while `live_verdict` never + contradicts an earlier decision and never varies for identical input. That contract + was documented on `LiveVerdict`/`BaseCriterion.live_verdict` but unenforced: a third + criterion implementing it non-monotonically would type-check, pass CE025, and + silently corrupt the stop logic. + + Monotonicity over arbitrary Python is undecidable, so there is no sound static rule + to write. This replays each criterion against every prefix of a recorded trajectory + and asserts the property directly. The fixture table lives in + `tests/lint/live_verdict_contract.py`; the coverage checks below are what stop it + from decaying into a vacuous always-"undecided" replay. + + Honest limit (documented on the helper module too): this proves the contract on the + trajectories an author supplied, not in general. + """ + + def test_real_criteria_honor_the_contract(self): + """Every case for every live criterion type, replayed prefix by prefix.""" + from coder_eval.criteria import CriterionRegistry, init_criteria + from tests.lint.live_verdict_contract import CASES, contract_violations + + init_criteria(validate=False) + violations = [ + violation + for criterion_type, cases in CASES.items() + for case in cases + for violation in contract_violations(CriterionRegistry.get_checker(criterion_type)(), case) + ] + assert not violations, "live_verdict contract violations:\n" + "\n".join(f" {v}" for v in violations) + + def test_every_live_criterion_type_has_cases(self): + """A new LiveSuccessCriterion with no fixtures enforces nothing — fail instead.""" + from tests.lint.live_verdict_contract import missing_case_types + + missing = missing_case_types() + assert not missing, ( + "live-observable criterion types with no live_verdict contract cases: " + + ", ".join(missing) + + "\n\nAdd ContractCase entries to CASES in tests/lint/live_verdict_contract.py demonstrating " + + "every polarity the type's instances can decide." + ) + + def test_fixtures_exercise_every_decidable_polarity(self): + """Claiming a polarity is live-decidable but never demonstrating it is a gap.""" + from tests.lint.live_verdict_contract import polarity_gaps + + gaps = polarity_gaps() + assert not gaps, "untested live_verdict decision paths:\n" + "\n".join(f" {g}" for g in gaps) + + # --- The harness must actually fire; a green replay proves nothing on its own --- # + + @staticmethod + def _positive_case(label: str, reaches: str): + from coder_eval.models import SkillTriggeredCriterion + from tests.lint.live_verdict_contract import ContractCase, cmd + + return ContractCase( + label=label, + criterion=SkillTriggeredCriterion( + type="skill_triggered", + description="synthetic", + skill_name="alpha", + expected_skill="alpha", + ), + commands=( + cmd("Bash", {"command": "ls"}, sequence_number=0), + cmd("Bash", {"command": "pwd"}, sequence_number=1), + ), + reaches=reaches, + ) + + @staticmethod + def _checker(live_verdict_impl): + from coder_eval.criteria.base import BaseCriterion + + class _Synthetic(BaseCriterion): + criterion_type = "synthetic_live" + + def _check_impl(self, criterion, sandbox, reference_code=None, *, turn_records=None, context=None): + raise NotImplementedError + + def live_verdict(self, criterion, turn_records): + return live_verdict_impl(turn_records) + + return _Synthetic() + + def test_detects_a_non_monotonic_live_verdict(self): + """Decides "pass" on a short prefix, then contradicts itself on a longer one.""" + from tests.lint.live_verdict_contract import contract_violations + + checker = self._checker(lambda records: "pass" if len(records[0].commands) == 1 else "undecided") + violations = contract_violations(checker, self._positive_case("synthetic", "undecided")) + assert any("NON-MONOTONIC" in v for v in violations), violations + + def test_detects_a_non_deterministic_live_verdict(self): + """Same input, different answer — e.g. a wall-clock or RNG read.""" + from tests.lint.live_verdict_contract import contract_violations + + flips = iter(range(1000)) + checker = self._checker(lambda _records: "pass" if next(flips) % 2 else "undecided") + violations = contract_violations(checker, self._positive_case("synthetic", "undecided")) + assert any("NON-DETERMINISTIC" in v for v in violations), violations + + def test_detects_a_raising_live_verdict(self): + """A raise mid-walk becomes ONE labeled violation (case + prefix length) and the + walk continues — the later prefixes still replay, so the terminal "pass" here is + judged normally and the raise is the only breach reported.""" + from tests.lint.live_verdict_contract import contract_violations + + def raises_mid_trajectory(records): + n = len(records[0].commands) + if n == 1: + raise ValueError("boom") + return "pass" if n == 2 else "undecided" + + checker = self._checker(raises_mid_trajectory) + violations = contract_violations(checker, self._positive_case("synthetic", "pass")) + assert len(violations) == 1, violations + assert "RAISED" in violations[0] and "prefix length 1" in violations[0], violations + + def test_a_raise_on_the_final_prefix_does_not_stack_a_phantom_reaches_breach(self): + """The terminal prefix has no verdict when it raises, so the `reaches` and + polarity checks are skipped rather than judging the PREVIOUS prefix's stale + verdict — which would report a second, derived breach on top of the real one.""" + from tests.lint.live_verdict_contract import contract_violations + + def raises_at_the_end(records): + if len(records[0].commands) == 2: + raise ValueError("boom") + return "undecided" + + checker = self._checker(raises_at_the_end) + # The case declares "pass"; the stale value from prefix 1 is "undecided", so the + # unguarded comparison would append a phantom "reaches" violation here. + violations = contract_violations(checker, self._positive_case("synthetic", "pass")) + assert len(violations) == 1, violations + assert "RAISED" in violations[0] and "prefix length 2" in violations[0], violations + + def test_detects_a_fixture_that_stopped_exercising_its_decision_path(self): + """Fixture rot: the case claims a decision the trajectory no longer reaches.""" + from tests.lint.live_verdict_contract import contract_violations + + checker = self._checker(lambda _records: "undecided") + violations = contract_violations(checker, self._positive_case("synthetic", "pass")) + assert any("declares 'pass'" in v for v in violations), violations + + def test_detects_a_verdict_outside_the_instance_declared_polarities(self): + """A positive skill_triggered instance can only live-pass; deciding "fail" means + the watcher would treat a live trigger as inert.""" + from tests.lint.live_verdict_contract import contract_violations + + checker = self._checker(lambda _records: "fail") + violations = contract_violations(checker, self._positive_case("synthetic", "fail")) + assert any("live_decidable_polarities" in v for v in violations), violations + + def test_detects_a_live_type_with_no_cases(self): + """The completeness check must fail on an empty table, not pass vacuously. + + Compared against the registry, not a hardcoded list: pinning today's type + names would red THIS test the moment someone adds a live criterion — at + exactly the moment `test_every_live_criterion_type_has_cases` is already + failing them with the actionable message, pointing at the wrong file. + """ + from tests.lint.live_verdict_contract import live_criterion_types, missing_case_types + + expected = sorted(live_criterion_types()) + assert expected, "the union walk found no live criterion types — the check would pass vacuously" + assert missing_case_types({}) == expected + + def test_detects_an_all_undecided_fixture_set(self): + """A type whose only case never decides claims coverage it does not have.""" + from tests.lint.live_verdict_contract import polarity_gaps + + gaps = polarity_gaps({"skill_triggered": (self._positive_case("synthetic", "undecided"),)}) + assert len(gaps) == 1 + assert "'pass'" in gaps[0] + + def test_real_criteria_hold_under_permutation(self): + """Determinism + monotonicity must survive seeded reorderings of every case.""" + from coder_eval.criteria import CriterionRegistry, init_criteria + from tests.lint.live_verdict_contract import CASES, permuted_violations + + init_criteria(validate=False) + violations = [ + violation + for criterion_type, cases in CASES.items() + for case in cases + for violation in permuted_violations(CriterionRegistry.get_checker(criterion_type)(), case) + ] + assert not violations, "live_verdict permutation violations:\n" + "\n".join(f" {v}" for v in violations) + + def test_permutation_layer_detects_an_order_sensitive_verdict(self): + """A recency bug (verdict read off the LATEST command) is monotone on an + ordering that happens to end with the match — only a reordering exposes it. + This is the exact bug shape the counterfactual experiment injected.""" + from coder_eval.models import SkillTriggeredCriterion + from tests.lint.live_verdict_contract import ContractCase, cmd, contract_violations, permuted_violations + + def recency_verdict(records): + commands = records[0].commands + if commands and commands[-1].parameters.get("command") == "pwd": + return "pass" + return "undecided" + + checker = self._checker(recency_verdict) + case = ContractCase( + label="synthetic recency", + criterion=SkillTriggeredCriterion( + type="skill_triggered", + description="synthetic", + skill_name="alpha", + expected_skill="alpha", + ), + commands=( + cmd("Bash", {"command": "ls"}, sequence_number=0), + cmd("Bash", {"command": "cat x"}, sequence_number=1), + cmd("Bash", {"command": "pwd"}, sequence_number=2), + ), + reaches="pass", + ) + # Clean on the authored ordering (it decides only on the final prefix)... + assert not [v for v in contract_violations(checker, case) if "NON-MONOTONIC" in v] + # ...caught under permutation. + violations = permuted_violations(checker, case) + assert any("NON-MONOTONIC" in v for v in violations), violations + + def test_permutation_renumbers_so_a_sequence_sorting_checker_is_still_probed(self): + """The watcher hands `live_verdict` a trajectory sorted by `sequence_number` + (`EarlyStopWatcher._collect_verdicts`), so a checker may legitimately sort by it + too. If the shuffle left the original numbers attached, that sort would undo + every permutation and this layer would silently probe nothing. Renumbering keeps + the same recency bug detectable through the sort.""" + from coder_eval.models import SkillTriggeredCriterion + from tests.lint.live_verdict_contract import ContractCase, cmd, permuted_violations + + def sorted_recency_verdict(records): + commands = sorted(records[0].commands, key=lambda c: c.sequence_number) + if commands and commands[-1].parameters.get("command") == "pwd": + return "pass" + return "undecided" + + checker = self._checker(sorted_recency_verdict) + case = ContractCase( + label="synthetic recency behind a sequence sort", + criterion=SkillTriggeredCriterion( + type="skill_triggered", + description="synthetic", + skill_name="alpha", + expected_skill="alpha", + ), + commands=( + cmd("Bash", {"command": "ls"}, sequence_number=0), + cmd("Bash", {"command": "cat x"}, sequence_number=1), + cmd("Bash", {"command": "pwd"}, sequence_number=2), + ), + reaches="pass", + ) + violations = permuted_violations(checker, case) + assert any("NON-MONOTONIC" in v for v in violations), violations diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 1ef55cae..72ec75f0 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -28,7 +28,7 @@ from datetime import datetime from pathlib import Path from types import SimpleNamespace -from typing import Any +from typing import Any, Literal from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -91,27 +91,30 @@ TurnEndStatus, TurnStartEvent, ) +from tests._fixtures.live_criteria import FROZEN_TS, make_command, make_turn # --------------------------------------------------------------------------- # # Helpers # --------------------------------------------------------------------------- # -_TS = datetime(2026, 1, 1, 0, 0, 0) +# Telemetry/turn primitives are shared with the CE036 contract-replay fixtures +# (tests/lint/live_verdict_contract.py); the thin wrappers below keep this file's +# historical call shape (tool- ids, no sequence numbers) at every call site. +_TS = FROZEN_TS -def _cmd(tool_name: str, parameters: dict[str, Any], *, result_status: str = "success") -> CommandTelemetry: - return CommandTelemetry( - tool_name=tool_name, - tool_id=f"tool-{tool_name}", - timestamp=_TS, - parameters=parameters, - result_status=result_status, - ) +def _cmd( + tool_name: str, + parameters: dict[str, Any], + *, + result_status: Literal["success", "error", "unknown"] = "success", +) -> CommandTelemetry: + return make_command(tool_name, parameters, tool_id=f"tool-{tool_name}", result_status=result_status) def _turn(*commands: CommandTelemetry) -> TurnRecord: - return TurnRecord(iteration=1, user_input="", agent_output="", commands=list(commands)) + return make_turn(*commands) def _task(