diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d4a332f3..59f866b1 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -397,6 +397,16 @@ jobs: # release that republished an old tag would leave no other trace. - name: Lint duplicate versions run: lua5.4 tests/check_duplicate_versions.lua pkgs/*/*.lua + # The compatibility measurement's classifier decides a PUBLISHED figure + # -- which members "run", which "build", and which were correctly + # "refused" -- and the only thing that exercised it was the four-hour + # openkal-compat matrix, on real members, where a misclassification is + # a number nobody can trace back to a rule. `classify_failure` is split + # out so it can be stated in a second, here, with no toolchain and no + # network. It runs in `lint` rather than in that workflow because its + # cases are about the rule and not about any member. + - name: The compatibility classifier agrees with its own rules + run: python3 tests/openkal/compat.py selftest # ── Single-source-of-truth grammar check ───────────────────────── # `mcpp xpkg parse` uses EXACTLY the resolver's parser, so what # passes here is what builds for users of the pinned MCPP_VERSION. diff --git a/docs/openkal-compat.md b/docs/openkal-compat.md index 59c3cd12..82eda1dc 100644 --- a/docs/openkal-compat.md +++ b/docs/openkal-compat.md @@ -45,10 +45,13 @@ judged by, and a score of that shape asks the engine to supply what the environment does not have -- which is how a build tool ends up simulating an operating system it is not running on. The summary counts the two apart. -The judge is mcpp's own refusal code (`interface-not-provided`) and not a -string in the diagnostic: the member DECLARED the requirement and the graph -answered. Matching prose would let a member fall into this status for saying -the right words in an ordinary compile error. No member carries it today -- +The judge is mcpp's reason token `[interface-not-provided]`, matched WITH its +brackets. mcpp prints it in the refusal's own message the way it prints +`E0006`, and it is an entry in `docs/50`'s token table -- a machine interface +this measurement may read, rather than a sentence that may be rewritten. The +brackets are part of the match: read as a bare word, the token is a hyphenated +phrase an ordinary compile error could contain, and a member that merely failed +while quoting it would be recorded as correctly refused. No member carries it today -- `requires-interfaces` reaches the index with mcpp 2026.9.20.1 and no third-party descriptor states it yet. diff --git a/tests/openkal/compat.py b/tests/openkal/compat.py index b8cad27f..efe497ea 100644 --- a/tests/openkal/compat.py +++ b/tests/openkal/compat.py @@ -208,6 +208,17 @@ def measure(member: str, target: str, pins: dict) -> dict: if proc.returncode == 0: status = "runs" if can_run else "builds" return {"status": status} + return classify_failure(out, can_run) + + +def classify_failure(out: str, can_run: bool) -> dict: + """Classify a member whose build or test command exited non-zero. + + Separated from `measure` so the rule below has a criterion that runs + without a toolchain, a network or a member: `compat.py selftest`. The + distinction it draws decides a published figure, and until this split + the only way to exercise it was a four-hour matrix. + """ built = can_run and re.search(r"^\s*Running bin/", out, re.M) is not None if built: return {"status": "builds", "diagnostic": first_diagnostic(out)} @@ -221,23 +232,32 @@ def measure(member: str, target: str, pins: dict) -> dict: # not have --- which is how a build tool ends up simulating an operating # system it is not running on. # - # THE JUDGE IS THE ENGINE'S OWN REFUSAL CODE AND NOT A STRING IN THE - # DIAGNOSTIC. mcpp emits `interface-not-provided` when a package's - # `[kernel-abi] requires-interfaces` names something the resolved - # implementation does not provide; the member DECLARED the requirement and - # the graph answered. Matching prose instead would let a member fall into - # this status for saying the right words in an ordinary compile error. + # THE JUDGE IS A REASON TOKEN, AND THE BRACKETS ARE WHAT MAKE IT ONE. + # mcpp prints `[interface-not-provided]` in the refusal's own message, in + # brackets, the way `E0006` does (docs/50 §"One token is also printed by + # `mcpp build` itself"); the token is an entry in that page's table, which + # is to say a machine interface this index may read, rather than a + # sentence that may be rewritten. + # + # THE BRACKETS ARE PART OF THE MATCH AND NOT DECORATION. Read as a bare + # word, `interface-not-provided` is a hyphenated phrase an ordinary + # compile error could contain --- a member whose own diagnostic quoted a + # manifest key, or an upstream error message using the same words, would + # be recorded as correctly refused when it had simply failed. Requiring + # the brackets is what distinguishes "the graph answered this member's + # declared requirement" from "the output happened to say so". # # No member carries this status today: `requires-interfaces` reaches the # index with mcpp 2026.9.20.1 and no third-party descriptor states it yet. # The path is here rather than added later because the figure it changes is # the one this file publishes, and a member that starts declaring its # requirements should not have to wait for this file to catch up. - if re.search(r"\binterface-not-provided\b", out): + if "[interface-not-provided]" in out: return {"status": "refused", "diagnostic": first_diagnostic(out)} return {"status": "fails", "diagnostic": first_diagnostic(out)} + def cmd_run(args: argparse.Namespace) -> int: pins = load_toml(os.path.join(HERE, "pins.toml")) members_file = load_toml(os.path.join(HERE, "members.toml")) @@ -330,6 +350,48 @@ def cmd_select(args: argparse.Namespace) -> int: return 0 +def cmd_selftest(_args: argparse.Namespace) -> int: + """Exercise `classify_failure`, whose distinctions decide a published + figure and which no other check reaches. + + Each case is one sentence about the rule, and each would have passed + before the rule it pins was written the way it is now. + """ + refusal = ( + "error: package 'x' requires the kernel-abi interface 'openkal.space',\n" + " which openkal-windows (14 interfaces) does not provide. " + "[interface-not-provided]\n") + # The same words, as prose, with no brackets: an upstream error quoting a + # manifest key, or a member's own diagnostic naming the condition. Before + # the brackets were part of the match this was recorded as a correct + # refusal, which is to say a member that merely failed improved the figure. + prose = ("error: no member named 'interface_not_provided'\n" + "note: the interface-not-provided condition is described in " + "the README\n") + ran = " Compiling x v0.1.0\n Running bin/x\n test failed\n" + + cases = [ + ("a bracketed token is a refusal", + classify_failure(refusal, False)["status"], "refused"), + ("the same token as prose is a failure", + classify_failure(prose, False)["status"], "fails"), + ("an ordinary compile error is a failure", + classify_failure("error: no such file\n", False)["status"], "fails"), + ("a member that ran and failed its tests still built", + classify_failure(ran, True)["status"], "builds"), + ("the same output without a runner is not evidence it built", + classify_failure(ran, False)["status"], "fails"), + ] + bad = 0 + for name, got, want in cases: + ok = got == want + bad += not ok + print(f"{'ok ' if ok else 'FAIL'} {name}: {got}" + + ("" if ok else f" (expected {want})")) + print(f"\n{len(cases) - bad} passed, {bad} failed") + return 1 if bad else 0 + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -344,8 +406,10 @@ def main() -> int: check.add_argument("--members", nargs="*") select = sub.add_parser("select") select.add_argument("files", nargs="*") + sub.add_parser("selftest") args = parser.parse_args() - return {"run": cmd_run, "check": cmd_check, "select": cmd_select}[args.command](args) + return {"run": cmd_run, "check": cmd_check, "select": cmd_select, + "selftest": cmd_selftest}[args.command](args) if __name__ == "__main__":