P68 (#2300) lexer observation totality + verify fixes: allowlists pin today's defect, scanner walrus arm observed

.github/workflows/dotnet10.yml

@@ -353,6 +353,12 @@ jobs:
     name: Test Compiler (shard ${{ matrix.shard }})
     needs: build
     runs-on: ubuntu-latest
+    env:
+      # The numeric-literal twin (Sharpy.Compiler.Tests.Lexer.NumericLiteralTwinTests, #2300) FAILS
+      # instead of skipping when no Python 3.12+ interpreter is found: a runner that lost the
+      # "Setup Python" step below would otherwise pass the twin vacuously. Job level, so every shard
+      # inherits it. Only the new twin reads it today; the older differential twins still soft-skip.
+      SHARPY_REQUIRE_PYTHON_ORACLE: "1"
     strategy:
       fail-fast: false
       matrix:

build_tools/differential_lex/__init__.py

@@ -0,0 +1 @@
+"""CPython numeric-literal oracle for the Sharpy lexer's numeric twin (#2300)."""

build_tools/differential_lex/number_oracle.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""CPython numeric-literal oracle for the Sharpy lexer's numeric twin (#2300, P68 row C1).
+
+``NumericLiteralTwinTests`` (src/Sharpy.Compiler.Tests/Lexer/) enumerates numeric-literal
+spellings in the grammar Sharpy shares with CPython and asks each side ONE question per
+spelling: "is this text a single numeric constant with value V, or is it refused?"
+
+The oracle sits at the PROGRAM altitude, not the token altitude: ``ast.parse(source,
+mode="eval")`` whose body is an ``ast.Constant`` holding an ``int`` or a ``float``. CPython's
+``tokenize`` splits ``.5e`` into NUMBER ``.5`` + NAME ``e`` and refuses only at parse time,
+while Sharpy refuses in the lexer; comparing token boundaries would manufacture divergences
+the language does not have, so token-boundary agreement is deliberately NOT asked.
+
+A ``complex`` constant (``1j``) is reported ``ok: false, error: "imaginary"`` — Sharpy has no
+complex literal, the twin excludes the imaginary family from its matrix, and the distinct
+error lets that exclusion be checked.
+
+Batch protocol (the C# twin spawns one ``python3`` per batch, not per spelling):
+
+    stdin  : one JSON object per line -> {"id": <int>, "source": "<spelling>"}
+    stdout : one JSON object per line -> {"id": <int>, "ok": true,
+                                          "kind": "int" | "float", "value": "<repr>"}
+                                     or  {"id": <int>, "ok": false, "error": "<message>"}
+
+``--batch <file>`` reads the JSONL from a file instead of stdin. With no input the module is
+import-only (used by build_tools/tests/test_number_oracle.py).
+
+Deterministic and dependency-free; pinned to CPython 3.12 by the caller.
+"""
+from __future__ import annotations
+
+import ast
+import json
+import sys
+import warnings
+from typing import Any, Dict, Iterable, List
+
+
+def classify(source: str) -> Dict[str, Any]:
+    """One spelling -> ``{"ok": True, "kind", "value"}`` or ``{"ok": False, "error"}``."""
+    try:
+        with warnings.catch_warnings():
+            # A SyntaxWarning (e.g. "invalid decimal literal" before a keyword) is not a
+            # refusal: CPython runs the program. Never let a warnings filter promote it.
+            warnings.simplefilter("ignore")
+            tree = ast.parse(source, mode="eval")
+    except SyntaxError as exc:
+        return {"ok": False, "error": str(exc.msg)}
+    except ValueError as exc:  # e.g. a null byte in the source
+        return {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
+
+    body = tree.body
+    if not isinstance(body, ast.Constant):
+        return {"ok": False, "error": f"not a constant: {type(body).__name__}"}
+    value = body.value
+    # Exact type checks: bool is an int subclass, and True/False are not numeric literals.
+    if type(value) is int:
+        return {"ok": True, "kind": "int", "value": str(value)}
+    if type(value) is float:
+        return {"ok": True, "kind": "float", "value": repr(value)}
+    if type(value) is complex:
+        return {"ok": False, "error": "imaginary"}
+    return {"ok": False, "error": f"not a numeric constant: {type(value).__name__}"}
+
+
+def run_batch(lines: Iterable[str]) -> List[str]:
+    """Map a JSONL request stream to a JSONL verdict stream (both keyed by ``id``)."""
+    out: List[str] = []
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        request = json.loads(line)
+        verdict = classify(request["source"])
+        verdict["id"] = request["id"]
+        out.append(json.dumps(verdict))
+    return out
+
+
+def main(argv: List[str]) -> int:
+    if len(argv) > 2 and argv[1] == "--batch":
+        with open(argv[2], "r", encoding="utf-8") as handle:
+            lines = handle.readlines()
+    elif not sys.stdin.isatty():
+        lines = sys.stdin.readlines()
+    else:
+        # Import-only / no input: nothing to do.
+        return 0
+
+    for verdict_line in run_batch(lines):
+        print(verdict_line)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

build_tools/stryker/lexer-ignore.json

@@ -0,0 +1,16 @@
+{
+  "$comment": "R-GB (#2289) ignore list for the Lexer mutation score, applied by build_tools/stryker/triage.py (P68, #2300). 'within' is a regex over the CALLEE names a mutant sits inside (never the declaring method); every '|' alternative must name a method invoked in the scanned sources or the script exits 2. Against the P68 plan's Design Decision 9 @ 220259b51: LogDebug is dropped (invoked nowhere in src/Sharpy.Compiler/Lexer/), AddError is added (two dedented-string message literals in Lexer.Literals.cs reach the diagnostic bag through it directly, the same message-text class).",
+  "entries": [
+    {
+      "mutator": "string",
+      "within": "ReportError|AddError|LogInfo|AddWarning|ReportUnclosedField|ReportFieldEndedByQuote",
+      "reason": "diagnostic/logger message text — DEVIATION by default (python-fidelity-scope.md); no program's control flow reads it"
+    },
+    {
+      "mutator": "string",
+      "within": "CreateToken",
+      "fieldOf": "operator Value",
+      "reason": "an operator token's Value is read only inside parser error-message text (grep-only @ 258a7b66a)"
+    }
+  ]
+}

build_tools/stryker/lexer-probe.json

@@ -0,0 +1,15 @@
+{
+  "stryker-config": {
+    "project": "Sharpy.Compiler.csproj",
+    "test-projects": ["src/Sharpy.Compiler.Tests/Sharpy.Compiler.Tests.csproj"],
+    "mutate": ["**/Lexer/Token.cs", "**/Lexer/LiteralSpans.cs", "**/Lexer/Lexer.Indentation.cs"],
+    "ignore-methods": ["ReportError", "LogInfo", "AddWarning", "AddError"],
+    "test-case-filter": "FullyQualifiedName~Sharpy.Compiler.Tests.Lexer.|FullyQualifiedName~Sharpy.Compiler.Tests.Properties.Lexer.",
+    "coverage-analysis": "perTest",
+    "mutation-level": "Standard",
+    "concurrency": 4,
+    "configuration": "Debug",
+    "reporters": ["dots", "json", "markdown", "cleartext"],
+    "thresholds": { "high": 80, "low": 60, "break": 0 }
+  }
+}