diff --git a/.github/workflows/dotnet10.yml b/.github/workflows/dotnet10.yml index 9495ca2d5..b35622a11 100644 --- a/.github/workflows/dotnet10.yml +++ b/.github/workflows/dotnet10.yml @@ -353,6 +353,12 @@ jobs: name: Test Compiler (shard ${{ matrix.shard }}) needs: build runs-on: ubuntu-latest + env: + # The numeric-literal twin (Sharpy.Compiler.Tests.Lexer.NumericLiteralTwinTests, #2300) FAILS + # instead of skipping when no Python 3.12+ interpreter is found: a runner that lost the + # "Setup Python" step below would otherwise pass the twin vacuously. Job level, so every shard + # inherits it. Only the new twin reads it today; the older differential twins still soft-skip. + SHARPY_REQUIRE_PYTHON_ORACLE: "1" strategy: fail-fast: false matrix: diff --git a/build_tools/differential_lex/__init__.py b/build_tools/differential_lex/__init__.py new file mode 100644 index 000000000..68771c3f6 --- /dev/null +++ b/build_tools/differential_lex/__init__.py @@ -0,0 +1 @@ +"""CPython numeric-literal oracle for the Sharpy lexer's numeric twin (#2300).""" diff --git a/build_tools/differential_lex/number_oracle.py b/build_tools/differential_lex/number_oracle.py new file mode 100644 index 000000000..b818bd852 --- /dev/null +++ b/build_tools/differential_lex/number_oracle.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""CPython numeric-literal oracle for the Sharpy lexer's numeric twin (#2300, P68 row C1). + +``NumericLiteralTwinTests`` (src/Sharpy.Compiler.Tests/Lexer/) enumerates numeric-literal +spellings in the grammar Sharpy shares with CPython and asks each side ONE question per +spelling: "is this text a single numeric constant with value V, or is it refused?" + +The oracle sits at the PROGRAM altitude, not the token altitude: ``ast.parse(source, +mode="eval")`` whose body is an ``ast.Constant`` holding an ``int`` or a ``float``. CPython's +``tokenize`` splits ``.5e`` into NUMBER ``.5`` + NAME ``e`` and refuses only at parse time, +while Sharpy refuses in the lexer; comparing token boundaries would manufacture divergences +the language does not have, so token-boundary agreement is deliberately NOT asked. + +A ``complex`` constant (``1j``) is reported ``ok: false, error: "imaginary"`` — Sharpy has no +complex literal, the twin excludes the imaginary family from its matrix, and the distinct +error lets that exclusion be checked. + +Batch protocol (the C# twin spawns one ``python3`` per batch, not per spelling): + + stdin : one JSON object per line -> {"id": , "source": ""} + stdout : one JSON object per line -> {"id": , "ok": true, + "kind": "int" | "float", "value": ""} + or {"id": , "ok": false, "error": ""} + +``--batch `` reads the JSONL from a file instead of stdin. With no input the module is +import-only (used by build_tools/tests/test_number_oracle.py). + +Deterministic and dependency-free; pinned to CPython 3.12 by the caller. +""" +from __future__ import annotations + +import ast +import json +import sys +import warnings +from typing import Any, Dict, Iterable, List + + +def classify(source: str) -> Dict[str, Any]: + """One spelling -> ``{"ok": True, "kind", "value"}`` or ``{"ok": False, "error"}``.""" + try: + with warnings.catch_warnings(): + # A SyntaxWarning (e.g. "invalid decimal literal" before a keyword) is not a + # refusal: CPython runs the program. Never let a warnings filter promote it. + warnings.simplefilter("ignore") + tree = ast.parse(source, mode="eval") + except SyntaxError as exc: + return {"ok": False, "error": str(exc.msg)} + except ValueError as exc: # e.g. a null byte in the source + return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + + body = tree.body + if not isinstance(body, ast.Constant): + return {"ok": False, "error": f"not a constant: {type(body).__name__}"} + value = body.value + # Exact type checks: bool is an int subclass, and True/False are not numeric literals. + if type(value) is int: + return {"ok": True, "kind": "int", "value": str(value)} + if type(value) is float: + return {"ok": True, "kind": "float", "value": repr(value)} + if type(value) is complex: + return {"ok": False, "error": "imaginary"} + return {"ok": False, "error": f"not a numeric constant: {type(value).__name__}"} + + +def run_batch(lines: Iterable[str]) -> List[str]: + """Map a JSONL request stream to a JSONL verdict stream (both keyed by ``id``).""" + out: List[str] = [] + for line in lines: + line = line.strip() + if not line: + continue + request = json.loads(line) + verdict = classify(request["source"]) + verdict["id"] = request["id"] + out.append(json.dumps(verdict)) + return out + + +def main(argv: List[str]) -> int: + if len(argv) > 2 and argv[1] == "--batch": + with open(argv[2], "r", encoding="utf-8") as handle: + lines = handle.readlines() + elif not sys.stdin.isatty(): + lines = sys.stdin.readlines() + else: + # Import-only / no input: nothing to do. + return 0 + + for verdict_line in run_batch(lines): + print(verdict_line) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build_tools/stryker/lexer-ignore.json b/build_tools/stryker/lexer-ignore.json new file mode 100644 index 000000000..65aa74d60 --- /dev/null +++ b/build_tools/stryker/lexer-ignore.json @@ -0,0 +1,16 @@ +{ + "$comment": "R-GB (#2289) ignore list for the Lexer mutation score, applied by build_tools/stryker/triage.py (P68, #2300). 'within' is a regex over the CALLEE names a mutant sits inside (never the declaring method); every '|' alternative must name a method invoked in the scanned sources or the script exits 2. Against the P68 plan's Design Decision 9 @ 220259b51: LogDebug is dropped (invoked nowhere in src/Sharpy.Compiler/Lexer/), AddError is added (two dedented-string message literals in Lexer.Literals.cs reach the diagnostic bag through it directly, the same message-text class).", + "entries": [ + { + "mutator": "string", + "within": "ReportError|AddError|LogInfo|AddWarning|ReportUnclosedField|ReportFieldEndedByQuote", + "reason": "diagnostic/logger message text — DEVIATION by default (python-fidelity-scope.md); no program's control flow reads it" + }, + { + "mutator": "string", + "within": "CreateToken", + "fieldOf": "operator Value", + "reason": "an operator token's Value is read only inside parser error-message text (grep-only @ 258a7b66a)" + } + ] +} diff --git a/build_tools/stryker/lexer-probe.json b/build_tools/stryker/lexer-probe.json new file mode 100644 index 000000000..f1a3fc9db --- /dev/null +++ b/build_tools/stryker/lexer-probe.json @@ -0,0 +1,15 @@ +{ + "stryker-config": { + "project": "Sharpy.Compiler.csproj", + "test-projects": ["src/Sharpy.Compiler.Tests/Sharpy.Compiler.Tests.csproj"], + "mutate": ["**/Lexer/Token.cs", "**/Lexer/LiteralSpans.cs", "**/Lexer/Lexer.Indentation.cs"], + "ignore-methods": ["ReportError", "LogInfo", "AddWarning", "AddError"], + "test-case-filter": "FullyQualifiedName~Sharpy.Compiler.Tests.Lexer.|FullyQualifiedName~Sharpy.Compiler.Tests.Properties.Lexer.", + "coverage-analysis": "perTest", + "mutation-level": "Standard", + "concurrency": 4, + "configuration": "Debug", + "reporters": ["dots", "json", "markdown", "cleartext"], + "thresholds": { "high": 80, "low": 60, "break": 0 } + } +} diff --git a/build_tools/stryker/triage.py b/build_tools/stryker/triage.py new file mode 100644 index 000000000..48a44107c --- /dev/null +++ b/build_tools/stryker/triage.py @@ -0,0 +1,828 @@ +#!/usr/bin/env python3 +"""Triage a Stryker.NET ``mutation-report.json`` against an ignore list kept as data (#2289 R-GB, #2300 P68). + +Usage:: + + python3 build_tools/stryker/triage.py --report \\ + --ignore build_tools/stryker/lexer-ignore.json [--classes] [--universe ...] + +The report follows the mutation-testing-elements schema: ``files: {: {language, source, +mutants: [{id, mutatorName, replacement, location: {start: {line, column}, end: {line, column}}, +status, statusReason?}]}}``. Lines and columns are 1-based (Roslyn's 0-based line span + 1); the +column counts UTF-16 code units, as Roslyn does. Each file entry carries its own ``source``; when it +is absent the path is read from disk. + +Every mutant is bucketed by (file, mutator, enclosing invocation, status): + +- ``mutator`` is Stryker's config identifier (``string``, ``equality``, ``statement``, …), derived + from the report's display name (``String mutation``, ``Equality mutation``, + ``PostIncrementExpression to PostDecrementExpression mutation``, …). A display name no rule knows + is kept as ``unknown:`` — counted, never dropped. +- ``enclosing invocation`` is the callee name of the innermost call whose argument list holds the + mutant (``-`` when none). It is read from the C# source with a small tokenizer (strings, verbatim, + interpolated and raw strings, char literals and comments are skipped, so ``'{'`` is not a brace). +- ``status`` is one of ``killed``, ``survived``, ``timeout``, ``noCoverage``, ``ignoredByStryker``, + ``compileError``, ``runtimeError``, ``pending`` — or ``ignoredByList`` when an ignore entry + matches a mutant Stryker scored (Killed/Survived/Timeout/NoCoverage). Any other status fails. + +Ignore list (``lexer-ignore.json``):: + + {"entries": [{"mutator": "string", "within": "", + "fieldOf": "", "reason": ""}]} + +An entry matches a mutant when its ``mutator`` equals the mutant's identifier and some enclosing +invocation's callee name fully matches ``within`` (and, when ``fieldOf`` is given, that invocation +satisfies the named predicate — see ``FIELD_PREDICATES``). ``within`` names a CALL the mutant sits +in, never the method declaring it: a message literal passed to ``ReportError`` inside ``ReadNumber`` +is ``within: ReportError``, and ``within: ReadNumber`` does not match it. Each top-level ``|`` +alternative of ``within`` must name a callee invoked somewhere in the report's sources (plus every +``*.cs`` under each ``--universe`` directory), otherwise the script exits 2: an entry that can +match nothing is a silent zero, not an ignore. + +Score (Stryker's definition, in percent, two decimals; ``null`` when nothing was scored):: + + score = (killed + timeout) / (killed + timeout + survived + noCoverage) * 100 + +CompileError, RuntimeError, Ignored and Pending mutants are outside both terms. ``scoreIgnoringListed`` +is the same formula after removing every ``ignoredByList`` mutant from the four counts. + +Classes (#2300 rows C1–C7) are assigned to every mutant left UNDETECTED after the ignore list +(survived + noCoverage), first matching rule wins: + + C7 enclosing member is ``Keywords`` or ``_keywordNames`` + C2 a non-block mutant whose original text names ``_column`` or ``SourceLength``/``sourceLength`` + C6 an equality mutant whose original text compares against a ``Length`` bound (``< _source.Length``) + C1 enclosing member in ReadNumber / ReadHexNumber / ReadBinaryNumber / ReadOctalNumber / + ReadPrefixedIntegerSuffix + C3 enclosing member is NextToken / ConsumeLineBreak / IsPreCloseNewline or a ``*HoleTrivia*`` method + C5 file ``Lexer.LiteralLoss.cs`` + C4 file ``Lexer.FStrings.cs`` (the rest of it) + unclassified everything else (counted, never dropped) + +``--classes`` adds ``classMutants``: per class, the mutants (file, line, column, mutator, member, status). +Exit codes: 0 ok, 2 bad input (unreadable report, unknown status, malformed ignore entry, a ``within`` +alternative that names no invoked method, a location outside its source). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import string +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + + +class TriageError(Exception): + """Bad input: the script refuses rather than report a number it cannot stand behind.""" + + +# --------------------------------------------------------------------------- # +# mutator display name -> Stryker config identifier +# --------------------------------------------------------------------------- # +# Display names read from dotnet-stryker 5.0.0's Stryker.Core.dll user strings; the composed ones +# ("{Kind} to {Kind} mutation", "{Kind} to un-{Kind} mutation") are matched by shape. +_MUTATOR_RULES: list[tuple[str, str]] = [ + (r"String mutation", "string"), + (r"String Method Mutation.*", "stringmethod"), + (r"Arithmetic mutation", "arithmetic"), + (r"Equality mutation", "equality"), + (r"Logical mutation", "logical"), + (r"Bitwise mutation", "bitwise"), + (r"Boolean mutation|Negate expression", "boolean"), + (r"Statement mutation", "statement"), + (r"Block removal mutation", "block"), + (r"(Array|Collection|Object) initializer mutation", "initializer"), + (r"Collection expression mutation", "collectionexpression"), + (r"Conditional \((true|false)\) mutation", "conditional"), + (r"Null coalescing mutation.*", "nullcoalescing"), + (r"Remove checked expression", "checked"), + (r"Linq method mutation.*", "linq"), + (r"Math method mutation.*", "math"), + (r"Regex.*", "regex"), + (r"\w*Assignment\w* to \w+ mutation", "assignment"), + (r"(Pre|Post)(Increment|Decrement)Expression to \w+ mutation", "update"), + (r"\w+Expression to (un-)?\w+ mutation", "unary"), +] + +KNOWN_MUTATORS = frozenset(ident for _, ident in _MUTATOR_RULES) + + +def mutator_id(display_name: str) -> str: + """Map a report ``mutatorName`` to its Stryker config identifier (``unknown:`` if none).""" + for pattern, ident in _MUTATOR_RULES: + if re.fullmatch(pattern, display_name): + return ident + if display_name in KNOWN_MUTATORS: # a report already spelled with identifiers + return display_name + return f"unknown:{display_name}" + + +# --------------------------------------------------------------------------- # +# status mapping +# --------------------------------------------------------------------------- # +_STATUS = { + "Killed": "killed", + "Survived": "survived", + "Timeout": "timeout", + "NoCoverage": "noCoverage", + "Ignored": "ignoredByStryker", + "CompileError": "compileError", + "RuntimeError": "runtimeError", + "Pending": "pending", +} +SCORED = frozenset({"killed", "survived", "timeout", "noCoverage"}) +UNDETECTED = frozenset({"survived", "noCoverage"}) + + +def score_of(counts: dict[str, int]) -> Optional[float]: + detected = counts.get("killed", 0) + counts.get("timeout", 0) + valid = detected + counts.get("survived", 0) + counts.get("noCoverage", 0) + if valid == 0: + return None + return round(detected * 100.0 / valid, 2) + + +# --------------------------------------------------------------------------- # +# C# tokenizer (enough structure for members and invocations; not a parser) +# --------------------------------------------------------------------------- # +@dataclass +class Tok: + kind: str # "id" | "str" | "char" | "num" | "punct" + text: str + start: int + end: int + interpolated: bool = False + value: Optional[str] = None # decoded content of a non-interpolated string + + +_TWO_CHAR = {"=>", "==", "!=", "<=", ">=", "&&", "||", "??", "::", "++", "--", "+=", "-=", "*=", + "/=", "%=", "&=", "|=", "^=", "?.", "->"} +_SIMPLE_ESCAPES = {"\\": "\\", '"': '"', "'": "'", "0": "\0", "a": "\a", "b": "\b", "f": "\f", + "n": "\n", "r": "\r", "t": "\t", "v": "\v", "e": "\x1b"} + + +class _Scanner: + def __init__(self, src: str): + self.src = src + self.toks: list[Tok] = [] + + def scan(self, pos: int, in_hole: bool = False) -> int: + """Tokenize code from ``pos``. In a hole, stop at a depth-0 ``}`` or ``:`` (format spec) + or ``,`` (alignment, scanned as code anyway) and return its index.""" + src, n = self.src, len(self.src) + depth = 0 + while pos < n: + c = src[pos] + if c.isspace(): + pos += 1 + continue + if src.startswith("//", pos): + nl = src.find("\n", pos) + pos = n if nl < 0 else nl + continue + if src.startswith("/*", pos): + close = src.find("*/", pos + 2) + pos = n if close < 0 else close + 2 + continue + m = re.match(r"[$@]*\"", src[pos:pos + 8]) if c in "$@\"" else None + if m: + pos = self._string(pos, m.group()[:-1]) + continue + if c == "'": + pos = self._char(pos) + continue + if c.isdigit(): + m2 = re.compile(r"[0-9][0-9A-Za-z_.]*").match(src, pos) + self.toks.append(Tok("num", m2.group(), pos, m2.end())) + pos = m2.end() + continue + if c.isalpha() or c == "_" or (c == "@" and pos + 1 < n and (src[pos + 1].isalpha() or src[pos + 1] == "_")): + m2 = re.compile(r"@?[A-Za-z_][A-Za-z0-9_]*").match(src, pos) + self.toks.append(Tok("id", m2.group().lstrip("@"), pos, m2.end())) + pos = m2.end() + continue + if in_hole: + if c in "([{": + depth += 1 + elif c in ")]}": + if depth == 0 and c == "}": + return pos + depth -= 1 + elif depth == 0 and c == ":" and not src.startswith("::", pos) and not src.startswith("::", pos - 1): + return pos + two = src[pos:pos + 2] + if two in _TWO_CHAR: + self.toks.append(Tok("punct", two, pos, pos + 2)) + pos += 2 + else: + self.toks.append(Tok("punct", c, pos, pos + 1)) + pos += 1 + if in_hole: + raise TriageError(f"unterminated interpolation hole at offset {pos}") + return pos + + def _char(self, pos: int) -> int: + src, i = self.src, pos + 1 + while i < len(src) and src[i] != "'": + i += 2 if src[i] == "\\" else 1 + self.toks.append(Tok("char", src[pos:i + 1], pos, i + 1)) + return i + 1 + + def _string(self, pos: int, prefix: str) -> int: + src, n = self.src, len(self.src) + dollars = prefix.count("$") + verbatim = "@" in prefix + q = pos + len(prefix) + quotes = 0 + while q + quotes < n and src[q + quotes] == '"': + quotes += 1 + holes_start = len(self.toks) + if quotes >= 3: # raw string literal: closes at the same run of quotes; holes use `dollars` braces + i = q + quotes + closer = '"' * quotes + while i < n and not src.startswith(closer, i): + if dollars and src.startswith("{" * dollars, i): + run = len(src[i:]) - len(src[i:].lstrip("{")) + i = self._hole(i + run, dollars) + continue + i += 1 + end = min(n, i + quotes) + self._emit_string(pos, end, bool(dollars), None, holes_start) + return end + i = q + 1 + content: list[str] = [] + while i < n: + ch = src[i] + if ch == '"': + if verbatim and src.startswith('""', i): + content.append('"') + i += 2 + continue + i += 1 + break + if ch == "\\" and not verbatim: + esc = src[i + 1] if i + 1 < n else "" + content.append(_SIMPLE_ESCAPES.get(esc, "\\" + esc)) + i += 2 + continue + if dollars and ch == "{": + if src.startswith("{{", i): + content.append("{") + i += 2 + continue + i = self._hole(i + 1, 1) + continue + if dollars and src.startswith("}}", i): + content.append("}") + i += 2 + continue + if ch == "\n" and not verbatim: + raise TriageError(f"newline inside a regular string literal at offset {pos}") + content.append(ch) + i += 1 + self._emit_string(pos, i, bool(dollars), None if dollars else "".join(content), holes_start) + return i + + def _hole(self, pos: int, braces: int) -> int: + stop = self.scan(pos, in_hole=True) + src = self.src + if src[stop] == ":": # format spec: literal text up to the closing brace(s) + close = src.find("}" * braces, stop) + if close < 0: + raise TriageError(f"unterminated format spec at offset {stop}") + stop = close + return stop + braces + + def _emit_string(self, start: int, end: int, interpolated: bool, value: Optional[str], at: int) -> None: + # The string precedes its hole tokens so the list stays in source order. + self.toks.insert(at, Tok("str", self.src[start:end], start, end, interpolated, value)) + + +def tokenize(src: str) -> list[Tok]: + sc = _Scanner(src) + sc.scan(0) + return sc.toks + + +# --------------------------------------------------------------------------- # +# structure: invocations and members +# --------------------------------------------------------------------------- # +_NOT_CALLEES = frozenset({ + "if", "while", "for", "foreach", "switch", "catch", "using", "lock", "typeof", "nameof", "sizeof", + "default", "checked", "unchecked", "fixed", "when", "return", "throw", "new", "is", "as", "in", + "await", "else", "case", "out", "ref", "var", "stackalloc", "not", "and", "or", +}) +_MODIFIERS = frozenset({"public", "private", "protected", "internal", "static", "readonly", "const", "volatile", + "override", "virtual", "abstract", "sealed", "async", "extern", "unsafe", "new", + "partial", "required", "ref", "file"}) +_EXPR_PREFIX_KEYWORDS = frozenset({"return", "throw", "new", "await", "in", "is", "as", "case", "when", + "yield", "else", "out", "ref", "typeof", "nameof", "not", "and", "or"}) +_TYPE_KEYWORDS = frozenset({"class", "struct", "interface", "record", "namespace"}) + + +@dataclass +class Invocation: + name: str + open_idx: int + close_idx: int + args_start: int # offset just after '(' + args_end: int # offset of ')' + + +@dataclass +class Member: + name: str + qualified: str + start: int + end: int + + +@dataclass +class SourceModel: + src: str + toks: list[Tok] + invocations: list[Invocation] + members: list[Member] + line_starts: list[int] = field(default_factory=list) + + @property + def callee_names(self) -> set[str]: + return {inv.name for inv in self.invocations} + + +def _match_brackets(toks: list[Tok]) -> dict[int, int]: + pairs: dict[int, int] = {} + stack: list[int] = [] + opens, closes = {"(": ")", "[": "]", "{": "}"}, {")", "]", "}"} + for i, t in enumerate(toks): + if t.kind != "punct": + continue + if t.text in opens: + stack.append(i) + elif t.text in closes: + if not stack or opens[toks[stack[-1]].text] != t.text: + raise TriageError(f"unbalanced '{t.text}' at offset {t.start}") + pairs[stack.pop()] = i + if stack: + raise TriageError(f"unclosed '{toks[stack[-1]].text}' at offset {toks[stack[-1]].start}") + return pairs + + +def _skip_generic_back(toks: list[Tok], i: int) -> int: + """``toks[i]`` is ``>``: return the index of its ``<`` (or ``i`` when it is not a generic list).""" + depth = 0 + for j in range(i, -1, -1): + t = toks[j] + if t.kind == "punct" and t.text == ">": + depth += 1 + elif t.kind == "punct" and t.text == "<": + depth -= 1 + if depth == 0: + return j + elif t.kind == "punct" and t.text in (";", "{", "}", "(", ")", "=>", "&&", "||"): + return i + return i + + +def _invocations(toks: list[Tok], pairs: dict[int, int]) -> list[Invocation]: + out: list[Invocation] = [] + for i, t in enumerate(toks): + if not (t.kind == "punct" and t.text == "(") or i == 0: + continue + j = i - 1 + if toks[j].kind == "punct" and toks[j].text == ">": + k = _skip_generic_back(toks, j) + if k == j or k == 0: + continue + j = k - 1 + name_tok = toks[j] + if name_tok.kind != "id" or name_tok.text in _NOT_CALLEES: + continue + close = pairs[i] + # A declaration (method, constructor, local function) is a type-ish token before the name and a + # body / expression body / constraint / ctor initializer / ';' after the parameter list. + prev = toks[j - 1] if j > 0 else None + after = toks[close + 1] if close + 1 < len(toks) else None + typeish_prev = prev is not None and ( + (prev.kind == "id" and prev.text not in _EXPR_PREFIX_KEYWORDS) + or (prev.kind == "punct" and prev.text in ("?", "]", ">"))) + bodyish_after = after is not None and ( + (after.kind == "punct" and after.text in ("{", "=>", ";")) + or (after.kind == "id" and after.text == "where") + or (after.kind == "punct" and after.text == ":" and close + 2 < len(toks) + and toks[close + 2].text in ("base", "this"))) + if typeish_prev and bodyish_after: + continue + out.append(Invocation(name_tok.text, i, close, t.end, toks[close].start)) + return out + + +def _members(toks: list[Tok], pairs: dict[int, int], lo: int, hi: int, prefix: str, out: list[Member]) -> None: + """Members of the container whose body tokens are ``toks[lo:hi]`` (exclusive of its braces).""" + i = lo + while i < hi: + t = toks[i] + if t.kind == "punct" and t.text == ";": + i += 1 + continue + # attributes + while i < hi and toks[i].kind == "punct" and toks[i].text == "[": + i = pairs[i] + 1 + if i >= hi: + break + start_idx = i + header_ids: list[Tok] = [] + name: Optional[str] = None + kind = "member" + end_idx = None + j = i + while j < hi: + u = toks[j] + if u.kind == "id": + header_ids.append(u) + if u.text in _TYPE_KEYWORDS: + kind = "container" + elif u.text == "enum": + kind = "enum" + if u.kind == "punct": + if u.text == "(" and name is None and kind == "member": + # `Name(` or `Name(` is a parameter list; any other `(` is a tuple type + # (`List<(int, int)> Of(…)`, `public (int, int) F()`) and names nothing. + k = j - 1 + if toks[k].kind == "punct" and toks[k].text == ">": + k = _skip_generic_back(toks, k) - 1 + if k >= start_idx and toks[k].kind == "id" and toks[k].text not in _MODIFIERS: + name = toks[k].text + j = pairs[j] + 1 + continue + if u.text in ("=", "=>"): + name = name or (header_ids[-1].text if header_ids else "?") + while j < hi and not (toks[j].kind == "punct" and toks[j].text == ";"): + j = pairs[j] + 1 if toks[j].kind == "punct" and toks[j].text in "([{" else j + 1 + end_idx = j + break + if u.text == ";": + name = name or (header_ids[-1].text if header_ids else "?") + end_idx = j + break + if u.text == "{": + close = pairs[j] + if kind == "container": + names = [h.text for h in header_ids] + kw = next(k for k, h in enumerate(names) if h in _TYPE_KEYWORDS) + cname = names[kw + 1] if kw + 1 < len(names) else "?" + if names[kw] == "record" and cname in ("struct", "class") and kw + 2 < len(names): + cname = names[kw + 2] + inner_prefix = prefix if names[kw] == "namespace" else f"{prefix}{cname}." + _members(toks, pairs, j + 1, close, inner_prefix, out) + end_idx = None + i = close + 1 + break + name = name or (header_ids[-1].text if header_ids else "?") + if kind == "enum": + names = [h.text for h in header_ids] + name = names[names.index("enum") + 1] if "enum" in names[:-1] else name + end_idx = close + # property with an initializer: `{ get; set; } = value;` + if close + 1 < hi and toks[close + 1].kind == "punct" and toks[close + 1].text == "=": + k = close + 1 + while k < hi and not (toks[k].kind == "punct" and toks[k].text == ";"): + k = pairs[k] + 1 if toks[k].kind == "punct" and toks[k].text in "([{" else k + 1 + end_idx = k + break + if u.text in "([" and u.text in "([{": + j = pairs[j] + 1 + continue + j += 1 + else: + # ran off the container (file-scoped namespace `namespace X;` handled by ';' above) + end_idx = hi - 1 + if end_idx is None: + continue + if kind == "container" and end_idx is not None and name is None: + name = "?" + end_idx = min(end_idx, len(toks) - 1) + out.append(Member(name or "?", f"{prefix}{name or '?'}", toks[start_idx].start, toks[end_idx].end)) + i = end_idx + 1 + + +_LINE_BREAK = re.compile(r"\r\n|\r|\n|
|
|\u0085") + + +def analyze(src: str) -> SourceModel: + toks = tokenize(src) + pairs = _match_brackets(toks) + members: list[Member] = [] + _members(toks, pairs, 0, len(toks), "", members) + starts = [0] + [m.end() for m in _LINE_BREAK.finditer(src)] + return SourceModel(src, toks, _invocations(toks, pairs), members, starts) + + +def offset_of(model: SourceModel, line: int, column: int, what: str) -> int: + """1-based (line, UTF-16 column) -> string offset.""" + if line < 1 or line > len(model.line_starts): + raise TriageError(f"{what}: line {line} is outside the source ({len(model.line_starts)} lines)") + i = model.line_starts[line - 1] + units = 0 + src = model.src + while units < column - 1: + if i >= len(src): + raise TriageError(f"{what}: column {column} runs past the end of the source") + units += 2 if ord(src[i]) > 0xFFFF else 1 + i += 1 + return i + + +# --------------------------------------------------------------------------- # +# ignore list +# --------------------------------------------------------------------------- # +def _args(model: SourceModel, inv: Invocation) -> list[list[Tok]]: + """The invocation's arguments as token lists split at depth-0 commas.""" + toks = model.toks + args: list[list[Tok]] = [[]] + depth = 0 + for t in toks[inv.open_idx + 1:inv.close_idx]: + if t.kind == "punct" and t.text in "([{": + depth += 1 + elif t.kind == "punct" and t.text in ")]}": + depth -= 1 + if depth == 0 and t.kind == "punct" and t.text == ",": + args.append([]) + continue + args[-1].append(t) + return args + + +def _operator_value(model: SourceModel, inv: Invocation, start: int, end: int) -> bool: + """``operator Value``: the mutant IS the ``value`` argument (index 1) of the call, that argument is a + single non-interpolated string literal spelling an operator or delimiter (non-empty, punctuation + only), and the token type (index 0) is a ``TokenType`` member outside the f-string family.""" + args = _args(model, inv) + if len(args) < 2 or len(args[1]) != 1: + return False + lit = args[1][0] + if lit.kind != "str" or lit.interpolated or not lit.value: + return False + if not (lit.start <= start and end <= lit.end): + return False + if any(ch not in string.punctuation for ch in lit.value): + return False + type_text = "".join(t.text for t in args[0]) + return re.fullmatch(r"TokenType\.(?!FString)\w+", type_text) is not None + + +FIELD_PREDICATES: dict[str, Callable[[SourceModel, Invocation, int, int], bool]] = { + "operator Value": _operator_value, +} + + +@dataclass +class IgnoreEntry: + mutator: str + within: str + reason: str + field_of: Optional[str] + regex: re.Pattern + + +def _top_level_alternatives(pattern: str) -> list[str]: + alts, depth, cur, i = [], 0, [], 0 + while i < len(pattern): + ch = pattern[i] + if ch == "\\" and i + 1 < len(pattern): + cur.append(pattern[i:i + 2]) + i += 2 + continue + if ch in "([": + depth += 1 + elif ch in ")]": + depth -= 1 + if ch == "|" and depth == 0: + alts.append("".join(cur)) + cur = [] + else: + cur.append(ch) + i += 1 + alts.append("".join(cur)) + return alts + + +def load_ignore(data: dict, universe: set[str]) -> list[IgnoreEntry]: + entries = data.get("entries") if isinstance(data, dict) else None + if not isinstance(entries, list): + raise TriageError("ignore list: expected an object with an 'entries' array") + out = [] + for n, e in enumerate(entries): + where = f"ignore entry {n}" + if not isinstance(e, dict): + raise TriageError(f"{where}: not an object") + unknown = set(e) - {"mutator", "within", "fieldOf", "reason"} + if unknown: + raise TriageError(f"{where}: unknown keys {sorted(unknown)}") + for key in ("mutator", "within", "reason"): + if not isinstance(e.get(key), str) or not e[key].strip(): + raise TriageError(f"{where}: '{key}' must be a non-empty string") + if e["mutator"] not in KNOWN_MUTATORS: + raise TriageError(f"{where}: mutator '{e['mutator']}' is not a Stryker identifier {sorted(KNOWN_MUTATORS)}") + field_of = e.get("fieldOf") + if field_of is not None and field_of not in FIELD_PREDICATES: + raise TriageError(f"{where}: fieldOf '{field_of}' is not one of {sorted(FIELD_PREDICATES)}") + try: + regex = re.compile(e["within"]) + except re.error as ex: + raise TriageError(f"{where}: within is not a regex: {ex}") from ex + for alt in _top_level_alternatives(e["within"]): + alt_re = re.compile(alt) + if not any(alt_re.fullmatch(name) for name in universe): + raise TriageError( + f"{where}: within alternative '{alt}' names no method invoked in any source " + f"({len(universe)} callee names scanned) — the entry would match nothing") + out.append(IgnoreEntry(e["mutator"], e["within"], e["reason"], field_of, regex)) + return out + + +# --------------------------------------------------------------------------- # +# classes +# --------------------------------------------------------------------------- # +_C1 = frozenset({"ReadNumber", "ReadHexNumber", "ReadBinaryNumber", "ReadOctalNumber", "ReadPrefixedIntegerSuffix"}) +_C3 = frozenset({"NextToken", "ConsumeLineBreak", "IsPreCloseNewline"}) +_C7 = frozenset({"Keywords", "_keywordNames"}) +CLASS_NAMES = ("C1", "C2", "C3", "C4", "C5", "C6", "C7", "unclassified") + + +def classify(file_name: str, member: str, mutator: str, original: str) -> str: + if member in _C7: + return "C7" + if mutator != "block" and re.search(r"\b(_column|[Ss]ourceLength)\b", original): + return "C2" + if mutator == "equality" and re.search(r"[<>]=?\s*[\w.]*\bLength\b", original): + return "C6" + if member in _C1: + return "C1" + if member in _C3 or "HoleTrivia" in member: + return "C3" + if file_name == "Lexer.LiteralLoss.cs": + return "C5" + if file_name == "Lexer.FStrings.cs": + return "C4" + return "unclassified" + + +# --------------------------------------------------------------------------- # +# triage +# --------------------------------------------------------------------------- # +def _cs_files(dirs: list[str]) -> list[Path]: + files: list[Path] = [] + for d in dirs: + p = Path(d) + if not p.is_dir(): + raise TriageError(f"--universe {d}: not a directory") + files.extend(sorted(p.rglob("*.cs"))) + return files + + +def _display_names(paths: list[str]) -> dict[str, str]: + if len(paths) <= 1: + return {p: Path(p).name for p in paths} + base = os.path.commonpath([str(Path(p).parent) for p in paths]) if paths else "" + return {p: os.path.relpath(p, base) if base else p for p in paths} + + +def triage(report: dict, ignore: dict, universe_dirs: Optional[list[str]] = None, with_classes: bool = False) -> dict: + files = report.get("files") if isinstance(report, dict) else None + if not isinstance(files, dict) or not files: + raise TriageError("report: no 'files' object (is this a mutation-testing-elements report?)") + + models: dict[str, SourceModel] = {} + for path, entry in files.items(): + src = entry.get("source") + if src is None: + try: + src = Path(path).read_text(encoding="utf-8-sig") + except OSError as ex: + raise TriageError(f"report: '{path}' has no source and cannot be read: {ex}") from ex + try: + models[path] = analyze(src) + except TriageError as ex: + raise TriageError(f"{path}: {ex}") from ex + + universe: set[str] = set() + for m in models.values(): + universe |= m.callee_names + for f in _cs_files(universe_dirs or []): + universe |= analyze(f.read_text(encoding="utf-8-sig")).callee_names + entries = load_ignore(ignore, universe) + + names = _display_names(list(files)) + raw = {s: 0 for s in _STATUS.values()} + listed = {s: 0 for s in SCORED} + buckets: dict[str, int] = {} + classes = {c: 0 for c in CLASS_NAMES} + class_mutants: dict[str, list[dict]] = {c: [] for c in CLASS_NAMES} + by_entry = [0] * len(entries) + mutants_out: list[dict] = [] + + for path, entry in files.items(): + model, fname = models[path], names[path] + for mut in entry.get("mutants", []): + mid = mut.get("id", "?") + what = f"{fname} mutant {mid}" + status = _STATUS.get(mut.get("status")) + if status is None: + raise TriageError(f"{what}: unknown status {mut.get('status')!r}") + raw[status] += 1 + try: + loc = mut["location"] + start = offset_of(model, int(loc["start"]["line"]), int(loc["start"]["column"]), what) + end = offset_of(model, int(loc["end"]["line"]), int(loc["end"]["column"]), what) + except (KeyError, TypeError, ValueError) as ex: + raise TriageError(f"{what}: malformed location: {ex}") from ex + mut_id = mutator_id(str(mut.get("mutatorName", ""))) + enclosing = sorted( + (inv for inv in model.invocations if inv.args_start <= start and end <= inv.args_end), + key=lambda inv: inv.args_end - inv.args_start) + member_hits = sorted((mb for mb in model.members if mb.start <= start and end <= mb.end), + key=lambda mb: mb.end - mb.start) + member = member_hits[0].name if member_hits else "-" + bucket_status = status + if status in SCORED: + for n, ig in enumerate(entries): + if ig.mutator != mut_id: + continue + hit = next((inv for inv in enclosing if ig.regex.fullmatch(inv.name)), None) + if hit is None: + continue + if ig.field_of and not FIELD_PREDICATES[ig.field_of](model, hit, start, end): + continue + bucket_status = "ignoredByList" + listed[status] += 1 + by_entry[n] += 1 + break + invocation = enclosing[0].name if enclosing else "-" + key = f"{fname}|{mut_id}|{invocation}|{bucket_status}" + buckets[key] = buckets.get(key, 0) + 1 + row = {"id": mid, "file": fname, "line": loc["start"]["line"], "column": loc["start"]["column"], + "mutator": mut_id, "member": member, "invocation": invocation, "status": bucket_status} + if bucket_status in UNDETECTED: + cls = classify(Path(fname).name, member, mut_id, model.src[start:end]) + classes[cls] += 1 + row["class"] = cls + if with_classes: + class_mutants[cls].append({k: row[k] for k in ("id", "file", "line", "column", "mutator", "member", "status")}) + mutants_out.append(row) + + total = sum(raw.values()) + after = {s: raw[s] - listed[s] for s in SCORED} + result = { + "score": score_of(raw), + "scoreIgnoringListed": score_of(after), + "total": total, + "killed": raw["killed"], + "survived": raw["survived"], + "timeout": raw["timeout"], + "noCoverage": raw["noCoverage"], + "compileError": raw["compileError"], + "runtimeError": raw["runtimeError"], + "pending": raw["pending"], + "ignoredByStryker": raw["ignoredByStryker"], + "ignoredByList": sum(listed.values()), + "afterList": after, + "listedByEntry": [{"within": e.within, "fieldOf": e.field_of, "matched": by_entry[n]} + for n, e in enumerate(entries)], + "buckets": dict(sorted(buckets.items())), + "classes": classes, + "mutants": mutants_out, + } + if with_classes: + result["classMutants"] = class_mutants + return result + + +def main(argv: Optional[list[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--report", required=True, help="Stryker mutation-report.json") + ap.add_argument("--ignore", required=True, help="ignore list (e.g. build_tools/stryker/lexer-ignore.json)") + ap.add_argument("--classes", action="store_true", help="also list the undetected mutants of each class") + ap.add_argument("--universe", action="append", default=[], + help="extra directory whose *.cs files count as sources for the within-name check") + args = ap.parse_args(argv) + try: + report = json.loads(Path(args.report).read_text(encoding="utf-8-sig")) + ignore = json.loads(Path(args.ignore).read_text(encoding="utf-8-sig")) + result = triage(report, ignore, args.universe, args.classes) + except (OSError, json.JSONDecodeError, TriageError) as ex: + print(f"triage: error: {ex}", file=sys.stderr) + return 2 + result.pop("mutants") + json.dump(result, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build_tools/tests/test_number_oracle.py b/build_tools/tests/test_number_oracle.py new file mode 100644 index 000000000..f52bd1a16 --- /dev/null +++ b/build_tools/tests/test_number_oracle.py @@ -0,0 +1,106 @@ +"""Tests for the CPython numeric-literal oracle (build_tools/differential_lex, #2300). + +The oracle is the CPython half of ``NumericLiteralTwinTests``. These pin the cells the P68 +plan measured on CPython 3.12 (its "Measured cells" table) at the oracle's altitude — +``ast.parse(source, mode="eval")`` is one numeric ``Constant`` or the spelling is refused — +plus the JSONL batch protocol that keeps one ``python3`` process serving every spelling. +""" + +from __future__ import annotations + +import json + +import pytest + +from build_tools.differential_lex import number_oracle + + +# --------------------------------------------------------------------------- # +# verdicts: the measured cells +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "source, kind, value", + [ + (".5e-3", "float", "0.0005"), + ("0x_FF", "int", "255"), + ("0_0", "int", "0"), + ("0XFF", "int", "255"), + ("1E+5", "float", "100000.0"), + ("1_000.000_1", "float", "1000.0001"), + ("00", "int", "0"), + # Trailing-dot floats are numbers to CPython; Sharpy refuses them by design + # (float_literals.md) and the twin routes them to its Sharpy-only rows. + ("1.", "float", "1.0"), + ("1.e5", "float", "100000.0"), + # Leading zeros are allowed in a float's integer part. + ("01.5", "float", "1.5"), + ("0B101", "int", "5"), + ("0O755", "int", "493"), + ("1e500", "float", "inf"), + # Integers of four or more digits: the value is plain decimal digits (no grouping), and + # a value past Int64 is exact. + ("1_000", "int", "1000"), + ("0xFFFF", "int", "65535"), + ("123_456_789_012_345_678_901", "int", "123456789012345678901"), + ], +) +def test_numeric_constant_is_ok_with_kind_and_repr(source, kind, value): + assert number_oracle.classify(source) == {"ok": True, "kind": kind, "value": value} + + +@pytest.mark.parametrize( + "source, message", + [ + (".1__2", "invalid decimal literal"), + ("0b1_", "invalid binary literal"), + ("0o8", "invalid digit '8' in octal literal"), + ("01", "leading zeros in decimal integer literals are not permitted"), + # ast.parse tokenizes `.5e` as NUMBER `.5` + NAME `e`; at the constant altitude it is + # simply refused, which is why the twin does not compare token boundaries. + (".5e", "invalid decimal literal"), + ("1e5.", "invalid syntax"), + ], +) +def test_refused_spelling_reports_the_syntax_error(source, message): + verdict = number_oracle.classify(source) + assert verdict["ok"] is False + assert verdict["error"].startswith(message) + + +def test_imaginary_constant_is_refused_as_imaginary(): + assert number_oracle.classify("1j") == {"ok": False, "error": "imaginary"} + + +@pytest.mark.parametrize("source", ["_1", "0xFFe+5", "-1", "True", "x"]) +def test_non_numeric_constant_programs_are_refused(source): + # A Name, a BinOp, a UnaryOp and a bool constant are not ONE numeric literal. + verdict = number_oracle.classify(source) + assert verdict["ok"] is False + assert verdict["error"] != "imaginary" + + +# --------------------------------------------------------------------------- # +# batch protocol +# --------------------------------------------------------------------------- # +def test_run_batch_preserves_ids_out_of_order_and_skips_blank_lines(): + lines = [ + json.dumps({"id": 7, "source": ".5e-3"}), + "", + json.dumps({"id": 3, "source": "01"}), + json.dumps({"id": 5, "source": "1j"}), + ] + verdicts = [json.loads(line) for line in number_oracle.run_batch(lines)] + + assert [v["id"] for v in verdicts] == [7, 3, 5] + by_id = {v["id"]: v for v in verdicts} + assert by_id[7] == {"id": 7, "ok": True, "kind": "float", "value": "0.0005"} + assert by_id[3]["ok"] is False + assert by_id[5] == {"id": 5, "ok": False, "error": "imaginary"} + + +def test_main_reads_a_batch_file(tmp_path, capsys): + batch = tmp_path / "batch.jsonl" + batch.write_text(json.dumps({"id": 1, "source": "0x_FF"}) + "\n", encoding="utf-8") + assert number_oracle.main(["number_oracle.py", "--batch", str(batch)]) == 0 + out = capsys.readouterr().out.strip().splitlines() + assert [json.loads(line) for line in out] == [{"id": 1, "ok": True, "kind": "int", "value": "255"}] diff --git a/build_tools/tests/test_stryker_triage.py b/build_tools/tests/test_stryker_triage.py new file mode 100644 index 000000000..765bd1f0a --- /dev/null +++ b/build_tools/tests/test_stryker_triage.py @@ -0,0 +1,536 @@ +"""Tests for build_tools/stryker/triage.py — Stryker survivor triage with the R-GB ignore list as data (#2300, P68). + +The fixture report is hand-built: five small C# sources shaped like the real Lexer partials, and one +mutant per bucket kind (each Stryker status, each ignore entry, each fieldOf arm, each C1–C7 class rule +and the unclassified fallback). Locations are computed here from the needle text, 1-based, independently +of the script's own offset arithmetic. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / "build_tools" / "stryker" / "triage.py" +IGNORE_FILE = REPO / "build_tools" / "stryker" / "lexer-ignore.json" +LEXER_DIR = REPO / "src" / "Sharpy.Compiler" / "Lexer" + +_spec = importlib.util.spec_from_file_location("stryker_triage", SCRIPT) +triage = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = triage # dataclasses resolve annotations through sys.modules +_spec.loader.exec_module(triage) + + +# --------------------------------------------------------------------------- # +# fixture sources +# --------------------------------------------------------------------------- # +LEXER_CS = """namespace Sharpy.Compiler.Lexer; + +public partial class Lexer +{ + private static readonly Dictionary Keywords = new() + { + { "union", TokenType.Union }, + { "def", TokenType.Def }, + }; + + private static readonly IReadOnlySet _keywordNames = new HashSet(Keywords.Keys) { "self" }; + + public Lexer(string source, ILogger logger) + { + _source = source; + _logger.LogInfo($"Lexer initialized, source length: {source.Length}"); + } + + private LexerAbortException ReportError(string message, int line, int column, string code) + { + _diagnostics.AddError(message, line, column, code); + return new LexerAbortException(); + } + + public List TokenizeAll() + { + if (_diagnostics.ErrorCount >= MaxErrors) + _diagnostics.AddWarning($"Too many errors ({MaxErrors}); further errors suppressed.", _line, _column); + return _tokens; + } + + public Token NextToken() + { + if (_position >= _source.Length) + return CreateToken(TokenType.Eof, "", _line, _column, _position); + var current = _source[_position]; + if (current == '\\n') + { + ConsumeLineBreak(); + return CreateToken(TokenType.Newline, "\\n", _line, _column, _position); + } + return char.IsDigit(current) ? ReadNumber() : ReadOperatorOrDelimiter(); + } + + private void ConsumeLineBreak() + { + if (_source[_position] == '\\r' && _position + 1 < _source.Length && _source[_position + 1] == '\\n') + _position++; + _position++; + _line++; + _column = 1; + } + + private Token ReadOperatorOrDelimiter() + { + var c = _source[_position]; + _position++; + _column++; + return c switch + { + '+' => CreateToken(TokenType.Plus, "+", _line, _column - 1, _position - 1), + '{' => CreateToken(TokenType.LeftBrace, "{", _line, _column - 1, _position - 1), + _ => throw ReportError($"Unexpected character: '{c}'", _line, _column, "SPY0001") + }; + } + + private Token CreateToken(TokenType type, string value, int startLine, int startColumn, int startPosition, + int? sourceLength = null) + { + return new Token(type, value, startLine, startColumn, startPosition) { SourceLength = sourceLength ?? value.Length }; + } +} +""" + +LITERALS_CS = """namespace Sharpy.Compiler.Lexer; + +public partial class Lexer +{ + private Token ReadNumber() + { + var sb = new StringBuilder(); + while (_position < _source.Length && (char.IsDigit(_source[_position]) || _source[_position] == '_')) + { + if (_source[_position] == '_' && _position + 1 < _source.Length && _source[_position + 1] == '_') + throw ReportError("Consecutive underscores in number literal", _line, _column, "SPY0007"); + sb.Append(_source[_position]); + _position++; + _column++; + } + return CreateToken(TokenType.Integer, sb.ToString(), _line, _column, _position); + } +} +""" + +FSTRINGS_CS = """namespace Sharpy.Compiler.Lexer; + +public partial class Lexer +{ + private class FStringContext + { + public bool IsTriple { get; init; } + public int Depth { get; set; } + public string Kind => IsTriple ? "f-string (triple)" : "f-string"; + } + + private bool IsPreCloseNewline(FStringContext context) + { + return context.IsTriple && _source[_position] == '\\n'; + } + + private void SkipHoleTrivia(FStringField field) + { + while (_source[_position] == ' ') + _position++; + } + + private Token NextFStringToken(FStringContext context) + { + if (context.Depth < 0) + throw ReportUnclosedField(context); + if (_source[_position] == '}' && context.Depth == 0) + throw ReportFieldEndedByQuote(context, $"{context.Kind}: expecting '}}'"); + return CreateToken(TokenType.FStringExprEnd, "}", _line, _column, _position); + } +} +""" + +LITERAL_LOSS_CS = """namespace Sharpy.Compiler.Lexer; + +public partial class Lexer +{ + private static bool DroppedLineContinues(string text) + { + var depth = 0; + foreach (var ch in text) + { + if (ch == '(') depth++; + } + return depth > 0; + } +} +""" + +INDENTATION_CS = """namespace Sharpy.Compiler.Lexer; + +public partial class Lexer +{ + private int MeasureIndentation() + { + var indent = 0; + while (_source[_position] == ' ') { indent++; _position++; } + if (indent % 4 != 0) + throw ReportError($"Indentation must be multiple of 4 spaces (found {indent})", _line, 1, "SPY0003"); + return indent; + } +} +""" + +ROOT = "/repo/src/Sharpy.Compiler/Lexer/" +SOURCES = { + "Lexer.cs": LEXER_CS, + "Lexer.Literals.cs": LITERALS_CS, + "Lexer.FStrings.cs": FSTRINGS_CS, + "Lexer.LiteralLoss.cs": LITERAL_LOSS_CS, + "Lexer.Indentation.cs": INDENTATION_CS, +} + + +def _loc(src: str, needle: str, nth: int = 0) -> dict: + at = -1 + for _ in range(nth + 1): + at = src.index(needle, at + 1) + end = at + len(needle) + + def lc(off: int) -> dict: + line = src.count("\n", 0, off) + 1 + col = off - (src.rfind("\n", 0, off) + 1) + 1 + return {"line": line, "column": col} + + return {"start": lc(at), "end": lc(end)} + + +# (id, file, needle, nth, mutatorName, status, expected bucket status, expected invocation, expected class) +MUTANTS = [ + ("killed", "Lexer.cs", "_line++;", 0, "Statement mutation", "Killed", "killed", "-", None), + ("timeout", "Lexer.FStrings.cs", "_position++;", 0, "Statement mutation", "Timeout", "timeout", "-", None), + ("nocov", "Lexer.LiteralLoss.cs", "depth++", 0, + "PostIncrementExpression to PostDecrementExpression mutation", "NoCoverage", "noCoverage", "-", "C5"), + ("ignored-by-stryker", "Lexer.Indentation.cs", "return indent;", 0, "Statement mutation", "Ignored", + "ignoredByStryker", "-", None), + ("compile-error", "Lexer.cs", "_column = 1;", 0, "Statement mutation", "CompileError", "compileError", "-", None), + ("runtime-error", "Lexer.cs", "_position++;", 1, "Statement mutation", "RuntimeError", "runtimeError", "-", None), + # ignore entry 0: message text (a Survived and a Killed one — both leave the score) + ("report-error-msg", "Lexer.Literals.cs", '"Consecutive underscores in number literal"', 0, "String mutation", + "Survived", "ignoredByList", "ReportError", None), + ("loginfo-msg", "Lexer.cs", '$"Lexer initialized, source length: {source.Length}"', 0, "String mutation", + "Killed", "ignoredByList", "LogInfo", None), + ("field-ended-msg", "Lexer.FStrings.cs", '$"{context.Kind}: expecting \'}}\'"', 0, "String mutation", + "Survived", "ignoredByList", "ReportFieldEndedByQuote", None), + # ignore entry 1: operator Value — and the two fieldOf arms that must NOT match + ("op-value", "Lexer.cs", '"+"', 0, "String mutation", "Survived", "ignoredByList", "CreateToken", None), + ("newline-value", "Lexer.cs", '"\\n"', 0, "String mutation", "Survived", "survived", "CreateToken", "C3"), + ("fstring-value", "Lexer.FStrings.cs", '"}"', 0, "String mutation", "Survived", "survived", "CreateToken", "C4"), + # classes + ("keyword", "Lexer.cs", '"union"', 0, "String mutation", "Survived", "survived", "-", "C7"), + ("column", "Lexer.Literals.cs", "_column++;", 0, "Statement mutation", "Survived", "survived", "-", "C2"), + ("eof-bound", "Lexer.Literals.cs", "_position < _source.Length", 0, "Equality mutation", "Survived", + "survived", "-", "C6"), + ("number", "Lexer.Literals.cs", "sb.Append(_source[_position]);", 0, "Statement mutation", "Survived", + "survived", "-", "C1"), + ("preclose", "Lexer.FStrings.cs", "context.IsTriple && _source[_position] == '\\n'", 0, "Logical mutation", + "Survived", "survived", "-", "C3"), + ("fstring-arm", "Lexer.FStrings.cs", "context.Depth == 0", 0, "Equality mutation", "Survived", "survived", "-", + "C4"), + ("literal-loss", "Lexer.LiteralLoss.cs", "depth > 0", 0, "Equality mutation", "Survived", "survived", "-", "C5"), + ("indent", "Lexer.Indentation.cs", "indent % 4", 0, "Arithmetic mutation", "Survived", "survived", "-", + "unclassified"), +] + + +def _report() -> dict: + files = {ROOT + name: {"language": "cs", "source": src, "mutants": []} for name, src in SOURCES.items()} + for mid, fname, needle, nth, mutator, status, *_ in MUTANTS: + files[ROOT + fname]["mutants"].append({ + "id": mid, "mutatorName": mutator, "replacement": "?", + "location": _loc(SOURCES[fname], needle, nth), "status": status, + }) + return {"schemaVersion": "2", "thresholds": {"high": 80, "low": 60}, "files": files} + + +def _ignore() -> dict: + return json.loads(IGNORE_FILE.read_text()) + + +def _by_id(result: dict) -> dict: + return {row["id"]: row for row in result["mutants"]} + + +# --------------------------------------------------------------------------- # +# the fixture report +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("mid,fname,needle,nth,mutator,status,bucket,invocation,cls", MUTANTS, + ids=[m[0] for m in MUTANTS]) +def test_bucket_invocation_and_class(mid, fname, needle, nth, mutator, status, bucket, invocation, cls): + row = _by_id(triage.triage(_report(), _ignore()))[mid] + assert row["status"] == bucket + assert row["invocation"] == invocation + assert row.get("class") == cls + key = f"{fname}|{triage.mutator_id(mutator)}|{invocation}|{bucket}" + assert triage.triage(_report(), _ignore())["buckets"][key] >= 1 + + +def test_every_bucket_kind_is_covered_by_the_fixture(): + statuses = {m[6] for m in MUTANTS} + assert statuses == {"killed", "survived", "timeout", "noCoverage", "ignoredByStryker", "compileError", + "runtimeError", "ignoredByList"} + assert {m[8] for m in MUTANTS} - {None} == set(triage.CLASS_NAMES) + + +def test_counts_and_score_arithmetic(): + r = triage.triage(_report(), _ignore()) + assert r["total"] == len(MUTANTS) == 20 + assert (r["killed"], r["survived"], r["timeout"], r["noCoverage"]) == (2, 13, 1, 1) + assert (r["compileError"], r["runtimeError"], r["ignoredByStryker"], r["pending"]) == (1, 1, 1, 0) + assert r["ignoredByList"] == 4 + assert r["afterList"] == {"killed": 1, "survived": 10, "timeout": 1, "noCoverage": 1} + # Stryker: (killed + timeout) / (killed + timeout + survived + noCoverage) + assert r["score"] == round(3 * 100 / 17, 2) == 17.65 + assert r["scoreIgnoringListed"] == round(2 * 100 / 13, 2) == 15.38 + assert sum(r["buckets"].values()) == r["total"] + assert [e["matched"] for e in r["listedByEntry"]] == [3, 1] + + +def test_classes_count_every_undetected_mutant_once(): + r = triage.triage(_report(), _ignore()) + assert r["classes"] == {"C1": 1, "C2": 1, "C3": 2, "C4": 2, "C5": 2, "C6": 1, "C7": 1, "unclassified": 1} + assert sum(r["classes"].values()) == r["afterList"]["survived"] + r["afterList"]["noCoverage"] + + +def test_classes_flag_lists_the_mutants_per_class(): + r = triage.triage(_report(), _ignore(), with_classes=True) + assert [m["id"] for m in r["classMutants"]["C7"]] == ["keyword"] + assert sorted(m["id"] for m in r["classMutants"]["C3"]) == ["newline-value", "preclose"] + assert "classMutants" not in triage.triage(_report(), _ignore()) + + +# --------------------------------------------------------------------------- # +# the guard: `within` names the CALL, both directions +# --------------------------------------------------------------------------- # +def test_within_report_error_lists_the_message_mutant_and_within_read_number_does_not(): + listed = _by_id(triage.triage(_report(), _ignore()))["report-error-msg"] + assert listed["status"] == "ignoredByList" + assert "class" not in listed + + mutated = _ignore() + mutated["entries"][0]["within"] = "ReadNumber" # ReadNumber is invoked (in NextToken), so the entry loads + row = _by_id(triage.triage(_report(), mutated))["report-error-msg"] + assert row["status"] == "survived" + assert row["class"] == "C1" # the mutant sits in ReadNumber's body, inside the ReportError call + + +def test_field_of_is_required_for_the_create_token_entry(): + unrestricted = _ignore() + del unrestricted["entries"][1]["fieldOf"] + rows = _by_id(triage.triage(_report(), unrestricted)) + assert rows["newline-value"]["status"] == "ignoredByList" + assert rows["fstring-value"]["status"] == "ignoredByList" + + +# --------------------------------------------------------------------------- # +# loud failures +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("within,absent", [("ReadNumbr", "ReadNumbr"), ("ReportError|LogDebug", "LogDebug")]) +def test_within_naming_no_invoked_method_fails_loudly(within, absent): + bad = _ignore() + bad["entries"][0]["within"] = within + with pytest.raises(triage.TriageError, match=f"'{absent}' names no method invoked"): + triage.triage(_report(), bad) + + +def test_cli_exits_nonzero_with_a_message_on_an_absent_within(tmp_path): + bad = _ignore() + bad["entries"][0]["within"] = "ReadNumbr" + (tmp_path / "r.json").write_text(json.dumps(_report())) + (tmp_path / "i.json").write_text(json.dumps(bad)) + proc = subprocess.run([sys.executable, str(SCRIPT), "--report", str(tmp_path / "r.json"), + "--ignore", str(tmp_path / "i.json")], capture_output=True, text=True) + assert proc.returncode == 2 + assert "ReadNumbr" in proc.stderr + assert proc.stdout == "" + + +def test_cli_prints_the_contract_keys(tmp_path): + (tmp_path / "r.json").write_text(json.dumps(_report())) + proc = subprocess.run([sys.executable, str(SCRIPT), "--report", str(tmp_path / "r.json"), + "--ignore", str(IGNORE_FILE), "--classes"], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + out = json.loads(proc.stdout) + for key in ("score", "scoreIgnoringListed", "total", "killed", "survived", "timeout", "ignoredByStryker", + "ignoredByList", "buckets", "classes", "classMutants"): + assert key in out + assert out["score"] == 17.65 + + +@pytest.mark.parametrize("mutate,message", [ + (lambda e: e.pop("reason"), "'reason' must be a non-empty string"), + (lambda e: e.update(mutator="strings"), "not a Stryker identifier"), + (lambda e: e.update(fieldOf="operator Text"), "fieldOf 'operator Text'"), + (lambda e: e.update(within_methods="x"), "unknown keys"), +]) +def test_malformed_ignore_entry_fails_loudly(mutate, message): + bad = _ignore() + mutate(bad["entries"][1]) + with pytest.raises(triage.TriageError, match=message): + triage.triage(_report(), bad) + + +def test_unknown_status_and_out_of_range_location_fail_loudly(): + r = _report() + r["files"][ROOT + "Lexer.cs"]["mutants"][0]["status"] = "Survivd" + with pytest.raises(triage.TriageError, match="unknown status"): + triage.triage(r, _ignore()) + r = _report() + r["files"][ROOT + "Lexer.cs"]["mutants"][0]["location"]["start"]["line"] = 999 + with pytest.raises(triage.TriageError, match="line 999 is outside"): + triage.triage(r, _ignore()) + + +def test_unknown_mutator_name_is_counted_not_dropped(): + r = _report() + r["files"][ROOT + "Lexer.cs"]["mutants"][0]["mutatorName"] = "Brand new mutation" + out = triage.triage(r, _ignore()) + assert out["total"] == 20 + assert "Lexer.cs|unknown:Brand new mutation|-|killed" in out["buckets"] + + +def test_missing_source_is_read_from_disk(tmp_path): + r = _report() + path = tmp_path / "Lexer.Indentation.cs" + path.write_text(INDENTATION_CS) + entry = r["files"].pop(ROOT + "Lexer.Indentation.cs") + del entry["source"] + r["files"][str(path)] = entry + rows = _by_id(triage.triage(r, _ignore())) + assert rows["indent"]["class"] == "unclassified" + + +# --------------------------------------------------------------------------- # +# mutator names and the committed data against the real Lexer sources +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("display,ident", [ + ("String mutation", "string"), + ("String Method Mutation (Replace Trim() with TrimStart())", "stringmethod"), + ("Equality mutation", "equality"), + ("Negate expression", "boolean"), + ("Block removal mutation", "block"), + ("Conditional (true) mutation", "conditional"), + ("Null coalescing mutation (remove left)", "nullcoalescing"), + ("AddAssignmentExpression to SubtractAssignmentExpression mutation", "assignment"), + ("PostIncrementExpression to PostDecrementExpression mutation", "update"), + ("PreDecrementExpression to PreIncrementExpression mutation", "update"), + ("UnaryMinusExpression to UnaryPlusExpression mutation", "unary"), + ("LogicalNotExpression to un-LogicalNotExpression mutation", "unary"), + ("Linq method mutation (First() to Last())", "linq"), +]) +def test_mutator_display_names_map_to_config_identifiers(display, ident): + assert triage.mutator_id(display) == ident + + +def _string_mutants_over_real_lexer() -> dict: + """A synthetic report: one Survived String mutation at every string literal in src/Sharpy.Compiler/Lexer.""" + import bisect + files = {} + for f in sorted(LEXER_DIR.glob("*.cs")): + src = f.read_text(encoding="utf-8-sig") + model = triage.analyze(src) + + def lc(off: int, starts=model.line_starts) -> dict: + i = bisect.bisect_right(starts, off) - 1 + return {"line": i + 1, "column": off - starts[i] + 1} + + mutants = [{"id": f"{f.name}:{k}", "mutatorName": "String mutation", "status": "Survived", + "location": {"start": lc(t.start), "end": lc(t.end)}} + for k, t in enumerate(model.toks) if t.kind == "str"] + files[str(f)] = {"language": "cs", "source": src, "mutants": mutants} + return {"files": files} + + +@pytest.mark.skipif(not LEXER_DIR.is_dir(), reason="needs the Sharpy.Compiler sources") +def test_committed_ignore_list_against_the_real_lexer(): + data = _ignore() + assert len(data["entries"]) == 2 + assert all(e["reason"].strip() for e in data["entries"]) + report = _string_mutants_over_real_lexer() + out = triage.triage(report, data) # every within alternative names a real callee, or this raises + rows = out["mutants"] + src = {Path(p).name: e["source"].splitlines() for p, e in report["files"].items()} + + def text(row): + return src[row["file"]][row["line"] - 1] + + listed = [r for r in rows if r["status"] == "ignoredByList"] + kept = [r for r in rows if r["status"] == "survived"] + # positive controls: an operator Value, a ReportError message, a LogInfo message + assert any("TokenType.Plus, \"+\"" in text(r) for r in listed) + assert any(r["invocation"] == "ReportError" for r in listed) + assert any(r["invocation"] == "LogInfo" for r in listed) + # negative controls: Eof's "", Newline's "\n", the f-string family's "}" and the keyword table stay scored + assert any("TokenType.Eof, \"\"" in text(r) for r in kept) + assert any("TokenType.Newline, \"\\n\"" in text(r) for r in kept) + assert any("TokenType.FStringExprEnd, \"}\"" in text(r) for r in kept) + assert any("{ \"union\", TokenType.Union }" in text(r) for r in kept) + assert all(r["invocation"] != "ReportError" for r in kept) + + +# --------------------------------------------------------------------------- # +# the guard: `within` is a FULL match over the callee name, and the list applies only to scored mutants +# --------------------------------------------------------------------------- # +_PREFIX_SRC = """namespace Sharpy.Compiler.Lexer; + +public partial class Lexer +{ + private void Trace() + { + LogInfo("listed message"); + LogInfoVerbose("not a LogInfo call"); + ReportError("compile-error message", 1, 1, "SPY0001"); + } +} +""" + + +def _single_file_report(mutants: list[tuple[str, str, str, str]]) -> dict: + """(id, needle, mutatorName, status) over _PREFIX_SRC.""" + rows = [{"id": mid, "mutatorName": mutator, "replacement": "?", "location": _loc(_PREFIX_SRC, needle), + "status": status} for mid, needle, mutator, status in mutants] + return {"schemaVersion": "2", "files": {ROOT + "Lexer.cs": {"language": "cs", "source": _PREFIX_SRC, + "mutants": rows}}} + + +def test_within_matches_the_whole_callee_name_never_a_prefix(): + report = _single_file_report([ + ("listed", '"listed message"', "String mutation", "Survived"), + ("prefix", '"not a LogInfo call"', "String mutation", "Survived"), + ]) + ignore = {"entries": [{"mutator": "string", "within": "LogInfo", "reason": "message text"}]} + rows = _by_id(triage.triage(report, ignore)) + assert rows["listed"]["status"] == "ignoredByList" + assert rows["prefix"]["status"] == "survived", "LogInfoVerbose is not LogInfo: `within` must fullmatch the callee" + assert rows["prefix"]["invocation"] == "LogInfoVerbose" + + +def test_ignore_list_never_reclassifies_an_unscored_mutant(): + report = _single_file_report([ + ("compile-error", '"compile-error message"', "String mutation", "CompileError"), + ("scored", '"listed message"', "String mutation", "Killed"), + ]) + ignore = {"entries": [{"mutator": "string", "within": "ReportError|LogInfo", "reason": "message text"}]} + r = triage.triage(report, ignore) + rows = _by_id(r) + assert rows["compile-error"]["status"] == "compileError", "a CompileError mutant is outside the score either way" + assert rows["scored"]["status"] == "ignoredByList" + assert (r["compileError"], r["ignoredByList"]) == (1, 1) + assert [e["matched"] for e in r["listedByEntry"]] == [1] diff --git a/docs/audits/stryker-lexer-pilot-2026-10-11.md b/docs/audits/stryker-lexer-pilot-2026-10-11.md index 38de00236..c54e3a8ac 100644 --- a/docs/audits/stryker-lexer-pilot-2026-10-11.md +++ b/docs/audits/stryker-lexer-pilot-2026-10-11.md @@ -111,3 +111,67 @@ Two C1 survivors were applied by hand in a scratch worktree at 258a7b66a, in the - Ignore-list the message-text and logger-string mutants (the B and C buckets) with a reason. - Remove the `TokenizeAll` CS0165 safe-mode hole, so the score is not inflated by what Stryker could not compile. - **Survivors:** classes C1–C7 go to the next `/batch-issues` as class issues, never one issue per mutant. + +## Ignore list (P68) + +P68 (#2300) answered two of the pilot's preconditions. It removed the `TokenizeAll` CS0165 hole in e621d9e94, and it turned the message-text buckets into data with a reason per entry. + +- **The data:** `build_tools/stryker/lexer-ignore.json`, applied by `build_tools/stryker/triage.py`. The triage script prints `score`, `scoreIgnoringListed`, the survivor buckets and the C1–C7 class of each mutated method. +- **Not used:** a global `ignore-mutations: ["string"]`, because it would also hide the `Keywords` table keys (C7) and the numeric-suffix roster (C1). `// Stryker disable` comments in production files are not used either. + +### What `ignore-methods` skips (measured) + +The Stryker docs do not say whether `ignore-methods` also skips mutations in the *arguments* of a matched call. A probe measured it. + +```bash +bash build_tools/stryker/run.sh build_tools/stryker/lexer-probe.json +``` + +- **Probe config:** `build_tools/stryker/lexer-probe.json` is `lexer.json` with these changes: + - `mutate` is `Token.cs`, `LiteralSpans.cs` and `Lexer.Indentation.cs`. Neither of the first two files calls `ReportError`, so the probe was widened to `Lexer.Indentation.cs`, as the plan allowed. + - `"ignore-methods": ["ReportError", "LogInfo", "AddWarning", "AddError"]`. +- **Where it ran:** a detached worktree at e621d9e94, through `run.sh`, holding the lock. +- **Result: argument mutations are skipped.** + - `Lexer.Indentation.cs` has four `throw ReportError("…", _line, 1, code)` sites (lines 47, 51, 55, 72). + - Each site's message-string mutant is reported `Ignored`, with the reason "Removed by method filter". That is 4 of 4. The json still records them, so the triage script can count them. + - Log line: `2282 mutants got status Ignored. Reason: Removed by method filter`. That count covers all of `Sharpy.Compiler`, because Stryker applies the method filter before the `mutate` filter. It is not a Lexer figure. +- **The `TokenizeAll` hole is closed.** Stryker compiles every mutant of `Sharpy.Compiler` before it applies `mutate` (73,308 created). The probe log has 85 "Safe Mode!" lines, and none names a method in `Lexer/`. At 258a7b66a, safe mode removed every mutation in `TokenizeAll` (CS0165 on `token`). After e621d9e94 moved the declaration inside the `try`, the method's mutants compile and the full re-run will score them. +- **Not skipped:** two `_logger.LogIndentChange(…)` statement-removal mutants (`Lexer.Indentation.cs:175`, `:191`) survived. `LogIndentChange` is not in the list. Whether `ignore-methods` also drops the statement-removal mutant of the matched call itself was not probed. That is the next entry to measure. +- **Score: 82.16 % @ e621d9e94 (measured)** over the three files. + - 144 killed, 31 timeout, 38 survived, 0 without coverage. + - 16 compile errors, 33 ignored: 29 block-already-covered and 4 method filter. + - `triage.py` gives `scoreIgnoringListed` 82.16 %. The data file's `within ReportError|…` entry matched 0, because Stryker had already removed those mutants. + +| File | Killed | Timeout | Survived | Ignored | Compile error | +|---|---|---|---|---|---| +| `Lexer.Indentation.cs` | 94 | 27 | 13 | 27 (4 method filter) | 6 | +| `LiteralSpans.cs` | 46 | 4 | 12 | 5 | 10 | +| `Token.cs` | 4 | 0 | 13 | 1 | 0 | + +- **Cost (measured):** + - **Wall clock:** 5,835 s (1 h 37 min), Stryker `Time Elapsed` 1:37:14. + - **Setup:** 24.5 min, including 17 min of per-test coverage capture over 2,217 tests. The pilot's whole setup was 12.5 min over 1,574 tests. + - **Testing:** 213 mutants in 72.8 min, about 20.5 s of wall clock per mutant at concurrency 4. The pilot's figure was about 8.8 s of wall clock per mutant (35 s per session ÷ 4). + - **Peak RSS:** 5.55 GB, the maximum single-process RSS that `/usr/bin/time -l` reports for the run. This is not the pilot's whole-machine sum. +- **What the cost means for the full re-run (derived, not measured).** + - The P68 harnesses cover most of the Lexer, so under `perTest` almost every mutant now runs the corpus twin, the injection differential and the CPython numeric twin. + - At the probe's rate, the pilot's 2,929 tested mutants would take about 16.7 h, against the pilot's 7 h 10 min. + - Timeouts are 15 % of the probe's tested mutants. Each one costs the full timeout. In `Lexer.Indentation.cs` they are mostly infinite loops from a deleted `tempPos++` or `_position++`. + - The full re-run (P68 Phase 5 Task 3) needs an owner slot sized from this, not from the pilot's 7 h. + +**Decision:** `lexer.json` will carry `ignore-methods` for the message and logger sinks. It is added in P68 Phase 5 Task 3, before the re-run; `lexer.json` has no `ignore-methods` key @ 0d58024dd. The key is measured to skip the argument mutations, and those mutants are then not tested at all. The data file stays authoritative for what `ignore-methods` cannot express: the `CreateToken` operator `Value` entry, and sinks the config does not list, such as `ReportUnclosedField` and `ReportFieldEndedByQuote`. + +### Per-class mutation control (P68 Phase 5 Task 2) + +This control applies one recorded survivor-class mutant at a time in a private worktree at e621d9e94, and runs the Lexer filter for each (`FullyQualifiedName~Sharpy.Compiler.Tests.Lexer.|FullyQualifiedName~Sharpy.Compiler.Tests.Properties.Lexer.`, 2217 tests). Each file is restored by a plain copy afterwards. Script and logs are in `.claude/tmp/p68/lead/ctl/` (measured). + +| Class | Mutant | Lexer filter | +|---|---|---| +| C1 numeric literals | `ReadHexNumber` trailing-underscore check off (`Lexer.Literals.cs`) | red, 1 failed (`NumericLiteralTwinTests.SharedGrammar_AgreesWithCPython`) | +| C2 token positions | `>>` advances `_column` by 1, not 2 (`Lexer.cs`) | red, 7 failed | +| C3 line endings | `ConsumeLineBreak` takes CRLF as one char (`Lexer.cs`) | red, 9 failed | +| C4 f/t/df arms | `IsPreCloseNewline` call inverted (`Lexer.FStrings.cs`) | red, 46 failed | +| C5 LiteralLoss scanner | the `{{` escaped-brace skip removed (`Lexer.LiteralLoss.cs`) | red, 2 failed | +| C6 end of file | single-line `dr"…"` loop bound `<` → `<=` (`Lexer.Literals.cs`) | red, 2 failed | +| C7 keyword roster | `"self"` removed from `_keywordNames` (`Lexer.cs`) | red, 1 failed | +| — | all restored | green, 2214 passed / 0 failed / 3 skipped (the three #2337 df skips) | diff --git a/docs/design/gap-discovery-contracts.md b/docs/design/gap-discovery-contracts.md index 9b68483bd..bf4cdf745 100644 --- a/docs/design/gap-discovery-contracts.md +++ b/docs/design/gap-discovery-contracts.md @@ -56,7 +56,11 @@ deployment closure (`StandaloneDeploymentTests` for the used stdlib closure; `Re | Contextual-keyword escape totality (#2166) | A backtick-escaped contextual keyword is an identifier at every site: an escaped soft keyword parses as a fresh identifier would. The predicate "this token is the contextual keyword *k*" is `Identifier ∧ ¬IsBacktickEscaped ∧ Value == k` (`Parser.IsContextualKeyword`) and "this node is the placeholder" is `Identifier { Name: "_", IsNameBacktickEscaped: false }` (`Parser.IsPlaceholder`), over ONE roster (`ContextualKeywords.All`, 13 words) | `ContextualKeywordEscapeSweepTests` (regular suite) — every corpus fixture of `FormatterMeaningPreservationSweepTests` × every unescaped roster-word identifier token at EVERY position (ordinary-identifier positions are the sweep's own green controls, so no position predicate can go stale): the escaped twin (that token in backticks) and the renamed twin (a fresh identifier of the escaped spelling's length, byte-aligned) must have the same `ContextualKeywordEscapeOracle.Outcome` — the parse-diagnostic `(code, line, column)` multiset, else the preorder AST shape (record types + enum-valued properties); one cell per (fixture, token value), census line `CKE-CENSUS`; ratchets `contextual-keyword-escape-allowlist.txt` (`stem token # #issue reason`, drain-on-fix) + `Parser/ContextualKeywordEscapeMatrixTests` — 26 (keyword, site) rows × {bare (construct present), renamed (ground truth), escaped (construct absent, named node kind or parse error, escaped ≡ renamed)} + binding-target/ordinary-read rows for every keyword, roster totality anchored to the literal 13, in-file `KnownRed` drain-on-fix | #2166 | | CLI input totality (#2173, #2223) | Every option every `sharpyc` command accepts — its own and the recursive globals — is honoured, refused for that input kind with a message, or documented as ignored in `--help`, never silently dropped; ONE registry (`CommandOptionEffects`, keyed by command path) generates the help notes and refusals, and the globals rule makes a command that compiles honour all nine globals and one that analyses honour the seven analysis-facing ones | `CliOptionEffectSweepTests` — reflective walk of the real root (every leaf command × own + recursive options, `compile` × {.spy, .spyproj}): refused cells are invoked, ignored cells' `--help` text is checked, honoured cells are observed by a named evidence test or a `GlobalOptionProbes` probe run on that command; synthetic-grammar positive controls; allowlist `src/Sharpy.Cli.Tests/cli-option-effect-allowlist.txt` | #2223 | | In-process isolation (#2250; P54) | A compile, and every semantic query on its result, is a function of its inputs alone — the same under 1, 2 or 8 concurrent compilations, on a cold or a warm `CompilerApi`, in a fresh or a long-lived process; every CLR name resolves against ONE `AssemblyVisibility` = a process-constant framework index ∪ the assemblies the compilation's own `ModuleRegistry` loaded, into a load context keyed by its reference set (`ReferenceLoadContext`); the master `BuiltinRegistry` is frozen after construction and a user definition never writes a master symbol; a stdlib module global CPython makes thread-safe is read and written under one module lock; a per-event artefact (crash bundle) is created exclusively | `InProcessIsolationSweepTests` (GapDiscovery, its own CI step: corpus × seeded schedule × sharing × threads × {Analyze, Compile}, allowlist `in-process-isolation-allowlist.txt`); `ProcessWideResolutionScanTests` (Roslyn symbol census of every process-wide resolution/load site: roster + drain-on-fix allowlist, literal anchors); `StaticStateConformanceTests` rule 2b (static `ConcurrentDictionary`/`Lazy`/`ConditionalWeakTable` rostered with a key-purity reason; static auto-properties included); `TpaNamespaceGateTests`, `RegistryIsolationTests`, `CompileServerE2ETests.RequestSequence_NoRef_Ref_NoRef…`, `WarmInstanceParityTests` (Lsp) + `MasterRegistryImmutabilityTests` (master symbols frozen, key-set snapshot); `StdlibModuleGlobalRosterTests` (written-after-init census) + `StdlibModuleGlobalConcurrencyTests` (CPython's process-wide model at 1/2/8 threads); `CrashBundleExclusivityTests`; `CliStaticStateConformanceTests` | #2250 | -| Typed-generator reach and refusal-oracle strength (#2288; P63) | Every construct the language spec defines has a roster row stating, per lane, what the typed property generator proves, and every verdict is measured: a generated-lane **Y** means every raw sample of the named generator member contains the construct and analyses clean (or is a pinned known red), an injected-lane **Y** means every applicable sample draws exactly ONE named code at the named severity (or is a pinned known red); a spec file with no row, a row whose anchor heading is gone, and a Y whose generator never emits the construct are red. Every refusal lane is judged by ONE seam (`ErrorInjector.JudgeDiagnostics`, fronted by `JudgeSource`/`JudgeProject`/`Judge`), and every soundness lane reads its generator through a `RejectionSink` that records each refused or throwing sample and fails on any rejection no declared pin explains | `TypedGeneratorRosterTests` — universe = directory scan of `docs/language_specification/*.md` minus README (125 files), literal rows anchored to a `##`/`#` heading, a RAW fixed-seed corpus per Y row (generator → Unparser → parser, AST predicate must hit), reflection-checked member labels, literal totality anchors, `NoConstruct` only for files with no compiled Sharpy block; `PropertyOracleConformanceTests` — Roslyn source scan of every `Properties/Semantic` file and every typed-generator consumer for the weak oracle spellings (prefix judging, thresholds, pass-counter lower bounds, any-failure, two-bucket sums, catch-as-pass, code comparison outside the seam), allowlist drained to 0; `GeneratorMemberSoundnessPropertyTests` (every valid member has a consumer); one shared pin roster `KnownGeneratorPins` (the #2169 Unparser cell); N/KnownRed rows are `KnownRedRow("#N", …)` so `check_allowlist_issue_state.sh` reddens when a cited issue closes, and the gate accepts only an OPEN issue | #2287 (N-generated: no generator emits the construct), #2324 (N-injected residue: generated, no injection breaks it), #2212/#2143 (KnownRed pins) | +| Typed-generator reach and refusal-oracle strength (#2288; P63) | Every construct the language spec defines has a roster row stating, per lane, what the typed property generator proves, and every verdict is measured: a generated-lane **Y** means every raw sample of the named generator member contains the construct and analyses clean (or is a pinned known red), an injected-lane **Y** means every applicable sample draws exactly ONE named code at the named severity (or is a pinned known red); a spec file with no row, a row whose anchor heading is gone, and a Y whose generator never emits the construct are red. Every refusal lane is judged by ONE seam (`ErrorInjector.JudgeDiagnostics`, fronted by `JudgeSource`/`JudgeProject`/`Judge`), and every soundness lane reads its generator through a `RejectionSink` that records each refused or throwing sample and fails on any rejection no declared pin explains | `TypedGeneratorRosterTests` — universe = directory scan of `docs/language_specification/*.md` minus README (125 files), literal rows anchored to a `##`/`#` heading, a RAW fixed-seed corpus per Y row (generator → Unparser → parser, AST predicate must hit), reflection-checked member labels, literal totality anchors, `NoConstruct` only for files with no compiled Sharpy block; `PropertyOracleConformanceTests` — Roslyn source scan of every `Properties/Semantic` file and every typed-generator consumer for the weak oracle spellings (prefix judging, thresholds, pass-counter lower bounds, any-failure, two-bucket sums, catch-as-pass, code comparison outside the seam), allowlist drained to 0; `GeneratorMemberSoundnessPropertyTests` (every valid member has a consumer); one shared pin roster `KnownGeneratorPins` (empty since its #2169 Unparser cell was fixed and drained, R-HP); N/KnownRed rows are `KnownRedRow("#N", …)` so `check_allowlist_issue_state.sh` reddens when a cited issue closes, and the gate accepts only an OPEN issue | #2287 (N-generated: no generator emits the construct), #2324 (N-injected residue: generated, no injection breaks it), #2212/#2143 (KnownRed pins) | +| Numeric-literal fidelity (#2300 C1; P68; cells #2339) | A numeric spelling in the grammar Sharpy shares with CPython is ONE constant with CPython's value (BigInteger for ints, IEEE bits for floats) or refused by both, judged at the program altitude (`ast.parse` constant), not at the token altitude; the Sharpy-only syntax (suffix roster, `j`, trailing-dot floats) is excluded only by a literal table citing the spec sentence, and every excluded row is pinned to Sharpy's own verdict | `NumericLiteralTwinTests` (regular suite) — form × separator placement × exponent × following character (49,432 spellings) against batched `build_tools/differential_lex/number_oracle.py` through ONE locator (`Sharpy.TestInfrastructure/Oracles/PythonInterpreter`, required under `SHARPY_REQUIRE_PYTHON_ORACLE=1` in CI); `SharpyOnlyRows_LexAsTheSpecSays` (83 hand rows); ratchet `Lexer/numeric-literal-twin-allowlist.txt` (`01`, `0_1` — the leading-zero DEVIATION-by-default cell, ruled at `/batch-issues`) | #2300 | +| Token position and line-ending totality (#2300 C2/C3/C6; P68; cells #2338) | Every token's `(Line, Column, Position, length)` names its own source slice (R1–R7 of `TokenPositionReconstruction`), and tokens, diagnostics and line facts are invariant under LF / CRLF / lone CR and a present / absent / doubled trailing newline | `LexerCorpusTwinTests` — every fixture × 9 twins (≈3.56 M tokens) through `Sharpy.TestInfrastructure/Lexing/TokenPositionReconstruction` (its own R-rule positive controls in `TokenPositionReconstructionTests`; P67 extends it with AST node ends, #2295); `EofConstructRosterTests` — 142 constructs (totality over operator/delimiter `TokenType`s) × 7 endings, no exception escapes; shared ratchet `Lexer/lexer-position-allowlist.txt` (rows cite #2338). Instrument limit: `FStringText`/`FStringFormatSpec`/`FStringConversion` carry no `SourceLength`, so their slice is read as the gap to the next token | #2300 #2338 | +| Literal-loss scanner agreement (#2300 C5; P68) | On text recovery drops, the literal-loss scanner's facts (opener crossing, continuation, open bracket, header colon, field state) equal the clean lexer's facts for the same lines | `LiteralLossScannerDifferentialTests` — `Injected_ScannerAgreesWithTheCleanLexersFacts`: the corpus prefix through line k × 4 error constructs (`$`, bad escape, bad conversion, tab indent) + backtick / escaped-quote / field-split / unbalanced-brace clean twins (≈284 k cells), against the clean-lexer oracle (`.CleanFacts.cs`); `Shapes_LexerFactsMatchTheirConstructionKnownTruth` — the P22 shape builders over every fixture; ratchet `Lexer/literal-loss-scanner-allowlist.txt` (0 rows) | #2300 (#2274 known limits) | +| F-string arm and keyword roster totality (#2300 C4/C7; P68; cells #2337, #2044) | Every replacement-field, format-spec, escape, dedent and refusal arm of the f/t/df lexer lexes as the spec and CPython say under every prefix × quoting, a refusal's FIRST diagnostic carrying its code at its line; every keyword in the spec's literal tables (hard, rejected, future, contextual `self`) lexes to its token type alone, between identifiers and at EOF without a newline, and `KeywordNames` IS the spec roster | `FStringArmMatrixTests` — 55 arms × prefix × quoting (309 cells, values verified with python3), `Roster_EveryArmHasACellAndEveryGapItsReason`, df dedent cells `Skip`-citing #2337 with live `d` controls; `KeywordRosterTests` — 54 rows × 3 sites + `KeywordNames_IsTheSpecRoster` | #2300 | ## Policy riders diff --git a/docs/language_specification/conversion_operators.md b/docs/language_specification/conversion_operators.md index d3afc6fd3..5b955106a 100644 --- a/docs/language_specification/conversion_operators.md +++ b/docs/language_specification/conversion_operators.md @@ -45,7 +45,7 @@ Emits: `public static implicit operator Celsius(double val) { ... }` ## Explicit Conversions (`__explicit__`) -Explicit conversions require an explicit cast (`as!`, or the transitional `to`): +Explicit conversions require an explicit cast (`as!`, or `as?` for a cast that yields `None` on failure): ```python temp: Celsius = Celsius(100.0) diff --git a/docs/language_specification/keywords.md b/docs/language_specification/keywords.md index 37e96c831..51902b65d 100644 --- a/docs/language_specification/keywords.md +++ b/docs/language_specification/keywords.md @@ -43,7 +43,6 @@ The following are reserved keywords in Sharpy: | `return` | Return statement | | `struct` | Struct declaration | | `True` | Boolean true literal | -| `to` | Type coercion operator | | `try` | Try block | | `type` | Type alias declaration | | `while` | While loop | diff --git a/docs/language_specification/operator_precedence.md b/docs/language_specification/operator_precedence.md index f25e40869..786d81396 100644 --- a/docs/language_specification/operator_precedence.md +++ b/docs/language_specification/operator_precedence.md @@ -15,7 +15,7 @@ Operators listed from highest to lowest precedence: | 9 | `^` | Bitwise XOR | Left-to-right | | 10 | `\|` | Bitwise OR | Left-to-right | | 11 | `\|>` | Pipe operator | Left-to-right | -| 12 | `as!`, `as?` (and transitional `to`) | Type cast | Left-to-right | +| 12 | `as!`, `as?` | Type cast | Left-to-right | | 13 | `in`, `not in`, `is`, `is not`, `<`, `<=`, `>`, `>=`, `!=`, `==` | Comparisons | **Chained** (see below) | | 14 | `not` | Logical NOT | Right-to-left (unary) | | 15 | `and` | Logical AND | Left-to-right | diff --git a/docs/language_specification/overload_resolution.md b/docs/language_specification/overload_resolution.md index 8a24b9f3c..0f14db17c 100644 --- a/docs/language_specification/overload_resolution.md +++ b/docs/language_specification/overload_resolution.md @@ -154,7 +154,7 @@ type *S* and target type *T*: `float` and `float64` are aliases for the 64-bit `double`. Narrowing conversions (e.g. `long → int`, `double → int`) are never implicit and never make a candidate applicable — they require an explicit -`to` conversion at the call site. +`as!` cast at the call site. If applicability leaves **zero** candidates, `SPY0354` (*no matching overload*) is reported. If it leaves exactly one, that candidate is selected. If it leaves more than one, betterness decides. diff --git a/docs/language_specification/try_expressions.md b/docs/language_specification/try_expressions.md index 5b1280e3e..b992760b6 100644 --- a/docs/language_specification/try_expressions.md +++ b/docs/language_specification/try_expressions.md @@ -53,7 +53,7 @@ exception. In such cases, the result type is always `Result[T, Exception]` where **Precedence Rules:** -The `try` expression has very low precedence (lower than `to`, arithmetic, comparisons, and logical operators), meaning it captures the entire following expression: +The `try` expression has very low precedence (lower than `as!`/`as?`, arithmetic, comparisons, and logical operators), meaning it captures the entire following expression: ```python # try captures the full expression including arithmetic diff --git a/src/Sharpy.Compiler.Tests/Lexer/EofConstructRosterTests.cs b/src/Sharpy.Compiler.Tests/Lexer/EofConstructRosterTests.cs new file mode 100644 index 000000000..56590d4a0 --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/EofConstructRosterTests.cs @@ -0,0 +1,207 @@ +using FluentAssertions; +using Sharpy.TestInfrastructure.Lexing; +using Xunit; +using Xunit.Abstractions; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// End of file × construct (#2300 row C6, with C2/C3 riding along, P68 Phase 2). Every construct of a closed literal +/// roster — every operator/delimiter spelling ReadOperatorOrDelimiter reads, every string prefix × quote closed +/// and unterminated, every number form, a comment, a backtick name closed and unterminated, a keyword, an identifier, +/// a lone backslash, an open block and the open f-string field/spec/conversion and escape-at-EOF shapes — is lexed +/// at the end of the source under seven endings: none, \n, \r\n, \r, \n\n, trailing +/// spaces and a trailing comment. Each construct sits after x = (statement constructs stand alone) and, where +/// a token can follow it on its line, is ALSO lexed followed by b — R1 of +/// is vacuous at column 1, so a dropped column advance inside the construct +/// is seen on that following token. Every cell must: +/// +/// let no exception escape (an IndexOutOfRangeException past the +/// source end is the class: the pilot's dr"abc mutant); +/// reconstruct ( empty); +/// agree with the same construct's \n cell on the (Type, Value) sequence — a Newline followed +/// only by Dedent/Eof dropped — and on the multiset of diagnostic codes. +/// +/// A construct whose text ends in a backslash (: the lone \, +/// "abc\, b"\x4) reads the ending's first character as its operand — \ + break is a +/// continuation, \ + space is not — so for it only the three single line-break endings are held equal (the C3 +/// invariance); its other endings still must not throw and must reconstruct. +/// +/// Roster totality: every operator/delimiter (Plus … Backslash) is the first +/// token of at least one construct lexed alone — the enum sweep is the positive control that the literal list is +/// complete; the list, not the enum, is the anchor. +/// +/// +public class EofConstructRosterTests +{ + private readonly ITestOutputHelper _output; + + public EofConstructRosterTests(ITestOutputHelper output) => _output = output; + + /// The construct, for the report. + /// Its source text, ending at the construct. + /// True when a token can follow it on its line (an unterminated literal or comment eats the rest). + /// True when it stands alone rather than after x = . + /// True when its text ends in a backslash that reads the ending's first character. + public sealed record Construct(string Name, string Text, bool Followable = true, bool Statement = false, bool EndingIsOperand = false); + + private static readonly string[] OperatorSpellings = + { + "+", "-", "*", "/", "//", "%", "**", "==", "!=", "!", "<", ">", "<=", ">=", "&", "|", "^", "~", "<<", ">>", + "=", ":=", "+=", "-=", "*=", "/=", "//=", "%=", "**=", "&=", "|=", "^=", "<<=", ">>=", "??=", "@=", + "?", "?.", "...", "|>", "(", ")", "[", "]", "{", "}", ",", ":", ";", ".", "->", "=>", "@", + }; + + private static readonly string[] StringPrefixes = { "", "f", "t", "r", "b", "d", "dr", "df" }; + private static readonly string[] Quotes = { "'", "\"", "'''", "\"\"\"" }; + + private static readonly string[] Numbers = { "42", "3.14", ".5", ".5e-3", "1e5", "0xFF", "0b101", "0o755", "42L", "3.14f", "0xFFu" }; + + public static readonly IReadOnlyList Constructs = BuildConstructs(); + + private static List BuildConstructs() + { + var constructs = new List(); + constructs.AddRange(OperatorSpellings.Select(op => new Construct($"op {op}", op))); + constructs.Add(new Construct("backslash", "\\", EndingIsOperand: true)); + foreach (var prefix in StringPrefixes) + { + foreach (var quote in Quotes) + { + constructs.Add(new Construct($"{prefix}{quote} closed", $"{prefix}{quote}abc{quote}")); + constructs.Add(new Construct($"{prefix}{quote} unterminated", $"{prefix}{quote}abc", Followable: false)); + } + } + constructs.Add(new Construct("d\"\"\" closed over lines", "d\"\"\"\n k\n \"\"\"")); + constructs.AddRange(Numbers.Select(n => new Construct($"number {n}", n))); + constructs.Add(new Construct("comment", "# comment", Followable: false, Statement: true)); + constructs.Add(new Construct("backtick name closed", "`a b`")); + constructs.Add(new Construct("backtick name unterminated", "`a b", Followable: false)); + constructs.Add(new Construct("keyword", "union")); + constructs.Add(new Construct("identifier", "name")); + constructs.Add(new Construct("open block", "if x:\n y", Statement: true)); + constructs.Add(new Construct("f-string open field", "f\"{x", Followable: false)); + constructs.Add(new Construct("f-string open spec", "f\"{x:>4", Followable: false)); + constructs.Add(new Construct("f-string open conversion", "f\"{x!r", Followable: false)); + constructs.Add(new Construct("escape at EOF", "\"abc\\", Followable: false, EndingIsOperand: true)); + constructs.Add(new Construct("short hex escape at EOF", "b\"\\x4", Followable: false, EndingIsOperand: true)); + constructs.Add(new Construct("dedented unterminated", "d\"\"\"\n k", Followable: false)); + return constructs; + } + + public static readonly IReadOnlyList<(string Name, string Text)> Endings = new[] + { + ("none", ""), ("LF", "\n"), ("CRLF", "\r\n"), ("CR", "\r"), ("LF LF", "\n\n"), ("trailing spaces", " "), ("trailing comment", " # c"), + }; + + private static readonly HashSet LineBreakEndings = new(StringComparer.Ordinal) { "LF", "CRLF", "CR" }; + + private sealed record Cell(string Name, string Source, LexerNs.Lexer Lexer, List? Tokens, Exception? Thrown); + + private static Cell LexCell(string name, string source) + { + var lexer = new LexerNs.Lexer(source); + try + { + return new Cell(name, source, lexer, lexer.TokenizeAll(), null); + } + catch (Exception ex) + { + return new Cell(name, source, lexer, null, ex); + } + } + + private static List TokenSequence(List tokens) + { + var sequence = tokens.Select(t => $"{t.Type}|{t.Value}").ToList(); + var newline = tokens.FindLastIndex(t => t.Type == TokenType.Newline); + if (newline >= 0 && tokens.Skip(newline + 1).All(t => t.Type is TokenType.Dedent or TokenType.Eof)) + sequence.RemoveAt(newline); + return sequence; + } + + private static List Codes(LexerNs.Lexer lexer) => + lexer.Diagnostics.GetAll().Select(d => d.Code ?? "-").OrderBy(c => c, StringComparer.Ordinal).ToList(); + + private static string Shown(string text) => + text.Replace("\r", "\\r", StringComparison.Ordinal).Replace("\n", "\\n", StringComparison.Ordinal); + + [Fact] + public void EveryConstruct_LexesTheSameAtEof_WithAndWithoutATrailingNewline() + { + var violations = new List<(string Scope, string Violation)>(); + var cells = 0; + foreach (var construct in Constructs) + { + var scope = $"roster {construct.Name}"; + var line = construct.Statement ? construct.Text : "x = " + construct.Text; + var placements = construct.Followable + ? new[] { ("at end", line), ("followed", line + " b") } + : new[] { ("at end", line) }; + foreach (var (placement, text) in placements) + { + var lexed = Endings.ToDictionary(e => e.Name, e => LexCell($"{construct.Name} [{placement}] {e.Name}", text + e.Text)); + cells += lexed.Count; + foreach (var cell in lexed.Values) + { + if (cell.Thrown != null) + { + violations.Add((scope, $"THROW {cell.Thrown.GetType().Name} {cell.Name} :: '{Shown(cell.Source)}': {cell.Thrown.Message}")); + continue; + } + violations.AddRange(TokenPositionReconstruction.Violations(cell.Source, cell.Lexer, cell.Tokens!) + .Select(v => (scope, $"{v} :: {cell.Name} '{Shown(cell.Source)}'"))); + } + + var reference = lexed["LF"]; + if (reference.Tokens == null) + continue; + foreach (var (endingName, _) in Endings) + { + var cell = lexed[endingName]; + if (cell.Tokens == null || endingName == "LF") + continue; + if (construct.EndingIsOperand && !LineBreakEndings.Contains(endingName)) + continue; + var expectedTokens = TokenSequence(reference.Tokens); + var actualTokens = TokenSequence(cell.Tokens); + if (!expectedTokens.SequenceEqual(actualTokens)) + violations.Add((scope, $"TOKENS {cell.Name} '{Shown(cell.Source)}': LF [{Shown(string.Join(" ", expectedTokens))}] this [{Shown(string.Join(" ", actualTokens))}]")); + var expectedCodes = Codes(reference.Lexer); + var actualCodes = Codes(cell.Lexer); + if (!expectedCodes.SequenceEqual(actualCodes)) + violations.Add((scope, $"CODES {cell.Name} '{Shown(cell.Source)}': LF [{string.Join(" ", expectedCodes)}] this [{string.Join(" ", actualCodes)}]")); + } + } + } + + var (rows, malformed) = LexerPositionAllowlist.Read(); + var (remaining, stale, allowlisted) = LexerPositionAllowlist.Apply(rows, "roster", violations); + var failures = malformed.Concat(stale).Concat(remaining).ToList(); + _output.WriteLine($"EOF-ROSTER constructs {Constructs.Count} endings {Endings.Count} cells {cells} violations {remaining.Count} allowlisted {allowlisted}"); + foreach (var failure in failures.Take(200)) + _output.WriteLine("EOF-ROSTER " + failure); + + failures.Should().BeEmpty("a position defect outside the contract is a findings-ledger line and a row in {0} citing its issue", + LexerPositionAllowlist.FileName); + } + + [Fact] + public void Roster_HasAConstructForEveryOperatorAndDelimiterTokenType() + { + var firstTypes = Constructs.Select(c => + { + var tokens = new LexerNs.Lexer(c.Text).TokenizeAll(); + return tokens[0].Type; + }).ToHashSet(); + + var operatorTypes = Enum.GetValues().Where(t => t >= TokenType.Plus && t <= TokenType.Backslash).ToList(); + operatorTypes.Should().Contain(new[] { TokenType.Plus, TokenType.RightShift, TokenType.NullCoalesceAssign, TokenType.Backslash }, + "the range Plus … Backslash is the operator/delimiter block of TokenType"); + + operatorTypes.Where(t => !firstTypes.Contains(t)).Should().BeEmpty( + "every operator/delimiter token type needs a roster construct that starts with it"); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/FStringArmMatrixTests.cs b/src/Sharpy.Compiler.Tests/Lexer/FStringArmMatrixTests.cs new file mode 100644 index 000000000..1e77d6986 --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/FStringArmMatrixTests.cs @@ -0,0 +1,415 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using Sharpy.Compiler.Diagnostics; +using Xunit; +using Xunit.Abstractions; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// #2300 row C4 (P68): every f/t/df-string lexer arm × prefix {f, t, df} × +/// quoting {single, triple}. Each cell lexes one source and asserts the WHOLE token (Type, Value) +/// sequence, or the code and line of the FIRST diagnostic it is refused with (later codes are +/// recovery resuming on the next line). The arm roster is read from +/// NextFStringToken, NextFStringSpecToken and PrescanDedentedTripleFString +/// (Lexer.FStrings.cs @ 220259b51). +/// +/// Why hand cells (plan Design Decision 5). The arms are PEP 701 holes, PEP 750 +/// t and PEP 822 df plus the SPY0020–SPY0022/SPY0030 refusals; CPython 3.12 (the +/// pinned oracle) has no t/d prefix, so a generator would reach only f. The +/// f-prefix VALUE surface already has generators (FormatSpecDifferentialTests, +/// GenExpressions.FStringLiteralExpr). +/// +/// Applicability is a literal. Every arm runs under all three prefixes (a t-string +/// "supports the same interpolation syntax as f-strings", template_strings.md; a single-quoted +/// df"…" is an f-string, dstrings.md "Single-line d-strings"). An arm's +/// or outcome is null only with a +/// reason, and overrides the triple +/// outcome for df""" where PEP 822 dedent makes it differ. +/// +/// Source markers. « opens, » closes, § is the cell's bare quote +/// (" or """), ⇥ is the cell's dedent indent. A df""" cell wraps the body +/// in the spec's canonical dedent shape — df"""\n BODY\n """ — so every df-triple cell +/// also exercises the dropped first line, the dedent skip and the swallowed pre-close newline +/// (dstrings.md §df). A arm writes the whole literal itself (« = prefix + +/// quote, » = quote): the PEP 822 contrast arms, whose f/t outcome keeps the indentation and whose +/// df outcome strips it. +/// +/// Oracles. Every f-prefix value is CPython 3.12.13's, probed before it was written with +/// python3 -c 'x=5;w=4;a=b=1;s="s";xs=[1,2,3];y=4;z="d";f=dict;print(repr(SRC))' and with +/// tokenize.generate_tokens for the FSTRING_MIDDLE text (the lexer's FStringText / +/// FStringFormatSpec value); each arm's records the result. t/df +/// outcomes come from executed spec examples (template_strings.md "Format Specs and Nested Fields" +/// and "Conversions and Self-Documenting Fields", dstrings.md §df — run with the built sharpyc @ +/// 220259b51, printed exactly the spec's output). +/// +/// Left to the findings ledger, not asserted (P68 Phase 4): {{ inside a spec +/// (the lexer reads a literal {; CPython 3.12 refuses it; fstrings.md silent) and a line break +/// in a SINGLE-quoted spec (the lexer keeps \n; CPython 3.12's FSTRING_MIDDLE drops it, 3.13 +/// refuses it) — the lexer's token differs from CPython's, so no cell pins it either way. The +/// \" spec cell IS asserted: its token matches CPython's FSTRING_MIDDLE (it is refused SPY0609 +/// downstream, a ledger line outside the lexer). +/// +/// PEP 822 refusal arms (#2337). dstrings.md refuses a content line shallower than the +/// close and a non-whitespace close line (SPY0029), and drops the first and pre-close newlines for any +/// strip amount. The d""" cells () are live; the +/// df""" twins assert the SPEC's verdict and are skipped citing #2337, so they un-skip when it +/// closes. +/// +public class FStringArmMatrixTests +{ + private readonly ITestOutputHelper _output; + + public FStringArmMatrixTests(ITestOutputHelper output) => _output = output; + + private sealed class Outcome + { + // The AsSingle sentinel carries neither; a real outcome carries exactly one. + public (TokenType Type, string Value)[]? Inner { get; private init; } + public string? Code { get; private init; } + + public static Outcome Lex(params (TokenType, string)[] inner) => new() { Inner = inner }; + public static Outcome Refuse(string code) => new() { Code = code }; + } + + private sealed record Arm( + string Name, + string Body, + Outcome? Single, + Outcome? Triple, + string Python, + Outcome? DfTriple = null, + bool Raw = false, + string? NotApplicable = null, + string? NestedPrefix = null); + + private static readonly string[] Prefixes = { "f", "t", "df" }; + private static readonly bool[] Quotings = { false, true }; + + private const string SPY0002 = DiagnosticCodes.Lexer.UnterminatedFString; + private const string SPY0020 = DiagnosticCodes.Lexer.UnterminatedFStringExpression; + private const string SPY0021 = DiagnosticCodes.Lexer.UnmatchedBraceInFString; + private const string SPY0022 = DiagnosticCodes.Lexer.UnterminatedFormatSpec; + private const string SPY0030 = DiagnosticCodes.Lexer.InvalidFStringConversion; + + private const string NoLineBreakInSingle = "a single-quoted string's text cannot hold a line break (the 'text: a line break' arm refuses it, SPY0002)"; + + private static (TokenType, string) T(TokenType type, string value) => (type, value); + private static readonly (TokenType, string) Es = (TokenType.FStringExprStart, "{"); + private static readonly (TokenType, string) Ee = (TokenType.FStringExprEnd, "}"); + private static (TokenType, string) Id(string name) => (TokenType.Identifier, name); + private static (TokenType, string) Text(string text) => (TokenType.FStringText, text); + private static (TokenType, string) Spec(string text) => (TokenType.FStringFormatSpec, text); + + /// A of AsSingle: the triple cell's outcome is the single cell's. + private static readonly Outcome AsSingle = new(); + + /// The arm roster. One row per arm; both outcomes, or a reason. + private static readonly Arm[] Arms = + { + // ── literal text ────────────────────────────────────────────────────────────────────── + new("text: {{ and }} escape to a brace", "«a{{b}}c»", + Lex(Text("a{b}c")), Lex(Text("a{b}c")), "f\"a{{b}}c\" → 'a{b}c'"), + new("text: a lone } is refused", "«a}b»", + Refuse(SPY0021), Refuse(SPY0021), "f\"a}b\" → SyntaxError: f-string: single '}' is not allowed"), + new("text: a backslash escape is decoded", "«a\\tb»", + Lex(Text("a\tb")), Lex(Text("a\tb")), "f\"a\\tb\" → 'a\\tb' (a TAB)"), + new("text: a backslash at EOF", "«abc\\", + Refuse(SPY0002), Refuse(SPY0002), "f\"abc\\ → SyntaxError: unterminated f-string literal"), + new("text: the other quote character is text", "«a'b»", + Lex(Text("a'b")), Lex(Text("a'b")), "f\"a'b\" → \"a'b\""), + new("text: one or two quotes do not end a triple-quoted string", "«a\"b\"\"c»", + null, Lex(Text("a\"b\"\"c")), "f\"\"\"a\"b\"\"c\"\"\" → 'a\"b\"\"c'", + NotApplicable: "a lone quote ends a single-quoted string — that close is every other arm's »"), + new("text: a line break", "«a\n⇥b»", + Refuse(SPY0002), Lex(Text("a\nb")), "f\"a⏎b\" → SyntaxError: unterminated; f\"\"\"a⏎b\"\"\" → 'a\\nb'"), + new("text: CRLF reads as \\n", "«a\r\n⇥b»", + Refuse(SPY0002), Lex(Text("a\nb")), "f\"\"\"a\\r\\nb\"\"\" → 'a\\nb'"), + new("text: a bare CR reads as \\n", "«a\r⇥b»", + Refuse(SPY0002), Lex(Text("a\nb")), "f\"\"\"a\\rb\"\"\" → 'a\\nb'"), + new("text: text after a hole continues across a line", "«a {x}\n⇥b»", + Refuse(SPY0002), Lex(Text("a "), Es, Id("x"), Ee, Text("\nb")), "f\"\"\"a {x}⏎b\"\"\" → 'a 5\\nb', MIDDLE ['a ', '\\nb']"), + + // ── replacement field: '=' and '!' ──────────────────────────────────────────────────── + new("field: '=' at the top level is self-doc", "«{x=}»", + Lex(Es, Id("x"), T(TokenType.FStringSelfDoc, "x="), Ee), AsSingle, "f\"{x=}\" → 'x=5'"), + new("field: self-doc keeps the whitespace around '='", "«{x = }»", + Lex(Es, Id("x"), T(TokenType.FStringSelfDoc, "x = "), Ee), AsSingle, "f\"{x = }\" → 'x = 5'"), + new("field: '=' inside () is a keyword argument", "«{f(a=1)}»", + Lex(Es, Id("f"), T(TokenType.LeftParen, "("), Id("a"), T(TokenType.Assign, "="), T(TokenType.Integer, "1"), T(TokenType.RightParen, ")"), Ee), AsSingle, + "f\"{f(a=1)}\" (f=dict) → \"{'a': 1}\""), + new("field: '==' is not self-doc", "«{a==b}»", + Lex(Es, Id("a"), T(TokenType.Equal, "=="), Id("b"), Ee), AsSingle, "f\"{a==b}\" → 'True'"), + new("field: !r", "«{x!r}»", + Lex(Es, Id("x"), T(TokenType.FStringConversion, "r"), Ee), AsSingle, "f\"{x!r}\" → '5'"), + new("field: !s followed by a spec", "«{x!s:>4}»", + Lex(Es, Id("x"), T(TokenType.FStringConversion, "s"), Spec(">4"), Ee), AsSingle, "f\"{x!s:>4}\" → ' 5', MIDDLE ['>4']"), + new("field: !a", "«{x!a}»", + Lex(Es, Id("x"), T(TokenType.FStringConversion, "a"), Ee), AsSingle, "f\"{x!a}\" → '5'"), + new("field: '!=' is an operator", "«{a!=b}»", + Lex(Es, Id("a"), T(TokenType.NotEqual, "!="), Id("b"), Ee), AsSingle, "f\"{a!=b}\" → 'False'"), + new("field: an invalid conversion !q", "«{x!q}»", + Refuse(SPY0030), AsSingle, "f\"{x!q}\" → SyntaxError: f-string: invalid conversion character 'q'"), + new("field: a valid conversion at EOF", "«{x!r", + Refuse(SPY0020), AsSingle, "f\"{x!r → SyntaxError: '{' was never closed"), + new("field: a valid conversion followed by neither ':' nor '}'", "«{x!r x}»", + Refuse(SPY0020), AsSingle, "f\"{x!r x}\" → SyntaxError: f-string: expecting ':' or '}'"), + + // ── format spec ─────────────────────────────────────────────────────────────────────── + new("spec: ':' starts a spec", "«{x:>4}»", + Lex(Es, Id("x"), Spec(">4"), Ee), AsSingle, "f\"{x:>4}\" → ' 5', MIDDLE ['>4']"), + new("spec: an empty spec", "«{x:}»", + Lex(Es, Id("x"), Spec(""), Ee), AsSingle, "f\"{x:}\" → '5', MIDDLE ['']"), + new("spec: a nested field", "«{x:{w}}»", + Lex(Es, Id("x"), Spec(""), Es, Id("w"), Ee, Ee), AsSingle, "f\"{x:{w}}\" → ' 5'"), + new("spec: '}}}' closes three fields — no '}}' escape in a spec (#1884)", "«{x:{y:{z}}}»", + Lex(Es, Id("x"), Spec(""), Es, Id("y"), Spec(""), Es, Id("z"), Ee, Ee, Ee), AsSingle, + "f\"{x:{y:{z}}}\" → ' 5' (y=4, z='d'), MIDDLE ['', '']"), + new("spec: a backslash-escaped quote is spec text", "«{x:\\\">4}»", + Lex(Es, Id("x"), Spec("\\\">4"), Ee), AsSingle, "f\"{x:\\\">4}\" → '\"\"\"5', MIDDLE ['\\\\\">4'] (single and triple)"), + new("spec: the enclosing quote ends the string inside a spec", "«{x:>4§", + Refuse(SPY0022), AsSingle, "f\"{x:>4\" → SyntaxError: f-string: expecting '}', or format specs"), + new("spec: a line break in a spec is spec text", "«{x:>4\n}»", + null, Lex(Es, Id("x"), Spec(">4\n"), Ee), "f\"\"\"{x:>4⏎}\"\"\" → MIDDLE ['>4\\n']", + NotApplicable: "single-quoted: the lexer keeps the \\n, CPython 3.12's FSTRING_MIDDLE drops it and 3.13 refuses it — a findings-ledger line, not pinned"), + new("spec: EOF inside a spec", "«{x:>4", + Refuse(SPY0022), AsSingle, "f\"{x:>4 → SyntaxError: unterminated f-string literal"), + + // ── a string literal inside a field: every prefix of the one dispatch (#2010) ───────── + new("field: a nested plain string", "«{'q'}»", + Lex(Es, T(TokenType.String, "q"), Ee), AsSingle, "f\"{'q'}\" → 'q'", NestedPrefix: ""), + new("field: a nested r-string", "«{r'\\d'}»", + Lex(Es, T(TokenType.RawString, "\\d"), Ee), AsSingle, "f\"{r'\\d'}\" → '\\\\d'", NestedPrefix: "r"), + new("field: a nested b-string", "«{b'ab'}»", + Lex(Es, T(TokenType.ByteString, "ab"), Ee), AsSingle, "f\"{b'ab'}\" → \"b'ab'\"", NestedPrefix: "b"), + new("field: a nested d-string", "«{d'q'}»", + Lex(Es, T(TokenType.String, "q"), Ee), AsSingle, "no d prefix in CPython 3.12; dstrings.md: d'…' is a plain string", NestedPrefix: "d"), + new("field: a nested dr-string", "«{dr'q'}»", + Lex(Es, T(TokenType.RawString, "q"), Ee), AsSingle, "no d prefix in CPython 3.12; dstrings.md: dr'…' is a raw string", NestedPrefix: "dr"), + new("field: a nested f-string", "«{f'{s}'}»", + Lex(Es, T(TokenType.FStringStart, "f'"), Es, Id("s"), Ee, T(TokenType.FStringEnd, "'"), Ee), AsSingle, + "f\"{f'{s}'}\" → 's'", NestedPrefix: "f"), + new("field: a nested t-string", "«{t'{s}'}»", + Lex(Es, T(TokenType.FStringStart, "t'"), Es, Id("s"), Ee, T(TokenType.FStringEnd, "'"), Ee), AsSingle, + "no t prefix in CPython 3.12; fstrings.md F-String Nesting Rules (executed)", NestedPrefix: "t"), + new("field: a nested df-string", "«{df'{s}'}»", + Lex(Es, T(TokenType.FStringStart, "df'"), Es, Id("s"), Ee, T(TokenType.FStringEnd, "'"), Ee), AsSingle, + "no d prefix in CPython 3.12; fstrings.md F-String Nesting Rules", NestedPrefix: "df"), + new("field: a nested literal reusing the enclosing quote (PEP 701)", "«{§q§}»", + Lex(Es, T(TokenType.String, "q"), Ee), AsSingle, "f\"{\"q\"}\" → 'q'; f\"\"\"{\"\"\"q\"\"\"}\"\"\" → 'q'"), + new("field: an unterminated literal opening with the enclosing quote is the missing '}'", "«{x§", + Refuse(SPY0020), AsSingle, "f\"{x\" and f\"\"\"{x\"\"\" → SyntaxError: f-string: expecting '}'"), + + // ── other expression tokens in a field ─────────────────────────────────────────────── + new("field: a backtick name", "«{`x`}»", + Lex(Es, Id("x"), Ee), AsSingle, "Sharpy-only (identifiers.md literal names); CPython refuses the backtick"), + new("field: a number", "«{42}»", + Lex(Es, T(TokenType.Integer, "42"), Ee), AsSingle, "f\"{42}\" → '42'"), + new("field: ':' inside [] is not a spec", "«{xs[0:2]}»", + Lex(Es, Id("xs"), T(TokenType.LeftBracket, "["), T(TokenType.Integer, "0"), T(TokenType.Colon, ":"), T(TokenType.Integer, "2"), T(TokenType.RightBracket, "]"), Ee), AsSingle, + "f\"{xs[0:2]}\" → '[1, 2]'"), + new("field: set/dict braces inside a field are tracked", "«{ {1} }»", + Lex(Es, T(TokenType.LeftBrace, "{"), T(TokenType.Integer, "1"), T(TokenType.RightBrace, "}"), Ee), AsSingle, "f\"{ {1} }\" → '{1}'"), + new("field: a ')' with nothing open", "«{x)}»", + Refuse(SPY0021), AsSingle, "f\"{x)}\" → SyntaxError: f-string: unmatched ')'"), + new("field: a ']' with nothing open", "«{x]}»", + Refuse(SPY0021), AsSingle, "f\"{x]}\" → SyntaxError: f-string: unmatched ']'"), + + // ── hole trivia (PEP 701, #2022) ───────────────────────────────────────────────────── + new("hole trivia: a comment", "«{x # c\n⇥}»", + Lex(Es, Id("x"), Ee), AsSingle, "f\"{x # c⏎}\" and the triple twin → '5'"), + new("hole trivia: line breaks and spaces", "«{ x\n⇥ }»", + Lex(Es, Id("x"), Ee), AsSingle, "f\"{ x⏎ }\" → '5'"), + new("hole trivia: a tab", "«{\tx}»", + Lex(Es, Id("x"), Ee), AsSingle, "f\"{\\tx}\" → '5'"), + + // ── end of file ────────────────────────────────────────────────────────────────────── + new("EOF: inside the text", "«abc", + Refuse(SPY0002), AsSingle, "f\"abc → SyntaxError: unterminated f-string literal"), + new("EOF: inside an open field", "«{x", + Refuse(SPY0020), AsSingle, "f\"{x → SyntaxError: '{' was never closed"), + + // ── PEP 822 dedent (Raw: the literal is written whole; f/t keep the indentation) ───── + new("dedent: the close line's indent is stripped, deeper indent kept", "«\n a\n b\n »", + null, Lex(Text("\n a\n b\n ")), "f\"\"\"⏎ a⏎ b⏎ \"\"\" → '\\n a\\n b\\n '", + DfTriple: Lex(Text("a\n b")), Raw: true, NotApplicable: NoLineBreakInSingle), + new("dedent: a tab counts as one character", "«\n\ta\n\t»", + null, Lex(Text("\n\ta\n\t")), "f\"\"\"⏎\\ta⏎\\t\"\"\" → '\\n\\ta\\n\\t'", + DfTriple: Lex(Text("a")), Raw: true, NotApplicable: NoLineBreakInSingle), + new("dedent: a blank line before the close is a trailing newline", "«\n a\n\n »", + null, Lex(Text("\n a\n\n ")), "f\"\"\"⏎ a⏎⏎ \"\"\" → '\\n a\\n\\n '", + DfTriple: Lex(Text("a\n")), Raw: true, NotApplicable: NoLineBreakInSingle), + new("dedent: CRLF line breaks", "«\r\n a\r\n »", + null, Lex(Text("\n a\n ")), "f\"\"\"\\r\\n a\\r\\n \"\"\" → '\\n a\\n '", + DfTriple: Lex(Text("a")), Raw: true, NotApplicable: NoLineBreakInSingle), + new("dedent: the pre-close newline after a hole is swallowed", "«\n {x}\n »", + null, Lex(Text("\n "), Es, Id("x"), Ee, Text("\n ")), "f\"\"\"⏎ {x}⏎ \"\"\" → '\\n 5\\n '", + DfTriple: Lex(Es, Id("x"), Ee), Raw: true, NotApplicable: NoLineBreakInSingle), + }; + + private static Outcome Lex(params (TokenType, string)[] inner) => Outcome.Lex(inner); + private static Outcome Refuse(string code) => Outcome.Refuse(code); + + private static Outcome? TripleOf(Arm arm) => ReferenceEquals(arm.Triple, AsSingle) ? arm.Single : arm.Triple; + + private static Outcome? OutcomeOf(Arm arm, string prefix, bool triple) + { + if (!triple) + return arm.Single; + return prefix == "df" && arm.DfTriple != null ? arm.DfTriple : TripleOf(arm); + } + + private static string SourceOf(Arm arm, string prefix, bool triple) + { + var quote = triple ? "\"\"\"" : "\""; + var dedented = !arm.Raw && triple && prefix == "df"; + var open = prefix + quote + (dedented ? "\n " : ""); + var close = (dedented ? "\n " : "") + quote; + return arm.Body + .Replace("«", open, StringComparison.Ordinal) + .Replace("»", close, StringComparison.Ordinal) + .Replace("§", quote, StringComparison.Ordinal) + .Replace("⇥", dedented ? " " : "", StringComparison.Ordinal); + } + + public static IEnumerable Cells() => + from arm in Arms + from prefix in Prefixes + from triple in Quotings + where OutcomeOf(arm, prefix, triple) != null + select new object[] { arm.Name, prefix, triple ? "triple" : "single" }; + + [Theory] + [MemberData(nameof(Cells))] + public void Arm_LexesAsTheSpecAndCPythonSay(string armName, string prefix, string quoting) + { + var arm = Arms.Single(a => a.Name == armName); + var triple = quoting == "triple"; + var source = SourceOf(arm, prefix, triple); + var outcome = OutcomeOf(arm, prefix, triple)!; + + var lexer = new LexerNs.Lexer(source); + var tokens = lexer.TokenizeAll(); + var errors = lexer.Diagnostics.GetErrors(); + var codes = errors.Select(d => d.Code).ToList(); + var actual = tokens.Select(t => (t.Type, t.Value)).ToList(); + var because = $"arm '{arm.Name}' × {prefix} × {quoting}\nsource: {Show(source)}\ntokens: {Show(actual)}\nerrors: [{string.Join(", ", errors.Select(e => $"{e.Code}@{e.Line}:{e.Column}"))}]\npython3: {arm.Python}"; + + if (outcome.Code != null) + { + // The FIRST diagnostic is the arm's refusal (CPython stops there), on the line the body + // starts on — so a recovery code cannot stand in for it. Later codes are recovery resuming + // on the next line and reading the rest as code (`b"` → SPY0026, ` """` → SPY0001; + // findings ledger, "lexer recovery cascade after a line-break refusal"). + var bodyLine = !arm.Raw && triple && prefix == "df" ? 2 : 1; + Assert.True(errors.Count > 0 && errors[0].Code == outcome.Code && errors[0].Line == bodyLine, + $"expected the first diagnostic to be {outcome.Code} on line {bodyLine}\n{because}"); + return; + } + + Assert.True(codes.Count == 0, $"expected no diagnostics\n{because}"); + var quote = triple ? "\"\"\"" : "\""; + var expected = new List<(TokenType, string)> { (TokenType.FStringStart, prefix + quote) }; + expected.AddRange(outcome.Inner!); + expected.Add((TokenType.FStringEnd, quote)); + expected.Add((TokenType.Eof, "")); + Assert.True(actual.SequenceEqual(expected), $"expected tokens: {Show(expected)}\n{because}"); + } + + [Fact] + public void Roster_EveryArmHasACellAndEveryGapItsReason() + { + Arms.Select(a => a.Name).Should().OnlyHaveUniqueItems(); + foreach (var arm in Arms) + { + var applicable = (from p in Prefixes from q in Quotings where OutcomeOf(arm, p, q) != null select 1).Count(); + Assert.True(applicable > 0, $"arm '{arm.Name}' must run in at least one cell"); + var full = applicable == Prefixes.Length * Quotings.Length; + Assert.True((arm.NotApplicable != null) == !full, + $"arm '{arm.Name}' runs in {applicable} of 6 cells — a gap needs a reason, a full row must not carry one"); + Assert.False(string.IsNullOrWhiteSpace(arm.Python), $"arm '{arm.Name}' records its python3 probe or why there is none"); + } + + var cells = Cells().Count(); + _output.WriteLine($"FSTRING-ARMS arms {Arms.Length} cells {cells}"); + } + + /// + /// PEP 822 refusal/normalization arms, d-string side: dstrings.md step 6 and Errors (SPY0029 for a + /// shallow content line and for a non-whitespace close line) and steps 4–5 (the empty first line + /// and the close line are dropped for ANY strip amount, zero included). Live — the control the + /// skipped df twins below must match. + /// + [Theory] + [InlineData("d\"\"\"\n a\n b\n \"\"\"", "SPY0029", null)] + [InlineData("d\"\"\"\n a\n b\"\"\"", "SPY0029", null)] + [InlineData("d\"\"\"\nabc\n\"\"\"", null, "abc")] + public void DedentRule_DString_IsTheSpecControl(string source, string? code, string? value) + { + var (codes, tokens) = LexAll(source); + if (code != null) + { + Assert.True(codes.SequenceEqual(new[] { code }), $"expected exactly [{code}] for {Show(source)}; got [{string.Join(", ", codes)}]"); + return; + } + Assert.True(codes.Count == 0, $"expected no diagnostics for {Show(source)}; got [{string.Join(", ", codes)}]"); + var expected = new List<(TokenType, string)> { (TokenType.String, value!), (TokenType.Eof, "") }; + Assert.True(tokens.SequenceEqual(expected), $"expected {Show(expected)} for {Show(source)}; got {Show(tokens)}"); + } + + [Fact(Skip = "#2337 df skips the SPY0029 dedent refusals that d applies")] + public void DedentRule_DfString_ShallowContentLine_IsSpy0029() + { + var (codes, _) = LexAll("df\"\"\"\n a {x}\n b\n \"\"\""); + Assert.True(codes.SequenceEqual(new[] { DiagnosticCodes.Lexer.DedentedStringIndentationError }), $"got [{string.Join(", ", codes)}]"); + } + + [Fact(Skip = "#2337 df skips the SPY0029 dedent refusals that d applies")] + public void DedentRule_DfString_NonWhitespaceCloseLine_IsSpy0029() + { + var (codes, _) = LexAll("df\"\"\"\n a {x}\n b\"\"\""); + Assert.True(codes.SequenceEqual(new[] { DiagnosticCodes.Lexer.DedentedStringIndentationError }), $"got [{string.Join(", ", codes)}]"); + } + + [Fact(Skip = "#2337 df keeps the first and pre-close newlines when the close has zero indent; d drops them")] + public void DedentRule_DfString_ZeroIndentClose_DropsTheFirstAndPreCloseNewlines() + { + var (codes, tokens) = LexAll("df\"\"\"\nabc\n\"\"\""); + Assert.True(codes.Count == 0, $"got [{string.Join(", ", codes)}]"); + var expected = new List<(TokenType, string)> + { + (TokenType.FStringStart, "df\"\"\""), (TokenType.FStringText, "abc"), (TokenType.FStringEnd, "\"\"\""), (TokenType.Eof, ""), + }; + Assert.True(tokens.SequenceEqual(expected), $"expected {Show(expected)}; got {Show(tokens)}"); + } + + private static (List Codes, List<(TokenType Type, string Value)> Tokens) LexAll(string source) + { + var lexer = new LexerNs.Lexer(source); + var tokens = lexer.TokenizeAll(); + return (lexer.Diagnostics.GetErrors().Select(d => d.Code ?? "").ToList(), tokens.Select(t => (t.Type, t.Value)).ToList()); + } + + /// + /// The nested-literal arms cover the lexer's ONE prefix dispatch (#2010) — read from the + /// lexer's table as the positive control that the literal arm list is complete, never as the + /// roster itself. + /// + [Fact] + public void NestedLiteralArms_CoverEveryStringPrefix() + { + Arms.Where(a => a.NestedPrefix != null).Select(a => a.NestedPrefix) + .Should().BeEquivalentTo(LexerNs.Lexer.StringLiteralPrefixes, + "every string prefix the lexer dispatches must have a nested-in-a-field arm"); + } + + private static string Show(string source) => + source.Replace("\r", "\\r", StringComparison.Ordinal).Replace("\n", "\\n", StringComparison.Ordinal).Replace("\t", "\\t", StringComparison.Ordinal); + + private static string Show(IEnumerable<(TokenType Type, string Value)> tokens) => + string.Join(" ", tokens.Select(t => $"{t.Type}'{Show(t.Value)}'")); +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/KeywordRosterTests.cs b/src/Sharpy.Compiler.Tests/Lexer/KeywordRosterTests.cs new file mode 100644 index 000000000..9e4804146 --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/KeywordRosterTests.cs @@ -0,0 +1,162 @@ +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// #2300 row C7 (P68): every keyword the SPEC lists is pinned to its inside +/// the Lexer namespaces. is a literal spelled from +/// docs/language_specification/keywords.md's three tables — Hard Keywords (49 rows; the stale +/// to row, removed in 0.8.0 per type_casting.md, is gone), Reserved Rejected Keywords +/// (global, nonlocal) and Future Keywords (defer, do) — plus the +/// contextual self, which is in Lexer.KeywordNames but always lexes as an +/// . It is anchored to the spec, not to the lexer's +/// Keywords table or the enum: a roster read from the table under +/// test would make a missing keyword invisible. +/// +/// Why self is a row. Its consumers read KeywordNames, not a token type: +/// Parser.IsReservedWordToken (Parser.cs:410) skips it because it lexes as an +/// Identifier, and the LSP rename handler (RenameHandler.cs:259) backtick-escapes a new name +/// that is in KeywordNames — dropping self from the set would let a rename write a bare +/// self. +/// +/// Whole-suite control. Integration.ReservedWordAsNameMatrixTests.Roster_IsTheLexerKeywordTable +/// pins the lexer's 53-name table (minus self) for the reserved-word-as-name matrix; it sits +/// outside the Stryker filter (Sharpy.Compiler.Tests.Lexer.|Sharpy.Compiler.Tests.Properties.Lexer.), +/// which is why this roster exists inside it. +/// +/// Next editors. P69 (out) and P70 (property) respell those keywords by +/// editing THIS table in one commit each, together with the spec row — they cite it; they never +/// widen it with an exemption. +/// +public class KeywordRosterTests +{ + private const string Hard = "Hard Keywords"; + private const string Rejected = "Reserved Rejected Keywords"; + private const string Future = "Future Keywords"; + private const string Contextual = "contextual — KeywordNames only"; + + private readonly ITestOutputHelper _output; + + public KeywordRosterTests(ITestOutputHelper output) => _output = output; + + /// keywords.md's tables, in the spec's row order. + private static readonly (string Keyword, TokenType Type, string SpecTable)[] Roster = + { + ("and", TokenType.And, Hard), + ("as", TokenType.As, Hard), + ("assert", TokenType.Assert, Hard), + ("auto", TokenType.Auto, Hard), + ("break", TokenType.Break, Hard), + ("case", TokenType.Case, Hard), + ("class", TokenType.Class, Hard), + ("const", TokenType.Const, Hard), + ("continue", TokenType.Continue, Hard), + ("def", TokenType.Def, Hard), + ("elif", TokenType.Elif, Hard), + ("else", TokenType.Else, Hard), + ("enum", TokenType.Enum, Hard), + ("event", TokenType.Event, Hard), + ("except", TokenType.Except, Hard), + ("False", TokenType.False, Hard), + ("finally", TokenType.Finally, Hard), + ("for", TokenType.For, Hard), + ("from", TokenType.From, Hard), + ("if", TokenType.If, Hard), + ("import", TokenType.Import, Hard), + ("in", TokenType.In, Hard), + ("interface", TokenType.Interface, Hard), + ("is", TokenType.Is, Hard), + ("lambda", TokenType.Lambda, Hard), + ("let", TokenType.Let, Hard), + ("match", TokenType.Match, Hard), + ("maybe", TokenType.Maybe, Hard), + ("None", TokenType.None, Hard), + ("not", TokenType.Not, Hard), + ("or", TokenType.Or, Hard), + ("pass", TokenType.Pass, Hard), + ("property", TokenType.Property, Hard), + ("raise", TokenType.Raise, Hard), + ("return", TokenType.Return, Hard), + ("struct", TokenType.Struct, Hard), + ("True", TokenType.True, Hard), + ("try", TokenType.Try, Hard), + ("type", TokenType.Type, Hard), + ("while", TokenType.While, Hard), + ("with", TokenType.With, Hard), + ("yield", TokenType.Yield, Hard), + ("async", TokenType.Async, Hard), + ("await", TokenType.Await, Hard), + ("del", TokenType.Del, Hard), + ("delegate", TokenType.Delegate, Hard), + ("super", TokenType.Super, Hard), + ("union", TokenType.Union, Hard), + ("Self", TokenType.SelfType, Hard), + + ("global", TokenType.Global, Rejected), + ("nonlocal", TokenType.Nonlocal, Rejected), + + ("defer", TokenType.Defer, Future), + ("do", TokenType.Do, Future), + + ("self", TokenType.Identifier, Contextual), + }; + + public static IEnumerable Rows() => + Roster.Select(r => new object[] { r.Keyword, r.Type, r.SpecTable }); + + private static List<(TokenType Type, string Value)> Lex(string source) + { + var lexer = new LexerNs.Lexer(source); + var tokens = lexer.TokenizeAll(); + lexer.Diagnostics.HasErrors.Should().BeFalse($"source {source.Replace("\n", "\\n", System.StringComparison.Ordinal)} lexes clean"); + return tokens.Select(t => (t.Type, t.Value)).ToList(); + } + + [Theory] + [MemberData(nameof(Rows))] + public void Keyword_Alone_LexesToItsTokenType(string keyword, TokenType type, string specTable) + { + Lex(keyword + "\n").Should().Equal( + new List<(TokenType, string)> { (type, keyword), (TokenType.Newline, "\n"), (TokenType.Eof, "") }, + $"'{keyword}' ({specTable}) on a line of its own"); + } + + [Theory] + [MemberData(nameof(Rows))] + public void Keyword_BetweenIdentifiers_LexesToItsTokenType(string keyword, TokenType type, string specTable) + { + Lex($"a {keyword} b\n").Should().Equal( + new List<(TokenType, string)> { (TokenType.Identifier, "a"), (type, keyword), (TokenType.Identifier, "b"), (TokenType.Newline, "\n"), (TokenType.Eof, "") }, + $"'{keyword}' ({specTable}) between two identifiers"); + } + + [Theory] + [MemberData(nameof(Rows))] + public void Keyword_AtEofWithoutNewline_LexesToItsTokenType(string keyword, TokenType type, string specTable) + { + Lex($"a {keyword}").Should().Equal( + new List<(TokenType, string)> { (TokenType.Identifier, "a"), (type, keyword), (TokenType.Eof, "") }, + $"'{keyword}' ({specTable}) as the last token of a source with no trailing newline"); + } + + [Fact] + public void KeywordNames_IsTheSpecRoster() + { + Roster.Select(r => r.Keyword).Should().OnlyHaveUniqueItems(); + Roster.Count(r => r.SpecTable == Hard).Should().Be(49, "keywords.md Hard Keywords"); + Roster.Count(r => r.SpecTable == Rejected).Should().Be(2, "keywords.md Reserved Rejected Keywords"); + Roster.Count(r => r.SpecTable == Future).Should().Be(2, "keywords.md Future Keywords"); + Roster.Count(r => r.SpecTable == Contextual).Should().Be(1, "self"); + + LexerNs.Lexer.KeywordNames.Should().BeEquivalentTo(Roster.Select(r => r.Keyword), + "the lexer's keyword set is exactly the spec's three tables plus the contextual self"); + + _output.WriteLine($"KEYWORD-ROSTER rows {Roster.Length}"); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/LexerCorpusTwinTests.cs b/src/Sharpy.Compiler.Tests/Lexer/LexerCorpusTwinTests.cs new file mode 100644 index 000000000..966ee0d3f --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/LexerCorpusTwinTests.cs @@ -0,0 +1,303 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using FluentAssertions; +using Sharpy.TestInfrastructure.Formatting; +using Sharpy.TestInfrastructure.Integration; +using Sharpy.TestInfrastructure.Lexing; +using Xunit; +using Xunit.Abstractions; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// The corpus position instrument (#2300 rows C2 + C3 + C6 at the corpus scale, P68 Phase 2). Over every fixture +/// under (build copies excluded, as the literal-loss differential does), each file is +/// lexed in nine twins — line ending {as-is, , } × +/// end of file {as-is, the trailing line break stripped, one extra trailing line break} — and every twin must: +/// +/// not throw (only the lexer's own recorded diagnostics end a lex); +/// reconstruct: +/// empty — every token's (line, column, position, length) maps back to its source slice; +/// agree with the as-is file on the token (Type, Value) sequence, the lexer diagnostics' (Code, Line, Column) +/// multiset, , the recovery resumes and their continued/possible-header +/// subsets (as LINE numbers — offsets move with the line break) and . +/// +/// The line-ending twins are compared exactly. The end-of-file twins are compared with three stated normalizations, +/// each the end of the source moving and nothing else: a Newline followed only by Dedent/Eof is +/// dropped (a stripped break removes it, an extra one adds it to a file without one; Newline's value is always +/// "\n"); a diagnostic positioned in the trailing line breaks or at the source end reads END (an +/// unterminated literal is reported where the source ends, which a stripped break moves up a line); and a recovery +/// resume in the trailing line breaks or at the source end is dropped (an abort on the last line of a file with no +/// final break stops without a recovery). A source whose text ends in a +/// backslash is compared across its line-ending twins only: its trailing break is the backslash's operand (a +/// continuation with one, SPY0016 without), so its EOF twins are different programs — the EOF roster's +/// EndingIsOperand rule; those twins must still not throw and must reconstruct. +/// +/// Expected 0-red at base: this is an instrument for a tests-only tracker, and its mutation record (the commit body: +/// a dropped _column advance after >>, ReadLiteralName's sourceLength: removed, +/// ConsumeLineBreak's \r\n two-step reduced to one) is what shows it is not vacuous. No GapDiscovery +/// trait: the Stryker filter runs the Lexer namespaces, and this must be inside it. +/// +/// +/// SHARPY_DUMP_TOKENS=<dir> (test-only; set it on one filtered invocation's command line, never in a +/// test) writes each file's as-is token dump and a SHA-256 per twin under that directory — the before/after +/// direction record for a behaviour-neutral lexer edit (P68 Design Decision 8: diff -r must be empty). +/// +/// +public class LexerCorpusTwinTests +{ + private const int FileFloor = 3_000; // 4,053 @ 220259b51 (3,809 Compiler + 244 Stdlib), measured + private const int CleanFloor = 2_500; // see the CORPUS-TWIN line for the measured count + + private static readonly string[] LineEndings = { "as-is", "crlf", "cr" }; + private static readonly string[] Eofs = { "as-is", "stripped", "extra" }; + + private readonly ITestOutputHelper _output; + + public LexerCorpusTwinTests(ITestOutputHelper output) => _output = output; + + private sealed record Lexed(string Source, LexerNs.Lexer Lexer, List Tokens); + + private static (Lexed? Lexed, Exception? Thrown) Lex(string source) + { + var lexer = new LexerNs.Lexer(source); + try + { + return (new Lexed(source, lexer, lexer.TokenizeAll()), null); + } + catch (Exception ex) + { + return (null, ex); + } + } + + internal static string EofTwin(string source, string eof) => eof switch + { + "stripped" => source.EndsWith("\r\n", StringComparison.Ordinal) ? source[..^2] + : source.EndsWith('\n') || source.EndsWith('\r') ? source[..^1] + : source, + "extra" => source + "\n", + _ => source, + }; + + internal static string LineEndingTwin(string source, string lineEnding) => lineEnding switch + { + "crlf" => FormatterTwins.Crlf(source), + "cr" => FormatterTwins.Cr(source), + _ => source, + }; + + /// Where the source's trailing line breaks start: an offset at or past it is "the end". + private static int ContentEnd(string source) => source.TrimEnd('\r', '\n').Length; + + private static int LineOf(IReadOnlyList lineStarts, int offset) + { + var line = 0; + while (line < lineStarts.Count && lineStarts[line] <= offset) + line++; + return line; + } + + /// What a twin must agree on, as comparable strings; applies the EOF normalizations. + private sealed record Signature(List Tokens, List Diagnostics, string Facts); + + private static Signature Sign(Lexed lexed, bool atEnd) + { + var tokens = lexed.Tokens.Select(t => $"{t.Type}|{t.Value}").ToList(); + if (atEnd) + { + var newline = lexed.Tokens.FindLastIndex(t => t.Type == TokenType.Newline); + if (newline >= 0 && lexed.Tokens.Skip(newline + 1).All(t => t.Type is TokenType.Dedent or TokenType.Eof)) + tokens.RemoveAt(newline); + } + + var lineStarts = TokenPositionReconstruction.LineStarts(lexed.Source); + var contentEnd = ContentEnd(lexed.Source); + var diagnostics = lexed.Lexer.Diagnostics.GetAll().Select(d => + { + if (atEnd && d.Line is { } line && d.Column is { } column && line >= 1 && line <= lineStarts.Count + && lineStarts[line - 1] + column - 1 >= contentEnd) + return $"{d.Code}@END"; + return $"{d.Code}@{d.Line}:{d.Column}"; + }).OrderBy(s => s, StringComparer.Ordinal).ToList(); + + string Lines(IEnumerable offsets) => string.Join(",", offsets + .Where(offset => !atEnd || offset < contentEnd) + .Select(offset => LineOf(lineStarts, offset)).OrderBy(l => l)); + + var lexer = lexed.Lexer; + var facts = $"loss={lexer.LiteralLoss} resumes=[{Lines(lexer.RecoveryResumes)}] " + + $"continued=[{Lines(lexer.RecoveryResumesAfterAContinuedLine)}] header=[{Lines(lexer.RecoveryResumesAfterAPossibleHeader)}] " + + $"bracketOpenAt={lexer.BracketLeftOpenAtLine?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "-"}"; + return new Signature(tokens, diagnostics, facts); + } + + private static IEnumerable Disagreements(string where, Signature expected, Signature actual) + { + if (!expected.Tokens.SequenceEqual(actual.Tokens)) + { + var at = 0; + while (at < expected.Tokens.Count && at < actual.Tokens.Count && expected.Tokens[at] == actual.Tokens[at]) + at++; + yield return $"TOKENS {where}: first difference at #{at}: as-is {Shown(expected.Tokens, at)} twin {Shown(actual.Tokens, at)} " + + $"(counts {expected.Tokens.Count}/{actual.Tokens.Count})"; + } + if (!expected.Diagnostics.SequenceEqual(actual.Diagnostics)) + yield return $"DIAGNOSTICS {where}: as-is [{string.Join(", ", expected.Diagnostics)}] twin [{string.Join(", ", actual.Diagnostics)}]"; + if (expected.Facts != actual.Facts) + yield return $"FACTS {where}: as-is {expected.Facts} twin {actual.Facts}"; + } + + private static string Shown(List tokens, int at) => + at < tokens.Count ? $"'{tokens[at].Replace("\n", "\\n", StringComparison.Ordinal).Replace("\r", "\\r", StringComparison.Ordinal)}'" : ""; + + private static string Dump(Lexed lexed) + { + var text = new StringBuilder(); + foreach (var t in lexed.Tokens) + text.Append(t.Type).Append('|').Append(Escape(t.Value)).Append('|').Append(t.Line).Append(':').Append(t.Column) + .Append('@').Append(t.Position).Append('+').Append(t.Length).Append('\n'); + foreach (var d in lexed.Lexer.Diagnostics.GetAll()) + text.Append("DIAG|").Append(d.Code).Append('|').Append(d.Line).Append(':').Append(d.Column).Append('|').Append(Escape(d.Message)).Append('\n'); + var lexer = lexed.Lexer; + text.Append("FACTS|loss=").Append(lexer.LiteralLoss) + .Append("|resumes=").Append(string.Join(",", lexer.RecoveryResumes)) + .Append("|continued=").Append(string.Join(",", lexer.RecoveryResumesAfterAContinuedLine.OrderBy(o => o))) + .Append("|header=").Append(string.Join(",", lexer.RecoveryResumesAfterAPossibleHeader.OrderBy(o => o))) + .Append("|bracketOpenAt=").Append(lexer.BracketLeftOpenAtLine).Append('\n'); + return text.ToString(); + } + + private static string Escape(string text) => + text.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\n", "\\n", StringComparison.Ordinal) + .Replace("\r", "\\r", StringComparison.Ordinal).Replace("|", "\\|", StringComparison.Ordinal); + + internal static List<(string Name, string Path)> Population() + { + var files = new List<(string, string)>(); + foreach (var root in FixtureRoots.All) + { + files.AddRange(Directory.EnumerateFiles(root.Path, "*.spy", SearchOption.AllDirectories) + .Select(f => (Relative: Path.GetRelativePath(root.Path, f), Full: f)) + .Where(f => !Sharpy.Compiler.Diagnostics.CrashBundleWriter.IsNonSourceSegment(f.Relative)) + .OrderBy(f => f.Relative, StringComparer.Ordinal) + .Select(f => (root.Label.Length == 0 ? f.Relative : $"{root.Label}/{f.Relative}", f.Full))); + } + return files; + } + + [Fact] + public void Corpus_PositionsReconstruct_InEveryLineEndingAndEofTwin() + { + var clock = Stopwatch.StartNew(); + var dumpRoot = Environment.GetEnvironmentVariable("SHARPY_DUMP_TOKENS"); + var files = Population(); + var violations = new List<(string Scope, string Violation)>(); + var clean = 0; + var operandEnds = 0; + long tokens = 0; + + foreach (var (name, path) in files) + { + var original = File.ReadAllText(path); + var (reference, thrown) = Lex(original); + if (reference == null) + { + violations.Add(($"corpus {name}", $"THROW {thrown!.GetType().Name} {name} [as-is/as-is]: {thrown.Message}")); + continue; + } + if (!reference.Lexer.Diagnostics.HasErrors) + clean++; + // A source whose text ends in a backslash reads its trailing break as the backslash's operand (a + // continuation; none or spaces = SPY0016): its EOF twins are different programs, so only its line-ending + // twins are compared — every twin must still not throw and must reconstruct (the roster's rule). + var endingIsOperand = original.TrimEnd('\r', '\n').EndsWith('\\'); + if (endingIsOperand) + operandEnds++; + var exact = Sign(reference, atEnd: false); + var atEnd = Sign(reference, atEnd: true); + var dump = dumpRoot == null ? null : new StringBuilder(); + + foreach (var eof in Eofs) + { + foreach (var lineEnding in LineEndings) + { + var twin = $"{lineEnding}/{eof}"; + var source = LineEndingTwin(EofTwin(original, eof), lineEnding); + var (lexed, error) = eof == "as-is" && lineEnding == "as-is" ? (reference, (Exception?)null) : Lex(source); + if (lexed == null) + { + violations.Add(($"corpus {name}", $"THROW {error!.GetType().Name} {name} [{twin}]: {error.Message}")); + dump?.Append(twin).Append(" THROW ").Append(error.GetType().Name).Append('\n'); + continue; + } + tokens += lexed.Tokens.Count; + + violations.AddRange(TokenPositionReconstruction.Violations(source, lexed.Lexer, lexed.Tokens) + .Select(v => ($"corpus {name}", $"{v} :: {name} [{twin}]"))); + var expected = eof == "as-is" ? exact : atEnd; + if (eof == "as-is" || !endingIsOperand) + violations.AddRange(Disagreements($"{name} [{twin}]", expected, Sign(lexed, atEnd: eof != "as-is")).Select(v => ($"corpus {name}", v))); + + if (dump != null) + { + var text = Dump(lexed); + if (twin == "as-is/as-is") + dump.Insert(0, text); + dump.Append(twin).Append(" sha256 ").Append(Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text)))).Append('\n'); + } + } + } + + if (dump != null) + { + var target = Path.Combine(dumpRoot!, name + ".tokens"); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.WriteAllText(target, dump.ToString()); + } + } + + var (rows, malformed) = LexerPositionAllowlist.Read(); + var (remaining, stale, allowlisted) = LexerPositionAllowlist.Apply(rows, "corpus", violations); + var failures = malformed.Concat(stale).Concat(remaining).ToList(); + _output.WriteLine($"CORPUS-TWIN files {files.Count} clean {clean} tokens {tokens} twins 9 backslashEnds {operandEnds} " + + $"violations {remaining.Count} allowlisted {allowlisted} elapsed {clock.Elapsed.TotalSeconds:F1}s"); + foreach (var failure in failures.Take(200)) + _output.WriteLine("CORPUS-TWIN " + failure); + + files.Count.Should().BeGreaterThanOrEqualTo(FileFloor, "the corpus is the instrument's population"); + clean.Should().BeGreaterThanOrEqualTo(CleanFloor, "most of the corpus must lex clean for the reconstruction to bite"); + failures.Should().BeEmpty("a position defect outside the contract is a findings-ledger line and a row in {0} citing its issue — never a changed rule", + LexerPositionAllowlist.FileName); + } + + [Theory] + [InlineData("x = 1\n", "stripped", "x = 1")] + [InlineData("x = 1\r\n", "stripped", "x = 1")] + [InlineData("x = 1\r", "stripped", "x = 1")] + [InlineData("x = 1", "stripped", "x = 1")] + [InlineData("x = 1\n\n", "stripped", "x = 1\n")] + [InlineData("x = 1\n", "extra", "x = 1\n\n")] + [InlineData("x = 1", "extra", "x = 1\n")] + public void EofTwin_MovesOnlyTheEnd(string source, string eof, string expected) => + EofTwin(source, eof).Should().Be(expected); + + [Fact] + public void EofTwins_AreDiscriminating() + { + // Positive control for the comparison: a twin that changes a token (not only the end) is a disagreement. + var (reference, _) = Lex("x = 1\ny = 2\n"); + var (changed, _) = Lex("x = 1\ny = 3\n"); + Disagreements("control", Sign(reference!, atEnd: true), Sign(changed!, atEnd: true)).Should().ContainSingle() + .Which.Should().StartWith("TOKENS"); + + // ... and a diagnostic that moves within the content is one too. + var (early, _) = Lex("x = $\ny = 2\n"); + var (late, _) = Lex("x = 1\ny = $\n"); + Disagreements("control", Sign(early!, atEnd: true), Sign(late!, atEnd: true)) + .Should().Contain(d => d.StartsWith("DIAGNOSTICS", StringComparison.Ordinal)); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/LexerPositionAllowlist.cs b/src/Sharpy.Compiler.Tests/Lexer/LexerPositionAllowlist.cs new file mode 100644 index 000000000..35725a73a --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/LexerPositionAllowlist.cs @@ -0,0 +1,172 @@ +using System.Text.RegularExpressions; +using FluentAssertions; +using Sharpy.TestInfrastructure.Integration; +using Xunit; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// The ratchet for the position instrument (, ; +/// #2300 C2/C3/C6): lexer-position-allowlist.txt, rows +/// <scope> <where> :: <TokenType> '<Value>'@L<line>:C<column> <R-rules> # #NNNN reason. +/// A row suppresses the violations whose SUBJECT is that one token — its type and Value at that (line, column) — and whose +/// rule is one the row names (R3,R7), in that one corpus file +/// (corpus <relative path>) or roster construct (roster <construct name>), in every twin or +/// ending — a lexer position defect found at base, filed by the lead, drained when the lexer is fixed. A row without a +/// #\d+ cite fails; a row that suppresses nothing, or names a rule that no longer fires, fails as stale (drain on +/// fix). Every other violation in the same file or construct still fails, including another token of the same type, the +/// same token with a different Value, and a rule the row does not name: the row admits today's defect, never a second one. +/// +public static partial class LexerPositionAllowlist +{ + public const string FileName = "lexer-position-allowlist.txt"; + + public sealed record Row(int Line, string Scope, string TokenType, IReadOnlySet Rules, string Rest); + + [GeneratedRegex(@"^(?(corpus|roster) .+?) :: (?[A-Za-z]+ '.*?'@L\d+:C\d+) (?R\d(?:,R\d)*) +(?#.*)$")] + private static partial Regex RowPattern(); + + [GeneratedRegex(@"#\d+")] + private static partial Regex Cite(); + + public static string Path => System.IO.Path.Combine(FixtureRoots.RepositoryRoot, "src", "Sharpy.Compiler.Tests", "Lexer", FileName); + + /// The rows, and one failure line per malformed row (no cite, or not the row shape). + public static (List Rows, List Malformed) Read(IEnumerable lines) + { + var rows = new List(); + var malformed = new List(); + var number = 0; + foreach (var raw in lines) + { + number++; + var line = raw.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + continue; + var match = RowPattern().Match(line); + if (!match.Success) + { + malformed.Add($"{FileName} line {number}: not ' :: ''@L:C # #NNNN reason': {line}"); + continue; + } + if (!Cite().IsMatch(match.Groups["rest"].Value)) + { + malformed.Add($"{FileName} line {number}: no #NNNN issue cite: {line}"); + continue; + } + var rules = match.Groups["rules"].Value.Split(',').ToHashSet(StringComparer.Ordinal); + rows.Add(new Row(number, match.Groups["scope"].Value, match.Groups["type"].Value, rules, match.Groups["rest"].Value)); + } + return (rows, malformed); + } + + public static (List Rows, List Malformed) Read() => Read(File.ReadAllLines(Path)); + + [GeneratedRegex(@"^(?[A-Za-z]+) (?'.*?') L(?\d+):C(?-?\d+)@")] + private static partial Regex Described(); + + /// + /// The token a violation line is about, as Type 'Value'@L<line>:C<column>: the token a rule line names + /// (R1 Eof '' L1:C11@10 …), the earlier token of an R7 gap (R7 gap […) between FStringEnd '…' L5:C24@88 …), + /// or the category of a twin disagreement (TOKENS/DIAGNOSTICS/FACTS/CODES/THROW, R6 — never a token, so never + /// allowlistable: they always fail). + /// + public static string Subject(string violation) + { + var words = violation.Split(' ', 3); + if (words.Length == 3 && words[0].Length == 2 && words[0][0] == 'R' && char.IsDigit(words[0][1])) + { + var described = words[1] == "gap" + ? violation.IndexOf(" between ", StringComparison.Ordinal) is var at and >= 0 ? violation[(at + " between ".Length)..] : "" + : violation[(words[0].Length + 1)..]; + var match = Described().Match(described); + if (match.Success) + return $"{match.Groups["type"].Value} {match.Groups["value"].Value}@L{match.Groups["line"].Value}:C{match.Groups["column"].Value}"; + } + return words[0]; + } + + /// + /// Splits (each paired with its scope) into the ones no row covers and the count each row + /// covered; a row covering none, or naming a rule that covered nothing, is stale. Only rows whose scope starts with + /// are judged. + /// + public static (List Remaining, List Stale, int Allowlisted) Apply( + IReadOnlyList rows, string scopeKind, IEnumerable<(string Scope, string Violation)> violations) + { + var mine = rows.Where(r => r.Scope.StartsWith(scopeKind + " ", StringComparison.Ordinal)).ToList(); + var used = new HashSet<(Row, string)>(); + var remaining = new List(); + var allowlisted = 0; + foreach (var (scope, violation) in violations) + { + var rule = violation.Split(' ', 2)[0]; + var row = mine.FirstOrDefault(r => r.Scope == scope && r.TokenType == Subject(violation) && r.Rules.Contains(rule)); + if (row == null) + { + remaining.Add(violation); + continue; + } + used.Add((row, rule)); + allowlisted++; + } + var stale = mine + .SelectMany(r => r.Rules.Where(rule => !used.Contains((r, rule))).Select(rule => (Row: r, Rule: rule))) + .Select(x => $"STALE {FileName} line {x.Row.Line}: '{x.Row.Scope} :: {x.Row.TokenType}' rule {x.Rule} suppresses nothing — " + + "delete the rule or the row (drain on fix)") + .ToList(); + return (remaining, stale, allowlisted); + } +} + +public class LexerPositionAllowlistTests +{ + [Fact] + public void EveryRow_HasTheShapeAndAnIssueCite() + { + var (_, malformed) = LexerPositionAllowlist.Read(); + malformed.Should().BeEmpty(); + } + + [Fact] + public void TheRatchet_RefusesAnUncitedRow_SuppressesOnlyItsSubject_AndFailsWhenStale() + { + var (rows, malformed) = LexerPositionAllowlist.Read(new[] + { + "corpus a.spy :: FStringEnd '\"\"\"'@L5:C24 R3,R7 # #2338 cited", + "corpus b.spy :: FStringEnd '\"\"\"'@L5:C24 R3 # no cite", + "corpus d.spy :: FStringEnd R3 # #2338 a kind, not a token", + "corpus e.spy :: FStringEnd '\"\"\"'@L5:C24 # #2338 no rule named", + "roster gone :: Eof ''@L1:C11 R1 # #2338 stale", + "roster half :: Eof ''@L1:C11 R1,R6 # #2338 R6 never fires", + }); + malformed.Should().HaveCount(3); + malformed.Should().ContainSingle(m => m.Contains("no #NNNN issue cite")); + malformed.Where(m => m.Contains("not '")).Should().HaveCount(2); + + var (remaining, stale, allowlisted) = LexerPositionAllowlist.Apply(rows, "corpus", new[] + { + ("corpus a.spy", "R3 FStringEnd '\"\"\"' L5:C24@88: slice ≠ Value"), + ("corpus a.spy", "R7 gap [91,100) between FStringEnd '\"\"\"' L5:C24@88 and Newline '\\n' L6:C12@100 is not trivia"), + ("corpus a.spy", "R1 Identifier 'x' L1:C2@0: off"), + ("corpus a.spy", "R3 FStringEnd '\"\"\"' L9:C24@130: slice ≠ Value"), + ("corpus a.spy", "R3 FStringEnd '}' L5:C24@88: slice ≠ Value"), + ("corpus a.spy", "R1 FStringEnd '\"\"\"' L5:C24@88: position 88 is not on line 5"), + ("corpus a.spy", "TOKENS a.spy [cr/as-is]: first difference"), + ("corpus c.spy", "R3 FStringEnd '\"\"\"' L5:C24@88: slice ≠ Value"), + }); + allowlisted.Should().Be(2, "the row suppresses its subject token type in its own file only"); + remaining.Should().HaveCount(6, "another FStringEnd in the same file, the same token with another Value, the same token " + + "under a rule the row does not name, another token, a disagreement and another file all still fail"); + stale.Should().BeEmpty(); + + var (_, rosterStale, rosterAllowlisted) = LexerPositionAllowlist.Apply(rows, "roster", new[] + { + ("roster half", "R1 Eof '' L1:C11@10: position 10 is not on line 1"), + }); + rosterAllowlisted.Should().Be(1); + rosterStale.Should().HaveCount(2, "the row that matched nothing, and the rule R6 that fired nowhere"); + rosterStale.Should().AllSatisfy(m => m.Should().StartWith("STALE")); + rosterStale.Should().ContainSingle(m => m.Contains("'roster half") && m.Contains("rule R6")); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.CleanFacts.cs b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.CleanFacts.cs new file mode 100644 index 000000000..ddc0a9ace --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.CleanFacts.cs @@ -0,0 +1,368 @@ +using FluentAssertions; +using Xunit; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +public partial class LiteralLossScannerDifferentialTests +{ + /// + /// The facts a CLEAN lex determines about each line of a document (P68 Phase 3, #2300 row C5): the oracle the + /// injection differential judges the literal-loss scanner's verdicts on dropped text against. Every fact is read + /// off the clean lexer's own tokens and , never off a second hand model of + /// the text, so a scanner arm that disagrees with the lexer cannot agree with this oracle by sharing its mistake. + /// + /// — a literal span starts in [at, lineEnd) and ends past the line's + /// break (the truth behind ). + /// — a real bracket (f-string contents excluded) is open at the line's end, or the + /// line ends in a backslash continuation (no Newline token at its end at depth 0; the backslash is checked + /// against the text — an inconsistency throws rather than guessing). + /// — the 1-based line of the outermost real bracket open at the line's end. + /// — a Colon token at bracket depth 0 outside every f-string on the + /// logical line containing the line, up to its end (:= is a different token). + /// — a token of that logical line was read before an offset (an abort there). + /// + /// Domain restriction (/verify-plan, Dimension 4): , + /// and are clean facts only for a line whose END is not inside a literal span — a line + /// where a multi-line literal opens also ends with no Newline at bracket depth 0, which the backslash reading + /// would misjudge. They THROW outside the domain, so a caller cannot assert them there by accident; on such a line + /// only is a clean fact, and the R-FR/R-FV rules (LiteralStateTests) own the rest. + /// + internal sealed class CleanLineFacts + { + private readonly LexerNs.Token[] _tokens; // non-synthetic tokens (Position ≥ 0), in source order + private readonly int[] _logical; // the logical line of each token: Newline tokens before it + private readonly bool[] _inFString; // inside an f-/t-string (FStringStart … FStringEnd), the delimiters included + private readonly int[] _depthBefore; // real bracket depth before each token + private readonly int[] _depthAfter; // ... and after it + private readonly int[] _outermostAfter; // 1-based line of the outermost real bracket open after it, or 0 + private readonly int[] _lastBeforeEnd; // per line: the last token index with Position < lineEnd, or -1 + private readonly HashSet _newlinePositions = new(); + + public string Source { get; } + + /// The lexer's literal spans (), the truth every literal fact reads. + public IReadOnlyList<(int Start, int End)> Spans { get; } + + /// Each line's [start, end), its break excluded (). + public IReadOnlyList<(int Start, int End)> Lines { get; } + + public IReadOnlyList Tokens => _tokens; + + private CleanLineFacts(string source, List tokens) + { + Source = source; + Spans = LexerNs.LiteralSpans.Of(tokens); + Lines = LineRanges(source); + _tokens = tokens.Where(t => t.Position >= 0).ToArray(); + var n = _tokens.Length; + _logical = new int[n]; + _inFString = new bool[n]; + _depthBefore = new int[n]; + _depthAfter = new int[n]; + _outermostAfter = new int[n]; + + var logical = 0; + var fstring = 0; + var brackets = new List(); // opener lines, outermost first + for (var i = 0; i < n; i++) + { + var token = _tokens[i]; + _logical[i] = logical; + _depthBefore[i] = brackets.Count; + if (token.Type == TokenType.FStringStart) + fstring++; + _inFString[i] = fstring > 0; + if (token.Type == TokenType.FStringEnd && fstring > 0) + fstring--; + else if (fstring == 0 && token.Type is TokenType.LeftParen or TokenType.LeftBracket or TokenType.LeftBrace) + brackets.Add(token.Line); + else if (fstring == 0 && token.Type is TokenType.RightParen or TokenType.RightBracket or TokenType.RightBrace && brackets.Count > 0) + brackets.RemoveAt(brackets.Count - 1); + _depthAfter[i] = brackets.Count; + _outermostAfter[i] = brackets.Count > 0 ? brackets[0] : 0; + if (token.Type == TokenType.Newline) + { + logical++; + _newlinePositions.Add(token.Position); + } + } + + _lastBeforeEnd = new int[Lines.Count]; + var t = 0; + for (var k = 0; k < Lines.Count; k++) + { + while (t < n && _tokens[t].Position < Lines[k].End) + t++; + _lastBeforeEnd[k] = t - 1; + } + } + + /// The facts of , or null when it does not lex clean (no oracle there). + public static CleanLineFacts? Of(string source) + { + var lexer = new LexerNs.Lexer(source); + var tokens = lexer.TokenizeAll(); + return lexer.Diagnostics.HasErrors ? null : new CleanLineFacts(source, tokens); + } + + /// The length of line 's break: 2 for \r\n, 1 for \n or \r, 0 at the end of the source. + public int LineBreakLength(int k) + { + var end = Lines[k].End; + if (end >= Source.Length) + return 0; + return Source[end] == '\r' && end + 1 < Source.Length && Source[end + 1] == '\n' ? 2 : 1; + } + + private static bool IsCode(LexerNs.Token token) + => token.Type is not (TokenType.Indent or TokenType.Dedent or TokenType.Newline or TokenType.Eof); + + /// A literal span crosses line 's break. + public bool EndInsideLiteral(int k) + { + var end = Lines[k].End; + return Spans.Any(s => s.Start < end && s.End > end); + } + + /// Line starts inside a literal span (a close line or an interior line of a literal that spans lines). + public bool StartsInsideLiteral(int k) + { + var start = Lines[k].Start; + return Spans.Any(s => s.Start < start && s.End > start); + } + + /// The CLOSE line of a literal that spanned lines: a span that started on an earlier line ends on line . + public bool IsCloseLine(int k) + { + var (start, end) = Lines[k]; + return Spans.Any(s => s.Start < start && s.End > start && s.End <= end); + } + + /// A literal span starts in [at, lineEnd) of line and ends past its break. + public bool OpenerCrossing(int k, int at) + { + var end = Lines[k].End; + return Spans.Any(s => s.Start >= at && s.Start < end && s.End > end); + } + + /// Line holds a code token (not Indent/Dedent/Newline/Eof). + public bool HasCode(int k) + { + var (start, end) = Lines[k]; + return _tokens.Any(t => t.Position >= start && t.Position < end && IsCode(t)); + } + + /// The first code-token start on line that lies outside every literal span (a span's own start counts as outside), or null. + public int? FirstCodeTokenStart(int k) + { + var (start, end) = Lines[k]; + foreach (var token in _tokens) + { + if (token.Position < start || !IsCode(token)) + continue; + if (token.Position >= end) + break; + var at = token.Position; + if (!Spans.Any(s => s.Start < at && at < s.End)) + return at; + } + return null; + } + + /// Line 's first code token is the first token of its logical line (no code token of the same logical line precedes it), and the line does not start inside a literal. + public bool StartsLogicalLine(int k) + { + if (StartsInsideLiteral(k)) + return false; + var (start, end) = Lines[k]; + for (var i = 0; i < _tokens.Length; i++) + { + if (_tokens[i].Position < start || !IsCode(_tokens[i])) + continue; + if (_tokens[i].Position >= end) + return false; + for (var j = i - 1; j >= 0; j--) + { + if (_tokens[j].Type is TokenType.Indent or TokenType.Dedent) + continue; + return _tokens[j].Type == TokenType.Newline; + } + return true; + } + return false; + } + + private void RequireDomain(int k, string fact) + { + if (EndInsideLiteral(k)) + throw new InvalidOperationException($"{fact}(L{k + 1}) is not a clean fact: a literal span crosses the line's end (domain restriction)"); + if (!HasCode(k)) + throw new InvalidOperationException($"{fact}(L{k + 1}) asked of a line with no code token"); + } + + private int DepthAtEnd(int k) => _lastBeforeEnd[k] < 0 ? 0 : _depthAfter[_lastBeforeEnd[k]]; + + /// + /// Line continues on the next: a real bracket is open at its end, or it ends in a backslash + /// continuation. Throws outside the domain (), and when the token reading (depth 0, + /// no Newline at the end, not the last line) is not confirmed by a backslash in the text. + /// + public bool Continued(int k) + { + RequireDomain(k, nameof(Continued)); + if (DepthAtEnd(k) > 0) + return true; + var end = Lines[k].End; + if (end >= Source.Length || _newlinePositions.Contains(end)) + return false; + var last = end - 1; + while (last >= Lines[k].Start && Source[last] is ' ' or '\t') + last--; + if (last < Lines[k].Start || Source[last] != '\\') + throw new InvalidOperationException($"oracle inconsistent at L{k + 1}: depth 0, no Newline at the line's end, yet no trailing backslash :: {Source[Lines[k].Start..end]}"); + return true; + } + + /// The 1-based line of the outermost real bracket open at line 's end, or null. Throws outside the domain. + public int? BracketOpenerLine(int k) + { + RequireDomain(k, nameof(BracketOpenerLine)); + var last = _lastBeforeEnd[k]; + return last < 0 || _outermostAfter[last] == 0 ? null : _outermostAfter[last]; + } + + /// The logical line of line 's first code token. + private int LogicalOf(int k) + { + var (start, end) = Lines[k]; + for (var i = 0; i < _tokens.Length; i++) + { + if (_tokens[i].Position >= start && _tokens[i].Position < end && IsCode(_tokens[i])) + return _logical[i]; + } + throw new InvalidOperationException($"L{k + 1} has no code token"); + } + + /// A Colon token at real bracket depth 0, outside every f-string, on the logical line containing line , up to its end. + public bool ColonOnLogicalLine(int k) + { + var logical = LogicalOf(k); + var end = Lines[k].End; + for (var i = 0; i < _tokens.Length && _tokens[i].Position < end; i++) + { + if (_logical[i] == logical && _tokens[i].Type == TokenType.Colon && !_inFString[i] && _depthBefore[i] == 0) + return true; + } + return false; + } + + /// A code token of the logical line containing line starts before (the lexer read it before aborting there). + public bool TokenBefore(int k, int abortAt) + { + var logical = LogicalOf(k); + for (var i = 0; i < _tokens.Length && _tokens[i].Position < abortAt; i++) + { + if (_logical[i] == logical && IsCode(_tokens[i])) + return true; + } + return false; + } + + /// + /// What RecoveryResumesAfterAPossibleHeader means for an abort at on line + /// whose given-up text is read cleanly (no unterminated literal: the end is readable): the + /// line does not continue, a token of its logical line was read before the abort (lead ruling L8: a line dropped + /// before any token is read is never one), and the logical line holds a depth-0 colon. Throws outside the domain. + /// + public bool PossibleHeader(int k, int abortAt) + => !Continued(k) && TokenBefore(k, abortAt) && ColonOnLogicalLine(k); + } + + /// + /// The oracle's own hand rows (it is an instrument; verification-contract §2 applies to it): one clean document per + /// fact arm, each with the expected value of every fact on its 1-based line. null in a domain-restricted + /// column means the line's end is inside a literal and the fact must THROW. + /// + public static TheoryData CleanFactRows => new() + { + // name, source, line, endInsideLiteral, closeLine, continued, bracketOpenerLine, colonOnLogicalLine + { "a statement", "x = 1\n", 1, false, false, false, null, false }, + { "a bracket open at the end", "x = foo(1,\n 2)\n", 1, false, false, true, 1, false }, + { "the bracket closed on the next line", "x = foo(1,\n 2)\n", 2, false, false, false, null, false }, + { "a backslash continuation", "y = 1 + \\\n 2\n", 1, false, false, true, null, false }, + { "a backslash continuation, CRLF", "y = 1 + \\\r\n 2\r\n", 1, false, false, true, null, false }, + { "the line after a backslash", "y = 1 + \\\n 2\n", 2, false, false, false, null, false }, + { "the last line with no break", "x = (1)", 1, false, false, false, null, false }, + { "the last line with no break, a bracket spanning lines", "x = (1,\n 2)", 1, false, false, true, 1, false }, + { "nested: the outermost opener", "a = [(1,\n 2), (3,\n 4)]\n", 2, false, false, true, 1, false }, + { "a header colon", "if x:\n y = 1\n", 1, false, false, false, null, true }, + { "a colon on the logical line BEFORE", "if x:\n y = 1\n", 2, false, false, false, null, false }, + { "a header colon after a bracket spanning lines", "def f(a,\n b):\n pass\n", 2, false, false, false, null, true }, + { "no colon yet on the header's first line", "def f(a,\n b):\n pass\n", 1, false, false, true, 1, false }, + { "a colon in a bracket", "d = {a: 1}\n", 1, false, false, false, null, false }, + { "a walrus is no colon", "y := 1\n", 1, false, false, false, null, false }, + { "a colon in a format spec", "s = f\"{x:>4}\"\n", 1, false, false, false, null, false }, + { "a lambda colon in a field", "s = f\"{(lambda: 1)()}\" + 'a'\n", 1, false, false, false, null, false }, + { "a field's bracket around a real one", "print(f\"{a[0]}\",\n 1)\n", 1, false, false, true, 1, false }, + { "a comment holding a bracket", "x = 1 # (\n", 1, false, false, false, null, false }, + { "a string holding a bracket and a colon", "x = \"(:\"\n", 1, false, false, false, null, false }, + { "a triple opener: the end is inside a literal", "s = \"\"\"\nabc\n\"\"\"\n", 1, true, false, null, null, false }, + { "the triple's close line", "s = \"\"\"\nabc\n\"\"\" + x\n", 3, false, true, false, null, false }, + { "a field spanning lines: the end is inside a literal", "x = f\"{foo(1,\n2)}\"\n", 1, true, false, null, null, false }, + { "a lone-CR triple opener", "s = \"\"\"\rabc\r\"\"\"\r", 1, true, false, null, null, false }, + }; + + [Theory] + [MemberData(nameof(CleanFactRows))] + public void CleanFacts_HandRows(string name, string source, int line, bool endInsideLiteral, bool closeLine, bool? continued, + int? bracketOpenerLine, bool colonOnLogicalLine) + { + var lexed = CleanLineFacts.Of(source); + lexed.Should().NotBeNull("{0} must lex clean to be inside the oracle", name); + var facts = lexed!; + var k = line - 1; + + facts.EndInsideLiteral(k).Should().Be(endInsideLiteral, name); + facts.IsCloseLine(k).Should().Be(closeLine, name); + facts.ColonOnLogicalLine(k).Should().Be(colonOnLogicalLine, name); + if (continued is { } expected) + { + facts.Continued(k).Should().Be(expected, name); + facts.BracketOpenerLine(k).Should().Be(bracketOpenerLine, name); + } + else + { + FluentActions.Invoking(() => facts.Continued(k)).Should().Throw("{0}: outside the domain", name); + FluentActions.Invoking(() => facts.BracketOpenerLine(k)).Should().Throw("{0}: outside the domain", name); + facts.OpenerCrossing(k, facts.Lines[k].Start).Should().BeTrue("{0}: the literal crossing the end starts on the line", name); + } + } + + /// + /// , , + /// and by offset: + /// an opener after the offset crosses, one before it does not; a token read before the abort is one of the logical + /// line's; a continuation line does not start its logical line. + /// + [Fact] + public void CleanFacts_OffsetFacts() + { + const string source = "x = 'a' + \"\"\"\nk\n\"\"\"\ny = foo(1,\n 2)\n"; + var facts = CleanLineFacts.Of(source)!; + var triple = source.IndexOf("\"\"\"", StringComparison.Ordinal); + + facts.OpenerCrossing(0, 0).Should().BeTrue(); + facts.OpenerCrossing(0, triple).Should().BeTrue("the span's own start is inside [at, lineEnd)"); + facts.OpenerCrossing(0, triple + 1).Should().BeFalse("the opener starts before the offset"); + facts.TokenBefore(0, 0).Should().BeFalse(); + facts.TokenBefore(0, source.IndexOf('\'')).Should().BeTrue("x and = were read"); + facts.FirstCodeTokenStart(0).Should().Be(0); + facts.FirstCodeTokenStart(2).Should().BeNull("the close line's only code is the literal that started above"); + facts.FirstCodeTokenStart(3).Should().Be(source.IndexOf('y')); + facts.StartsLogicalLine(3).Should().BeTrue(); + facts.StartsLogicalLine(4).Should().BeFalse("the line continues the bracket opened above"); + facts.TokenBefore(4, facts.Lines[4].Start).Should().BeTrue("the logical line's tokens on the line above were read"); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.Injection.cs b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.Injection.cs new file mode 100644 index 000000000..aa2911ee3 --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.Injection.cs @@ -0,0 +1,547 @@ +using System.Diagnostics; +using System.Text; +using System.Text.RegularExpressions; +using FluentAssertions; +using Sharpy.TestInfrastructure.Integration; +using Xunit; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +public partial class LiteralLossScannerDifferentialTests +{ + /// + /// Every -th BULK cell of the generated sequence is lexed (fixed, never random); every + /// other cell is lexed always. Bulk = the dollar and tab constructs on the as-is document and its + /// @backtick twin (one per line each: 184,641 of 284,952 generated cells @ 0d58024dd + the @walrus twin, + /// measured); the sparse constructs (escape, + /// conversion) and the targeted content twins (@escaped-quote, @unbalanced-brace, + /// @field-split, @walrus) — the cells that alone kill the scanner-arm mutants — are never sampled. + /// Each bulk construct has its own sequence (the ordered corpus × twins × lines), so the sample is even across the + /// two and a pure function of the tree. Stride 4 keeps 146,471 of the 284,952 cells (51 %, measured) and halves the + /// CPU the test costs — it runs for nearly every lexer mutant under Stryker's perTest coverage. It does NOT bound the + /// in-gate wall clock: the gate measured 96 s @ 2c15c47a0 (stride 1) and 102 s @ be4664e8f (stride 4), while the + /// untouched Shapes test went 28 s → 83 s in the same runs, so in-gate durations are load, not cells. Quiet (the + /// Lexer filter alone): 6.3 s at stride 4 — the figure that measures the class against Design Decision 10's 60 s. + /// + private const int InjectionStride = 4; + + private static bool IsBulk(string twin, Injection cell) => + cell.Construct is ("dollar" or "tab") && twin is ("" or "@backtick"); + + /// The error constructs, in roster order (the K of the output line). + private static readonly IReadOnlyList<(string Name, string Code)> InjectionConstructs = new[] + { + ("dollar", "SPY0015"), + ("escape", "SPY0004"), + ("conversion", "SPY0030"), + ("tab", "SPY0012"), + }; + + /// + /// One injected cell: the error construct, the clean document's 0-based line, where the text the lexer gives up + /// starts in the CLEAN document (: the oracle's at), where it aborts + /// (: tokens before it were read), and the edit — characters at + /// replaced by . + /// + internal sealed record Injection(string Construct, int Line, int DroppedFrom, int AbortAt, int EditAt, int Remove, string Insert, string Code); + + /// + /// The roster of error constructs injected on line of a clean document (verification-contract + /// §1's matrix: dropped-line text × injected construct; the text's content is the corpus and its twins): + /// + /// dollar — $ at the line's first token start outside every literal (SPY0015): the given-up text + /// is [at, lineEnd). + /// escape — \q right after the opening quote of each non-raw short string token on the line that + /// is a literal span of its own (not a literal nested in an f-string field) and closes on the line (SPY0004): the + /// abort is INSIDE the literal, and UnreadTextStart starts the given-up text at its prefix. + /// conversion — !q before the } of each outermost replacement field of a single-line, + /// top-level f-/t-string that has no conversion, format spec or self-documenting = (SPY0030): the given-up + /// text starts at the f-string's prefix; the tokens of the f-string up to the abort were read. + /// tab — the line's leading whitespace replaced by a tab, when the line starts its logical line + /// (SPY0012, an indentation error): the whole line is given up and no token of it was read. + /// + /// A line that starts inside a literal is skipped: an interior line has no site, and a CLOSE line's truth is the + /// RePairedCloser rule (R-FR/R-FV, LiteralStateTests' hand rows), not a clean fact. + /// + internal static IEnumerable Injections(CleanLineFacts facts) + { + var source = facts.Source; + for (var k = 0; k < facts.Lines.Count; k++) + { + if (!facts.HasCode(k) || facts.StartsInsideLiteral(k)) + continue; + var (lineStart, lineEnd) = facts.Lines[k]; + + if (facts.FirstCodeTokenStart(k) is { } at) + yield return new Injection("dollar", k, at, at, at, 0, "$", "SPY0015"); + + foreach (var token in facts.Tokens) + { + if (token.Position < lineStart) + continue; + if (token.Position >= lineEnd) + break; + if (token.Type is not (TokenType.String or TokenType.ByteString)) + continue; + var span = (token.Position, token.Position + token.Length); + if (!facts.Spans.Contains(span) || span.Item2 > lineEnd) + continue; // nested in an f-string field (no span of its own), or not closed on its line + var quote = source.IndexOfAny(new[] { '"', '\'' }, token.Position); + if (quote + 2 < span.Item2 && source[quote + 1] == source[quote] && source[quote + 2] == source[quote]) + continue; // triple-quoted + yield return new Injection("escape", k, token.Position, token.Position, quote + 1, 0, "\\q", "SPY0004"); + } + + foreach (var (fstringStart, exprEnd) in PlainOuterFields(facts, k, allowConversion: false)) + yield return new Injection("conversion", k, fstringStart, exprEnd, exprEnd, 0, "!q", "SPY0030"); + + if (facts.StartsLogicalLine(k)) + { + var indentEnd = lineStart; + while (indentEnd < lineEnd && source[indentEnd] is ' ' or '\t') + indentEnd++; + yield return new Injection("tab", k, lineStart, lineStart, lineStart, indentEnd - lineStart, "\t", "SPY0012"); + } + } + } + + /// + /// The outermost replacement fields of the single-line, top-level f-/t-strings that start on line : + /// (the f-string's start, the position of the field's closing }) for each field with no format spec and no + /// self-documenting = (and no conversion unless ). + /// + private static IEnumerable<(int FStringStart, int ExprEnd)> PlainOuterFields(CleanLineFacts facts, int k, bool allowConversion) + { + var (lineStart, lineEnd) = facts.Lines[k]; + var tokens = facts.Tokens; + for (var i = 0; i < tokens.Count; i++) + { + var start = tokens[i]; + if (start.Position >= lineEnd) + yield break; + if (start.Type != TokenType.FStringStart || start.Position < lineStart) + continue; + if (!facts.Spans.Any(s => s.Start == start.Position && s.End <= lineEnd)) + continue; // nested in a field (no span of its own), or not closed on its line + var fdepth = 0; + var field = 0; + var plain = true; + for (var j = i; j < tokens.Count; j++) + { + var token = tokens[j]; + if (token.Type == TokenType.FStringStart) + fdepth++; + else if (token.Type == TokenType.FStringEnd && --fdepth == 0) + { + i = j; + break; + } + if (fdepth != 1) + continue; + if (token.Type == TokenType.FStringExprStart && field++ == 0) + plain = true; + else if (token.Type == TokenType.FStringExprEnd && --field == 0 && plain) + yield return (start.Position, token.Position); + else if (field == 1 && (token.Type is TokenType.FStringFormatSpec or TokenType.FStringSelfDoc + || (token.Type == TokenType.FStringConversion && !allowConversion))) + plain = false; + } + } + } + + /// + /// The CLEAN twins each corpus document is judged in, beside itself — corpus-derived constructs that put scanner arms + /// the fixtures never reach into the given-up text (the C5 content axis), each a document the lexer must still read + /// clean (a twin that does not is reported, never skipped): + /// + /// @backtick — every identifier outside an f-string respelled as a backtick name whose text holds + /// :("""'#: the name must stay opaque to the scanner (DroppedLineContinues' and WalkFrom's + /// SkipBacktickName arms; no fixture spells a backtick name holding a bracket, colon, quote or #). + /// @escaped-quote — every top-level, non-raw short string gets its OWN quote, escaped, right after its + /// opening quote ("\"abc"): the scanner must read the escape as the lexer does (DroppedLineContinues' + /// escape arm — no fixture's short string holds an odd number of escaped own quotes, so without the twin a scanner + /// that closed the string at \" agreed on every cell). Built only when the document has such a string. + /// @unbalanced-brace — {{# right after the opening quote(s) of every single-line, top-level + /// f-/t-string: an escaped brace with no }} to pair it, then a # that is TEXT there. A scanner that + /// read the {{ as a field opener (WalkFrom's escaped-brace arm dropped) reads the # as a comment + /// in a field and the line as ending inside one. Every fixture's {{ is balanced by a later }}, which + /// pairs the misread out. A lone {{ followed by plain text (f"a{{b") pairs out too: the + /// enclosing quote in a field closes the literal when no literal opened there closes on its line. + /// Built only when the document has such an f-string. + /// @field-split — a line break inserted before the } of every outermost field of each + /// single-line, top-level single-quoted f-/t-string with no spec or self-documenting = (PEP 701 whitespace in a + /// hole, fstrings/multiline_hole_2022): the f-string's line now ends inside an open field, so the scanner's + /// field reading — nested literals, the enclosing quote (ClosesOnItsLine), escapes and braces in the text — + /// decides DroppedOpener there. Built only when the document has such a field. + /// @walrus — ; w__ := 0 appended after the last code token of every line that ends its logical + /// line with no comment, in a document that already spells a walrus: a depth-0 := at the END of the given-up + /// text, after tokens of the line were read (the escape/conversion cells), that is NOT a header colon + /// (DroppedLineContinues' depth-0 colon arm must skip it — every fixture's walrus sits inside brackets or on a + /// header line, so a scanner that read := as a colon agreed on every cell). Built only when the document has a + /// := token. + /// + /// + internal static IEnumerable<(string Twin, string Source)> Twins(CleanLineFacts facts) + { + var source = facts.Source; + yield return ("", source); + + var backtick = new StringBuilder(source.Length * 2); + var at = 0; + var fdepth = 0; + foreach (var token in facts.Tokens) + { + if (token.Type == TokenType.FStringStart) + fdepth++; + else if (token.Type == TokenType.FStringEnd && fdepth > 0) + fdepth--; + else if (fdepth == 0 && token.Type == TokenType.Identifier) + { + backtick.Append(source, at, token.Position - at).Append('`').Append(token.Value).Append(":(\"\"\"'#`"); + at = token.Position + token.Length; + } + } + yield return ("@backtick", backtick.Append(source, at, source.Length - at).ToString()); + + var escaped = new StringBuilder(source.Length + 64); + at = 0; + foreach (var token in facts.Tokens) + { + if (token.Type is not (TokenType.String or TokenType.ByteString) || !facts.Spans.Contains((token.Position, token.Position + token.Length))) + continue; + var quote = source.IndexOfAny(new[] { '"', '\'' }, token.Position); + if (quote + 2 < token.Position + token.Length && source[quote + 1] == source[quote] && source[quote + 2] == source[quote]) + continue; // triple-quoted + escaped.Append(source, at, quote + 1 - at).Append('\\').Append(source[quote]); + at = quote + 1; + } + if (at > 0) + yield return ("@escaped-quote", escaped.Append(source, at, source.Length - at).ToString()); + + var unbalanced = new StringBuilder(source.Length + 64); + at = 0; + fdepth = 0; + foreach (var token in facts.Tokens) + { + if (token.Type == TokenType.FStringEnd && fdepth > 0) + fdepth--; + if (token.Type != TokenType.FStringStart || fdepth++ != 0) + continue; + var span = facts.Spans.FirstOrDefault(s => s.Start == token.Position); + if (span.End <= span.Start || source.AsSpan(span.Start, span.End - span.Start).IndexOfAny('\n', '\r') >= 0) + continue; // not a top-level span, or it spans lines + var textStart = token.Position + token.Value.Length; // past the prefix and the opening quote(s) + unbalanced.Append(source, at, textStart - at).Append("{{#"); + at = textStart; + } + if (at > 0) + yield return ("@unbalanced-brace", unbalanced.Append(source, at, source.Length - at).ToString()); + + var lineBreak = source.Contains("\r\n", StringComparison.Ordinal) ? "\r\n" : source.Contains('\r') ? "\r" : "\n"; + var splits = new List(); + for (var k = 0; k < facts.Lines.Count; k++) + { + foreach (var (fstringStart, exprEnd) in PlainOuterFields(facts, k, allowConversion: true)) + { + var quote = source.IndexOfAny(new[] { '"', '\'' }, fstringStart); + if (!(quote + 2 < source.Length && source[quote + 1] == source[quote] && source[quote + 2] == source[quote])) + splits.Add(exprEnd); + } + } + if (splits.Count > 0) + { + var split = new StringBuilder(source.Length + (splits.Count * 2)); + at = 0; + foreach (var offset in splits) + { + split.Append(source, at, offset - at).Append(lineBreak); + at = offset; + } + yield return ("@field-split", split.Append(source, at, source.Length - at).ToString()); + } + + if (facts.Tokens.Any(t => t.Type == TokenType.ColonAssign)) + { + var walrus = new StringBuilder(source.Length + (facts.Lines.Count * 12)); + at = 0; + for (var k = 0; k < facts.Lines.Count; k++) + { + var (lineStart, lineEnd) = facts.Lines[k]; + if (!facts.HasCode(k) || facts.EndInsideLiteral(k)) + continue; + var codeEnd = -1; + var endsLogicalLine = false; + foreach (var token in facts.Tokens) + { + if (token.Position < lineStart) + continue; + if (token.Position > lineEnd) + break; + if (token.Type == TokenType.Newline) + endsLogicalLine = true; + else if (token.Type is not (TokenType.Indent or TokenType.Dedent or TokenType.Eof) && token.Position < lineEnd) + codeEnd = token.Position + token.Length; + } + if (!endsLogicalLine || codeEnd < 0 || codeEnd > lineEnd || source[codeEnd..lineEnd].Trim().Length > 0) + continue; // the logical line goes on, or a comment follows the code + walrus.Append(source, at, codeEnd - at).Append("; w__ := 0"); + at = codeEnd; + } + if (at > 0) + yield return ("@walrus", walrus.Append(source, at, source.Length - at).ToString()); + } + } + + /// ' source through line .Line's break, with the cell's edit. + private static string InjectedPrefix(CleanLineFacts facts, Injection cell) + { + var cut = facts.Lines[cell.Line].End + facts.LineBreakLength(cell.Line); + var source = facts.Source; + return string.Concat(source.AsSpan(0, cell.EditAt), cell.Insert, source.AsSpan(cell.EditAt + cell.Remove, cut - cell.EditAt - cell.Remove)); + } + + /// + /// The verdicts of one injected cell against the clean facts: a misfire (the construct did not abort as designed — + /// an instrument defect, never allowlistable) or the facts that disagree (empty when every fact agrees). The injected + /// document is the clean PREFIX through the line (): error recovery resets the indent + /// stack and the bracket depth, so on a whole document the lines below would abort again (cascade), and each later + /// recovery would add its own flags to the cumulative LiteralLoss — the prefix makes every fact the injection's. + /// + internal static (string? Misfire, List Disagreements, bool OpenerLine) Judge(CleanLineFacts facts, Injection cell) + { + var text = InjectedPrefix(facts, cell); + var lexer = new LexerNs.Lexer(text); + lexer.TokenizeAll(); + var errors = lexer.Diagnostics.GetErrors().ToList(); + if (errors.Count != 1 || errors[0].Code != cell.Code || errors[0].Line != cell.Line + 1 + || lexer.RecoveryResumes.Count != 1 || lexer.RecoveryResumes[0] != text.Length) + { + return ($"{cell.Code} expected once on L{cell.Line + 1}, got [{string.Join("; ", errors.Select(d => $"{d.Code}@{d.Line}:{d.Column}"))}] " + + $"resumes [{string.Join(",", lexer.RecoveryResumes)}] of {text.Length}", new List(), false); + } + + var disagreements = new List(); + var resume = lexer.RecoveryResumes[0]; + var dropped = (lexer.LiteralLoss & LexerNs.LiteralLoss.DroppedOpener) != 0; + var crossing = facts.OpenerCrossing(cell.Line, cell.DroppedFrom); + if (dropped != crossing) + disagreements.Add($"DroppedOpener lexer={dropped} oracle={crossing}"); + if (facts.EndInsideLiteral(cell.Line)) + return (null, disagreements, true); // the domain restriction: only DroppedOpener is a clean fact here + + var continued = lexer.RecoveryResumesAfterAContinuedLine.Contains(resume); + var wantContinued = facts.Continued(cell.Line); + if (continued != wantContinued) + disagreements.Add($"Continued lexer={continued} oracle={wantContinued}"); + var header = lexer.RecoveryResumesAfterAPossibleHeader.Contains(resume); + var wantHeader = facts.PossibleHeader(cell.Line, cell.AbortAt); + if (header != wantHeader) + disagreements.Add($"PossibleHeader lexer={header} oracle={wantHeader}"); + var bracket = lexer.BracketLeftOpenAtLine; + var wantBracket = facts.BracketOpenerLine(cell.Line); + if (bracket != wantBracket) + disagreements.Add($"BracketLeftOpenAtLine lexer={bracket?.ToString() ?? "null"} oracle={wantBracket?.ToString() ?? "null"}"); + return (null, disagreements, false); + } + + /// The clean-fact oracle's population: every .spy under , build output excluded, in ordinal order. + private static List<(string Name, string Source)> CorpusDocuments() + { + var documents = new List<(string, string)>(); + foreach (var root in FixtureRoots.All) + { + var files = Directory.EnumerateFiles(root.Path, "*.spy", SearchOption.AllDirectories) + .Where(f => !Sharpy.Compiler.Diagnostics.CrashBundleWriter.IsNonSourceSegment(Path.GetRelativePath(root.Path, f))) + .OrderBy(f => f, StringComparer.Ordinal); + foreach (var file in files) + documents.Add((Path.GetRelativePath(FixtureRoots.RepositoryRoot, file).Replace('\\', '/'), File.ReadAllText(file))); + } + return documents; + } + + /// + /// The scanner ratchet (Lexer/literal-loss-scanner-allowlist.txt): rows key # #NNNN reason, the key + /// file:line:construct[@twin] (injection cells) or file:line:shape:NAME (shape cells). A row with no + /// #\d+ cite fails; a row whose key the generated universe of its kind does not hold fails; a row whose cell + /// agrees fails as stale (drain on fix). Every row is a findings-ledger line (plan Design Decision 4). + /// + internal sealed class ScannerAllowlist + { + public const string RelativePath = "src/Sharpy.Compiler.Tests/Lexer/literal-loss-scanner-allowlist.txt"; + + private static readonly Regex Cite = new(@"#\d+", RegexOptions.Compiled); + + public Dictionary Rows { get; } = new(StringComparer.Ordinal); + + public List Malformed { get; } = new(); + + public static ScannerAllowlist Load() + { + var path = Path.Combine(FixtureRoots.RepositoryRoot, RelativePath); + File.Exists(path).Should().BeTrue("the ratchet's file arms it: {0}", path); + var allowlist = new ScannerAllowlist(); + foreach (var raw in File.ReadAllLines(path)) + { + var line = raw.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + continue; + var hash = line.IndexOf(" #", StringComparison.Ordinal); + var key = (hash < 0 ? line : line[..hash]).Trim(); + if (hash < 0 || !Cite.IsMatch(line[hash..])) + allowlist.Malformed.Add($"no #NNNN issue cite: {raw}"); + else if (!allowlist.Rows.TryAdd(key, raw)) + allowlist.Malformed.Add($"duplicate key: {raw}"); + } + return allowlist; + } + + public static bool IsShapeKey(string key) => key.Contains(":shape:", StringComparison.Ordinal); + + /// + /// The ratchet's verdicts over one kind of key: the disagreeing keys the allowlist does not hold (failures), the + /// rows of that kind whose key is outside the or whose cell agrees (stale). + /// + public (List Unlisted, List Stale, int Allowlisted) Ratchet( + Func ofKind, IReadOnlySet universe, IReadOnlyDictionary disagreements) + { + var unlisted = new List(); + var stale = new List(); + var allowlisted = 0; + foreach (var (key, detail) in disagreements) + { + if (Rows.ContainsKey(key)) + allowlisted++; + else + unlisted.Add($"{key} :: {detail}"); + } + foreach (var (key, raw) in Rows.Where(r => ofKind(r.Key))) + { + if (!universe.Contains(key)) + stale.Add($"not in the generated universe: {raw}"); + else if (!disagreements.ContainsKey(key)) + stale.Add($"the cell agrees now (drain it): {raw}"); + } + return (unlisted, stale, allowlisted); + } + } + + /// + /// P68 Phase 3 (#2300 row C5): the literal-loss scanner judged on the text the lexer GIVES UP — the only text it is + /// for, which the clean-corpus differential above never shows it. Over every clean corpus document and its clean + /// twins (), each line × each error construct () is injected into the clean + /// prefix through that line, and the lexer's recorded facts must equal the clean lexer's facts for that line + /// (): DroppedOpener ≡ a literal span opens in the given-up text and crosses the + /// line's end; and, on a line whose end is not inside a literal (the domain restriction), the resume is a continued + /// one ≡ , a possible header ≡ , and + /// BracketLeftOpenAtLine ≡ . A cell whose construct does not abort + /// exactly once on its line is a misfire (instrument defect, fails). Disagreements are ratcheted by + /// . + /// + [Fact] + public void Injected_ScannerAgreesWithTheCleanLexersFacts() + { + var clock = Stopwatch.StartNew(); + var allowlist = ScannerAllowlist.Load(); + var corpus = CorpusDocuments(); + var universe = new HashSet(StringComparer.Ordinal); + var disagreements = new Dictionary(StringComparer.Ordinal); + var misfires = new List(); + var rejectedTwins = new List(); + var perConstruct = InjectionConstructs.ToDictionary(c => c.Name, _ => 0); + var perTwin = new SortedDictionary(StringComparer.Ordinal); + var clean = 0; + var documents = 0; + var sites = 0; + var cells = 0; + var openerLines = 0; + var closeLines = 0; + var bulkSequence = new Dictionary(StringComparer.Ordinal); // per construct, so the sample is even + var sampledOut = 0; + + foreach (var (name, source) in corpus) + { + var facts = CleanLineFacts.Of(source); + if (facts == null) + continue; + clean++; + foreach (var (twin, twinSource) in Twins(facts)) + { + var twinFacts = twin.Length == 0 ? facts : CleanLineFacts.Of(twinSource); + if (twinFacts == null) + { + rejectedTwins.Add($"{name}{twin}"); + continue; + } + documents++; + perTwin[twin.Length == 0 ? "as-is" : twin[1..]] = perTwin.GetValueOrDefault(twin.Length == 0 ? "as-is" : twin[1..]) + 1; + for (var k = 0; k < twinFacts.Lines.Count; k++) + { + if (twinFacts.IsCloseLine(k) && twinFacts.HasCode(k)) + closeLines++; + } + + var lastSite = -1; + foreach (var cell in Injections(twinFacts)) + { + if (IsBulk(twin, cell) && (bulkSequence[cell.Construct] = bulkSequence.GetValueOrDefault(cell.Construct) + 1) % InjectionStride != 1) + { + sampledOut++; + continue; + } + var offset = cell.Construct is "escape" or "conversion" ? $"@{cell.EditAt - twinFacts.Lines[cell.Line].Start}" : ""; + var key = $"{name}:{cell.Line + 1}:{cell.Construct}{offset}{twin}"; + universe.Add(key); + cells++; + perConstruct[cell.Construct]++; + if (cell.Line != lastSite) + { + sites++; + lastSite = cell.Line; + } + + var (misfire, wrong, openerLine) = Judge(twinFacts, cell); + var text = Escape(twinFacts.Source[twinFacts.Lines[cell.Line].Start..twinFacts.Lines[cell.Line].End]); + if (misfire != null) + { + misfires.Add($"{key} :: {misfire} :: {text}"); + continue; + } + if (openerLine) + openerLines++; + if (wrong.Count > 0) + disagreements[key] = $"{string.Join("; ", wrong)} :: {text}"; + } + } + } + + var (unlisted, stale, allowlisted) = allowlist.Ratchet(k => !ScannerAllowlist.IsShapeKey(k), universe, disagreements); + clock.Stop(); + _output.WriteLine($"SCANNER-INJECT documents {documents} sites {sites} constructs {InjectionConstructs.Count} cells {cells} " + + $"disagreements {disagreements.Count} allowlisted {allowlisted} stride {InjectionStride} (bulk only; sampled out {sampledOut})"); + _output.WriteLine($"SCANNER-INJECT corpus {corpus.Count} clean {clean}; twins {string.Join(", ", perTwin.Select(t => $"{t.Key}={t.Value}"))} " + + $"(rejected {rejectedTwins.Count}); cells per construct {string.Join(", ", perConstruct.Select(c => $"{c.Key}={c.Value}"))}; " + + $"opener-crossing cells (DroppedOpener only) {openerLines}; close lines excluded {closeLines}; misfires {misfires.Count}; " + + $"wall {clock.Elapsed.TotalSeconds:F1}s"); + foreach (var line in misfires.Take(50)) + _output.WriteLine("SCANNER-INJECT MISFIRE " + line); + foreach (var line in rejectedTwins.Take(50)) + _output.WriteLine("SCANNER-INJECT TWIN-NOT-CLEAN " + line); + foreach (var line in unlisted.Take(200)) + _output.WriteLine("SCANNER-INJECT DISAGREE " + line); + foreach (var line in stale.Concat(allowlist.Malformed)) + _output.WriteLine("SCANNER-INJECT ALLOWLIST " + line); + + clean.Should().BeGreaterThan(3000, "the oracle's population is the fixture corpus"); + perConstruct.Where(c => c.Value == 0).Select(c => c.Key).Should().BeEmpty("every construct must reach the corpus"); + perTwin.Keys.Should().Contain(new[] { "as-is", "backtick", "escaped-quote", "unbalanced-brace", "field-split", "walrus" }, "every twin must be built"); + misfires.Should().BeEmpty("a construct that does not abort exactly once on its line is outside the oracle"); + rejectedTwins.Should().BeEmpty("every twin is a clean document by construction"); + allowlist.Malformed.Should().BeEmpty(); + stale.Should().BeEmpty(); + unlisted.Should().BeEmpty(); + } + + private static string Escape(string text) => text.Replace("\r", "\\r", StringComparison.Ordinal).Replace("\t", "\\t", StringComparison.Ordinal); +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.Shapes.cs b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.Shapes.cs new file mode 100644 index 000000000..054c5a037 --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.Shapes.cs @@ -0,0 +1,218 @@ +using System.Diagnostics; +using FluentAssertions; +using Sharpy.TestInfrastructure.Formatting; +using Xunit; +using LexerNs = Sharpy.Compiler.Lexer; + +namespace Sharpy.Compiler.Tests.Lexer; + +public partial class LiteralLossScannerDifferentialTests +{ + /// Every -th clean corpus document (ordinal order) parents the shapes (fixed, never random). + private const int ShapeStride = 1; + + /// + /// The continuation shapes' construction-known truth (; plan-f92797 + /// Design 5, lead rulings L8/L10/L11): for each shape, the recovery that drops its injected line resumes as a + /// CONTINUED line (a bracket left open at the line's end), as a POSSIBLE HEADER (a colon at depth 0 read or given up, + /// or a line end the lexer cannot read), and whether BracketLeftOpenAtLine is the injected line. -nested + /// drops two header lines; -cont drops the header's continuation line (); + /// bracket-eof has no recovery (the lexer follows the bracket to the end of the source). + /// + private static readonly IReadOnlyDictionary ContinuationTruth = + new Dictionary(StringComparer.Ordinal) + { + [FormatterTwins.BracketEofShape] = (false, false, true), + [FormatterTwins.BracketStringShape] = (true, false, true), + [FormatterTwins.BracketBacktickShape] = (true, false, true), + [FormatterTwins.BracketDroppedShape] = (true, false, true), + [FormatterTwins.RecoveryShape] = (false, false, false), + [FormatterTwins.RecoveryMisindentedShape] = (false, false, false), + [FormatterTwins.RecoveryHeaderPrefix + "2"] = (false, true, false), + [FormatterTwins.RecoveryHeaderPrefix + "4"] = (false, true, false), + [FormatterTwins.RecoveryHeaderPrefix + "6"] = (false, true, false), + [FormatterTwins.RecoveryHeaderPrefix + "tab"] = (false, true, false), + [FormatterTwins.RecoveryHeaderPrefix + "string"] = (false, true, false), + [FormatterTwins.RecoveryHeaderPrefix + "backtick"] = (false, true, false), + [FormatterTwins.RecoveryHeaderNestedShape] = (false, true, false), + [FormatterTwins.RecoveryHeaderSiblingShape] = (false, true, false), + [FormatterTwins.RecoveryHeaderContShape] = (false, true, false), + }; + + /// The offset where 0-based of starts (\r\n, \r and \n each end a line). + private static int LineStartOf(string text, int line) + { + var i = 0; + for (var l = 0; l < line && i < text.Length; l++) + { + while (i < text.Length && text[i] is not ('\n' or '\r')) + i++; + if (i < text.Length && text[i] == '\r' && i + 1 < text.Length && text[i + 1] == '\n') + i++; + i++; + } + return i; + } + + /// + /// The verdicts of one literal-loss shape against its construction-known truth, or null when the shape's truth is not + /// a construction fact (the repair*/closeline* shapes: R-FR/R-FV, owned by LiteralStateTests' rows + /// and Corpus_EveryAbortFreeLossShape_SetsItsMechanismsFlag) or its construct did not fire where it was built + /// (observed, never assumed — the dropped shapes on an opener line inside a bracket measure no indentation). + /// + /// A budget stop () leaves ALL of Q unread, so LiteralLoss is + /// exactly UnreadRemainder when Q holds a literal spanning lines (Kind not null) and exactly None + /// otherwise — the scanner's verdict on the unread text, read from the aborted construct's own start (an unterminated + /// short string, a string aborting at an invalid escape, an f-string aborting in a field, a line dropped whole). + /// A dropped-SPY00NN shape whose FIRST error is its code on a line where one of Q's literals spanning lines + /// opens drops text holding that opener — DroppedOpener must be set. + /// + /// + internal static (string Key, string? Disagreement)? JudgeLossShape(string name, LiteralLossShape shape, LexerNs.Lexer lexer, IReadOnlyList<(int Start, int End)> qSpans, string q) + { + if (FormatterTwins.IsBudgetShape(shape.Shape)) + { + var want = shape.Kind == null ? LexerNs.LiteralLoss.None : LexerNs.LiteralLoss.UnreadRemainder; + var key = $"{name}:{shape.LineShift}:shape:{shape.Shape}"; + return (key, lexer.LiteralLoss == want ? null : $"LiteralLoss lexer={lexer.LiteralLoss} truth={want} (Kind {shape.Kind ?? "none"})"); + } + if (!shape.Shape.StartsWith(FormatterTwins.DroppedShapePrefix, StringComparison.Ordinal)) + return null; + + var first = lexer.Diagnostics.GetErrors().FirstOrDefault(); + if (first == null || first.Code != shape.Shape[FormatterTwins.DroppedShapePrefix.Length..] || first.Line is not { } errorLine) + return null; + var qLines = LineRanges(q); + var line = errorLine - 1; + if (line < 0 || line >= qLines.Count) + return null; + var (lineStart, lineEnd) = qLines[line]; + if (!qSpans.Any(s => s.Start >= lineStart && s.Start < lineEnd && s.End > lineEnd)) + return null; // not an opener line of Q's (the dropped shapes keep Q's lines in place: LineShift 0) + var dropped = (lexer.LiteralLoss & LexerNs.LiteralLoss.DroppedOpener) != 0; + return ($"{name}:{errorLine}:shape:{shape.Shape}", dropped ? null : $"DroppedOpener lexer=false truth=true ({lexer.LiteralLoss})"); + } + + /// The verdicts of one continuation shape against . + internal static (string Key, string? Disagreement) JudgeContinuationShape(string name, ContinuationShape shape, LexerNs.Lexer lexer) + { + var injected = shape.InsertedAt + 1; + var key = $"{name}:{injected}:shape:{shape.Shape}"; + var truth = ContinuationTruth[shape.Shape]; + var wrong = new List(); + var bracketAtInjected = lexer.BracketLeftOpenAtLine == injected; + if (bracketAtInjected != truth.BracketAtInjected) + wrong.Add($"BracketLeftOpenAtLine lexer={lexer.BracketLeftOpenAtLine?.ToString() ?? "null"} truth={(truth.BracketAtInjected ? injected.ToString() : "not " + injected)}"); + if (lexer.BracketLeftOpenAtLine is { } opener && opener < injected) + wrong.Add($"BracketLeftOpenAtLine {opener} is above the injected line {injected}: Q is clean there"); + if (shape.Shape != FormatterTwins.BracketEofShape) + { + var dropped = shape.Shape == FormatterTwins.RecoveryHeaderNestedShape ? new[] { 0, 1 } : new[] { shape.DroppedOffset }; + foreach (var offset in dropped) + { + var resume = LineStartOf(shape.Text, shape.InsertedAt + offset + 1); + if (!lexer.RecoveryResumes.Contains(resume)) + { + wrong.Add($"no recovery resumes after injected line {injected + offset} (resumes {string.Join(",", lexer.RecoveryResumes)})"); + continue; + } + var continued = lexer.RecoveryResumesAfterAContinuedLine.Contains(resume); + if (continued != truth.Continued) + wrong.Add($"Continued@{injected + offset} lexer={continued} truth={truth.Continued}"); + var header = lexer.RecoveryResumesAfterAPossibleHeader.Contains(resume); + if (header != truth.Header) + wrong.Add($"PossibleHeader@{injected + offset} lexer={header} truth={truth.Header}"); + } + } + return (key, wrong.Count == 0 ? null : string.Join("; ", wrong)); + } + + /// + /// P68 Phase 3 Task 3 (#2300 row C5): the P22 shape builders ( with + /// , and ) over the clean + /// corpus, each document's lexer facts judged against the truth its CONSTRUCTION fixes (, + /// ) — the per-document exact verdicts the two shape sweeps in LiteralStateTests + /// leave unasserted (the budget stop's exact flag, the dropped opener on every document its construct fired on, the + /// possible-header and continued facts of every continuation shape). Loss shapes over Q and its wide twin (the SPY0014 + /// shape needs every level a multiple of 8); continuation shapes over Q and its lone-CR twin. Disagreements are + /// ratcheted by under file:line:shape:NAME keys. + /// + [Fact] + public void Shapes_LexerFactsMatchTheirConstructionKnownTruth() + { + var clock = Stopwatch.StartNew(); + var allowlist = ScannerAllowlist.Load(); + var corpus = CorpusDocuments(); + var universe = new HashSet(StringComparer.Ordinal); + var disagreements = new Dictionary(StringComparer.Ordinal); + var judged = new SortedDictionary(StringComparer.Ordinal); + var parents = 0; + var clean = 0; + var documents = 0; + + foreach (var (name, source) in corpus) + { + if (CleanLineFacts.Of(source) == null) + continue; + if (clean++ % ShapeStride != 0) + continue; + parents++; + + foreach (var (q, twin) in new[] { (source, ""), (FormatterTwins.WideIndented(source), "@wide") }) + { + var parent = CleanLineFacts.Of(q); + if (parent == null) + continue; + foreach (var shape in FormatterTwins.LiteralLossShapes(q, includeMismatch: twin.Length > 0)) + { + var lexer = new LexerNs.Lexer(shape.Text); + lexer.TokenizeAll(); + documents++; + if (JudgeLossShape(name + twin, shape, lexer, parent.Spans, q) is not { } verdict) + continue; + judged[shape.Shape] = judged.GetValueOrDefault(shape.Shape) + 1; + universe.Add(verdict.Key); + if (verdict.Disagreement != null) + disagreements[verdict.Key] = verdict.Disagreement; + } + } + + foreach (var (q, twin) in new[] { (source, ""), (FormatterTwins.Cr(source), "@cr") }) + { + if (CleanLineFacts.Of(q) == null) + continue; + foreach (var shape in FormatterTwins.ContinuationShapes(q)) + { + var lexer = new LexerNs.Lexer(shape.Text); + lexer.TokenizeAll(); + documents++; + var (key, disagreement) = JudgeContinuationShape(name + twin, shape, lexer); + judged[shape.Shape] = judged.GetValueOrDefault(shape.Shape) + 1; + universe.Add(key); + if (disagreement != null) + disagreements[key] = disagreement; + } + } + } + + var (unlisted, stale, allowlisted) = allowlist.Ratchet(ScannerAllowlist.IsShapeKey, universe, disagreements); + clock.Stop(); + _output.WriteLine($"SCANNER-SHAPES parents {parents} of clean {clean} stride {ShapeStride} documents {documents} cells {universe.Count} " + + $"disagreements {disagreements.Count} allowlisted {allowlisted} wall {clock.Elapsed.TotalSeconds:F1}s"); + _output.WriteLine($"SCANNER-SHAPES judged per shape: {string.Join(", ", judged.Select(j => $"{j.Key}={j.Value}"))}"); + foreach (var line in unlisted.Take(200)) + _output.WriteLine("SCANNER-SHAPES DISAGREE " + line); + foreach (var line in stale.Concat(allowlist.Malformed)) + _output.WriteLine("SCANNER-SHAPES ALLOWLIST " + line); + + parents.Should().BeGreaterThanOrEqualTo(250, "the shapes' parents are a stride over the clean corpus"); + var expectedShapes = FormatterTwins.BudgetStopLines.Select(b => b.Shape) + .Concat(FormatterTwins.DroppedCodes.Select(c => FormatterTwins.DroppedShapePrefix + c)) + .Concat(ContinuationTruth.Keys); + expectedShapes.Where(s => judged.GetValueOrDefault(s) == 0).Should().BeEmpty("every judged shape must reach ≥ 1 document"); + ContinuationTruth.Keys.Should().BeEquivalentTo(FormatterTwins.ContinuationShapeNames, "the truth table covers every continuation shape the builder names"); + allowlist.Malformed.Should().BeEmpty(); + stale.Should().BeEmpty(); + unlisted.Should().BeEmpty(); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.cs b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.cs index 104ec8fb3..fd3d43e51 100644 --- a/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.cs +++ b/src/Sharpy.Compiler.Tests/Lexer/LiteralLossScannerDifferentialTests.cs @@ -16,7 +16,7 @@ namespace Sharpy.Compiler.Tests.Lexer; /// spans lines". A document with a lexer error is outside the oracle (the lexer's spans stop at the abort), /// so the hand rows assert a clean lex first — a row that errors is a test defect, not a vacuous pass. /// -public class LiteralLossScannerDifferentialTests +public partial class LiteralLossScannerDifferentialTests { private readonly ITestOutputHelper _output; diff --git a/src/Sharpy.Compiler.Tests/Lexer/NumericLiteralTwinTests.cs b/src/Sharpy.Compiler.Tests/Lexer/NumericLiteralTwinTests.cs new file mode 100644 index 000000000..d44cbbbed --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/NumericLiteralTwinTests.cs @@ -0,0 +1,636 @@ +using System.Diagnostics; +using System.Globalization; +using System.Numerics; +using System.Text.Json; +using System.Text.RegularExpressions; +using Sharpy.TestInfrastructure.Oracles; +using Xunit; +using Xunit.Abstractions; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// The CPython numeric-literal twin (#2300 row C1, P68 Phase 1): every numeric-literal spelling +/// in the grammar Sharpy SHARES with CPython lexes to CPython's verdict and value. +/// +/// +/// Oracle altitude. Each side answers ONE question per spelling: "a single numeric +/// constant with value V, or refused?" CPython: ast.parse(s, mode="eval") whose body is an +/// int/float Constant (build_tools/differential_lex/number_oracle.py). Sharpy: +/// Lexer.TokenizeAll() yields exactly [Integer|Float, Eof] with no error. Token +/// boundaries are deliberately NOT compared: CPython tokenizes .5e as NUMBER .5 + +/// NAME e and refuses at parse time, Sharpy refuses in the lexer — both "refused". +/// Matrix. form {decimal int, decimal float, trailing-dot float, leading-dot float, +/// hex, binary, octal} × separator placement {none, interior, consecutive, interior of the +/// fraction, interior of the exponent, leading, after the base prefix, trailing, before/after +/// ., before/after e, after the exponent sign} × exponent {none, e/E × +/// sign {none, +, -} × digits, digits missing} × following character. A following +/// character that cannot glue onto a number (end of file, newline, space, )) is a Sharpy +/// self-consistency axis: the spelling followed by it must lex to the bare spelling's verdict +/// followed by that character's own token. A following character that CAN glue (_, +/// ., the non-suffix letters g/x/z) makes a new spelling, judged by +/// CPython like any other. +/// Sharpy-only syntax is never compared with CPython. The suffix roster and the +/// imaginary j are excluded by construction (no generated spelling contains a roster +/// letter after a number); the trailing-dot floats are excluded by , +/// a literal (predicate, spec sentence) table whose cells are still CHECKED — Sharpy must refuse +/// them, and CPython must accept them (an exclusion CPython agrees with is over-broad). Every +/// excluded family is pinned to Sharpy's own token or diagnostic by +/// . +/// Ratchet. A disagreement fails unless its spelling is a row of +/// numeric-literal-twin-allowlist.txt (spelling => pinned-verdict # #NNNN reason); a +/// disagreement whose Sharpy verdict differs from the row's pin fails. A row without a +/// #NNNN cite fails, a row naming a spelling the matrix does not compare fails, and a row +/// whose spelling now agrees fails as stale (drain on fix). +/// Python absent: a loud skip, unless SHARPY_REQUIRE_PYTHON_ORACLE=1 (CI's +/// Compiler shards), where it fails. +/// +public class NumericLiteralTwinTests +{ + private const string AllowlistFileName = "numeric-literal-twin-allowlist.txt"; + + private readonly ITestOutputHelper _output; + + public NumericLiteralTwinTests(ITestOutputHelper output) => _output = output; + + // ===================================================================== // + // The closed enumeration + // ===================================================================== // + + /// One numeric spelling split at the points a separator can be placed. + private sealed record Parts(string Form, string Prefix, string Main, string Dot, string Fraction, + string Mark, string Sign, string ExponentDigits); + + private sealed record Spelling(string Text, string Origin); + + private static readonly string[] DecimalDigits = { "0", "1", "10", "12", "123", "1000", "00", "01" }; + private static readonly string[] FloatIntegerParts = { "0", "1", "12", "00", "01" }; + private static readonly string[] FractionDigits = { "5", "12", "05" }; + private static readonly string[] LeadingDotFractions = { "5", "12", "125", "05" }; + private static readonly string[] TrailingDotIntegerParts = { "0", "1", "12", "01" }; + private static readonly string[] HexPrefixes = { "0x", "0X" }; + private static readonly string[] HexDigits = { "F", "FF", "fF", "10" }; + private static readonly string[] BinaryPrefixes = { "0b", "0B" }; + private static readonly string[] BinaryDigits = { "1", "101" }; + private static readonly string[] OctalPrefixes = { "0o", "0O" }; + private static readonly string[] OctalDigits = { "7", "755" }; + private static readonly string[] ExponentMarks = { "e", "E" }; + private static readonly string[] ExponentSigns = { "", "+", "-" }; + private static readonly string[] ExponentDigitStrings = { "1", "3", "10" }; + + /// Exponent missing its digits: (mark, sign). + private static readonly (string Mark, string Sign)[] ExponentsMissingDigits = { ("e", ""), ("E", "+"), ("e", "-") }; + + /// Following characters that glue onto a number: the spelling + char is a NEW spelling. + private static readonly string[] GluedFollowers = { "_", ".", "g", "x", "z" }; + + /// + /// Following characters that end a number: (name, text, the tokens the text itself lexes to). + /// End of file is the bare spelling. + /// + private static readonly (string Name, string Text, TokenType[] Tokens)[] SeparatingFollowers = + { + ("newline", "\n", new[] { TokenType.Newline }), + ("space", " ", Array.Empty()), + ("rparen", ")", new[] { TokenType.RightParen }), + }; + + /// + /// The suffix roster (integer_literals.md, float_literals.md) plus the imaginary j: no + /// generated spelling may hold one of these letters, so the shared matrix never meets them. + /// + private static readonly string SharpyOnlyLetters = "fFdDmMlLuUjJ"; + + /// + /// Shared-grammar spellings whose Sharpy verdict is fixed by the spec, not by CPython. Each + /// cell matching a predicate must be refused by Sharpy AND accepted by CPython (else the + /// exclusion is over-broad); pins the tokens. + /// + private static readonly (string Name, Regex Predicate, string SpecSentence)[] SharpyOnly = + { + ("trailing-dot float", + new Regex(@"^[0-9](?:_?[0-9])*\.(?:[eE][+-]?[0-9](?:_?[0-9])*)?$", RegexOptions.CultureInvariant), + "float_literals.md: \"Invalid - trailing decimal point without digit … # invalid = 5. # ERROR: use 5.0 instead\""), + }; + + private static IEnumerable BaseParts() + { + foreach (var d in DecimalDigits) + yield return new Parts("decimal-int", "", d, "", "", "", "", ""); + foreach (var i in FloatIntegerParts) + foreach (var f in FractionDigits) + yield return new Parts("decimal-float", "", i, ".", f, "", "", ""); + foreach (var i in TrailingDotIntegerParts) + yield return new Parts("trailing-dot-float", "", i, ".", "", "", "", ""); + foreach (var f in LeadingDotFractions) + yield return new Parts("leading-dot-float", "", "", ".", f, "", "", ""); + foreach (var p in HexPrefixes) + foreach (var d in HexDigits) + yield return new Parts("hex", p, d, "", "", "", "", ""); + foreach (var p in BinaryPrefixes) + foreach (var d in BinaryDigits) + yield return new Parts("binary", p, d, "", "", "", "", ""); + foreach (var p in OctalPrefixes) + foreach (var d in OctalDigits) + yield return new Parts("octal", p, d, "", "", "", "", ""); + } + + private static IEnumerable WithExponents(Parts b) + { + yield return b; + foreach (var mark in ExponentMarks) + foreach (var sign in ExponentSigns) + foreach (var digits in ExponentDigitStrings) + yield return b with { Mark = mark, Sign = sign, ExponentDigits = digits }; + foreach (var (mark, sign) in ExponentsMissingDigits) + yield return b with { Mark = mark, Sign = sign, ExponentDigits = "" }; + } + + private static string Join(Parts p) => p.Prefix + p.Main + p.Dot + p.Fraction + p.Mark + p.Sign + p.ExponentDigits; + + private static string InsertInterior(string run, string separator) => run[..1] + separator + run[1..]; + + /// Every separator placement that applies to : (name, spelling). + private static IEnumerable<(string Name, string Text)> WithSeparators(Parts p) + { + yield return ("none", Join(p)); + // The "main" digit run is the integer part, or the fraction of a leading-dot float. + bool mainIsFraction = p.Main.Length == 0; + string main = mainIsFraction ? p.Fraction : p.Main; + if (main.Length >= 2) + { + yield return ("interior", Join(mainIsFraction ? p with { Fraction = InsertInterior(main, "_") } : p with { Main = InsertInterior(main, "_") })); + yield return ("consecutive", Join(mainIsFraction ? p with { Fraction = InsertInterior(main, "__") } : p with { Main = InsertInterior(main, "__") })); + } + if (!mainIsFraction && p.Fraction.Length >= 2) + yield return ("interior-fraction", Join(p with { Fraction = InsertInterior(p.Fraction, "_") })); + if (p.ExponentDigits.Length >= 2) + yield return ("interior-exponent", Join(p with { ExponentDigits = InsertInterior(p.ExponentDigits, "_") })); + yield return ("leading", "_" + Join(p)); + if (p.Prefix.Length > 0) + yield return ("after-prefix", Join(p with { Prefix = p.Prefix + "_" })); + yield return ("trailing", Join(p) + "_"); + if (p.Dot.Length > 0 && p.Main.Length > 0) + yield return ("before-dot", Join(p with { Main = p.Main + "_" })); + if (p.Dot.Length > 0) + yield return ("after-dot", Join(p with { Dot = "._" })); + if (p.Mark.Length > 0) + { + yield return ("before-exponent", Join(p with { Mark = "_" + p.Mark })); + yield return ("after-exponent", Join(p with { Mark = p.Mark + "_" })); + } + if (p.Sign.Length > 0) + yield return ("after-sign", Join(p with { Sign = p.Sign + "_" })); + } + + /// + /// The closed enumeration: every bare spelling, then every bare spelling + a glued follower, + /// deduplicated by text (first origin wins), in a deterministic order. + /// + private static List Spellings() + { + var bare = new List(); + foreach (var b in BaseParts()) + foreach (var p in WithExponents(b)) + foreach (var (sep, text) in WithSeparators(p)) + bare.Add(new Spelling(text, $"{p.Form} sep={sep} exp={p.Mark}{p.Sign}{p.ExponentDigits}")); + + var all = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var s in bare) + { + if (seen.Add(s.Text)) + all.Add(s); + } + foreach (var s in bare) + { + foreach (var follower in GluedFollowers) + { + var text = s.Text + follower; + if (seen.Add(text)) + all.Add(new Spelling(text, $"{s.Origin} follow='{follower}'")); + } + } + return all; + } + + private static string? ExclusionOf(string text) + { + foreach (var (name, predicate, _) in SharpyOnly) + { + if (predicate.IsMatch(text)) + return name; + } + return null; + } + + // ===================================================================== // + // Verdicts + // ===================================================================== // + + /// + /// A side's answer for one spelling: a numeric constant ( is canonical: + /// int:<decimal> or float:<IEEE bits>) or refused. + /// + private sealed record Verdict(bool Ok, string Value, string Detail) + { + public static Verdict Refused(string detail) => new(false, "", detail); + + public string Show() => Ok ? $"{Value} ({Detail})" : $"refused ({Detail})"; + + public bool SameAs(Verdict other) => Ok == other.Ok && (!Ok || Value == other.Value); + } + + /// Sharpy's verdict for , whose tail is expected to lex to . + private static Verdict SharpyVerdict(string text, IReadOnlyList tail, List throws) + { + var lexer = new LexerNs.Lexer(text); + List tokens; + try + { + tokens = lexer.TokenizeAll(); + } + catch (Exception ex) + { + // An escaped exception is never a refusal: it is an instrument-level failure. + throws.Add($"THROW {ex.GetType().Name} on {Show(text)}: {ex.Message}"); + return Verdict.Refused("threw"); + } + + if (lexer.Diagnostics.HasErrors) + return Verdict.Refused(string.Join(",", lexer.Diagnostics.GetErrors().Select(d => d.Code))); + + string shape = string.Join(" ", tokens.Select(t => t.Type)); + if (tokens.Count != tail.Count + 2 + || (tokens[0].Type != TokenType.Integer && tokens[0].Type != TokenType.Float) + || tokens[^1].Type != TokenType.Eof + || !tokens.Skip(1).Take(tail.Count).Select(t => t.Type).SequenceEqual(tail)) + { + return Verdict.Refused($"tokens {shape}"); + } + + var number = tokens[0]; + return number.Type == TokenType.Integer + ? new Verdict(true, CanonicalInteger(number.Value), $"Integer '{number.Value}'") + : new Verdict(true, CanonicalFloat(number.Value), $"Float '{number.Value}'"); + } + + private static string CanonicalInteger(string value) + { + BigInteger result; + if (value.StartsWith("0x", StringComparison.Ordinal)) + { + if (!BigInteger.TryParse("0" + value[2..], NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out result)) + return $"unparseable-int:{value}"; + } + else if (value.StartsWith("0b", StringComparison.Ordinal) || value.StartsWith("0o", StringComparison.Ordinal)) + { + int radix = value[1] == 'b' ? 2 : 8; + result = BigInteger.Zero; + foreach (var c in value[2..]) + { + int digit = c - '0'; + if (digit < 0 || digit >= radix) + return $"unparseable-int:{value}"; + result = result * radix + digit; + } + } + else if (!BigInteger.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out result)) + { + return $"unparseable-int:{value}"; + } + return "int:" + result.ToString(CultureInfo.InvariantCulture); + } + + private static string CanonicalFloat(string value) + { + double d; + switch (value) + { + case "inf": + d = double.PositiveInfinity; + break; + case "-inf": + d = double.NegativeInfinity; + break; + case "nan": + d = double.NaN; + break; + default: + // Only digits, one '.', an exponent and its sign: a value like M2's "0.5-3" is no number. + if (!Regex.IsMatch(value, @"^[0-9]*\.?[0-9]*(?:[eE][+-]?[0-9]+)?$", RegexOptions.CultureInvariant) + || !double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out d)) + { + return $"unparseable-float:{value}"; + } + break; + } + return "float:" + BitConverter.DoubleToInt64Bits(d).ToString(CultureInfo.InvariantCulture); + } + + private static Verdict PythonVerdict(JsonElement root) + { + if (!root.GetProperty("ok").GetBoolean()) + return Verdict.Refused(root.GetProperty("error").GetString() ?? ""); + string kind = root.GetProperty("kind").GetString() ?? ""; + string value = root.GetProperty("value").GetString() ?? ""; + return kind switch + { + "int" => new Verdict(true, CanonicalInteger(value), $"int {value}"), + "float" => new Verdict(true, CanonicalFloat(value), $"float {value}"), + _ => throw new InvalidOperationException($"number_oracle.py returned kind '{kind}'"), + }; + } + + private static string Show(string text) => JsonSerializer.Serialize(text); + + // ===================================================================== // + // Allowlist + // ===================================================================== // + + /// + /// One allowlist row: the spelling, the Sharpy verdict it pins (), and the cite. + /// The pin keeps a row from absorbing a SECOND change on its own spelling: the row admits + /// today's disagreement, not any disagreement. + /// + private sealed record AllowlistRow(int Line, string Key, string Pinned, string Rest); + + /// The verdict an allowlist row pins: the canonical value, or refused. + private static string Pin(Verdict v) => v.Ok ? v.Value : "refused"; + + private static string AllowlistPath() + { + return PythonInterpreter.FindRepoFile("src", "Sharpy.Compiler.Tests", "Lexer", AllowlistFileName) + ?? throw new FileNotFoundException( + $"{AllowlistFileName} not found under the repository root (src/Sharpy.Compiler.Tests/Lexer/)."); + } + + private static List ReadAllowlist(string path) + { + var rows = new List(); + var lines = File.ReadAllLines(path); + for (int i = 0; i < lines.Length; i++) + { + var line = lines[i].Trim(); + if (line.Length == 0 || line.StartsWith('#')) + continue; + int hash = line.IndexOf('#'); + string head = (hash < 0 ? line : line[..hash]).Trim(); + string rest = hash < 0 ? "" : line[hash..]; + int arrow = head.IndexOf("=>", StringComparison.Ordinal); + string key = (arrow < 0 ? head : head[..arrow]).Trim(); + string pinned = arrow < 0 ? "" : head[(arrow + 2)..].Trim(); + rows.Add(new AllowlistRow(i + 1, key, pinned, rest)); + } + return rows; + } + + /// The cite and universe rules — checkable without python. + private static List AllowlistShapeViolations(IReadOnlyList rows, ISet compared) + { + var violations = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var row in rows) + { + if (!Regex.IsMatch(row.Rest, @"^#\s*#\d+\b", RegexOptions.CultureInvariant)) + violations.Add($"line {row.Line}: '{row.Key}' has no '# #NNNN reason' issue cite"); + if (!Regex.IsMatch(row.Pinned, @"^(?:refused|int:-?[0-9]+|float:-?[0-9]+)$", RegexOptions.CultureInvariant)) + violations.Add($"line {row.Line}: '{row.Key}' pins no Sharpy verdict (' => int: | float: | refused # #NNNN reason')"); + if (!compared.Contains(row.Key)) + violations.Add($"line {row.Line}: '{row.Key}' is not a spelling the twin compares (not enumerated, or Sharpy-only)"); + if (!seen.Add(row.Key)) + violations.Add($"line {row.Line}: '{row.Key}' is listed twice"); + } + return violations; + } + + // ===================================================================== // + // Tests + // ===================================================================== // + + [Fact] + public void Enumeration_HoldsTheIssuesRepresentativeCells_AndNoSharpyOnlyLetter() + { + var spellings = Spellings(); + var texts = spellings.Select(s => s.Text).ToHashSet(StringComparer.Ordinal); + + // #2300's C1 cells and the plan's measured cells, anchored as literals (not derived from the generator). + string[] representative = + { + ".5e3", ".5e-3", ".1__2", ".12_", ".5e", ".5e1_", "0xF_", "01", "00", "0_0", "0x_FF", "0XFF", + "1E+3", "1_0", "12.", "1.e3", "0b1_", "1._5", "1_.5", "1e_3", "1e+_3", "0xFF.", "12g", "0xFFx", + }; + var missing = representative.Where(r => !texts.Contains(r)).ToList(); + Assert.True(missing.Count == 0, "enumeration lacks: " + string.Join(" ", missing)); + + // The suffix/imaginary families are excluded by construction: no roster letter right after a digit. + var leaked = spellings + .Where(s => Regex.IsMatch(s.Text, $"[0-9_.][{SharpyOnlyLetters}]", RegexOptions.CultureInvariant) + && !s.Text.StartsWith("0x", StringComparison.Ordinal) && !s.Text.StartsWith("0X", StringComparison.Ordinal) + && !s.Text.StartsWith("_0x", StringComparison.Ordinal) && !s.Text.StartsWith("_0X", StringComparison.Ordinal)) + .Select(s => s.Text) + .ToList(); + Assert.True(leaked.Count == 0, "Sharpy-only suffix letters leaked into the shared matrix: " + string.Join(" ", leaked.Take(20))); + + Assert.Equal("trailing-dot float", ExclusionOf("12.")); + Assert.Equal("trailing-dot float", ExclusionOf("1_0.e-3")); + Assert.Null(ExclusionOf("1._5")); + Assert.Null(ExclusionOf("1.5.")); + Assert.Null(ExclusionOf("0xFF.")); + } + + [Fact] + public void Allowlist_EveryRowCitesAnIssue_AndNamesAComparedSpelling() + { + var compared = Spellings().Select(s => s.Text).Where(t => ExclusionOf(t) is null).ToHashSet(StringComparer.Ordinal); + var violations = AllowlistShapeViolations(ReadAllowlist(AllowlistPath()), compared); + Assert.True(violations.Count == 0, string.Join("\n", violations)); + + // Positive control: the rules fire on rows built to break them. + var broken = AllowlistShapeViolations(new[] + { + new AllowlistRow(1, "01", "int:1", "# no cite here"), + new AllowlistRow(2, "0x1p3", "int:8", "# #2300 hex float is not in the matrix"), + new AllowlistRow(3, "12.", "refused", "# #2300 Sharpy-only spelling"), + new AllowlistRow(4, "00", "", "# #2300 no pinned verdict"), + }, compared); + Assert.Equal(4, broken.Count); + } + + [Fact] + public void SharedGrammar_AgreesWithCPython() + { + var stopwatch = Stopwatch.StartNew(); + var spellings = Spellings(); + var compared = spellings.Select(s => s.Text).Where(t => ExclusionOf(t) is null).ToHashSet(StringComparer.Ordinal); + var allowlist = ReadAllowlist(AllowlistPath()); + var shapeViolations = AllowlistShapeViolations(allowlist, compared); + Assert.True(shapeViolations.Count == 0, string.Join("\n", shapeViolations)); + + var python = PythonInterpreter.Locate(); + var script = PythonInterpreter.FindRepoFile("build_tools", "differential_lex", "number_oracle.py"); + if (python is null || script is null) + { + string why = python is null ? "no Python 3.12+ interpreter (python3/python) on PATH" : "build_tools/differential_lex/number_oracle.py not found"; + if (PythonInterpreter.Required) + Assert.Fail($"{why}, and {PythonInterpreter.RequireEnvironmentVariable}=1 makes the numeric twin's oracle required."); + _output.WriteLine($"SKIP: {why}; the numeric twin did not run. Set {PythonInterpreter.RequireEnvironmentVariable}=1 to make this a failure."); + return; + } + + var requests = spellings.Select((s, i) => JsonSerializer.Serialize(new { id = i, source = s.Text })); + var replies = python.RunBatch(script, requests); + var pythonVerdicts = new Dictionary(); + foreach (var reply in replies) + { + using var doc = JsonDocument.Parse(reply); + pythonVerdicts[doc.RootElement.GetProperty("id").GetInt32()] = PythonVerdict(doc.RootElement); + } + Assert.Equal(spellings.Count, pythonVerdicts.Count); + + var allowed = allowlist.ToDictionary(r => r.Key, StringComparer.Ordinal); + var failures = new List(); + var throws = new List(); + var stillDisagreeing = new HashSet(StringComparer.Ordinal); + int agree = 0, refusedBoth = 0, allowlisted = 0, sharpyOnly = 0, followCells = 0; + + for (int i = 0; i < spellings.Count; i++) + { + var s = spellings[i]; + var sharpy = SharpyVerdict(s.Text, Array.Empty(), throws); + var cpython = pythonVerdicts[i]; + + // Separating followers: Sharpy must give the bare spelling's verdict, then the follower's token. + foreach (var (name, text, tail) in SeparatingFollowers) + { + followCells++; + var followed = SharpyVerdict(s.Text + text, tail, throws); + if (!followed.SameAs(sharpy)) + failures.Add($"FOLLOW {Show(s.Text)} + {name}: bare {sharpy.Show()} but followed {followed.Show()} [{s.Origin}]"); + } + + var exclusion = ExclusionOf(s.Text); + if (exclusion is not null) + { + sharpyOnly++; + if (sharpy.Ok) + failures.Add($"SHARPY-ONLY {Show(s.Text)} ({exclusion}): the spec refuses it, Sharpy lexes {sharpy.Show()}"); + if (!cpython.Ok) + failures.Add($"OVER-BROAD {Show(s.Text)} ({exclusion}): CPython refuses it too ({cpython.Detail}) — compare it instead of excluding it"); + continue; + } + + if (sharpy.SameAs(cpython)) + { + if (sharpy.Ok) + agree++; + else + refusedBoth++; + continue; + } + + stillDisagreeing.Add(s.Text); + if (allowed.TryGetValue(s.Text, out var row)) + { + if (Pin(sharpy) != row.Pinned) + failures.Add($"PIN allowlist line {row.Line}: {Show(s.Text)} pins Sharpy {row.Pinned} but Sharpy now gives {sharpy.Show()} — a second change on an allowlisted spelling is not absorbed [{s.Origin}]"); + allowlisted++; + continue; + } + failures.Add($"DISAGREE {Show(s.Text)}: CPython {cpython.Show()}, Sharpy {sharpy.Show()} [{s.Origin}]"); + } + + foreach (var row in allowlist) + { + if (!stillDisagreeing.Contains(row.Key)) + failures.Add($"STALE allowlist line {row.Line}: '{row.Key}' now agrees with CPython — delete the row (drain on fix)"); + } + + failures.InsertRange(0, throws); + stopwatch.Stop(); + _output.WriteLine($"NUMERIC-TWIN spellings {compared.Count} agree {agree} refusedBoth {refusedBoth} allowlisted {allowlisted}"); + _output.WriteLine($"NUMERIC-TWIN python {python.Version} ({python.Executable}) sharpyOnly {sharpyOnly} followCells {followCells} elapsed {stopwatch.Elapsed.TotalSeconds:F1}s"); + foreach (var f in failures.Take(200)) + _output.WriteLine(f); + + Assert.True(failures.Count == 0, + $"{failures.Count} numeric-twin failure(s); first {Math.Min(failures.Count, 40)}:\n" + string.Join("\n", failures.Take(40)) + + $"\nA disagreement outside the contract is a findings-ledger line and an allowlist row in {AllowlistFileName} citing its issue — never a changed expectation."); + } + + // ===================================================================== // + // The Sharpy-only families, pinned to the spec + // ===================================================================== // + + private static readonly string[] FloatSuffixes = { "f", "F", "d", "D", "m", "M" }; + private static readonly string[] IntegerSuffixes = { "l", "L", "u", "U", "ul", "UL", "uL", "Ul" }; + private static readonly string[] InvalidSuffixes = { "lu", "x", "j" }; + + /// + /// (source, expected): expected is the token list without Eof as Type 'Value' + /// joined by ", ", or error SPY####. + /// + public static TheoryData SharpyOnlyCases() + { + var data = new TheoryData(); + const string Refused = "error SPY0023"; + + // integer_literals.md: L/l, u/U, ul/UL on decimal integers; float_literals.md: f/F, d/D, m/M. + foreach (var s in IntegerSuffixes) + data.Add("42" + s, $"Integer '42{s}'"); + foreach (var s in FloatSuffixes) + data.Add("42" + s, $"Float '42{s}'"); + // float_literals.md: "3.14f" / "3.14d" / "3.14m". (An INTEGER suffix on a float is spec-silent + // and accepted by the lexer today — a findings-ledger line, deliberately not pinned here.) + foreach (var s in FloatSuffixes) + data.Add("3.14" + s, $"Float '3.14{s}'"); + + // integer_literals.md: "The integer suffixes work on every base"; "A hex digit is never read as a + // suffix … 0xFFf is 0xFFF"; "on a binary or octal literal, and m on a hex literal, they are refused (SPY0023)". + foreach (var s in IntegerSuffixes) + { + data.Add("0xFF" + s, $"Integer '0xFF{s}'"); + data.Add("0b101" + s, $"Integer '0b101{s}'"); + data.Add("0o17" + s, $"Integer '0o17{s}'"); + } + foreach (var s in FloatSuffixes) + { + data.Add("0b101" + s, Refused); + data.Add("0o17" + s, Refused); + data.Add("0xFF" + s, s is "m" or "M" ? Refused : $"Integer '0xFF{s}'"); + } + + // Not in the roster: refused on every base. + foreach (var s in InvalidSuffixes) + { + foreach (var number in new[] { "42", "3.14", "0xFF", "0b101", "0o17" }) + data.Add(number + s, Refused); + } + + // float_literals.md: "Invalid - trailing decimal point without digit … ERROR: use 5.0 instead" — + // the lexer leaves the dot to the parser. + data.Add("1.", "Integer '1', Dot '.'"); + data.Add("12.", "Integer '12', Dot '.'"); + data.Add("1_000.", "Integer '1000', Dot '.'"); + data.Add("1.e5", "Integer '1', Dot '.', Identifier 'e5'"); + + // No complex literal: the imaginary j is an invalid suffix. + data.Add("1j", Refused); + data.Add("1.5j", Refused); + return data; + } + + [Theory] + [MemberData(nameof(SharpyOnlyCases))] + public void SharpyOnlyRows_LexAsTheSpecSays(string source, string expected) + { + var lexer = new LexerNs.Lexer(source); + var tokens = lexer.TokenizeAll(); + string actual = lexer.Diagnostics.HasErrors + ? string.Join(", ", lexer.Diagnostics.GetErrors().Select(d => "error " + d.Code)) + : string.Join(", ", tokens.Where(t => t.Type != TokenType.Eof).Select(t => $"{t.Type} '{t.Value}'")); + Assert.Equal(expected, actual); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/PythonInterpreterTests.cs b/src/Sharpy.Compiler.Tests/Lexer/PythonInterpreterTests.cs new file mode 100644 index 000000000..853022a31 --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/PythonInterpreterTests.cs @@ -0,0 +1,110 @@ +using System.Text; +using Sharpy.TestInfrastructure.Oracles; +using Xunit; +using Xunit.Abstractions; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// The shared python-oracle locator the lexer's numeric twin uses (#2300, P68 Phase 1). It sits +/// in the Lexer namespace so the Stryker filter that scores the lexer runs it with the twin; it +/// mutates nothing in the lexer. +/// +/// +/// The required-mode arm is exercised through the explicit required overload with a +/// candidate list that cannot resolve — never by setting SHARPY_REQUIRE_PYTHON_ORACLE +/// inside this testhost, which every other oracle consumer shares. +/// +public class PythonInterpreterTests +{ + private static readonly string[] NoSuchInterpreter = { "sharpy-p68-no-such-python-interpreter" }; + + private readonly ITestOutputHelper _output; + + public PythonInterpreterTests(ITestOutputHelper output) => _output = output; + + [Fact] + public void Locate_RequiredAndMissing_Fails() + { + var ex = Assert.Throws( + () => PythonInterpreter.Locate(NoSuchInterpreter, required: true)); + Assert.Contains(PythonInterpreter.RequireEnvironmentVariable, ex.Message); + Assert.Contains(NoSuchInterpreter[0], ex.Message); + } + + [Fact] + public void Locate_NotRequiredAndMissing_ReturnsNull() + { + // The skip arm: the same unresolvable list without the required flag is a loud skip + // for the caller (null), not a throw — the control for the failing arm above. + Assert.Null(PythonInterpreter.Locate(NoSuchInterpreter, required: false)); + } + + [Fact] + public void TryLocate_FindsAPinnedInterpreter_AndRefusesAnOlderMinimum() + { + var python = PythonInterpreter.Locate(PythonInterpreter.DefaultCandidates, PythonInterpreter.Required); + if (python is null) + { + _output.WriteLine("SKIP: no Python 3.12+ interpreter (python3/python) on PATH; set SHARPY_REQUIRE_PYTHON_ORACLE=1 to make this a failure."); + return; + } + + _output.WriteLine($"python oracle: {python.Executable} → {python.Version}"); + Assert.Matches(@"^Python \d+\.\d+", python.Version); + Assert.True(python.Major > 3 || (python.Major == 3 && python.Minor >= 12), python.Version); + + // The version gate is load-bearing: the same interpreter fails a minimum it does not meet. + Assert.Null(PythonInterpreter.TryLocate(new[] { python.Executable }, minMinor: python.Minor + 1000)); + Assert.NotNull(PythonInterpreter.TryLocate(new[] { python.Executable }, minMinor: python.Minor)); + } + + [Fact] + public void RunBatch_EchoesStdinLines_AndReportsStderrOnFailure() + { + var python = PythonInterpreter.Locate(PythonInterpreter.DefaultCandidates, PythonInterpreter.Required); + if (python is null) + { + _output.WriteLine("SKIP: no Python 3.12+ interpreter (python3/python) on PATH."); + return; + } + + string dir = Path.Combine(Path.GetTempPath(), $"sharpy-p68-pyinterp-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + try + { + string echo = Path.Combine(dir, "echo.py"); + File.WriteAllText(echo, + "import sys\nfor line in sys.stdin:\n line = line.strip()\n if line:\n print(line[::-1])\n", + new UTF8Encoding(false)); + var lines = python.RunBatch(echo, new[] { "{\"id\": 1}", "", "abc" }); + Assert.Equal(new[] { "}1 :\"di\"{", "cba" }, lines); + + string fail = Path.Combine(dir, "fail.py"); + File.WriteAllText(fail, "import sys\nsys.stderr.write('p68-oracle-broke')\nsys.exit(3)\n", new UTF8Encoding(false)); + var ex = Assert.Throws(() => python.RunBatch(fail, new[] { "x" })); + Assert.Contains("exited 3", ex.Message); + Assert.Contains("p68-oracle-broke", ex.Message); + + // A script that exits without reading stdin, sent a batch larger than any pipe buffer: the write breaks the + // pipe, and the failure must still be the script's exit code and stderr, never a bare IOException. + var large = Enumerable.Repeat(new string('x', 1000), 2000); + var broken = Assert.Throws(() => python.RunBatch(fail, large)); + Assert.Contains("exited 3", broken.Message); + Assert.Contains("p68-oracle-broke", broken.Message); + } + finally + { + try + { Directory.Delete(dir, recursive: true); } + catch (IOException) { } + } + } + + [Fact] + public void FindRepoFile_ResolvesAFileUnderTheRepositoryRoot_AndNullForAMissingOne() + { + Assert.NotNull(PythonInterpreter.FindRepoFile("sharpy.sln")); + Assert.Null(PythonInterpreter.FindRepoFile("build_tools", "sharpy-p68-no-such-file.py")); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/TokenPositionReconstructionTests.cs b/src/Sharpy.Compiler.Tests/Lexer/TokenPositionReconstructionTests.cs new file mode 100644 index 000000000..de0032f1a --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/TokenPositionReconstructionTests.cs @@ -0,0 +1,263 @@ +using FluentAssertions; +using Sharpy.TestInfrastructure.Lexing; +using Xunit; +using LexerNs = Sharpy.Compiler.Lexer; +using TokenType = Sharpy.Compiler.Lexer.TokenType; + +namespace Sharpy.Compiler.Tests.Lexer; + +/// +/// The position instrument (, #2300 C2) is itself a guard, so each of its +/// rules is shown to fire: a clean lex reconstructs (the baseline), and one hand-broken token per rule yields that +/// rule's line. A rule that never fires here would let the corpus twin pass vacuously. +/// +public class TokenPositionReconstructionTests +{ + private const string Mixed = + "if a >> b:\n y = `na me` + 0X_FF + .5e-3 + 1_000\n z = f\"{x!r:>4}{y=}\" + dr'''\n k\n ''' + b\"q\" # c\nw = \\\n 1\n"; + + private static List Lex(string source) + { + var lexer = new LexerNs.Lexer(source); + var tokens = lexer.TokenizeAll(); + lexer.Diagnostics.HasErrors.Should().BeFalse("the baseline must lex clean: {0}", + string.Join("; ", lexer.Diagnostics.GetErrors().Select(d => $"{d.Code}@{d.Line}:{d.Column}"))); + return tokens; + } + + private static List Violations(string source, List tokens) => + TokenPositionReconstruction.Violations(source, tokens).ToList(); + + private static List Replace(List tokens, Func which, + Func change) + { + var index = tokens.FindIndex(t => which(t)); + index.Should().BeGreaterThanOrEqualTo(0, "the hand-broken token must exist in the baseline"); + var copy = tokens.ToList(); + copy[index] = change(copy[index]); + return copy; + } + + [Theory] + [InlineData("", new[] { 0 })] + [InlineData("a", new[] { 0 })] + [InlineData("a\n", new[] { 0, 2 })] + [InlineData("a\nb\r\nc\rd", new[] { 0, 2, 5, 7 })] + [InlineData("\r\r\n\n", new[] { 0, 1, 3, 4 })] + public void LineStarts_EveryBreakKindEndsALine(string source, int[] expected) => + TokenPositionReconstruction.LineStarts(source).Should().Equal(expected); + + [Theory] + [InlineData("lf")] + [InlineData("crlf")] + [InlineData("cr")] + [InlineData("no final break")] + public void CleanLex_Reconstructs(string twin) + { + var source = twin switch + { + "crlf" => Mixed.Replace("\n", "\r\n", StringComparison.Ordinal), + "cr" => Mixed.Replace('\n', '\r'), + "no final break" => Mixed.TrimEnd('\n'), + _ => Mixed, + }; + var tokens = Lex(source); + tokens.Should().Contain(t => t.Type == TokenType.Indent).And.Contain(t => t.Type == TokenType.Dedent) + .And.Contain(t => t.Type == TokenType.FStringConversion).And.Contain(t => t.Type == TokenType.FStringSelfDoc) + .And.Contain(t => t.Type == TokenType.FStringFormatSpec).And.Contain(t => t.IsBacktickEscaped) + .And.Contain(t => t.Type == TokenType.RawString).And.Contain(t => t.Type == TokenType.ByteString); + + Violations(source, tokens).Should().BeEmpty(); + } + + [Fact] + public void R1_AColumnOneShortAfterAConstruct_IsSeenOnTheFollowingToken() + { + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.Value == "b" && t.Line == 1, t => t with { Column = t.Column - 1 }); + + Violations(Mixed, broken).Should().ContainSingle().Which.Should().StartWith("R1 Identifier 'b'"); + } + + [Fact] + public void R1_ALineOffByOne_IsSeen() + { + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.Value == "w", t => t with { Line = t.Line + 1 }); + + Violations(Mixed, broken).Should().Contain(v => v.StartsWith("R1 Identifier 'w'", StringComparison.Ordinal)); + } + + [Fact] + public void R1_AColumnRunPastItsLinesEnd_IsSeen() + { + // L1:C3 sums to offset 2, which is where line 2 starts: right offset, wrong line (a break read without _line++). + const string source = "x\ny\n"; + var tokens = Lex(source); + var broken = Replace(tokens, t => t.Value == "y", t => t with { Line = 1, Column = 3 }); + + Violations(source, broken).Should().ContainSingle().Which.Should().StartWith("R1 Identifier 'y'").And.Contain("is not on line 1"); + } + + [Fact] + public void R1_ATokenWithoutAPosition_IsSeen() + { + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.Value == "w", t => t with { Position = -1 }); + + Violations(Mixed, broken).Should().Contain(v => v.StartsWith("R1 Identifier 'w'", StringComparison.Ordinal) && v.EndsWith("no position", StringComparison.Ordinal)); + } + + [Fact] + public void R1_AnIndentWithTextBeforeItsPosition_IsSeen() + { + // The Indent exemption (column 1, position after the indentation) admits spaces only. + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.Type == TokenType.Indent, t => t with { Position = t.Position + 1 }); + + Violations(Mixed, broken).Should().Contain(v => v.StartsWith("R1 Indent", StringComparison.Ordinal)); + } + + [Fact] + public void R2_ASpanPastTheSourceEnd_IsSeen() + { + const string source = "x = \"abc\""; + var tokens = Lex(source); + var broken = Replace(tokens, t => t.Type == TokenType.String, t => t with { SourceLength = t.SourceLength + 3 }); + + Violations(source, broken).Should().Contain(v => v.StartsWith("R2 String", StringComparison.Ordinal)); + } + + [Fact] + public void R3_AVerbatimTokenWhoseSliceIsNotItsValue_IsSeen() + { + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.Type == TokenType.RightShift, t => t with { Value = ">=" }); + + Violations(Mixed, broken).Should().Contain(v => v.StartsWith("R3 RightShift", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("0xFF", "0X_FF")] + [InlineData("0.5e-3", ".5e-3")] + [InlineData("1000", "1_000")] + public void R3_ANumberIsReadThroughItsNormalization(string value, string spelling) + { + var source = "x = " + spelling; + var tokens = Lex(source); + tokens.Should().Contain(t => t.Value == value, "the lexer normalizes {0} to {1}", spelling, value); + Violations(source, tokens).Should().BeEmpty(); + + var broken = Replace(tokens, t => t.Value == value, t => t with { Value = value + "1" }); + Violations(source, broken).Should().Contain(v => v.StartsWith("R3 ", StringComparison.Ordinal) && v.Contains("normalized slice", StringComparison.Ordinal)); + } + + [Fact] + public void R4_ABacktickNameWithoutItsSourceLength_IsSeen() + { + // The ReadLiteralName shape: SourceLength dropped → Length = Value.Length, two short. + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.IsBacktickEscaped, t => t with { SourceLength = null }); + + Violations(Mixed, broken).Should().Contain(v => v.StartsWith("R4 Identifier 'na me'", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(TokenType.String, "x = \"abc\"")] + [InlineData(TokenType.RawString, "x = dr'''\n k\n '''")] + [InlineData(TokenType.ByteString, "x = b\"q\"")] + public void R4_AStringSpanThatStopsShortOfItsCloser_IsSeen(TokenType type, string source) + { + var tokens = Lex(source); + var broken = Replace(tokens, t => t.Type == type, t => t with { SourceLength = t.SourceLength - 1 }); + + Violations(source, broken).Should().Contain(v => v.StartsWith("R4 " + type, StringComparison.Ordinal)); + } + + [Fact] + public void R4_AConversionNotAtItsBang_IsSeen() + { + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.Type == TokenType.FStringConversion, t => t with { Position = t.Position + 1, Column = t.Column + 1 }); + + Violations(Mixed, broken).Should().Contain(v => v.StartsWith("R4 FStringConversion", StringComparison.Ordinal)); + } + + [Fact] + public void R4_ASelfDocSpanHoldingMoreThanTheEquals_IsSeen() + { + var tokens = Lex(Mixed); + var broken = Replace(tokens, t => t.Type == TokenType.FStringSelfDoc, t => t with { SourceLength = t.SourceLength + 1 }); + + Violations(Mixed, broken).Should().Contain(v => v.StartsWith("R4 FStringSelfDoc", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("lf")] + [InlineData("crlf")] + public void R5_ANewlineNotAtALineBreak_IsSeen(string twin) + { + var source = twin == "crlf" ? "x = 1\r\ny = 2\r\n" : "x = 1\ny = 2\n"; + var tokens = Lex(source); + var broken = Replace(tokens, t => t.Type == TokenType.Newline, t => t with { Position = t.Position - 1, Column = t.Column - 1 }); + + Violations(source, broken).Should().Contain(v => v.StartsWith("R5 Newline", StringComparison.Ordinal)); + } + + [Fact] + public void R5_ANewlineWhoseNextLineStartsElsewhere_IsSeen() + { + // A CRLF break read as one character: the slice is a break, but line 2 starts after "\r\n". + const string source = "x = 1\r\ny = 2\r\n"; + var tokens = Lex(source); + var broken = Replace(tokens, t => t.Type == TokenType.Newline, t => t with { Line = 2 }); + + Violations(source, broken).Should().Contain(v => v.StartsWith("R5 Newline", StringComparison.Ordinal)); + } + + [Fact] + public void R6_AnEofShortOfTheSourceEnd_IsSeen() + { + const string source = "x = 1"; + var tokens = Lex(source); + var broken = Replace(tokens, t => t.Type == TokenType.Eof, t => t with { Position = t.Position - 1, Column = t.Column - 1 }); + + Violations(source, broken).Should().Contain(v => v.StartsWith("R6 Eof", StringComparison.Ordinal)); + TokenPositionReconstruction.Violations(source, tokens.Take(tokens.Count - 1).ToList()).Should() + .Contain("R6 the token list does not end with Eof"); + } + + [Fact] + public void R7_TextBetweenTokensThatIsNotTrivia_IsSeen() + { + const string source = "x = a + b"; + var tokens = Lex(source); + var dropped = tokens.Where(t => t.Type != TokenType.Plus).ToList(); + + Violations(source, dropped).Should().ContainSingle().Which.Should().StartWith("R7 gap"); + } + + [Fact] + public void R7_OverlappingTokens_AreSeen() + { + const string source = "x = ab + c"; + var tokens = Lex(source); + var broken = Replace(tokens, t => t.Value == "ab", t => t with { Value = "ab +" }); + + Violations(source, broken).Should().Contain(v => v.StartsWith("R7 Plus", StringComparison.Ordinal) && v.Contains("overlaps", StringComparison.Ordinal)); + } + + [Fact] + public void R7_DroppedTextIsExemptOnlyWhereRecoveryResumed() + { + // `$` aborts the first line; recovery resumes at line 2. That gap is dropped text, the rest is not. + const string source = "x = $ junk\ny = 2\n"; + var lexer = new LexerNs.Lexer(source); + var tokens = lexer.TokenizeAll(); + lexer.Diagnostics.HasErrors.Should().BeTrue(); + lexer.RecoveryResumes.Should().ContainSingle(); + + TokenPositionReconstruction.Violations(source, lexer, tokens).Should().BeEmpty(); + TokenPositionReconstruction.Violations(source, tokens).Should().ContainSingle().Which.Should().StartWith("R7 gap"); + } +} diff --git a/src/Sharpy.Compiler.Tests/Lexer/lexer-position-allowlist.txt b/src/Sharpy.Compiler.Tests/Lexer/lexer-position-allowlist.txt new file mode 100644 index 000000000..ff23fff8b --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/lexer-position-allowlist.txt @@ -0,0 +1,28 @@ +# LexerCorpusTwinTests / EofConstructRosterTests allowlist (P68 Phase 2, rows C2/C3/C6 of the tests-only tracker). +# +# One row per TOKEN whose (line, column, position, length) does not map back to its source slice +# because of a lexer position defect found at base. Format: +# +# corpus :: ''@L:C # #NNNN +# roster :: ''@L:C # #NNNN +# +# A row suppresses only the violations whose subject is that one token (type and Value at that +# line and column) AND whose rule (R1..R7, TokenPositionReconstruction) the row names, in that one +# fixture or construct, in every twin or ending; any other violation there still fails — another +# token of the same type, the same token with another Value, or a rule the row does not name (the +# row admits today's defect, never a second one). A row without an issue cite fails; a row, or a +# named rule, that suppresses nothing fails as stale (drain on fix). Rows leave when the cited +# issue is fixed. + +# df""" closed on its own line: FStringEnd is stamped at the dropped pre-close newline +# (L5:C24, slice "\n ") instead of the closing quotes on line 6. + +corpus strings/d_string_fstring.spy :: FStringEnd '"""'@L5:C24 R3,R7 # #2338 df pre-close newline: FStringEnd positioned at the dropped newline, not the closing quotes + +# A line break read as an escape's operand is consumed without advancing the line: Eof claims +# line 1 at the offset where line 2 starts (LF, CR endings) or line 2 where line 3 starts (LF LF). + +roster escape at EOF :: Eof ''@L1:C11 R1 # #2338 line break consumed as an escape operand without advancing the line +roster escape at EOF :: Eof ''@L2:C1 R1 # #2338 line break consumed as an escape operand without advancing the line (LF LF ending) +roster short hex escape at EOF :: Eof ''@L1:C11 R1 # #2338 line break consumed inside a short hex escape without advancing the line +roster short hex escape at EOF :: Eof ''@L2:C1 R1 # #2338 line break consumed inside a short hex escape without advancing the line (LF LF ending) diff --git a/src/Sharpy.Compiler.Tests/Lexer/literal-loss-scanner-allowlist.txt b/src/Sharpy.Compiler.Tests/Lexer/literal-loss-scanner-allowlist.txt new file mode 100644 index 000000000..01beba512 --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/literal-loss-scanner-allowlist.txt @@ -0,0 +1,10 @@ +# The literal-loss scanner ratchet (P68 Phase 3, #2300 row C5), read by +# LiteralLossScannerDifferentialTests.Injected_ScannerAgreesWithTheCleanLexersFacts and +# LiteralLossScannerDifferentialTests.Shapes_LexerFactsMatchTheirConstructionKnownTruth. +# +# Row format: key # #NNNN reason +# injection cell: :<1-based line>:[@][@backtick|@escaped-quote|@unbalanced-brace|@field-split|@walrus] +# shape cell: [@wide|@cr]:<1-based line>:shape: +# A row without an issue cite fails; a row whose key is not in the generated universe fails; a row whose +# cell agrees fails as stale (drain on fix). Every row is also a findings-ledger line. The file's presence +# arms the ratchet: it is drained to empty, never deleted. diff --git a/src/Sharpy.Compiler.Tests/Lexer/numeric-literal-twin-allowlist.txt b/src/Sharpy.Compiler.Tests/Lexer/numeric-literal-twin-allowlist.txt new file mode 100644 index 000000000..5688ba16b --- /dev/null +++ b/src/Sharpy.Compiler.Tests/Lexer/numeric-literal-twin-allowlist.txt @@ -0,0 +1,17 @@ +# NumericLiteralTwinTests allowlist (#2300, P68 Phase 1 — row C1). +# +# One row per numeric-literal spelling on which CPython 3.12 and the Sharpy lexer disagree at the +# program altitude ("one numeric constant with the same value, or refused"). Format: +# +# => | float: | refused> # #NNNN +# +# The pin is checked: if Sharpy's verdict on the spelling changes, the row fails (it admits +# today's disagreement, never a second change on the same spelling). +# +# A row without a `# #NNNN` issue cite fails; a row naming a spelling the twin does not compare +# fails; a row whose spelling now agrees fails as stale (drain on fix). A row leaves only by an +# owner ruling: a deviation-ledger entry (the spelling moves to the twin's Sharpy-only table) +# or a promoted lexer-defect issue (the row re-cites it and drains when the fix lands). + +01 => int:1 # #2300 leading-zero decimal integer — CPython refuses ("leading zeros in decimal integer literals are not permitted"), Sharpy lexes Integer 1; spec silent; DEVIATION-by-default (python-fidelity-scope D5 a) until ruled at /batch-issues +0_1 => int:1 # #2300 the same leading-zero decimal integer cell spelled with a digit separator diff --git a/src/Sharpy.Compiler.Tests/Pretty/UnparserParenthesizationMatrixTests.cs b/src/Sharpy.Compiler.Tests/Pretty/UnparserParenthesizationMatrixTests.cs index 76d5342c3..075b4ffa9 100644 --- a/src/Sharpy.Compiler.Tests/Pretty/UnparserParenthesizationMatrixTests.cs +++ b/src/Sharpy.Compiler.Tests/Pretty/UnparserParenthesizationMatrixTests.cs @@ -26,8 +26,9 @@ namespace Sharpy.Compiler.Tests.PrettyTests; /// Python shares, the table's necessity verdicts were checked against python3's parser. /// /// Axes: binary operator (outer) × binary operator (inner) × side; unary operator × binary -/// operator × side (and binary inside unary); postfix form × receiver kind. Hand-written axes, not -/// generated: the ternary, lambda, walrus and cast forms (their operands are not plain +/// operator × side (and binary inside unary); postfix form × receiver kind; and the conditional's +/// operand kind × position × source (, #2169). Hand-written axes, not +/// generated: the lambda, walrus and cast forms as outer nodes (their operands are not plain /// precedence slots) and unary-in-unary — covered by UnparserTests and the corpus sweeps. /// public class UnparserParenthesizationMatrixTests @@ -161,6 +162,82 @@ public void Matrix_PinsTheLeftOperandOfPower(string label, string minimal) Cells().Select(c => ((string)c[0], (string)c[2])).Should().Contain((label, minimal)); } + /// + /// The conditional a if t else e (#2169). Its then-value and test are ??-level operands + /// (the parser reads both with ParseNullCoalesce, as CPython reads them as or_test); + /// only its else-value is a full test, where a nested conditional or a bare lambda sits bare. + /// A walrus, try or maybe is grouped in every position. Two sources per cell: the + /// SYNTHESIZED tree (normalised — no Parenthesized node, as a generator builds it) must + /// unparse to exactly the minimal spelling; the PARSED tree (it keeps the user's + /// Parenthesized node) must unparse to source that re-parses to the same tree. Necessity + /// verdicts for the forms Python shares (conditional, lambda, walrus, or) were checked + /// against python3's parser and ast.unparse. + /// + public static TheoryData ConditionalCells() + { + // (label, minimal spelling, grouped in the then-value and test, grouped in the else-value) + var operands = new (string Label, string Text, bool OperandNeeds, bool ElseNeeds)[] + { + ("conditional", "p if q else r", true, false), + // A parameter, so the cell is not the parameterless `lambda :` spacing cell (#2169, P22d). + ("lambda", "lambda z: p", true, false), + ("walrus", "y := p", true, true), + ("try", "try p", true, true), + ("maybe", "maybe p", true, true), + ("??", "p ?? r", false, false), + ("or", "p or r", false, false), + }; + var positions = new (string Label, string Template, bool IsElse)[] + { + ("then", "{0} if t else e", false), + ("test", "a if {0} else e", false), + ("else", "a if t else {0}", true), + }; + + var cells = new TheoryData(); + foreach (var (label, text, operandNeeds, elseNeeds) in operands) + { + foreach (var (position, template, isElse) in positions) + { + var full = string.Format(template, $"({text})"); + var minimal = (isElse ? elseNeeds : operandNeeds) ? full : string.Format(template, text); + foreach (var synthesized in new[] { true, false }) + cells.Add($"{label} in {position} ({(synthesized ? "synthesized" : "parsed")})", full, minimal, synthesized); + } + } + return cells; + } + + [Theory] + [MemberData(nameof(ConditionalCells))] + public void Unparse_ConditionalOperand_RoundTrips(string label, string full, string minimal, bool synthesized) + { + var fullModule = Parse($"x = {full}\n", out var fullErrors); + fullErrors.Should().BeEmpty($"{label}: the fully parenthesised spelling must parse"); + var minimalModule = Parse($"x = {minimal}\n", out var minimalErrors); + minimalErrors.Should().BeEmpty($"{label}: the minimal spelling must parse — else the table is wrong about this cell"); + + var normalized = AstNormalizer.Instance.NormalizeModule(fullModule); + StructuralEqualityComparer.Instance.Equals(normalized, AstNormalizer.Instance.NormalizeModule(minimalModule)) + .Should().BeTrue($"{label}: `{minimal}` must parse to the same tree as `{full}` — the grammar and the table disagree"); + + var written = Unparser.Unparse(synthesized ? normalized : fullModule); + if (synthesized) + written.Should().Be($"x = {minimal}\n", $"{label}: the unparser must write exactly the parentheses the grammar needs"); + + var reparsed = Parse(written, out var reparseErrors); + reparseErrors.Should().BeEmpty($"{label}: `{written.TrimEnd()}` must re-parse"); + StructuralEqualityComparer.Instance.Equals(normalized, AstNormalizer.Instance.NormalizeModule(reparsed)) + .Should().BeTrue($"{label}: `{written.TrimEnd()}` must re-parse to the tree of `{full}`"); + } + + [Fact] + public void ConditionalMatrix_CoversEveryAxis() + { + // 7 operand kinds × 3 positions × 2 sources, anchored literally. + ConditionalCells().Count.Should().Be(7 * 3 * 2); + } + private static SModule Parse(string source, out List errors) { var lexer = new SLexer(source); diff --git a/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/KnownGeneratorPins.cs b/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/KnownGeneratorPins.cs index c0545d732..ffde8448c 100644 --- a/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/KnownGeneratorPins.cs +++ b/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/KnownGeneratorPins.cs @@ -12,9 +12,10 @@ namespace Sharpy.Compiler.Tests.Properties.Generators.Typed; /// /// A sample is explained by a pin only when its PRE-unparse AST has the pinned shape AND re-parsing /// the unparsed source yields parse errors that ALL carry the pin's code: any other parse error, any -/// semantic refusal of a parseable program, and any throw stay unexplained. Each pin must keep -/// firing (the roster's ParsePins_StillFire), so it drains with its fix instead of going -/// stale silently. Pins are constructed as new KnownRedRefusal("#N", …) so +/// semantic refusal of a parseable program, and any throw stay unexplained. The roster is empty +/// since its one pin, the #2169 Unparser cell (a conditional in a conditional's test position +/// written without parentheses), was fixed and deleted with its staleness guard (ruling R-HP). Pins +/// are constructed as new KnownRedRefusal("#N", …) so /// build_tools/allowlist_issue_state.py's KNOWNRED_CELL_RE sees the cite — a pin /// citing a CLOSED issue reddens the /push gate. /// @@ -25,11 +26,7 @@ internal static class KnownGeneratorPins /// produces, the pre-unparse node shape that produces it. public sealed record ParsePin(KnownRedRefusal Pin, string Code, Func AstShape); - public static readonly IReadOnlyList ParsePins = new ParsePin[] - { - new(new KnownRedRefusal("#2169", "Unparser prints a conditional in a conditional's test position without parentheses → SPY0104"), - "SPY0104", n => n is ConditionalExpression { Test: ConditionalExpression }), - }; + public static readonly IReadOnlyList ParsePins = Array.Empty(); /// The pin that explains — the re-parse errors of the /// source unparsed from — or null. An empty error list is never diff --git a/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/SemanticFilter.cs b/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/SemanticFilter.cs index 39a3b7df2..3be2bedde 100644 --- a/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/SemanticFilter.cs +++ b/src/Sharpy.Compiler.Tests/Properties/Generators/Typed/SemanticFilter.cs @@ -200,7 +200,7 @@ internal void Reject(Rejection rejection) Interlocked.Increment(ref _drawn); } - /// label: rejections k/N (pinned p [#2169 ×p], unexplained u). + /// label: rejections k/N (pinned p [#N ×p], unexplained u). public string Summary { get diff --git a/src/Sharpy.Compiler.Tests/Properties/Semantic/ErrorCompletenessPropertyTests.cs b/src/Sharpy.Compiler.Tests/Properties/Semantic/ErrorCompletenessPropertyTests.cs index dca36f99d..e598774be 100644 --- a/src/Sharpy.Compiler.Tests/Properties/Semantic/ErrorCompletenessPropertyTests.cs +++ b/src/Sharpy.Compiler.Tests/Properties/Semantic/ErrorCompletenessPropertyTests.cs @@ -152,15 +152,15 @@ public void UseInFailedNarrowingBranch_IsRejected() /// GenTyped.NarrowedBlock's first caller is sound by construction (#2288 Phase 4): every /// RAW sample of TypedProgram(WithOptionals, …, withStatements: true) analyses clean — /// judged per sample, never filtered — and the share that carries a narrowing block is printed - /// and must be non-zero, so the soundness claim covers the block. Fuel 1: at fuel 2 the generator - /// can nest a conditional in a conditional's test, which the Unparser prints unparenthesized - /// (#2169) — a cell of that pin, not of this generator. + /// and must be non-zero, so the soundness claim covers the block. Fuel 2, so the generator nests + /// conditionals (held at fuel 1 while the Unparser wrote a conditional in a conditional's test + /// position without parentheses, #2169). /// [Fact] public void NarrowedProgram_AnalysesClean() { int narrowedTested = 0; - var verdicts = ErrorInjector.JudgeSamples(ProgramWithNarrowing(fuel: 1), module => + var verdicts = ErrorInjector.JudgeSamples(ProgramWithNarrowing(fuel: 2), module => { if (HasNarrowedBlock(module)) Interlocked.Increment(ref narrowedTested); diff --git a/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedAnalyzePropertyTests.cs b/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedAnalyzePropertyTests.cs index 8c924e542..753a33083 100644 --- a/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedAnalyzePropertyTests.cs +++ b/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedAnalyzePropertyTests.cs @@ -20,7 +20,7 @@ public TypedAnalyzePropertyTests(ITestOutputHelper output) /// /// GenTyped.TypedProgram over three bindings: every raw sample analyses clean (or is a - /// pinned known red — the #2169 nested-conditional cell). Before #2288 the lane passed with half + /// pinned known red in KnownGeneratorPins). Before #2288 the lane passed with half /// its samples refused, and a throw was swallowed. /// [Fact] diff --git a/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedGeneratorRosterTests.cs b/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedGeneratorRosterTests.cs index 765bfa065..0cb69b859 100644 --- a/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedGeneratorRosterTests.cs +++ b/src/Sharpy.Compiler.Tests/Properties/Semantic/TypedGeneratorRosterTests.cs @@ -1206,8 +1206,7 @@ void CheckLane(RosterRow row, string laneName, Lane lane) // The compiler-side defects the raw corpus draws are pinned ONCE, in KnownGeneratorPins (shared with // the generated lane's RejectionSink, #2288): a sample is tolerated (excluded from the corpus, counted, // printed) only when its PRE-unparse AST has the pinned shape AND every re-parse error carries the - // pin's code; any other parse error stays red. ParsePins_StillFire requires every pin to tolerate at - // least one sample of the fixed-seed corpus, so a pin drains with its fix instead of going stale. + // pin's code; any other parse error stays red. The roster is empty since the #2169 pin drained. private sealed record RawControlResult(int RowsTested, IReadOnlyList Failures, IReadOnlyDictionary PinHits, IReadOnlyList<(string Row, int Hits)> RowHits, string CorpusHash, TimeSpan Elapsed); @@ -1283,21 +1282,6 @@ public void YGenerated_RawCorpusContainsTheConstruct() $"{result.Failures.Count} Y-generated row(s) fail the raw control:\n " + string.Join("\n ", result.Failures)); } - /// Every parse pin still tolerates at least one sample: when the pinned defect is fixed the - /// pin goes red here and is dropped, never left behind as a silent exemption. - [Fact] - public void ParsePins_StillFire() - { - var result = RawControl.Value; - PrintRawControl(result); - var stale = result.PinHits.Where(kv => kv.Value == 0) - .Select(kv => $"{kv.Key.Pin.Issue} ({kv.Key.Code}): {kv.Key.Pin.Observed}") - .ToList(); - Assert.True(stale.Count == 0, - "parse pin(s) tolerated no sample of the fixed-seed corpus — the defect is fixed; drop the pin:\n " - + string.Join("\n ", stale)); - } - private sealed class RawSampleException(string message) : Exception(message); private static string Unwrap(Exception e) => diff --git a/src/Sharpy.Compiler/Lexer/Lexer.cs b/src/Sharpy.Compiler/Lexer/Lexer.cs index 5193e80a3..c35eefcd3 100644 --- a/src/Sharpy.Compiler/Lexer/Lexer.cs +++ b/src/Sharpy.Compiler/Lexer/Lexer.cs @@ -399,10 +399,16 @@ public List TokenizeAll() { _cancellationToken.ThrowIfCancellationRequested(); - Token token; try { - token = NextToken(); + // The token is read and kept inside the try, so no statement after it reads a variable the + // catch path leaves unassigned (Stryker's statement-removal mutant otherwise hits CS0165 and + // safe mode drops every mutation in this method — P68 Design Decision 8, #2300). + var token = NextToken(); + tokens.Add(token); + + if (token.Type == TokenType.Eof) + break; } catch (LexerAbortException) { @@ -454,11 +460,6 @@ public List TokenizeAll() RecoverFromError(droppedFrom, resumesAfterUnclosedField); continue; } - - tokens.Add(token); - - if (token.Type == TokenType.Eof) - break; } if (_preserveTrivia) diff --git a/src/Sharpy.Compiler/Pretty/UnparseVisitor.Expressions.cs b/src/Sharpy.Compiler/Pretty/UnparseVisitor.Expressions.cs index 42d57a589..770f0cba2 100644 --- a/src/Sharpy.Compiler/Pretty/UnparseVisitor.Expressions.cs +++ b/src/Sharpy.Compiler/Pretty/UnparseVisitor.Expressions.cs @@ -286,9 +286,12 @@ public override void VisitComparisonChain(ComparisonChain node) public override void VisitConditionalExpression(ConditionalExpression node) { - VisitExprInContext(node.ThenValue, PrecConditional, isRightChild: false, parentIsRightAssoc: false); + // The then-value and the test are `??`-level operands (the parser reads both with + // ParseNullCoalesce, as CPython reads them as `or_test`), so a nested conditional, a lambda, + // a walrus or a `try`/`maybe` there is grouped; only the else-value is a full `test` (#2169). + VisitExprInContext(node.ThenValue, PrecNullCoalesce, isRightChild: false, parentIsRightAssoc: false); _w.Write(" if "); - Visit(node.Test); + VisitExprInContext(node.Test, PrecNullCoalesce, isRightChild: false, parentIsRightAssoc: false); _w.Write(" else "); VisitExprInContext(node.ElseValue, PrecConditional, isRightChild: true, parentIsRightAssoc: true); } diff --git a/src/Sharpy.TestInfrastructure/Lexing/TokenPositionReconstruction.cs b/src/Sharpy.TestInfrastructure/Lexing/TokenPositionReconstruction.cs new file mode 100644 index 000000000..01ac375ab --- /dev/null +++ b/src/Sharpy.TestInfrastructure/Lexing/TokenPositionReconstruction.cs @@ -0,0 +1,391 @@ +using System.Globalization; +using Sharpy.Compiler.Lexer; + +namespace Sharpy.TestInfrastructure.Lexing; + +/// +/// The token half of the position instrument (#2300 row C2, P68): every token's (line, column, position, length) +/// must map back to the source slice it was read from. Built here, in TestInfrastructure, so P67 (#2295 — +/// node end-position authority) extends it with NODE ends rather than writing a second model. +/// +/// yields one line per broken +/// rule; an empty result is the claim. The rules, by token kind: +/// +/// +/// R1 — LineStarts[Line-1] + Column - 1 == Position for every token, the position lying on that +/// line (before the next line's start — a column run past the line's end sums to the right offset on the wrong +/// line, the shape of a line break consumed without _line++). A token with no position +/// (Position < 0) or a line outside the source is itself a violation. Indent/Dedent are +/// written at column 1 with the position AFTER the indentation (Lexer.Indentation.cs), so for them R1 also +/// accepts column 1 with only spaces between the line start and the position. R1 is vacuous for a token at column 1 +/// (a dropped _column++ inside a construct is seen by the FOLLOWING token on the same line) — callers that +/// want a construct's column advance observed must put a token after it on its line. +/// R2 — Position + span ≤ source.Length (span per R7). +/// R3 — verbatim kinds (keywords, non-backtick identifiers, operators, delimiters, f-string +/// start/end/brace tokens): the slice equals Value. Integer/Float carry a NORMALIZED value +/// and no SourceLength: the source run from Position with underscores removed, the base prefix's case +/// folded (0X→0x, 0B→0b, 0O→0o) and a leading . read as 0. +/// must spell Value; that run is the number's span. +/// R4 — String/RawString/ByteString: the slice (SourceLength) starts with +/// a prefix the type admits plus a quote run (one or three) and ends with the same quote run; a backtick +/// Identifier's slice is ` + Value + `. FStringConversion (no +/// SourceLength; Value is the flag letter) spans ! + Value; FStringSelfDoc's slice is +/// = followed by hole trivia only. +/// R5 — Newline: Value is "\n", the slice at Position is a line break +/// (\r\n, \n or a lone \r) and the next line starts right after it. +/// R6 — the last token is Eof and Eof.Position == source.Length (≤ when the end was +/// given up — a budget stop reads no further). +/// R7 — tiling: consecutive non-synthetic tokens (all but Indent/Dedent/Eof) do +/// not overlap, and the gap text between them — and before the first and after the last — is whitespace, +/// comments, line breaks and backslash continuations only; the gap before the first FStringFormatSpec of a +/// field may end in the spec's : (no token carries it). A synthetic token lies between its neighbours. +/// Recorded limit: FStringText/FStringFormatSpec carry no SourceLength (their Value is +/// decoded text), so their span is read as next.Position - Position — only the (line, column) ↔ position +/// identity and the tiling order are asserted for them, never slice == Value (a cell for #2300's C2 row). +/// +/// A gap holding a recovery resume () is text error recovery dropped, and the +/// last gap is the text an abort at the end gave up; /endGivenUp exempt +/// exactly those gaps' CONTENT from R7 (never their order). +/// reads both from the lexer. +/// +public static class TokenPositionReconstruction +{ + /// + /// The 0-based offset where each 1-based line starts (index k is line k + 1). \r\n, + /// \n and a lone \r each end a line, as in the lexer (LiteralSpans.LinesStartingInside); + /// a source ending in a line break has a last, empty line starting at source.Length. + /// + public static IReadOnlyList LineStarts(string source) + { + var starts = new List { 0 }; + for (var i = 0; i < source.Length; i++) + { + var c = source[i]; + if (c != '\n' && c != '\r') + continue; + if (c == '\r' && i + 1 < source.Length && source[i + 1] == '\n') + i++; + starts.Add(i + 1); + } + return starts; + } + + /// + /// with the dropped text read + /// from : its , and the end given up when it reported + /// any error (an abort on the last line or a budget stop runs no recovery, so no resume marks that text). + /// + public static IEnumerable Violations(string source, Lexer lexer, IReadOnlyList tokens) => + Violations(source, tokens, lexer.RecoveryResumes, endGivenUp: lexer.Diagnostics.HasErrors); + + /// One line per violated rule (R1–R7, see the type's summary); empty when every token reconstructs. + public static IEnumerable Violations(string source, IReadOnlyList tokens, + IReadOnlyList? recoveryResumes = null, bool endGivenUp = false) + { + var lineStarts = LineStarts(source); + var resumes = recoveryResumes ?? Array.Empty(); + + // R1 / R4 / R5 / R3 per token; spans for R2 / R7. + var spans = new int[tokens.Count]; + for (var i = 0; i < tokens.Count; i++) + { + var token = tokens[i]; + foreach (var violation in PositionRule(source, lineStarts, token)) + yield return violation; + if (token.Position < 0 || token.Position > source.Length) + { + spans[i] = 0; + continue; + } + + var (span, kindViolation) = SpanOf(source, lineStarts, tokens, i); + spans[i] = span; + if (kindViolation != null) + yield return kindViolation; + if (token.Position + span > source.Length) + yield return $"R2 {Describe(token)}: span {span} runs past the source end {source.Length}"; + } + + // R6 + if (tokens.Count == 0 || tokens[^1].Type != TokenType.Eof) + yield return "R6 the token list does not end with Eof"; + else if (endGivenUp ? tokens[^1].Position > source.Length : tokens[^1].Position != source.Length) + yield return $"R6 {Describe(tokens[^1])}: Eof is not at the source end {source.Length}"; + + // R7 tiling over the non-synthetic tokens; synthetic tokens lie between their neighbours. + var end = 0; // end of the previous non-synthetic token + var previous = "start of source"; + var pendingSynthetic = new List(); + for (var i = 0; i < tokens.Count; i++) + { + var token = tokens[i]; + if (token.Position < 0 || token.Position > source.Length) + continue; + if (IsSynthetic(token.Type)) + { + if (token.Position < end) + yield return $"R7 {Describe(token)}: synthetic token before the end {end} of {previous}"; + pendingSynthetic.Add(token); + if (token.Type != TokenType.Eof) + continue; + foreach (var violation in Gap(source, end, token.Position, previous, Describe(token), resumes, exempt: endGivenUp, specColon: false)) + yield return violation; + continue; + } + + if (token.Position < end) + { + yield return $"R7 {Describe(token)}: overlaps {previous}, which ends at {end}"; + } + else + { + foreach (var violation in Gap(source, end, token.Position, previous, Describe(token), resumes, exempt: false, + specColon: token.Type == TokenType.FStringFormatSpec)) + yield return violation; + } + foreach (var synthetic in pendingSynthetic) + { + if (synthetic.Position > token.Position) + yield return $"R7 {Describe(synthetic)}: synthetic token after the start of {Describe(token)}"; + } + pendingSynthetic.Clear(); + end = System.Math.Max(end, token.Position + spans[i]); + previous = Describe(token); + } + } + + private static bool IsSynthetic(TokenType type) => type is TokenType.Indent or TokenType.Dedent or TokenType.Eof; + + private static string Describe(Token token) => + $"{token.Type} '{Escape(token.Value)}' L{token.Line}:C{token.Column}@{token.Position}"; + + private static string Escape(string text) + { + var shown = text.Length > 24 ? text[..24] + "…" : text; + return shown.Replace("\r", "\\r", StringComparison.Ordinal).Replace("\n", "\\n", StringComparison.Ordinal); + } + + /// R1: the token's (line, column) is its position read through the line-start table. + private static IEnumerable PositionRule(string source, IReadOnlyList lineStarts, Token token) + { + if (token.Position < 0) + { + yield return $"R1 {Describe(token)}: no position"; + yield break; + } + if (token.Position > source.Length) + { + yield return $"R1 {Describe(token)}: position past the source end {source.Length}"; + yield break; + } + if (token.Line < 1 || token.Line > lineStarts.Count) + { + yield return $"R1 {Describe(token)}: line outside 1..{lineStarts.Count}"; + yield break; + } + + var lineStart = lineStarts[token.Line - 1]; + var expected = lineStart + token.Column - 1; + if (token.Line < lineStarts.Count && token.Position >= lineStarts[token.Line]) + { + // A column past the line's end still sums to the position; the position must lie on the token's line. + yield return $"R1 {Describe(token)}: position {token.Position} is not on line {token.Line} (line {token.Line + 1} starts at {lineStarts[token.Line]})"; + yield break; + } + if (expected == token.Position) + yield break; + + // Indent/Dedent: column 1, position after the line's indentation. + if (token.Type is TokenType.Indent or TokenType.Dedent && token.Column == 1 && token.Position >= lineStart + && source.AsSpan(lineStart, token.Position - lineStart).TrimStart(' ').IsEmpty) + yield break; + + yield return $"R1 {Describe(token)}: line start {lineStart} + column {token.Column} - 1 = {expected} ≠ position {token.Position}"; + } + + /// The token's source span (for R2/R7) and the kind rule it breaks (R3/R4/R5), if any. + private static (int Span, string? Violation) SpanOf(string source, IReadOnlyList lineStarts, IReadOnlyList tokens, int index) + { + var token = tokens[index]; + var at = token.Position; + var rest = source.Length - at; + switch (token.Type) + { + case TokenType.Indent or TokenType.Dedent or TokenType.Eof: + return (0, token.Value.Length == 0 ? null : $"R3 {Describe(token)}: a synthetic token carries text"); + + case TokenType.Newline: + { + if (rest < 1 || source[at] is not ('\n' or '\r')) + return (token.Length, $"R5 {Describe(token)}: no line break at the position"); + var length = source[at] == '\r' && rest > 1 && source[at + 1] == '\n' ? 2 : 1; + if (token.Value != "\n") + return (length, $"R5 {Describe(token)}: Newline value is not \"\\n\""); + if (token.Line >= lineStarts.Count || lineStarts[token.Line] != at + length) + return (length, $"R5 {Describe(token)}: the line after it does not start at {at + length}"); + return (length, null); + } + + case TokenType.Integer or TokenType.Float: + return NumberSpan(source, token); + + case TokenType.String or TokenType.RawString or TokenType.ByteString: + return (token.Length, QuotedRule(source, token)); + + case TokenType.Identifier when token.IsBacktickEscaped: + { + var length = token.Length; + var expected = "`" + token.Value + "`"; + return (length, length <= rest && length == expected.Length && string.CompareOrdinal(source, at, expected, 0, length) == 0 + ? null + : $"R4 {Describe(token)}: slice '{Escape(Slice(source, at, length))}' is not '{Escape(expected)}'"); + } + + case TokenType.FStringConversion: + { + var expected = "!" + token.Value; + return (expected.Length, expected.Length <= rest && string.CompareOrdinal(source, at, expected, 0, expected.Length) == 0 + ? null + : $"R4 {Describe(token)}: slice '{Escape(Slice(source, at, expected.Length))}' is not '{Escape(expected)}'"); + } + + case TokenType.FStringSelfDoc: + { + var length = token.Length; + if (length < 1 || length > rest || source[at] != '=') + return (length, $"R4 {Describe(token)}: slice '{Escape(Slice(source, at, length))}' does not start with '='"); + return (length, TriviaEnd(source, at + 1, at + length) == at + length + ? null + : $"R4 {Describe(token)}: slice '{Escape(Slice(source, at, length))}' holds more than '=' and trivia"); + } + + case TokenType.FStringText or TokenType.FStringFormatSpec: + { + // Recorded limit: decoded Value, no SourceLength — the span is up to the next token. + var next = index + 1 < tokens.Count ? tokens[index + 1].Position : source.Length; + return (System.Math.Max(0, next - at), null); + } + + default: + { + var length = token.Value.Length; + if (token.SourceLength is { } sourceLength && sourceLength != length) + return (sourceLength, $"R3 {Describe(token)}: verbatim kind with SourceLength {sourceLength} ≠ Value length {length}"); + return (length, length <= rest && string.CompareOrdinal(source, at, token.Value, 0, length) == 0 + ? null + : $"R3 {Describe(token)}: slice '{Escape(Slice(source, at, length))}' ≠ Value"); + } + } + } + + private static string Slice(string source, int at, int length) => + at < 0 || at > source.Length ? "" : source.Substring(at, System.Math.Max(0, System.Math.Min(length, source.Length - at))); + + /// R3 for numbers: the normalized source run from the position spells Value; that run is the span. + private static (int Span, string? Violation) NumberSpan(string source, Token token) + { + var at = token.Position; + var i = at; + var spelled = new System.Text.StringBuilder(); + if (i < source.Length && source[i] == '.') + { + spelled.Append("0."); + i++; + } + else if (i + 1 < source.Length && source[i] == '0' && source[i + 1] is 'x' or 'X' or 'b' or 'B' or 'o' or 'O') + { + spelled.Append('0').Append(char.ToLower(source[i + 1], CultureInfo.InvariantCulture)); + i += 2; + } + while (spelled.Length < token.Value.Length && i < source.Length) + { + if (source[i] != '_') + spelled.Append(source[i]); + i++; + } + + var span = i - at; + return (span, spelled.ToString() == token.Value + ? null + : $"R3 {Describe(token)}: normalized slice '{Escape(spelled.ToString())}' ≠ Value"); + } + + private static readonly IReadOnlyDictionary QuotedPrefixes = new Dictionary + { + [TokenType.String] = new[] { "", "d" }, + [TokenType.RawString] = new[] { "r", "dr" }, + [TokenType.ByteString] = new[] { "b" }, + }; + + /// R4 for string literals: prefix + quote run … same quote run. + private static string? QuotedRule(string source, Token token) + { + var at = token.Position; + var length = token.Length; + if (length > source.Length - at) + return $"R4 {Describe(token)}: span {length} runs past the source end"; + var slice = source.Substring(at, length); + foreach (var prefix in QuotedPrefixes[token.Type].OrderByDescending(p => p.Length)) + { + if (!slice.StartsWith(prefix, StringComparison.Ordinal) || slice.Length <= prefix.Length) + continue; + var quote = slice[prefix.Length]; + if (quote is not ('"' or '\'')) + continue; + var run = new string(quote, 3); + var triple = slice.Length >= prefix.Length + 6 && string.CompareOrdinal(slice, prefix.Length, run, 0, 3) == 0; + var delimiter = triple ? run : quote.ToString(); + if (slice.Length >= prefix.Length + (2 * delimiter.Length) && slice.EndsWith(delimiter, StringComparison.Ordinal)) + return null; + } + return $"R4 {Describe(token)}: slice '{Escape(slice)}' is not a {token.Type} prefix + quote … quote"; + } + + /// R7: the text between two tokens is trivia (or exempt as dropped text). + private static IEnumerable Gap(string source, int from, int to, string previous, string next, + IReadOnlyList resumes, bool exempt, bool specColon) + { + if (from >= to) + yield break; + if (exempt || resumes.Any(r => r > from && r <= to)) + yield break; + + var triviaEnd = TriviaEnd(source, from, to); + if (triviaEnd == to) + yield break; + if (specColon && triviaEnd == to - 1 && source[triviaEnd] == ':') + yield break; + yield return $"R7 gap [{from},{to}) between {previous} and {next} is not trivia: '{Escape(source.Substring(from, to - from))}'"; + } + + /// + /// The first offset in [from, to) that is not trivia — spaces, tabs, form feeds, line breaks, a comment + /// running to its line's end, a backslash immediately followed by a line break — or . + /// + private static int TriviaEnd(string source, int from, int to) + { + var i = from; + while (i < to) + { + var c = source[i]; + if (c is ' ' or '\t' or '\f' or '\n' or '\r') + { + i++; + } + else if (c == '#') + { + while (i < to && source[i] is not ('\n' or '\r')) + i++; + } + else if (c == '\\' && i + 1 < to && source[i + 1] is '\n' or '\r') + { + i += 2; + } + else + { + break; + } + } + return i; + } +} diff --git a/src/Sharpy.TestInfrastructure/Oracles/PythonInterpreter.cs b/src/Sharpy.TestInfrastructure/Oracles/PythonInterpreter.cs new file mode 100644 index 000000000..63d3e059c --- /dev/null +++ b/src/Sharpy.TestInfrastructure/Oracles/PythonInterpreter.cs @@ -0,0 +1,251 @@ +using System.Diagnostics; +using System.Text; +using System.Text.RegularExpressions; + +namespace Sharpy.TestInfrastructure.Oracles; + +/// +/// The python3 interpreter a differential twin uses as its oracle (#2300, P68). One locator so +/// a new twin does not add another private copy (CPythonDifferentialParseTests.PythonOracle, +/// FormatSpecDifferentialTests.PyOracle and DifferentialExecutionTests each carry +/// one today; retrofitting them is a chore note on #2300, not this helper's seam). +/// +/// +/// Absence is a SKIP on a developer machine and a FAIL where the oracle is required: +/// reads SHARPY_REQUIRE_PYTHON_ORACLE=1, which CI sets on the +/// Compiler shards, so a runner that lost its setup-python step cannot pass the twin +/// vacuously. The overload takes the +/// required flag explicitly so a test can exercise the failing arm without setting the +/// process-global environment variable inside a testhost it shares with every other oracle +/// consumer. +/// +public sealed class PythonInterpreter +{ + /// The environment variable that turns a missing interpreter into a failure. + public const string RequireEnvironmentVariable = "SHARPY_REQUIRE_PYTHON_ORACLE"; + + /// The executables tried, in order, when no candidate list is given. + public static readonly IReadOnlyList DefaultCandidates = new[] { "python3", "python" }; + + private static readonly Regex VersionPattern = new(@"Python (\d+)\.(\d+)(?:\.(\d+))?", RegexOptions.CultureInvariant); + + private PythonInterpreter(string executable, string version, int major, int minor) + { + Executable = executable; + Version = version; + Major = major; + Minor = minor; + } + + /// The candidate that answered, e.g. python3. + public string Executable { get; } + + /// The interpreter's own version line, e.g. Python 3.12.13. + public string Version { get; } + + /// Major version number. + public int Major { get; } + + /// Minor version number. + public int Minor { get; } + + /// True when SHARPY_REQUIRE_PYTHON_ORACLE=1: a missing oracle fails, never skips. + public static bool Required => Environment.GetEnvironmentVariable(RequireEnvironmentVariable) == "1"; + + /// The first default candidate reporting Python 3. or later, or null. + public static PythonInterpreter? TryLocate(int minMinor = 12) => TryLocate(DefaultCandidates, minMinor); + + /// The first of reporting Python 3. or later, or null. + public static PythonInterpreter? TryLocate(IReadOnlyList candidates, int minMinor = 12) + { + foreach (var candidate in candidates) + { + var located = Probe(candidate); + if (located is null) + continue; + if (located.Major > 3 || (located.Major == 3 && located.Minor >= minMinor)) + return located; + } + return null; + } + + /// + /// over the default candidates with read + /// from the environment: null means "skip loudly", a missing interpreter in required mode + /// throws . + /// + public static PythonInterpreter? Locate(int minMinor = 12) => Locate(DefaultCandidates, Required, minMinor); + + /// + /// ; when nothing answers and + /// is set, throws + /// instead of returning null. + /// + public static PythonInterpreter? Locate(IReadOnlyList candidates, bool required, int minMinor = 12) + { + var located = TryLocate(candidates, minMinor); + if (located is null && required) + { + throw new PythonOracleMissingException( + $"No Python 3.{minMinor}+ interpreter among [{string.Join(", ", candidates)}], and " + + $"{RequireEnvironmentVariable}=1 makes the python oracle required: the twin FAILS rather than skips."); + } + return located; + } + + /// + /// The absolute path of under the repository root (the first + /// ancestor of the test binary holding sharpy.sln), or null when the root or the file + /// is not found. + /// + public static string? FindRepoFile(params string[] segments) + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + if (File.Exists(Path.Combine(dir.FullName, "sharpy.sln"))) + { + var candidate = Path.Combine(new[] { dir.FullName }.Concat(segments).ToArray()); + return File.Exists(candidate) ? candidate : null; + } + dir = dir.Parent; + } + return null; + } + + /// + /// Runs ONCE with on stdin and + /// returns its non-empty stdout lines. A non-zero exit, an exit before the whole batch was read, or a timeout throws + /// with stderr in the message. + /// + public List RunBatch(string scriptPath, IEnumerable jsonlLines, int timeoutMilliseconds = 120_000) + { + using var proc = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = Executable, + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + StandardInputEncoding = new UTF8Encoding(false), + StandardOutputEncoding = new UTF8Encoding(false), + StandardErrorEncoding = new UTF8Encoding(false), + CreateNoWindow = true, + }, + }; + proc.StartInfo.ArgumentList.Add(scriptPath); + proc.StartInfo.Environment["PYTHONUTF8"] = "1"; + proc.StartInfo.Environment["PYTHONIOENCODING"] = "utf-8"; + + var stdout = new List(); + var stderr = new StringBuilder(); + proc.OutputDataReceived += (_, e) => + { + if (!string.IsNullOrWhiteSpace(e.Data)) + { + lock (stdout) + stdout.Add(e.Data); + } + }; + proc.ErrorDataReceived += (_, e) => + { + if (e.Data != null) + { + lock (stderr) + stderr.AppendLine(e.Data); + } + }; + + proc.Start(); + proc.BeginOutputReadLine(); + proc.BeginErrorReadLine(); + + // A script that exits before reading all of stdin (a crash, an import error) breaks the pipe; the write then + // throws IOException with no stderr. Stop writing and let the exit-code check below report the script's stderr. + var stdinBroken = false; + try + { + foreach (var line in jsonlLines) + proc.StandardInput.WriteLine(line); + proc.StandardInput.Close(); + } + catch (IOException) + { + stdinBroken = true; + } + + if (!proc.WaitForExit(timeoutMilliseconds)) + { + try + { proc.Kill(entireProcessTree: true); } + catch (InvalidOperationException) { } + throw new InvalidOperationException( + $"{Path.GetFileName(scriptPath)} timed out after {timeoutMilliseconds} ms under {Version}. stderr:\n{stderr}"); + } + proc.WaitForExit(); // drain the async readers + + if (proc.ExitCode != 0) + { + throw new InvalidOperationException( + $"{Path.GetFileName(scriptPath)} exited {proc.ExitCode} under {Version}. stderr:\n{stderr}"); + } + if (stdinBroken) + { + throw new InvalidOperationException( + $"{Path.GetFileName(scriptPath)} exited 0 under {Version} before reading the whole batch. stderr:\n{stderr}"); + } + + lock (stdout) + return new List(stdout); + } + + private static PythonInterpreter? Probe(string executable) + { + try + { + using var proc = Process.Start(new ProcessStartInfo + { + FileName = executable, + Arguments = "--version", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }); + if (proc is null) + return null; + var errTask = proc.StandardError.ReadToEndAsync(); + string outText = proc.StandardOutput.ReadToEnd(); + if (!proc.WaitForExit(10_000)) + { + try + { proc.Kill(entireProcessTree: true); } + catch (InvalidOperationException) { } + return null; + } + outText += errTask.Result; + var match = VersionPattern.Match(outText); + if (!match.Success) + return null; + return new PythonInterpreter( + executable, + match.Value, + int.Parse(match.Groups[1].Value, System.Globalization.CultureInfo.InvariantCulture), + int.Parse(match.Groups[2].Value, System.Globalization.CultureInfo.InvariantCulture)); + } + catch (System.ComponentModel.Win32Exception) + { + // The candidate is not on PATH. + return null; + } + } +} + +/// Thrown when the python oracle is required () but absent. +public sealed class PythonOracleMissingException : Exception +{ + /// Creates the exception with its loud message. + public PythonOracleMissingException(string message) : base(message) { } +}