diff --git a/.github/workflows/code-projects-type-check.yml b/.github/workflows/code-projects-type-check.yml deleted file mode 100644 index 515b98a85..000000000 --- a/.github/workflows/code-projects-type-check.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: code_projects Type Check - -on: - push: - paths: - - 'frontend/python/rst_code_example_pipeline/**' - pull_request: - branches: - - main - paths: - - 'frontend/python/rst_code_example_pipeline/**' - -jobs: - pyright: - - runs-on: ubuntu-26.04 - - strategy: - matrix: - python-version: ['3.14'] - - steps: - - uses: actions/checkout@v7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 - with: - python-version: ${{ matrix.python-version }} - - name: Install pyright - run: pip install pyright - - name: Install rst_code_example_pipeline - run: pip install -e frontend/python/rst_code_example_pipeline - - name: Run pyright on rst_code_example_pipeline - working-directory: frontend/python/rst_code_example_pipeline - run: pyright . diff --git a/.github/workflows/rst-code-example-pipeline-ci.yml b/.github/workflows/rst-code-example-pipeline-ci.yml new file mode 100644 index 000000000..c6a9ce591 --- /dev/null +++ b/.github/workflows/rst-code-example-pipeline-ci.yml @@ -0,0 +1,62 @@ +name: rst_code_example_pipeline CI + +on: + push: + paths: + - 'frontend/python/rst_code_example_pipeline/**' + pull_request: + branches: + - main + paths: + - 'frontend/python/rst_code_example_pipeline/**' + +jobs: + pyright: + + runs-on: ubuntu-26.04 + + strategy: + matrix: + python-version: ['3.14'] + + steps: + - uses: actions/checkout@v7 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + - name: Install pyright + run: pip install pyright + - name: Install rst_code_example_pipeline + run: pip install -e 'frontend/python/rst_code_example_pipeline[test]' + - name: Run pyright on rst_code_example_pipeline + working-directory: frontend/python/rst_code_example_pipeline + run: pyright . + + pytest: + + runs-on: ubuntu-26.04 + + strategy: + matrix: + python-version: ['3.14'] + + steps: + - uses: actions/checkout@v7 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + - name: Install OS deps + run: | + sudo apt-get update && \ + sudo apt-get install -y \ + crudini + - name: Install GNAT FSF + run: | + ${GITHUB_WORKSPACE}/.github/workflows/install_toolchain.sh --gnat --gnatprove --gprbuild + - name: Install rst_code_example_pipeline + run: pip install -e 'frontend/python/rst_code_example_pipeline[test]' + - name: Run rst_code_example_pipeline test suite + working-directory: frontend + run: make test_rst_pipeline diff --git a/.gitignore b/.gitignore index 50ead7d5e..656572878 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store *.pyc +*.egg-info/ env .idea .vagrant* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec6e35c45..8572f99be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -310,6 +310,11 @@ The following button-related parameters are available for this directive: | `prove_flow_report_all_button` | examine SPARK data and control flow and report all findings | | `submit_button` | submit code for a lab | +When `compile_button` is used and nothing else asks for the code to be run — +no `run_button`, and no class that asks for a run — the code is compiled but +not linked. No executable is produced, so such a code block does not have to +contain a main subprogram (in Ada) or a `main` function (in C). + ### Project parameter and code accumulation A `project` parameter must be provided. For this parameter, we use the @@ -586,7 +591,8 @@ block in the generated HTML or e-book output. The following classes are available for the testing phase: - - `ada-nocheck`: testing of this specific code block is completely skipped. + - `ada-nocheck` and `c-nocheck`: testing of this specific code block is + completely skipped, for Ada or C code respectively. - `nosyntax-check`: code must not be checked for syntax errors. (Note that the code block is still compiled in the testing phase.) @@ -597,9 +603,21 @@ The following classes are available for the testing phase: If an error is expected during the testing phase, one of the following classes must be used: - - `ada-expect-compile-error`: a compilation error is expected. + - `ada-expect-compile-error` and `c-expect-compile-error`: a compilation + error is expected, in Ada or C code respectively. + + - `ada-run-expect-failure` and `c-run-expect-failure`: a run-time error is + expected, in Ada or C code respectively. - - `ada-run-expect-failure`: a run-time error is expected. + - `ada-expect-prove-error`: a proof error is expected. The code block must + also be proved, either through one of the prove buttons or through one of + the `ada-prove` classes listed below. + +These classes state a requirement on the testing phase, not a hint about the +generated output: the expected error has to actually occur. If it does not — +the code compiles, runs or proves cleanly — that absence is reported as an +error and fails the check, so one of these classes left behind after the code +example was fixed makes the testing phase fail rather than passing quietly. When the `no_button` parameter is used, the following classes are available to compile or run the code examples: @@ -618,7 +636,13 @@ output. When the `run_button` parameter is used, the following classes are available: - `ada-norun` and `c-norun`: to explicitly deactivate the run of Ada or C - code, respectively, during the testing phase. + code, respectively, during the testing phase. These classes also take + precedence over a class that asks for a run, such as `ada-run` or `c-run`. + + The code is built in order to be run, so deactivating the run also + deactivates the build unless something else asks for the code to be + compiled — a `compile_button`, or the `ada-compile` or `c-compile` class. + Without one of those, the code block is only checked for syntax errors. ## Lab exercises diff --git a/README.md b/README.md index 774bc42c6..bba0fc0f1 100644 --- a/README.md +++ b/README.md @@ -224,3 +224,71 @@ check-code \ For more examples and alternative configurations, please refer to the [README of the rst_code_example_pipeline package](frontend/python/rst_code_example_pipeline/README.md) + +#### Running the package's unit tests + +The package also has its own pytest-based unit test suite. Run the whole +suite with `make test_rst_pipeline`, from the `frontend/` directory: + +```sh +# Full suite -- the validation run, and the only one +make test_rst_pipeline +``` + +This needs an Ada toolchain *installation*, not merely a compiler on `PATH`. +The package exists to extract, build and run source-code examples, so the +tests covering that work invoke the toolchain for real -- and some of them +create and remove symlinks under the installation tree configured in the +package's `src/rst_code_example_pipeline/data/toolchain.ini` (`/opt/ada` and +below). A distribution-packaged GNAT on `PATH`, without that tree, is not +enough: the suite goes red. This is the only run that validates the +module (the `rst_code_example_pipeline` package). The epub VM has such an +installation; so does the `pytest` job in +`.github/workflows/rst-code-example-pipeline-ci.yml`, which provisions one +and then runs this same target on a GitHub runner. + +For developers working on a machine without an Ada toolchain, a second +target runs just the subset of tests that need no toolchain: + +```sh +# Toolchain-free subset -- does not validate the module +make test_rst_pipeline_smoke +``` + +**This smoke run does not validate the module.** It compiles nothing, so it +proves nothing about the module's actual job. Its reach is narrower than +"every test that does not need a toolchain" may suggest: it executes roughly +half of the module's lines and leaves `check_code_block.py`, the file that +drives the toolchain, almost entirely unexecuted. Ordinary Python inside +the toolchain-facing files -- diagnostic parsing, message formatting, +writing the JSON check report -- is deselected wholesale along with the +tests that cover it, so breaking any of that still leaves the smoke run +green. A green smoke run must never be mistaken for a passing suite. +Coverage is switched off for it, because a coverage figure measured from a +subset would invite the same misreading. + +So before considering a change tested, run the full suite where a toolchain +installation exists: on the epub VM, or on a machine provisioned the way the +CI runner is. `.github/workflows/install_toolchain.sh` is the specification +for that provisioning -- it reads every path and version from the package's +`toolchain.ini` and unpacks the GNAT FSF builds into that tree. It is +written for the CI runner (it expects `GITHUB_WORKSPACE`, needs `crudini`, +and exports the `bin` directories through `GITHUB_PATH`), so read it rather +than run it unchanged on a workstation. + +Both targets are thin wrappers around `pytest`. `make` itself runs from +`frontend/`, where the `Makefile` lives -- there is no `Makefile` in the +package directory. It is the recipes that `cd` into +`frontend/python/rst_code_example_pipeline`; `pytest` runs there. Without +the Makefile, run the equivalent commands from that directory: + +```sh +# Full suite -- the validation run, and the only one +pytest + +# Smoke subset -- does not validate the module +pytest -m "not toolchain" --no-cov +``` + +See the "Development" section of the package README for installation and +usage details. diff --git a/frontend/Makefile b/frontend/Makefile index e255c56e5..66872ce72 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -215,6 +215,12 @@ test_parser: # @coverage run --source=widget,rst_code_example_pipeline.chop,rst_code_example_pipeline.resource -m unittest discover --start-directory sphinx @coverage report --fail-under=90 -m +test_rst_pipeline: ## Test the rst_code_example_pipeline package (epub VM). + @cd python/rst_code_example_pipeline && pytest + +test_rst_pipeline_smoke: ## Toolchain-free smoke test; does NOT validate the package. + @cd python/rst_code_example_pipeline && pytest -m "not toolchain" --no-cov + ##@ Build website publish: ## [DEPRECATED] Publish contents to the learn website. @echo "Publishing current branch to learn..." diff --git a/frontend/python/rst_code_example_pipeline/README.md b/frontend/python/rst_code_example_pipeline/README.md index 5009f5e01..4a4efdd52 100644 --- a/frontend/python/rst_code_example_pipeline/README.md +++ b/frontend/python/rst_code_example_pipeline/README.md @@ -2,9 +2,8 @@ ## Introduction -The [rst_code_example_pipeline](frontend/python/rst_code_example_pipeline) package contains -scripts to extract, build and run the code blocks from the ReST files. These are the main -entry points: +The `rst_code_example_pipeline` package contains scripts to extract, build and +run the code blocks from the ReST files. These are the main entry points: - `extract-code` extracts all code blocks and stores into the specified build directory; @@ -13,18 +12,23 @@ entry points: - `check-block` checks a single (previously extracted) code block. -The package is installed in editable mode as part of the VM provisioning: +Install the package from the repository, in editable mode: ```sh pip install -e frontend/python/rst_code_example_pipeline ``` +The entry points drive an Ada toolchain directly and expect it on `PATH`: +`extract-code` splits an Ada code block with `gnatchop`, and the two checking +commands syntax-check and compile with `gcc`, build with `gprbuild`, clean up +with `gprclean`, and prove with `gnatprove`. + ## Simple usage To build and run the source-code examples from a course, just run `extract-code` followed by `check-code`. For example, to test the source-code examples from the -[Introduction to Ada course](content/courses/intro-to-ada), run: +[Introduction to Ada course](../../../content/courses/intro-to-ada), run: ```sh extract-code \ @@ -41,6 +45,26 @@ for each code block (source-code example) that is extracted from the ReST files. and checks the source-code example described in each of those JSON files. +## Exit status + +All three entry points report the outcome of a run through their exit status: + +- `check-code` and `check-block` exit `1` if any checked code block failed a + check, and `0` otherwise. A code block that could not be read or checked + also counts as a failure. + +- `extract-code` exits `1` when it cannot run at all — for example, a code + block has no project name, or neither `--build-dir` nor + `--extracted_projects` was given — and `0` otherwise. + +An invalid command line is rejected before any work is done, with exit +status `2`. + +Some malformed code blocks are reported only through an `ERROR` line in the +output, while `extract-code` itself still exits `0`. Check the output, not +only the exit code, to catch these. + + ## Verbose mode All the scripts have a `--verbose` / `-v` switch. For example: @@ -62,7 +86,7 @@ check-code \ It's possible to store the list of extracted projects into a JSON file and use that file for checking the projects. For example, to build the source-code examples from the -[Introduction to Ada course](content/courses/intro-to-ada), run: +[Introduction to Ada course](../../../content/courses/intro-to-ada), run: ```sh extract-code \ @@ -168,3 +192,35 @@ check-block \ --max-columns 80 \ test_output/projects/Courses/Intro_To_Ada/Imperative_Language/Greet/cba89a34b87c9dfa71533d982d05e6ab/block_info.json ``` + + +## Development + +### Installing with test dependencies + +The package declares an optional `test` extras group that installs +[pytest](https://docs.pytest.org/) and +[pytest-cov](https://pytest-cov.readthedocs.io/). +Install the package in editable mode together with those extras: + +```sh +pip install -e ".[test]" +``` + +### Running the unit tests + +Coverage options and test paths are configured in `pyproject.toml`, so a plain +`pytest` invocation from the package root is enough: + +```sh +pytest +``` + +Some modules require an Ada toolchain (GNAT) to be on `PATH`; run the full +suite in an environment where GNAT is available. + +To pass coverage options explicitly: + +```sh +pytest --cov=rst_code_example_pipeline --cov-report=term-missing tests/ +``` diff --git a/frontend/python/rst_code_example_pipeline/pyproject.toml b/frontend/python/rst_code_example_pipeline/pyproject.toml index 69ba8bf23..289b7869f 100644 --- a/frontend/python/rst_code_example_pipeline/pyproject.toml +++ b/frontend/python/rst_code_example_pipeline/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rst-code-example-pipeline" -version = "0.2.0" +version = "0.3.0" requires-python = ">=3.10" [project.scripts] @@ -12,11 +12,35 @@ extract-code = "rst_code_example_pipeline.cli.extract:main" check-code = "rst_code_example_pipeline.cli.check:main" check-block = "rst_code_example_pipeline.cli.check_block:main" +[project.optional-dependencies] +test = ["pytest", "pytest-cov"] + [tool.setuptools.packages.find] where = ["src"] [tool.setuptools.package-data] rst_code_example_pipeline = ["data/*.ini"] +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--cov=rst_code_example_pipeline --cov-report=term-missing --strict-markers" +markers = [ + "toolchain: test needs the Ada toolchain -- it either invokes a binary (gcc, gprbuild, gnatprove, gnatchop) on PATH, or reaches code that creates/removes symlinks under the toolchain installation tree", +] + +[tool.coverage.run] +source = ["rst_code_example_pipeline"] +branch = true + +[tool.coverage.report] +show_missing = true +fail_under = 99 +exclude_lines = [ + # Standard pragma for uncoverable lines + "pragma: no cover", + # CLI __main__ entry points are not exercised by unit tests + "if __name__ == .__main__.:", +] + [tool.pyright] pythonVersion = "3.10" diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/__init__.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/__init__.py index 24131d7f4..bc24cd28c 100644 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/__init__.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/__init__.py @@ -1,2 +1,2 @@ __title__ = 'rst_code_example_pipeline' -__version__ = '0.2.0' +__version__ = '0.3.0' diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/blocks.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/blocks.py index b618cef12..2bdf95858 100644 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/blocks.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/blocks.py @@ -7,6 +7,7 @@ from typing import Any from . import colors as C +from . import constants from . import toolchain_info class Block(object): @@ -197,21 +198,102 @@ def to_json_file(self, json_filename: str | None = None) -> None: block_info = vars(self) if json_filename is None: - json_filename = "block_info.json" + json_filename = constants.BLOCK_INFO_FILENAME with open(json_filename, u'w') as f: json.dump(block_info, f, indent=4) class CodeBlock(Block): + """A single code block extracted from a ReST file + + Note: + ``text_hash`` and ``text_hash_short`` are derived from the block's + text whenever the constructor is not handed them. What this package + asks of them is exactly three things: + + * **determinism** -- the same text hashes the same way in every run, + or a block's directory moves and the result cached in it is never + found again; + * **distinctness** -- two different texts do not collide, or one + block's extracted project overwrites another's and one of the two + silently stops being checked; + * **hexadecimal shape** -- the short hash is used verbatim as a + directory name, so it must hold nothing a path would have to + escape. + + What this package does **not** ask of them is any particular digest. + Neither hash is compared against a value computed anywhere else in + the package, so SHA-512 and MD5 are a choice made here, not a + promise made to a caller. Tests belong on the three properties above + and never on a literal digest: pinning one turns a correct change of + algorithm into a test failure, which is the opposite of what such a + test is for. + """ + @staticmethod def from_json_file(json_filename: str | None = None) -> CodeBlock | None: + """Reads a code block back from the block info file written for it + + Args: + json_filename (str, optional): The file to read. Defaults to + ``block_info.json`` in the current working directory, which + is the name the extraction step writes and the place the + checking step changes into. + + Returns: + CodeBlock, optional: The code block the file describes, or None + when it does not describe one. + + Note: + **A file that cannot be turned into a code block yields None + rather than an exception**, and that is the part callers build + on. Nothing on this side decides what an unreadable code block + means for a run -- the callers do, and they differ: the + checking commands leave the code block unchecked and fail the + run over it, while extraction warns and rewrites the record + from the ReST source. What is decided here is that the *reason* + is reported before it is lost, since only this side has it. + + Two files reduce to None without a word: one that is not there, + and one whose name is not a regular file at all. Neither is a + complaint worth making -- the first is the ordinary way to ask + whether a code block has been extracted yet. + + The reasons that *are* reported are a file that does not decode + as UTF-8, one that does not parse as JSON, and one that parses + into something that is not a code block record. **That list is + not exhaustive, and it is a list of exception types rather than + of intent**: it names the ways a file has actually been seen to + be unusable, so a file unusable in some other way still raises + out of here. Reading a deeply enough nested JSON array is the + known example, and opening the file is outside the guard + entirely, so a file whose permissions forbid reading raises as + well. + """ if json_filename is None: - json_filename = "block_info.json" + json_filename = constants.BLOCK_INFO_FILENAME if os.path.isfile(json_filename): with open(json_filename, u'r') as f: - block_info_json = json.load(f) - return CodeBlock(**block_info_json) + try: + block_info_json = json.load(f) + return CodeBlock(**block_info_json) + except (json.JSONDecodeError, UnicodeDecodeError, + TypeError) as e: + # A file that is present but cannot be turned into a + # block is reported and treated as no block at all. The + # callers already say what that means for them; only the + # reason is known here, and it is the part that would + # otherwise be lost. + # + # UnicodeDecodeError is listed separately on purpose: it + # is a *sibling* of JSONDecodeError under ValueError, not + # a subclass, so a record holding bytes that are not + # valid UTF-8 would otherwise escape -- and a hand edit + # in an editor defaulting to another encoding produces + # exactly that. + print("{}: cannot read block info from {}: {}".format( + C.col("ERROR", C.Colors.RED), json_filename, e)) return None @@ -259,30 +341,40 @@ def __init__(self, self.active: bool = active if active is not None else True self.no_check: bool = no_check if no_check is not None else \ - any(sphinx_class in ["ada-nocheck", "c-nocheck"] + any(sphinx_class in [constants.CLASS_ADA_NOCHECK, constants.CLASS_C_NOCHECK] for sphinx_class in self.classes) self.syntax_only: bool = syntax_only if syntax_only is not None else \ - 'ada-syntax-only' in self.classes + constants.CLASS_ADA_SYNTAX_ONLY in self.classes + # The C spellings are paired with the language the way compile_it + # pairs its own, so that asking for a run by class alone works for C + # as it already does for Ada. Without them a c-run block was never + # run and the check still reported success, and the branch handling + # c-run-expect-failure could only be reached through a run button. self.run_it: bool = run_it if run_it is not None else \ - (('ada-run' in self.classes - or 'ada-run-expect-failure' in self.classes + ((((constants.CLASS_ADA_RUN in self.classes + or constants.CLASS_ADA_RUN_EXPECT_FAILURE in self.classes) + and self.language == 'ada') + or ((constants.CLASS_C_RUN in self.classes + or constants.CLASS_C_RUN_EXPECT_FAILURE in self.classes) + and self.language == 'c') or 'run' in self.buttons) - and not 'ada-norun' in self.classes) + and not (constants.CLASS_ADA_NORUN in self.classes + and self.language == 'ada') + and not (constants.CLASS_C_NORUN in self.classes + and self.language == 'c')) self.compile_it: bool = compile_it if compile_it is not None else \ self.run_it or \ - (('ada-compile' in self.classes and self.language == 'ada') - or ('c-compile' in self.classes and self.language == 'c') + ((constants.CLASS_ADA_COMPILE in self.classes and self.language == 'ada') + or (constants.CLASS_C_COMPILE in self.classes and self.language == 'c') or 'compile' in self.buttons) prove_buttons: list[str] = ["prove", "prove_flow", "prove_flow_report_all", "prove_report_all"] - prove_classes: list[str] = ["ada-prove", "ada-prove-flow", "ada-prove-flow-report-all", - "ada-prove-report-all"] self.prove_it: bool = prove_it if prove_it is not None else \ - (any(b in prove_classes for b in self.classes) + (any(b in constants.PROVE_CLASSES for b in self.classes) or any(b in prove_buttons for b in self.buttons)) self.source_files: list[str] = source_files if source_files is not None else \ @@ -300,13 +392,39 @@ def __init__(self, class ConfigBlock(Block): + """Settings read from a ReST source, held as boolean attributes + + Every keyword argument becomes an attribute of the same name, whose + value is coerced to a boolean by a deliberately asymmetric rule: a real + ``bool`` is kept as it stands, and anything else is true unless it is + exactly the string ``"False"``. So ``"false"``, ``"0"`` and the empty + string are all true, and so is any value that is not a string at all. + + The asymmetry follows the two kinds of caller. Settings normally arrive + as strings, parsed out of a ``:code-config:`` directive, where + ``"False"`` is the only spelling of false the directive has; that is + where the string comparison comes from. A caller that hands over a real + boolean -- as the extractor does for the settings it starts from -- + means that boolean literally, and comparing it against a string it can + never equal would silently turn every such setting true. + + Args: + rst_file (str, optional): The ReST source the settings were read + from. + **opts (Any): The settings themselves, coerced as described above. + """ + def __init__(self, rst_file: str | None = None, **opts: Any) -> None: self.rst_file: str | None = rst_file self._opts: dict[str, Any] = opts for k, v in opts.items(): - setattr(self, k, False if v == "False" else True) + # Values normally arrive as strings from a code-config directive, + # where only "False" means false. A caller passing a real + # boolean means it literally, so pass it through instead of + # comparing it against a string it can never equal. + setattr(self, k, v if isinstance(v, bool) else v != "False") def update(self, other_config: ConfigBlock) -> None: self.__init__(**other_config._opts) diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_code_block.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_code_block.py index 36a53277b..5134135fe 100755 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_code_block.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_code_block.py @@ -1,16 +1,32 @@ #! /usr/bin/env python3 """ -This program will try to compile and execute code blocks. -The default behavior is to: -- If the user indicated that the example should be ran (more on that later): - a. Run gnatmake on the unit named 'main' if there are several, or on the - first and only one if there is only one - b. Run the resulting program and check the return code -- Else: - a. Run gcc on every Ada file +Check code blocks that were previously extracted from the ReST sources, one +block_info.json record at a time. What runs for a code block is decided by +what the code block itself declares. A code block declaring 'ada-nocheck' or +'c-nocheck' is skipped entirely, before anything runs, and so is one that +already carries a recorded result, unless --force is given. Every other code +block is syntax-checked unless it declares 'nosyntax-check', and one +declaring 'ada-syntax-only' stops there; the syntax check invokes a compiler +for Ada and for C only, so a record naming any other language passes it +having parsed nothing. A code block that asks to be compiled or to be run is +built (gprbuild for Ada, gcc for C), and the resulting program is run, with +its exit status checked, only after a build that succeeded. A code block +that asks to be proved is proved with gnatprove independently of the build, +so a proof needs no build and does not trigger one. A code block may also +declare that its compilation, its run or its proof is expected to fail; the +failure is then the passing outcome, and its absence is reported. A run +class names a language and applies only to a code block written in that +language; one naming the other language is reported and fails the check. The +outcome is recorded next to the code block as block_checks.json. """ +# The text above is what argparse prints as this command's help +# description. It is deliberately free of ReST markup and of any layout +# worth preserving: the default help formatter re-wraps a description into a +# single filled paragraph, so a list would arrive as a run-on sentence and +# inline literals would arrive with their backquotes intact. + import argparse import os import subprocess as S @@ -20,6 +36,7 @@ from . import blocks from . import checks +from . import constants from . import fmt_utils from . import toolchain_setup @@ -52,6 +69,89 @@ def check_block(block: blocks.CodeBlock, all_diagnostics: bool = all_diagnostics, max_columns: int = max_columns, force_checks: bool = force_checks) -> bool: + """Runs the checks a single code block asks for + + A code block declares what is to be done with it, through its buttons + and its ``:class:`` values, and this function turns that declaration + into checks. The order below is part of the contract rather than an + accident of the code, because the later checks depend on the earlier + ones having run. + + Two returns come before any check at all, and they are part of that + order too. A code block declaring ``ada-nocheck`` or ``c-nocheck`` + returns first, with nothing done to it and nothing recorded. A code + block that already carries a recorded result returns next, handing back + that result, unless ``force_checks`` asks for the checks to be re-run. + + A **syntax check** comes first among the checks themselves, over every + source file of the code block, and it runs for every code block that + got past those two returns -- including one that asks for nothing else + at all -- unless the code block declares ``nosyntax-check``. It invokes + a compiler for ``ada`` and for ``c`` only, so a code block whose record + names any other language reaches the end of it having parsed nothing. + + A code block declared **syntax-only** returns right after that check, + so it never reaches the build. It is still cleaned up and its result + still recorded; what it skips is every check below. + + A **build** follows for a code block that asks to be compiled, which + includes every code block that asks to be run, since asking for a run + implies asking for a compile. + + The **run** is nested inside that build step, not placed beside it: a + code block cannot be run without having been built, and a build that + did not succeed suppresses the run. That holds for a build that failed + and was reported, and equally for one that failed the way the code + block said it would -- an expected compile error is still a program + that was not produced. + + A **proof** is a sibling of the build rather than part of it. A code + block that asks only to be proved is therefore never built, and one + that asks for both gets both, independently of each other. + + A check of the code block's **own declarations** runs last, after + everything that could satisfy them. It has to: what it reports is a + compile error, a proof error or a run failure that the code block + declared it expected and that then did not happen, and that is only + knowable once the checks above have had their turn. + + The same check also reports a **run class that names the other + language** -- ``ada-run`` on a C code block, say. Unlike the reports + beside it, this one is knowable from the declaration alone; it is + reported here because it too is a declaration that was not honored, not + because it had to wait. A run class naming the other language has no + effect at all, so a code block asking for a run that way would otherwise + be neither built nor run and still recorded as a success. + + Args: + block (blocks.CodeBlock): The code block to check. + json_file (str): The block info file the code block was read from. + Only its directory is used, as the place the extracted sources + and the generated project were written to. + verbose (bool): Reports each command as it runs, plus toolchain + versions and paths. + all_diagnostics (bool): Reports the diagnostics collected over the + whole check, in addition to those reported per failing check. + max_columns (int): Maximum source line length the syntax check + enforces for Ada; zero leaves the length unchecked. + force_checks (bool): Re-runs the checks for a code block that + already carries a result from an earlier run, which is + otherwise reused. + + Returns: + bool: True if any check failed. Note the polarity: this is an error + flag, not a success flag, and the callers OR it across code blocks. + + Note: + The outcome is written next to the code block as + ``block_checks.json``, and a later run reuses it instead of + checking again. Only the overall status survives that round trip: + the per-check entries recorded here are written to the file but are + dropped when it is read back, so nothing acts on them. They are a + record for whoever reads the file, not an interface: a reader + wanting an example's log files finds them by globbing the code + block's directory, not by reading their names from here. + """ def run(*run_args): if verbose: @@ -131,6 +231,9 @@ def cleanup_project(language, project_filename, main_file): run("gnatprove", "-P", project_filename, "--clean") except S.CalledProcessError as e: out = str(e.output.decode("utf-8")) + print_error(loc, + "Failed to clean-up example (gnatprove --clean)") + print(out) elif language == "c": try: cmd = ["rm", "-f"] + glob.glob('*.o') + glob.glob('*.gch') @@ -140,7 +243,6 @@ def cleanup_project(language, project_filename, main_file): except S.CalledProcessError as e: print_error(loc, "Failed to clean-up example") print(e.output) - has_error = True toolchain_setup.set_toolchain(block) @@ -152,6 +254,31 @@ def cleanup_project(language, project_filename, main_file): print("Skipping code block {}".format(loc)) return has_error + # A run class names a language, and is honored only for a code block + # written in it. Reported rather than passed over: the code block would + # otherwise be built by nothing and still recorded as a success, which is + # the one outcome this checker exists to prevent. + # + # Read from the declaration before the recorded result below is consulted, + # because that record is keyed on a hash of the code block's text alone. + # Editing only the class leaves the text, and so the key, unchanged, so a + # mis-declared code block would otherwise reuse the success recorded for + # the declaration it had before the edit. + # + # Reporting here and carrying the failure to each return separately, + # rather than setting has_error now: has_error also decides whether the + # code block is run at all, and a code block whose run button asks for + # the run its class did not is still to be built and run. + wrong_language_classes = [ + code_class for code_class in block.classes + if constants.RUN_CLASS_LANGUAGES.get(code_class) not in ( + None, block.language)] + + for code_class in wrong_language_classes: + print_error(loc, + "Wrong language selected for run class '{}'".format( + code_class)) + if LOOK_FOR_PREVIOUS_CHECKS: ref_block_check = None @@ -164,13 +291,13 @@ def cleanup_project(language, project_filename, main_file): has_error = not ref_block_check.status_ok if verbose: print("Code block {} already checked. Skipping...".format(loc)) - if __name__ == '__main__': + if __name__ == '__main__': # pragma: no cover print("WARNING: Code block {} already checked: use '--force' to re-run the check. Skipping...".format(loc)) if has_error: print_error( loc, "Previous check of example has failed" ) - return has_error + return has_error or bool(wrong_language_classes) if verbose: print(fmt_utils.header("Checking code block {}".format(loc))) @@ -198,7 +325,7 @@ def cleanup_project(language, project_filename, main_file): block_check.status_ok = True # Syntax check - if 'nosyntax-check' not in block.classes: + if constants.CLASS_NOSYNTAX_CHECK not in block.classes: check_error = False for source_file in block.source_files: @@ -233,6 +360,9 @@ def cleanup_project(language, project_filename, main_file): cleanup_project(block.language, block.project_filename, block.project_main_file) + # Reported above; carried into the result here, because this + # return comes before the declaration checks that would carry it. + has_error = has_error or bool(wrong_language_classes) block_check.status_ok = not has_error block_check.to_json_file() return has_error @@ -260,7 +390,7 @@ def cleanup_project(language, project_filename, main_file): out = run(*cmdline) except S.CalledProcessError as e: - if 'ada-expect-compile-error' in block.classes: + if constants.CLASS_ADA_EXPECT_COMPILE_ERROR in block.classes: compile_error = True else: print_error(loc, "Failed to compile example") @@ -285,12 +415,20 @@ def cleanup_project(language, project_filename, main_file): elif block.language == "c": cmdline = None try: - assert block.project_main_file is not None - cmdline = ["gcc", "-o", - P.splitext(block.project_main_file)[0]] + glob.glob('*.c') + sources = glob.glob('*.c') + if block.project_main_file is not None: + cmdline = ["gcc", "-o", + P.splitext(block.project_main_file)[0]] + sources + else: + # A compile button asks for a compile and not a link, and + # a block that is not also run has no main file resolved + # for it -- it may hold no main at all. Compiling without + # linking is what was asked for, and needs no name for an + # executable that is not being produced. + cmdline = ["gcc", "-c"] + sources out = run(*cmdline) except S.CalledProcessError as e: - if 'c-expect-compile-error' in block.classes: + if constants.CLASS_C_EXPECT_COMPILE_ERROR in block.classes: compile_error = True else: print_error(loc, "Failed to compile example") @@ -313,21 +451,23 @@ def cleanup_project(language, project_filename, main_file): if not compile_error and not has_error and block.run_it: check_error = False cmdline = None + run_attempted = False if block.language == "ada": + run_attempted = True try: assert block.project_main_file is not None cmdline = ["./{}".format(P.splitext(block.project_main_file)[0])] out = run(*cmdline) - if 'ada-run-expect-failure' in block.classes: + if constants.CLASS_ADA_RUN_EXPECT_FAILURE in block.classes: print_error( loc, "Running of example should have failed" ) check_error = True except S.CalledProcessError as e: - if 'ada-run-expect-failure' in block.classes: + if constants.CLASS_ADA_RUN_EXPECT_FAILURE in block.classes: if verbose: print("Running of example expectedly failed") else: @@ -335,75 +475,55 @@ def cleanup_project(language, project_filename, main_file): check_error = True out = str(e.output.decode("utf-8")) + except FileNotFoundError as e: + print_error(loc, "Running of example failed: " + "no executable to run") + check_error = True + out = str(e) with open("run.log", u"w") as logfile: logfile.write(out) elif block.language == "c": + run_attempted = True try: assert block.project_main_file is not None cmdline = ["./{}".format(P.splitext(block.project_main_file)[0])] out = run(*cmdline) - if 'c-run-expect-failure' in block.classes: + if constants.CLASS_C_RUN_EXPECT_FAILURE in block.classes: print_error( loc, "Running of example should have failed" ) check_error = True except S.CalledProcessError as e: - if 'c-run-expect-failure' in block.classes: + if constants.CLASS_C_RUN_EXPECT_FAILURE in block.classes: if verbose: print("Running of example expectedly failed") else: print_error(loc, "Running of example failed") check_error = True out = str(e.output.decode("utf-8")) + except FileNotFoundError as e: + print_error(loc, "Running of example failed: " + "no executable to run") + check_error = True + out = str(e) with open("run.log", u"w") as logfile: logfile.write(out) - code_check = checks.CodeCheck(status_ok=(not check_error), - logfile="run.log", - cmdline=str(cmdline)) + # Only a language the checker actually runs gets a RUN phase. + # Recording one for any other language claimed a successful run + # of a command that was never built, naming a log file that was + # never written. + if run_attempted: + code_check = checks.CodeCheck(status_ok=(not check_error), + logfile="run.log", + cmdline=str(cmdline)) - block_check.add_check("RUN", code_check) - - if check_error: - has_error = True - - if False: - check_error = False - - for source_file in block.source_files: - if block.language == "ada": - try: - out = run("gcc", "-c", "-gnatc", "-gnatyg0-s", - source_file) - except S.CalledProcessError as e: - if 'ada-expect-compile-error' in block.classes: - compile_error = True - else: - print_error(loc, "Failed to compile example") - check_error = True - out = str(e.output.decode("utf-8")) - - with open("compile.log", u"w+") as logfile: - logfile.write(out) - - elif block.language == "c": - try: - out = run("gcc", "-c", source_file) - except S.CalledProcessError as e: - if 'c-expect-compile-error' in block.classes: - compile_error = True - else: - print_error(loc, "Failed to compile example") - check_error = True - out = str(e.output.decode("utf-8")) - - with open("compile.log", u"w+") as logfile: - logfile.write(out) + block_check.add_check("RUN", code_check) if check_error: has_error = True @@ -413,20 +533,20 @@ def cleanup_project(language, project_filename, main_file): if block.language == "ada": - is_prove_error_class = any(c in ['ada-expect-prove-error', - 'ada-expect-compile-error', - 'ada-run-expect-failure'] + is_prove_error_class = any(c in [constants.CLASS_ADA_EXPECT_PROVE_ERROR, + constants.CLASS_ADA_EXPECT_COMPILE_ERROR, + constants.CLASS_ADA_RUN_EXPECT_FAILURE] for c in block.classes) extra_args = [] if 'prove_flow' in block.buttons \ - or 'ada-prove-flow' in block.classes: + or constants.CLASS_ADA_PROVE_FLOW in block.classes: extra_args = ["--mode=flow"] elif 'prove_flow_report_all' in block.buttons \ - or 'ada-prove-flow-report-all' in block.classes: + or constants.CLASS_ADA_PROVE_FLOW_REPORT_ALL in block.classes: extra_args = ["--mode=flow", "--report=all"] elif 'prove_report_all' in block.buttons \ - or 'ada-report-all' in block.classes: + or constants.CLASS_ADA_PROVE_REPORT_ALL in block.classes: extra_args = ["--report=all"] # Default switches for GNATprove 14 and above @@ -473,52 +593,69 @@ def cleanup_project(language, project_filename, main_file): has_error = True - if True: - check_error = False + check_error = False + + if len(block.buttons) == 0: + print_error(loc, "Expected at least 'no_button' indicator, got none!") + check_error = True + + if ((block.gnat_version[0] == 'selected' or + block.gnatprove_version[0] == 'selected' or + block.gprbuild_version[0] == 'selected') and + block.buttons != ['no']): + print_error(loc, "Only 'no_button' is allowed when selecting a specific toolchain!") + check_error = True - if len(block.buttons) == 0: - print_error(loc, "Expected at least 'no_button' indicator, got none!") + if constants.CLASS_ADA_EXPECT_COMPILE_ERROR in block.classes: + if (not (any(b in ['compile', 'run'] for b in block.buttons) or + any(c in [constants.CLASS_ADA_COMPILE, + constants.CLASS_ADA_RUN] + for c in block.classes))): + print_error(loc, "Expected compile or run button/class, got none!") + check_error = True + if not compile_error: + print_error(loc, "Expected compile error, got none!") check_error = True - if ((block.gnat_version[0] == 'selected' or - block.gnatprove_version[0] == 'selected' or - block.gprbuild_version[0] == 'selected') and - block.buttons != ['no']): - print_error(loc, "Only 'no_button' is allowed when selecting a specific toolchain!") + # The C spelling is checked on its own rather than beside the Ada one, + # because only this half of the pair was missing: a C block declaring an + # expected compile error that compiled cleanly was reported as a success. + # The compile step sets the same flag for either language, so the test + # for an expectation that went unmet is the same test. + if constants.CLASS_C_EXPECT_COMPILE_ERROR in block.classes: + if not compile_error: + print_error(loc, "Expected compile error, got none!") check_error = True - if 'ada-expect-compile-error' in block.classes: - if (not (any(b in ['compile', 'run'] for b in block.buttons) or - any(c in ['ada-compile', 'ada-run'] for c in block.classes))): - print_error(loc, "Expected compile or run button/class, got none!") - check_error = True - if not compile_error: - print_error(loc, "Expected compile error, got none!") - check_error = True + if constants.CLASS_ADA_EXPECT_PROVE_ERROR in block.classes: + if not block.prove_it: + print_error(loc, "Expected prove button, got none!") + check_error = True - if 'ada-expect-prove-error' in block.classes: - if not block.prove_it: - print_error(loc, "Expected prove button, got none!") - check_error = True + if block.prove_it: + if is_prove_error_class and not prove_error: + print_error(loc, "Expected prove error, got none!") + check_error = True - if block.prove_it: - if is_prove_error_class and not prove_error: - print_error(loc, "Expected prove error, got none!") - check_error = True + if (any (c in [constants.CLASS_ADA_RUN_EXPECT_FAILURE, + constants.CLASS_ADA_NORUN] + for c in block.classes) + and not ('run' in block.buttons or + constants.CLASS_ADA_RUN in block.classes)): + print_error(loc, "Expected run button, got none!") + check_error = True - if (any (c in ['ada-run-expect-failure','ada-norun'] for - c in block.classes) - and not ('run' in block.buttons or - 'ada-run' in block.classes)): - print_error(loc, "Expected run button, got none!") - check_error = True + # Already reported above, before the recorded result was consulted; this + # only carries it into the record the code block leaves behind. + if wrong_language_classes: + check_error = True - code_check = checks.CodeCheck(status_ok=(not check_error)) + code_check = checks.CodeCheck(status_ok=(not check_error)) - block_check.add_check("BUTTONS", code_check) + block_check.add_check("BUTTONS", code_check) - if check_error: - has_error = True + if check_error: + has_error = True if not has_error and verbose: fmt_utils.simple_success("SUCCESS") @@ -556,8 +693,12 @@ def check_code_block_json(json_file: str) -> bool: return has_error -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) +if __name__ == "__main__": # pragma: no cover + # prog is the name this command is installed under. Without it, + # argparse advertises the module path instead, which is not what a + # user types, and which is long enough to distort the usage line. + parser = argparse.ArgumentParser(prog='check-block', + description=__doc__) parser.add_argument('json_files', type=str, nargs="+", help="The JSON file for each code block") parser.add_argument('--verbose', '-v', action='store_true', diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_projects.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_projects.py index 6672b031f..2fc450545 100755 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_projects.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/check_projects.py @@ -10,6 +10,7 @@ from . import blocks from . import check_code_block +from . import constants from . import extract_projects from . import fmt_utils @@ -19,7 +20,8 @@ force_checks: bool = False -def get_blocks(json_files_regex_list: list[str]) -> dict[str, list[tuple[blocks.CodeBlock, str]]]: +def get_blocks(json_files_regex_list: list[str], + skipped: list[str] | None = None) -> dict[str, list[tuple[blocks.CodeBlock, str]]]: projects: dict[str, list[tuple[blocks.CodeBlock, str]]] = dict() for json_regex in json_files_regex_list: @@ -29,10 +31,14 @@ def get_blocks(json_files_regex_list: list[str]) -> dict[str, list[tuple[blocks. if b is None: print("ERROR: Could not load block info from {}".format(json_file_path)) + if skipped is not None: + skipped.append(json_file_path) continue if b.project is None: print("ERROR: Block has no project in {}".format(json_file_path)) + if skipped is not None: + skipped.append(json_file_path) continue if not b.project in projects: @@ -42,7 +48,8 @@ def get_blocks(json_files_regex_list: list[str]) -> dict[str, list[tuple[blocks. return projects -def get_projects(build_dir: str, projects_list_file: str | None = None) -> dict[str, list[tuple[blocks.CodeBlock, str]]]: +def get_projects(build_dir: str, projects_list_file: str | None = None, + skipped: list[str] | None = None) -> dict[str, list[tuple[blocks.CodeBlock, str]]]: json_files_regex_list: list[str] = list() os.chdir(build_dir) @@ -54,13 +61,13 @@ def get_projects(build_dir: str, projects_list_file: str | None = None) -> dict[ if extracted_projects: for prj in extracted_projects.projects: json_files_regex_list.append(extract_projects.get_project_dir(prj) + - "/**/block_info.json") + "/**/" + constants.BLOCK_INFO_FILENAME) else: print("WARNING: no projects found in file: " + projects_list_file) else: - json_files_regex_list.append("./**/block_info.json") + json_files_regex_list.append("./**/" + constants.BLOCK_INFO_FILENAME) - projects = get_blocks(json_files_regex_list) + projects = get_blocks(json_files_regex_list, skipped) return projects @@ -79,7 +86,16 @@ def check_projects(build_dir: str, projects_list_file: str | None = None) -> boo work_dir = os.getcwd() - projects = get_projects(build_dir, projects_list_file) + # Every skip above describes a block that was not checked -- one whose + # info file could not be read, and one that names no project. Reporting + # either and then exiting 0 would claim a clean run over an example + # nothing looked at. + skipped: list[str] = [] + + projects = get_projects(build_dir, projects_list_file, skipped) + + if skipped: + check_error = True for project in projects: @@ -104,10 +120,15 @@ def check_projects(build_dir: str, projects_list_file: str | None = None) -> boo return check_error -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover import argparse - parser = argparse.ArgumentParser(description=__doc__) + # prog is the name this command is installed under. Without it, + # argparse derives a name long enough that the usage line has to + # be broken after it, leaving every option on its own deeply + # indented line. + parser = argparse.ArgumentParser(prog='check-code', + description=__doc__) parser.add_argument('--build-dir', '-B', type=str, default=None, help='Dir in which to build code') parser.add_argument('--extracted_projects', type=str, default=None, diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/checks.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/checks.py index 1ee73edf7..1cad5c556 100644 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/checks.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/checks.py @@ -6,6 +6,8 @@ import json import time +from . import constants + class CodeCheck(object): def __init__(self, timestamp: float | None = None, @@ -26,7 +28,7 @@ class BlockCheck(object): def from_json_file(json_filename: str | None = None) -> BlockCheck | None: if json_filename is None: - json_filename = "block_checks.json" + json_filename = constants.BLOCK_CHECKS_FILENAME if os.path.isfile(json_filename): with open(json_filename, u'r') as f: @@ -52,7 +54,7 @@ def to_json_file(self, json_filename: str | None = None) -> None: block_checks = self.__dict__ if json_filename is None: - json_filename = "block_checks.json" + json_filename = constants.BLOCK_CHECKS_FILENAME with open(json_filename, u'w') as f: json.dump(block_checks, f, indent=4, default=lambda __o: __o.__dict__) diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/colors.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/colors.py index e88cdd207..d6a30b728 100644 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/colors.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/colors.py @@ -44,10 +44,15 @@ def disable_colors(cls) -> None: def no_colors() -> Iterator[None]: """ Context manager to disable colors for a given scope. + + The previous setting is restored when the scope ends, including when it + ends by raising an exception. """ old_val, Colors._enabled = Colors._enabled, False - yield - Colors._enabled = old_val + try: + yield + finally: + Colors._enabled = old_val def col(msg: str, color: str) -> str: diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/constants.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/constants.py new file mode 100644 index 000000000..6717e2ec0 --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/constants.py @@ -0,0 +1,99 @@ +"""Names the pipeline's modules have to agree on. + +The pipeline is three commands that talk to each other through files on +disk: the extraction step writes an artifact, and the checking step goes +looking for it by name. Nothing checks that the two names match -- a +mismatch produces no error, only a check that quietly finds nothing to do. +Keeping the names here means the commands in this package cannot disagree. + +Two limits are worth knowing before renaming anything here. + +The guarantee stops at the package boundary. These names are part of an +on-disk contract: whatever reads the artifacts this package writes carries +its own copy of the names, and a reader that misses a file may well treat it +as absent metadata rather than as an error. Renaming one is therefore not a +local change, and nothing here will say so. + +And the two project file names are not free even inside the package: the +templates below name the project units ``Main`` and ``Main_Spark``, which +the builder requires to match the file names. Renaming those two constants +alone produces a project whose unit name does not match its file, which the +builder reports; the unit names have to move with them. +""" + +# The per-block file the extraction step writes and the checking step reads. +BLOCK_INFO_FILENAME = "block_info.json" + +# The record of what was checked for a block, written after the checks run. +BLOCK_CHECKS_FILENAME = "block_checks.json" + +# The generated project file, and the configuration pragmas it refers to. +# The two are a pair: the project names the pragma file, so the name used +# when writing the file and the name written into the project have to be the +# same one. +PROJECT_FILENAME = "main.gpr" +PROJECT_PRAGMAS_FILENAME = "main.adc" + +# The SPARK variants of the same pair, generated instead of the above when a +# block is proved rather than merely built. +SPARK_PROJECT_FILENAME = "main_spark.gpr" +SPARK_PROJECT_PRAGMAS_FILENAME = "main_spark.adc" + + +# The ``:class:`` values a course author writes on a code block, which are +# what the checker reads to decide what to do with it. They arrive as plain +# strings from the RST source, so a misspelling here would not raise -- the +# comparison would simply never match and the check would be skipped in +# silence, on a block that looks checked. Naming them turns that typo into +# an AttributeError at the point of use. +# +# What a course author may actually write is fixed elsewhere -- +# ``CONTRIBUTING.md`` documents it, and the code-block directive rejects any +# class it does not recognize -- so read this list as the names the checker +# acts on, not as the reference for the RST source. +CLASS_ADA_NOCHECK = "ada-nocheck" +CLASS_C_NOCHECK = "c-nocheck" + +CLASS_ADA_SYNTAX_ONLY = "ada-syntax-only" +CLASS_NOSYNTAX_CHECK = "nosyntax-check" + +CLASS_ADA_COMPILE = "ada-compile" +CLASS_C_COMPILE = "c-compile" + +CLASS_ADA_RUN = "ada-run" +CLASS_ADA_NORUN = "ada-norun" +CLASS_C_RUN = "c-run" +CLASS_C_NORUN = "c-norun" +CLASS_ADA_RUN_EXPECT_FAILURE = "ada-run-expect-failure" +CLASS_C_RUN_EXPECT_FAILURE = "c-run-expect-failure" + +CLASS_ADA_EXPECT_COMPILE_ERROR = "ada-expect-compile-error" +CLASS_C_EXPECT_COMPILE_ERROR = "c-expect-compile-error" +CLASS_ADA_EXPECT_PROVE_ERROR = "ada-expect-prove-error" + +CLASS_ADA_PROVE = "ada-prove" +CLASS_ADA_PROVE_FLOW = "ada-prove-flow" +CLASS_ADA_PROVE_FLOW_REPORT_ALL = "ada-prove-flow-report-all" +CLASS_ADA_PROVE_REPORT_ALL = "ada-prove-report-all" + +# The run classes paired with the language each one names. A run class is +# honored only for a code block written in that language; one naming the +# other language is reported rather than quietly doing nothing, so that a +# mis-typed class cannot leave a code block unbuilt and still passing. +RUN_CLASS_LANGUAGES = { + CLASS_ADA_RUN: "ada", + CLASS_ADA_NORUN: "ada", + CLASS_ADA_RUN_EXPECT_FAILURE: "ada", + CLASS_C_RUN: "c", + CLASS_C_NORUN: "c", + CLASS_C_RUN_EXPECT_FAILURE: "c", +} + +# The classes that ask for a proof. Grouped here because the check that +# reads them treats them as one set rather than testing each in turn. +PROVE_CLASSES = [ + CLASS_ADA_PROVE, + CLASS_ADA_PROVE_FLOW, + CLASS_ADA_PROVE_FLOW_REPORT_ALL, + CLASS_ADA_PROVE_REPORT_ALL, +] diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/extract_projects.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/extract_projects.py index 8129b0066..50e38c8d8 100755 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/extract_projects.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/extract_projects.py @@ -1,21 +1,38 @@ #! /usr/bin/env python3 """ -This program will extract every Ada code block in an Ada source file -The default behavior is to: -- Split the block with ``gnatchop`` +Extract the code blocks of the ReST sources into a build directory, ready for +the checking commands to pick up. Only Ada and C code blocks are extracted, +and each one must name a project; a code block that names none ends the run. +A code block is split into individual source files -- with gnatchop for Ada, +or by a leading filename marker for C and for an Ada code block that asks to +be chopped manually -- and those files are written to a directory of their +own, named after a hash of the code block's text, under a directory named +after the project. Beside them goes a record of what the code block declares, +as block_info.json, which is what a checking command reads to decide what to +run. A code block that asks to be compiled or run also gets a project file +written for it, and one that asks to be proved gets a second project file in +SPARK mode; only an Ada code block can ask to be proved. The list of the +projects extracted can also be collected into a JSON file, so that a later +check can be limited to exactly those projects. """ +# The text above is what argparse prints as this command's help +# description. It is deliberately free of ReST markup and of any layout +# worth preserving: the default help formatter re-wraps a description into a +# single filled paragraph, so a list would arrive as a run-on sentence and +# inline literals would arrive with their backquotes intact. + from __future__ import annotations import os import shutil -import re import json from .chop import manual_chop, real_gnatchop from . import blocks +from . import constants from . import fmt_utils from . import toolchain_setup @@ -79,11 +96,11 @@ def get_project_dir(project: str) -> str: package Builder is for Default_Switches ("Ada") use ("-g"); - for Global_Configuration_Pragmas use "main.adc"; + for Global_Configuration_Pragmas use "{}"; end Builder; end Main; -""" +""".format(constants.PROJECT_PRAGMAS_FILENAME) MAIN_SPARK_GPR=""" project Main_Spark is @@ -97,22 +114,50 @@ def get_project_dir(project: str) -> str: package Builder is for Default_Switches ("Ada") use ("-g"); - for Global_Configuration_Pragmas use "main_spark.adc"; + for Global_Configuration_Pragmas use "{}"; end Builder; end Main_Spark; -""" +""".format(constants.SPARK_PROJECT_PRAGMAS_FILENAME) def write_project_file(main_file: str | None, compiler_switches: list[str], spark_mode: bool) -> str: - gpr_filename = "main.gpr" - adc_filename = "main.adc" + """Writes the project file for a code block, and its pragmas file + + Both files are written into the current working directory, which the + caller has already changed to the block's own directory. + + Args: + main_file (str, optional): The source file holding the main + procedure, or None to generate a project that names no main. + compiler_switches (list[str]): Switches added to the ``Compiler`` + package of the generated project. + spark_mode (bool): Selects the SPARK variants of the project file + and of the configuration pragmas file. + + Returns: + str: The name of the project file that was written. + + Note: + The project gets a ``for Main use`` attribute only when a main file + is passed, and the caller passes one only for a code block that is + meant to be run. That restriction is deliberate rather than + incidental: a code block that is only compiled may legitimately have + no main procedure at all -- a package spec and body on their own are + a complete example -- and naming a main for such a block would send + the builder looking for something to link that the block does not + contain. The extraction tests pin both halves of the distinction: + the attribute is present for a runnable code block and absent + otherwise. + """ + gpr_filename = constants.PROJECT_FILENAME + adc_filename = constants.PROJECT_PRAGMAS_FILENAME main_gpr = MAIN_GPR if spark_mode: - gpr_filename = "main_spark.gpr" - adc_filename = "main_spark.adc" + gpr_filename = constants.SPARK_PROJECT_FILENAME + adc_filename = constants.SPARK_PROJECT_PRAGMAS_FILENAME main_gpr = MAIN_SPARK_GPR adc_content = COMMON_ADC @@ -171,6 +216,48 @@ def add(self, project: str) -> None: def analyze_file(rst_file: str, extracted_projects_list_file: str | None = None) -> bool: + """Extracts the code blocks of a single ReST file + + Each active code block is written to its own project directory below the + current working directory, together with the ``block_info.json`` file that + describes it for the checking stage. + + Args: + rst_file (str): The ReST file to extract the code blocks from + extracted_projects_list_file (str, optional): JSON file the names of + the extracted projects are added to. Defaults to None. + + Returns: + bool: The error flag for this file. The extraction command turns a + true value into a non-zero exit status. + + Note: + That flag is effectively the constant ``False`` today, so the exit + status derived from it never becomes non-zero: + + * The single assignment that would set it sits in the nested + ``expand_source_files()``. Without a ``nonlocal`` declaration it + binds a fresh local there rather than the flag defined in this + function, so the chopping failure it records dies with the nested + scope. + * The remaining per-block errors printed here never touch the flag at + all: a block whose button and language do not go together, and a + block with no button indicator. + * The one condition this function treats as fatal for the whole run, + a code block with no project name, calls ``exit(1)`` directly and so + bypasses the flag too. + + A caller that inspects only the returned value therefore always + concludes the file was extracted cleanly. In the extraction command + this leaves the failure branch unreachable; that branch also announces + ``TEST ERROR`` through ``fmt_utils.simple_success()``, the formatter + for success messages. + + Not every ``ERROR`` line printed here marks a failure either. Removing + a per-block directory left over from an earlier run whose info JSON + file has gone missing is reported the same way, and that is a recovery + on the success path. + """ analysis_error = False @@ -190,9 +277,6 @@ def analyze_file(rst_file: str, extracted_projects_list_file: str | None = None) if block.line_start < code_block_at < block.line_end: block.active = True - def remove_string(some_text, rem): - return re.sub(".*" + rem + ".*\n?","", some_text) - projects = dict() extr_prjs = None @@ -248,7 +332,7 @@ def init_project_dir(project): print("Number of code blocks: {}".format(len(projects[project]))) for i, block in projects[project]: - if isinstance(block, blocks.ConfigBlock): + if isinstance(block, blocks.ConfigBlock): # pragma: no cover current_config.update(block) toolchain_setup.reset_toolchain() continue @@ -264,6 +348,9 @@ def init_project_dir(project): def print_error(*error_args): fmt_utils.error(*error_args) + def print_warning(*warning_args): + fmt_utils.warning(*warning_args) + def chdir_project(): # combining path to work directory (absolute path) # and current project directory @@ -309,11 +396,34 @@ def prepare_project_block_dir(latest_project_dir): copytree_latest = True if os.path.exists(project_block_dir): - json_filename = "block_info.json" + json_filename = constants.BLOCK_INFO_FILENAME json_file = project_block_dir + "/" + json_filename - if os.path.exists(json_file): + # isfile, not exists, to match the guard the reader uses: + # anything else here would trip the warning below over a + # file the reader never attempted and could not report on. + if os.path.isfile(json_file): copytree_latest = False ref_block = blocks.CodeBlock.from_json_file(json_file) + if ref_block is None: + # The file is there, so it is present but + # unreadable. Extraction rewrites the record, so + # nothing is dropped and the run still succeeds + # -- but something damaged this file earlier, and + # a kept build directory carries it between runs. + # Say so where it cannot be mistaken for the + # fatal case. + # + # The message does not promise the block is + # checked: a block carrying a no-check class is + # extracted and then deliberately skipped, so + # that would be false for it. + print_warning( + loc, + "Block info file could not be read and is " + "being rebuilt: {}. The example is still " + "extracted and the run was not cut short, " + "but something damaged this file " + "earlier".format(json_file)) else: print_error(loc, "Directory exists, but no JSON info file: removing it...\n") shutil.rmtree(project_block_dir, @@ -397,10 +507,14 @@ def get_main_filename(block): return analysis_error -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover import argparse - parser = argparse.ArgumentParser(description=__doc__) + # prog is the name this command is installed under. Without it, + # argparse advertises the module path instead, which is not what a + # user types, and which is long enough to distort the usage line. + parser = argparse.ArgumentParser(prog='extract-code', + description=__doc__) parser.add_argument('rst_files', type=str, nargs="+", help="The rst file from which to extract doc") parser.add_argument('--build-dir', '-B', type=str, default=None, diff --git a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/fmt_utils.py b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/fmt_utils.py index f5caa5f93..0f1dc86ed 100644 --- a/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/fmt_utils.py +++ b/frontend/python/rst_code_example_pipeline/src/rst_code_example_pipeline/fmt_utils.py @@ -8,6 +8,9 @@ def header(strn: str) -> str: def error(loc: str, strn: str) -> None: print("{} {}: {}".format(C.col("ERROR", C.Colors.RED), loc, strn)) +def warning(loc: str, strn: str) -> None: + print("{} {}: {}".format(C.col("WARNING", C.Colors.YELLOW), loc, strn)) + def simple_error(msg: str) -> None: print(C.col(msg, C.Colors.RED)) diff --git a/frontend/python/rst_code_example_pipeline/tests/conftest.py b/frontend/python/rst_code_example_pipeline/tests/conftest.py new file mode 100644 index 000000000..1f6b95e2e --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/conftest.py @@ -0,0 +1,100 @@ +""" +Fixtures shared by the whole rst_code_example_pipeline test suite. + +The package keeps its settings in module-level globals, and its entry points +change the process working directory without changing it back. Whatever a +test does to either one is therefore still in place when the next test runs, +and containing that is not specific to any one test module -- so it is done +here once instead of being re-implemented, differently, in each of them. + +- ``restore_cwd`` puts the working directory back after every test. Several + entry points chdir and never chdir back: check_block() moves into the block + directory it is checking, and get_projects() moves into the build directory + it is scanning. A test that reaches either one would otherwise leave the + whole session pointing at a temporary directory that is deleted soon + afterwards, and every later test that uses a relative path would fail for + reasons that have nothing to do with what it is testing. +- ``reset_pipeline_globals`` puts the settings globals of the three entry-point + modules back to the values their modules declare, around every test. Those + globals are what the command-line switches assign to, so a test that sets one + is changing the setting for the rest of the session. The values are read + back from the modules rather than written down here, so that a default which + changes in the source is followed instead of being quietly overridden with a + stale copy of it for the whole suite. +- ``restore_color_state`` puts ``Colors._enabled`` back after every test, so a + test that turns colors on or off cannot change what a later test finds in its + captured output. +- ``work_dir`` is opt-in rather than autouse: it enters a fresh temporary + directory for the duration of the test and hands it back, for the many tests + whose subject reads or writes relative to the working directory. +""" +import copy +import os + +import pytest + +from rst_code_example_pipeline import check_code_block +from rst_code_example_pipeline import check_projects +from rst_code_example_pipeline import extract_projects +from rst_code_example_pipeline.colors import Colors + + +@pytest.fixture(autouse=True) +def restore_cwd(): + """Restore the working directory after each test.""" + original = os.getcwd() + yield + os.chdir(original) + + +# The settings globals of each entry-point module, captured as those modules +# declare them. A conftest is imported before any test module, so nothing has +# had the chance to assign to one of these yet and what is captured here is the +# declared value. +_DECLARED_SETTINGS = { + module: {name: getattr(module, name) for name in names} + for module, names in ( + (check_code_block, + ("verbose", "all_diagnostics", "max_columns", "force_checks")), + (check_projects, + ("verbose", "all_diagnostics", "max_columns", "force_checks")), + (extract_projects, + ("verbose", "code_block_at", "current_config")), + ) +} + + +def _reset_pipeline_globals() -> None: + """Assign the settings globals the values their own modules declare. + + Each value is handed out as a copy. One of them is a configuration block + the package updates in place, so assigning the captured object itself would + give every test the same one to mutate and lose the declared value with the + first test that did. + """ + for module, declared in _DECLARED_SETTINGS.items(): + for name, value in declared.items(): + setattr(module, name, copy.deepcopy(value)) + + +@pytest.fixture(autouse=True) +def reset_pipeline_globals(): + """Reset the entry-point modules' settings globals around each test.""" + _reset_pipeline_globals() + yield + _reset_pipeline_globals() + + +@pytest.fixture(autouse=True) +def restore_color_state(): + """Restore Colors._enabled after each test.""" + original = Colors._enabled + yield + Colors._enabled = original + + +@pytest.fixture() +def work_dir(tmp_path, monkeypatch): + """Change to a fresh temporary directory and restore cwd on teardown.""" + monkeypatch.chdir(tmp_path) + return tmp_path diff --git a/frontend/python/rst_code_example_pipeline/tests/test_blocks.py b/frontend/python/rst_code_example_pipeline/tests/test_blocks.py new file mode 100644 index 000000000..1e38f1255 --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_blocks.py @@ -0,0 +1,1125 @@ +""" +Unit tests for rst_code_example_pipeline.blocks. + +Covers: +- Block.get_blocks_from_rst(): RST parser (all attributes, derived fields) +- CodeBlock constructor derived fields (no_check, syntax_only, run_it, compile_it, + prove_it), including the C run classes, which ask for a run only on a C block + and are suppressed by c-norun +- every run class is honored only for a block written in the language it names: + the Ada run classes ask for no run, and therefore no build, on a C block, and + neither norun class takes a run away from a block of the other language -- + with the controls that say so, since a derivation refusing every run, or + suppressing nothing anywhere, satisfies those on its own. The run button, the + syntax-only class and the two no-check classes are deliberately not paired + with any language, and are pinned as such +- text_hash / text_hash_short: deterministic, distinct per text, usable as a + directory name +- CodeBlock.to_json_file() + from_json_file() round-trip +- CodeBlock.from_json_file() on a record that is present but cannot be turned + into a block: read back as no block, and reported with the file name and the + reason, rather than left as an exception for the caller to trip over +- ConfigBlock.__init__ and update(), for the strings a code-config directive + produces and for the real booleans a caller may hand over instead +- Adversarial: empty RST, missing json file, exit(1) path + +NOTE: get_blocks_from_rst() calls toolchain_info.get_toolchain_default_version() +at parse time; requires the Ada toolchain .ini +is present and toolchain_info initializes correctly. + +NOTE: the version strings written inside the RST fixtures below, and the values +the parser is expected to produce from them, are deliberately spelled out. They +stand for what a course author types in a real .rst file, and the parser never +validates them against the configured toolchains -- it round-trips the string +verbatim. Driving both the input and the expected output from the toolchain +configuration would make the pair self-referential and hide a parsing error. +Version strings passed straight to the CodeBlock constructor are a different +matter: those are copies of configuration data and are read back from it. +""" +import json +import re +import subprocess +import sys +import textwrap + +import pytest + +from rst_code_example_pipeline.blocks import Block, CodeBlock, ConfigBlock +import rst_code_example_pipeline.toolchain_info as info + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +RST_FILE = "test.rst" + + +def minimal_rst(body: str) -> str: + """Wrap body in a minimal RST file so there is a trailing explanatory + paragraph to close the code block.""" + return body + "\n\nExplanatory paragraph.\n" + + +# --------------------------------------------------------------------------- +# T-blocks-01: minimal Ada block +# --------------------------------------------------------------------------- + +class TestMinimalAdaBlock: + RST = minimal_rst("""\ +.. code:: ada + + with Ada.Text_IO; use Ada.Text_IO; + procedure Main is + begin + Put_Line ("Hello"); + end Main; +""") + + def test_returns_one_block(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert len(blocks) == 1 + + def test_rst_file_stored(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].rst_file == RST_FILE + + def test_language_is_ada(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].language == "ada" + + def test_project_is_none(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].project is None + + def test_main_file_is_none(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].main_file is None + + def test_manual_chop_false(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].manual_chop is False + + def test_default_compiler_switches_includes_gnata(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert "-gnata" in blocks[0].compiler_switches + + def test_gnat_version_default(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].gnat_version[0] == "default" + + def test_gnatprove_version_default(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].gnatprove_version[0] == "default" + + def test_gprbuild_version_default(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].gprbuild_version[0] == "default" + + def test_line_span_and_text_are_exact(self): + """The parser must report the block's span in the RST file and hand + back its body with the directive indentation removed. + + Counting lines from zero, ``line_start`` is the first line after the + ``.. code::`` directive -- which makes it equal to the directive's own + 1-based line number -- and ``line_end`` is the line that closed the + block. The body is everything between the two, so it keeps the blank + lines separating the block from what follows it. + + The expected values are spelled out rather than derived from the + parser: every consumer of a block reports diagnostics against these + line numbers, so an off-by-one here misdirects a course author to the + wrong line. Recomputing them the way the parser does would make the + test agree with whatever the parser produced. + """ + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].line_start == 1 + assert blocks[0].line_end == 9 + assert blocks[0].text == ( + 'with Ada.Text_IO; use Ada.Text_IO;\n' + 'procedure Main is\n' + 'begin\n' + ' Put_Line ("Hello");\n' + 'end Main;\n' + '\n' + ) + + def test_active_defaults_to_true(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].active is True + + +# --------------------------------------------------------------------------- +# T-blocks-02: project and main_file attributes +# --------------------------------------------------------------------------- + +class TestProjectAndMainFile: + RST = minimal_rst("""\ +.. code:: ada project=MyProject main=main.adb + + procedure Main is + begin + null; + end Main; +""") + + def test_project(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].project == "MyProject" + + def test_main_file(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].main_file == "main.adb" + + +# --------------------------------------------------------------------------- +# T-blocks-03: compiler switches +# --------------------------------------------------------------------------- + +class TestCompilerSwitches: + RST = minimal_rst("""\ +.. code:: ada switches=Compiler(-gnatwa,-gnatwe) + + procedure P is null; +""") + + def test_custom_switches_present(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + switches = blocks[0].compiler_switches + assert "-gnatwa" in switches + assert "-gnatwe" in switches + + def test_default_gnata_also_present(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert "-gnata" in blocks[0].compiler_switches + + +# --------------------------------------------------------------------------- +# T-blocks-04: gnat version selected +# --------------------------------------------------------------------------- + +class TestGnatVersionSelected: + RST = minimal_rst("""\ +.. code:: ada gnat=12.2.0-1 + + procedure P is null; +""") + + def test_gnat_version_is_selected(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].gnat_version == ["selected", "12.2.0-1"] + + +# --------------------------------------------------------------------------- +# T-blocks-05: language=c sets manual_chop=True +# --------------------------------------------------------------------------- + +class TestLanguageC: + RST = minimal_rst("""\ +.. code:: c + + #include + int main() { return 0; } +""") + + def test_manual_chop_true_for_c(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].manual_chop is True + + def test_language_is_c(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].language == "c" + + +# --------------------------------------------------------------------------- +# T-blocks-06: explicit manual_chop keyword +# --------------------------------------------------------------------------- + +class TestManualChopKeyword: + RST = minimal_rst("""\ +.. code:: ada manual_chop + + procedure P is null; +""") + + def test_manual_chop_true(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].manual_chop is True + + +# --------------------------------------------------------------------------- +# T-blocks-07: buttons +# --------------------------------------------------------------------------- + +class TestButtons: + def test_run_button(self): + rst = minimal_rst("""\ +.. code:: ada run_button + + procedure P is null; +""") + blocks = Block.get_blocks_from_rst(RST_FILE, rst) + assert isinstance(blocks[0], CodeBlock) + assert "run" in blocks[0].buttons + + def test_compile_button(self): + rst = minimal_rst("""\ +.. code:: ada compile_button + + procedure P is null; +""") + blocks = Block.get_blocks_from_rst(RST_FILE, rst) + assert isinstance(blocks[0], CodeBlock) + assert "compile" in blocks[0].buttons + + +# --------------------------------------------------------------------------- +# T-blocks-08: :code-config: line produces ConfigBlock +# --------------------------------------------------------------------------- + +class TestCodeConfig: + RST = """\ +:code-config:`run_button=False;prove_button=True;accumulate_code=False` + +Some paragraph. +""" + + def test_config_block_in_list(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + config_blocks = [b for b in blocks if isinstance(b, ConfigBlock)] + assert len(config_blocks) == 1 + + def test_config_attributes(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + cb = [b for b in blocks if isinstance(b, ConfigBlock)][0] + # run_button/prove_button/accumulate_code are set dynamically via + # setattr() in ConfigBlock.__init__, so they are looked up with + # getattr() rather than direct attribute access. + assert getattr(cb, "run_button") is False + assert getattr(cb, "prove_button") is True + assert getattr(cb, "accumulate_code") is False + + +# --------------------------------------------------------------------------- +# T-blocks-09: two consecutive code blocks +# --------------------------------------------------------------------------- + +class TestTwoConsecutiveBlocks: + RST = """\ +.. code:: ada + + procedure A is null; + +Some text. + +.. code:: ada + + procedure B is null; + +More text. +""" + + def test_two_code_blocks(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + code_blocks = [b for b in blocks if isinstance(b, CodeBlock)] + assert len(code_blocks) == 2 + + def test_line_spans_are_exact_and_ordered(self): + """Each block must carry its own span, in file order and without + overlapping the other one.""" + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + code_blocks = [b for b in blocks if isinstance(b, CodeBlock)] + assert [(b.line_start, b.line_end) for b in code_blocks] == [(1, 4), (7, 10)] + assert [b.text for b in code_blocks] == [ + "procedure A is null;\n", + "procedure B is null;\n", + ] + + +# --------------------------------------------------------------------------- +# T-blocks-10: block at end of file +# --------------------------------------------------------------------------- + +class TestBlockAtEndOfFile: + RST_WITH_CONTENT = """\ +.. code:: ada + + procedure P is null; +""" + # Block with content but no trailing explanatory paragraph. + # process_block() can still extract the block when called with "END" at + # indent=0, so no exit(1) — just a WARNING printed. + + RST_EMPTY_BODY = ".. code:: ada\n" + # Block with NO content at all — cb_indent stays -1, so process_block() + # cannot set the indent and the block is not created. exit(1) is called. + + def test_block_with_content_no_trailing_paragraph_succeeds(self): + """A block at end-of-file that has content produces a WARNING but + is successfully parsed (no SystemExit). + + With no explanatory paragraph to close the block, the end of the file + closes it instead, so the span ends one line past the last line of the + file -- pinned because this path computes it differently from the + ordinary one, and because nothing follows the body here the text + carries no trailing blank line. + """ + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST_WITH_CONTENT) + assert len(blocks) == 1 + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].line_start == 1 + assert blocks[0].line_end == 3 + assert blocks[0].text == "procedure P is null;" + + def test_empty_block_body_raises_system_exit(self): + """A code-block directive with an empty body (no content lines at all) + cannot be processed and triggers exit(1).""" + with pytest.raises(SystemExit): + Block.get_blocks_from_rst(RST_FILE, self.RST_EMPTY_BODY) + + +# --------------------------------------------------------------------------- +# T-blocks-11: empty RST returns empty list +# --------------------------------------------------------------------------- + +class TestEmptyRst: + def test_empty_string(self): + blocks = Block.get_blocks_from_rst(RST_FILE, "") + assert blocks == [] + + def test_only_text_no_code_blocks(self): + blocks = Block.get_blocks_from_rst(RST_FILE, "Just some text.\n\nNo code here.\n") + assert blocks == [] + + +# --------------------------------------------------------------------------- +# T-blocks-12: CodeBlock derived fields from classes +# --------------------------------------------------------------------------- + +class TestCodeBlockDerivedFields: + def _make_block(self, classes, buttons=None, language="ada", + text="procedure P is null;"): + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + return CodeBlock( + rst_file="test.rst", + line_start=0, + line_end=5, + text=text, + language=language, + project=None, + main_file=None, + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=["-gnata"], + classes=classes, + manual_chop=False, + buttons=buttons or [], + ) + + def test_no_check_from_ada_nocheck_class(self): + b = self._make_block(["ada-nocheck"]) + assert b.no_check is True + + def test_no_check_from_c_nocheck_class(self): + b = self._make_block(["c-nocheck"], language="c") + assert b.no_check is True + + def test_no_check_false_default(self): + b = self._make_block([]) + assert b.no_check is False + + def test_syntax_only_from_class(self): + b = self._make_block(["ada-syntax-only"]) + assert b.syntax_only is True + + def test_syntax_only_false_default(self): + b = self._make_block([]) + assert b.syntax_only is False + + def test_run_it_from_ada_run_class(self): + b = self._make_block(["ada-run"]) + assert b.run_it is True + + def test_run_it_from_run_button(self): + b = self._make_block([], buttons=["run"]) + assert b.run_it is True + + def test_run_it_false_when_ada_norun(self): + # ada-norun overrides even when "run" is in buttons + b = self._make_block(["ada-norun"], buttons=["run"]) + assert b.run_it is False + + # The C run classes, which a course author may write and CONTRIBUTING.md + # documents. They are asserted one class at a time and with no button + # present, because a button would make every one of these pass on its own + # and say nothing about the class. Their Ada counterparts are covered + # above; what is new here is that the C spellings are read at all, and + # that they are read only on a C block. + + def test_run_it_from_c_run_class_on_a_c_block(self): + """c-run alone must ask for a run, the way ada-run does.""" + b = self._make_block(["c-run"], language="c") + assert b.run_it is True + + def test_run_it_from_c_run_expect_failure_class_on_a_c_block(self): + """c-run-expect-failure alone must ask for a run. + + Nothing can expect a run to fail without a run happening, so a class + that declares the expectation and does not cause the run leaves the + handling of that expectation unreachable. + """ + b = self._make_block(["c-run-expect-failure"], language="c") + assert b.run_it is True + + def test_run_it_false_for_a_c_block_declaring_nothing(self): + """A C block that asks for nothing must not be run. + + The control for the two above: without it they would pass equally + well against a derivation that ran every C block. + """ + b = self._make_block([], language="c") + assert b.run_it is False + + def test_run_it_false_when_c_norun_suppresses_a_run_button(self): + """c-norun must suppress a run the button asked for, as ada-norun + does.""" + b = self._make_block(["c-norun"], buttons=["run"], language="c") + assert b.run_it is False + + def test_run_it_false_when_c_norun_suppresses_the_c_run_class(self): + """Asking for a run and suppressing it in the same breath must + suppress: the two C classes are not read independently of each + other.""" + b = self._make_block(["c-run", "c-norun"], language="c") + assert b.run_it is False + + def test_run_it_false_for_c_run_class_on_an_ada_block(self): + """A C run class on an Ada block must not cause a run. + + The class is paired with the language the way the compile classes + already are, so writing the wrong language's spelling asks for + nothing rather than for a run of a block it does not describe. + """ + b = self._make_block(["c-run"], language="ada") + assert b.run_it is False + + def test_run_it_false_for_c_run_expect_failure_class_on_an_ada_block(self): + """Same pairing for the expect-failure spelling.""" + b = self._make_block(["c-run-expect-failure"], language="ada") + assert b.run_it is False + + # The Ada run classes read on a block of the other language, and the two + # norun classes read on a block they do not describe. The C positives + # above were already paired with their language; these are the remaining + # four spellings, so that every class naming a language is honored only + # for a block written in it. + # + # The compile is asserted beside the run wherever the run is taken away, + # because it is the consequence that matters: compile_it is derived as + # "run_it or ...", so a class that stops asking for a run also stops the + # block from being built, and a block that is never built is checked by + # nothing at all. + + def test_ada_run_class_on_a_c_block_asks_for_no_run_and_no_compile(self): + """ada-run on a C block must ask for nothing. + + The class names Ada, so it does not describe this block. Before the + pairing it asked for a run, and the build dispatches on the block's + own language, so the block really was built with gcc and run -- a + visible mistake rather than a silent one. + """ + b = self._make_block(["ada-run"], language="c") + assert b.run_it is False + assert b.compile_it is False + + def test_ada_run_expect_failure_class_on_a_c_block_asks_for_nothing(self): + """Same pairing for the expect-failure spelling. + + Written separately from the plain spelling rather than left to it: + the two class names are read as one set, so a derivation that stopped + pairing this one would still satisfy the test above. + """ + b = self._make_block(["ada-run-expect-failure"], language="c") + assert b.run_it is False + assert b.compile_it is False + + @pytest.mark.parametrize("code_class", + ["ada-run", "ada-run-expect-failure"]) + def test_the_ada_run_classes_still_ask_for_a_run_on_an_ada_block( + self, code_class): + """The control for the two tests above. + + Without it they are equally well satisfied by a derivation that + refused every run, which would take the Ada classes away from the + blocks they do describe. + """ + b = self._make_block([code_class], language="ada") + assert b.run_it is True + + def test_a_c_compile_class_on_an_ada_block_asks_for_no_compile(self): + """The control for "no compile" above. + + The compile classes have been paired with the block's language all + along, so this is the shape the run classes now follow. It says that + a compile_it of False is attributable to the class naming the other + language, rather than to some other route through the derivation that + would leave every block of this shape unbuilt. + """ + b = self._make_block(["c-compile"], language="ada") + assert b.compile_it is False + + def test_ada_norun_on_a_c_block_does_not_suppress_a_run_button(self): + """ada-norun must not take a run away from a C block. + + Suppressing a run is the direction where the old, unpaired reading + was itself the silent skip: a stray Ada norun on a C block took away + a run the author had asked for, and nothing said so. + """ + b = self._make_block(["ada-norun"], buttons=["run"], language="c") + assert b.run_it is True + + def test_ada_norun_on_a_c_block_does_not_suppress_the_c_run_class(self): + """The same, where the run was asked for by a class rather than by a + button -- the two are separate terms of the derivation.""" + b = self._make_block(["ada-norun", "c-run"], language="c") + assert b.run_it is True + + def test_c_norun_on_an_ada_block_does_not_suppress_a_run_button(self): + """The mirror of the ada-norun case, on an Ada block.""" + b = self._make_block(["c-norun"], buttons=["run"], language="ada") + assert b.run_it is True + + def test_c_norun_on_an_ada_block_does_not_suppress_ada_run(self): + """c-norun must leave ada-run alone, and the block must still be + built. + + This is the combination in which the unpaired reading did the most + damage: the run was canceled, so the compile went with it, and an + Ada example nobody built was recorded as having passed. + """ + b = self._make_block(["ada-run", "c-norun"], language="ada") + assert b.run_it is True + assert b.compile_it is True + + def test_c_norun_on_an_ada_block_does_not_suppress_the_expect_failure_class( + self): + """The same for the expect-failure spelling of the Ada run class.""" + b = self._make_block(["ada-run-expect-failure", "c-norun"], + language="ada") + assert b.run_it is True + + def test_the_norun_classes_still_suppress_on_their_own_language(self): + """The control for the four tests above. + + Asserted as one test over both spellings so that a pairing widened + until it never suppresses anything reddens something that names the + property, rather than only the C case or only the Ada one. + """ + ada = self._make_block(["ada-norun"], buttons=["run"], language="ada") + c = self._make_block(["c-norun"], buttons=["run"], language="c") + assert (ada.run_it, c.run_it) == (False, False) + + def test_a_run_button_asks_for_a_run_whatever_the_language_is(self): + """A run button is not paired with any language, deliberately. + + Only the classes name a language; the button says "run this" about + whatever the block happens to be written in. Pinned here so that a + later completion of the pairing, applied to the button as well, + cannot silently stop running every block of a language the classes do + not spell. + """ + b = self._make_block([], buttons=["run"], language="cpp") + assert b.run_it is True + + def test_ada_syntax_only_on_a_c_block_is_still_syntax_only(self): + """The syntax-only class is not paired with a language either. + + It is the one class/language mismatch the material really carries: a + C block declaring ada-syntax-only, which stops at the syntax check + and is meant to. Pinned so that a pairing widened to this class is + caught here rather than by a content build. + """ + b = self._make_block(["ada-syntax-only"], language="c") + assert b.syntax_only is True + + def test_the_nocheck_classes_are_not_paired_with_a_language(self): + """Either spelling of the no-check class suppresses the check on + either language. + + Also deliberate, and asserted over both spellings at once for the + same reason the norun control is. The two names are documented as + the Ada one and the C one, and that difference is recorded rather + than acted on -- so a pairing applied here would quietly take the + opposite decision. + """ + ada_on_c = self._make_block(["ada-nocheck"], language="c") + c_on_ada = self._make_block(["c-nocheck"], language="ada") + assert (ada_on_c.no_check, c_on_ada.no_check) == (True, True) + + def test_compile_it_true_when_a_c_block_is_run_by_class(self): + """A run implies a compile for the C classes too, so a C block asking + to be run by class alone has something to run.""" + b = self._make_block(["c-run"], language="c") + assert b.compile_it is True + + def test_compile_it_true_when_run_it_true(self): + b = self._make_block(["ada-run"]) + assert b.compile_it is True + + def test_compile_it_from_ada_compile_class(self): + b = self._make_block(["ada-compile"]) + assert b.compile_it is True + + def test_compile_it_false_default(self): + b = self._make_block([]) + assert b.compile_it is False + + def test_prove_it_from_ada_prove_class(self): + b = self._make_block(["ada-prove"]) + assert b.prove_it is True + + def test_prove_it_from_prove_button(self): + b = self._make_block([], buttons=["prove"]) + assert b.prove_it is True + + def test_prove_it_false_default(self): + b = self._make_block([]) + assert b.prove_it is False + + # The two hashes are tested for the properties the rest of the package + # relies on, not against a fixed digest: the short hash names a block's + # project directory and the long one keys its check cache, so nothing + # outside this package requires any particular algorithm, and a pinned + # digest would freeze one for no benefit. + + # Hash the given text in a fresh interpreter, in a block whose every other + # field differs from the one the test builds in process. Two things have + # to be true at once and neither alone is enough: the hash must survive a + # process boundary -- one that folds in a value drawn per process is + # perfectly stable within a single run, and still moves the project + # directory and orphans the cached check result on the next one -- and it + # must be a function of the block text alone, or moving a block to another + # file, or editing the line above it, has the same effect. + _HASH_PROBE = textwrap.dedent( + """ + import json, sys + from rst_code_example_pipeline.blocks import CodeBlock + + block = CodeBlock( + rst_file="other.rst", + line_start=42, + line_end=99, + text=sys.argv[1], + language="c", + project="OtherProject", + main_file="other.c", + gnat_version=["selected", "1.2.3-4"], + gnatprove_version=["selected", "1.2.3-4"], + gprbuild_version=["selected", "1.2.3-4"], + compiler_switches=["-gnatwa"], + classes=["c-nocheck"], + manual_chop=True, + buttons=["run"], + ) + print(json.dumps([block.text_hash, block.text_hash_short])) + """ + ) + + def test_text_hashes_are_deterministic_across_runs(self): + """The same block text must hash the same way on every run and in every + block that carries it, or a block's project directory moves and its + cached check result is never found again.""" + b = self._make_block([]) + output = subprocess.check_output( + [sys.executable, "-c", self._HASH_PROBE, b.text], text=True) + fresh_hash, fresh_hash_short = json.loads(output) + assert fresh_hash == b.text_hash + assert fresh_hash_short == b.text_hash_short + + def test_text_hashes_distinguish_different_text(self): + """Two blocks with different text must hash differently, or one + block's extracted project overwrites the other's and one of the two + is silently never checked.""" + b1 = self._make_block([], text="procedure P is null;") + b2 = self._make_block([], text="procedure Q is null;") + assert b1.text_hash != b2.text_hash + assert b1.text_hash_short != b2.text_hash_short + + def test_text_hashes_are_usable_as_directory_names(self): + """The short hash is used verbatim as a directory name, so both + hashes must be non-empty lowercase hexadecimal with nothing in them + that a path would have to escape.""" + b = self._make_block([]) + assert re.fullmatch(r"[0-9a-f]+", b.text_hash) + assert re.fullmatch(r"[0-9a-f]+", b.text_hash_short) + + +# --------------------------------------------------------------------------- +# T-blocks-13: CodeBlock JSON round-trip +# --------------------------------------------------------------------------- + +class TestCodeBlockJsonRoundTrip: + def _make_block(self): + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + return CodeBlock( + rst_file="foo.rst", + line_start=1, + line_end=10, + text="procedure P is null;", + language="ada", + project="MyProj", + main_file="main.adb", + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=["-gnata"], + classes=[], + manual_chop=False, + buttons=[], + ) + + def test_round_trip_basic_fields(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + b = self._make_block() + b.to_json_file() + b2 = CodeBlock.from_json_file() + assert b2 is not None + assert b2.rst_file == "foo.rst" + assert b2.language == "ada" + assert b2.project == "MyProj" + assert b2.main_file == "main.adb" + + def test_round_trip_active_true(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + b = self._make_block() + b.to_json_file() + b2 = CodeBlock.from_json_file() + assert b2 is not None + assert b2.active is True + + def test_round_trip_explicit_filename(self, tmp_path): + b = self._make_block() + f = str(tmp_path / "info.json") + b.to_json_file(f) + b2 = CodeBlock.from_json_file(f) + assert b2 is not None + assert b2.text == "procedure P is null;" + + def test_from_json_file_nonexistent(self, tmp_path): + f = str(tmp_path / "no_such.json") + assert CodeBlock.from_json_file(f) is None + + +# --------------------------------------------------------------------------- +# A block record that is present but cannot be turned into a block +# --------------------------------------------------------------------------- + +class TestCodeBlockRecordThatCannotBeRead: + """A block record file that exists but does not describe a block. + + The reader used to check only that the file was there, so anything past + that point left the reader as an exception -- and it is the reader both + commands go through, so the traceback came out of whichever one was + running. Each case below is a different way for the file to be + unusable, and each must come back as no block at all, with a message + saying which file it was and why it could not be used. + + A record written by the extraction step is never in any of these states. + These are the file after something else has been at it: a truncated + write, a hand edit, a merge that went wrong. + """ + + # The text of a record that is present and unusable, one entry per way of + # being unusable. The first two never parse; the third parses into + # something that is not a record; the fourth is a record with none of the + # fields a block is made of. + UNUSABLE_TEXTS = { + "truncated": '{"rst_file": "test.rst", "line_start": 1', + "not_json_at_all": "this file is not JSON", + "json_but_not_an_object": "[1, 2, 3]", + "an_object_with_none_of_the_fields": '{"something": "else"}', + } + + # The fifth way, built from a real block at test time rather than written + # out here: a complete, valid record of a real block, carrying one field + # a block is not made of -- a record written by a later version of the + # package than the one reading it. It is valid JSON and an object of the + # right shape, so it gets as far as being handed to the block, which is + # where it is refused. This is the case that shows the guard is not + # merely a check that the text parses. + A_RECORD_FROM_A_LATER_FORMAT = "a_record_from_a_later_format" + + ALL_CASES = sorted(UNUSABLE_TEXTS) + [A_RECORD_FROM_A_LATER_FORMAT] + + def _record_text(self, case: str) -> str: + if case in self.UNUSABLE_TEXTS: + return self.UNUSABLE_TEXTS[case] + + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + block = CodeBlock( + rst_file="foo.rst", + line_start=1, + line_end=10, + text="procedure P is null;", + language="ada", + project="MyProj", + main_file="main.adb", + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=["-gnata"], + classes=[], + manual_chop=False, + buttons=[], + ) + record = json.loads(json.dumps(block, default=lambda o: o.__dict__)) + record["a_field_this_version_does_not_know"] = "from a later format" + return json.dumps(record) + + @pytest.mark.parametrize("case", ALL_CASES) + def test_an_unusable_record_is_reported_as_no_block(self, case, tmp_path, + capsys): + """Reading an unusable record must come back as no block, and must + say which file could not be read and what was wrong with it. + + The reason is asserted separately from the file name, because the + name alone is what the callers already print for themselves -- the + reader is the only place that knows why. + """ + json_file = str(tmp_path / "block_info.json") + (tmp_path / "block_info.json").write_text(self._record_text(case)) + + assert CodeBlock.from_json_file(json_file) is None, \ + "a record that cannot be turned into a block must read back as " \ + "no block rather than as an exception" + + out = capsys.readouterr().out + assert "ERROR" in out, \ + "an unusable record must be reported: {}".format(out) + assert json_file in out, \ + "the report must name the file it could not read: {}".format(out) + assert out.split(json_file, 1)[1].strip(" :\n"), \ + "the report must say why the file could not be used, not only " \ + "which file it was: {}".format(out) + + def test_bytes_that_are_not_valid_utf8_are_reported_as_no_block( + self, tmp_path, capsys): + """A record file holding bytes that are not valid UTF-8 must also + come back as no block, reported the same way, not as an exception. + + Every case above is written with ``Path.write_text()``, which is + UTF-8 by construction and so cannot exercise this: the file it + produces is always decodable. This case writes raw bytes instead -- + a lead byte with no valid meaning in UTF-8, the kind a hand edit in + an editor defaulting to another encoding leaves behind. Decoding it + raises ``UnicodeDecodeError``, which is a *sibling* of + ``json.JSONDecodeError`` under ``ValueError`` rather than a subclass, + so the reader has to name it separately or this case would escape as + an uncaught exception instead of the reported failure the other + unusable records get. + """ + json_file = str(tmp_path / "block_info.json") + (tmp_path / "block_info.json").write_bytes(b"\xff\xfe not valid utf-8") + + assert CodeBlock.from_json_file(json_file) is None, \ + "a record that is not valid UTF-8 must read back as no block " \ + "rather than as an exception" + + out = capsys.readouterr().out + assert "ERROR" in out, \ + "an unreadable record must be reported: {}".format(out) + assert json_file in out, \ + "the report must name the file it could not read: {}".format(out) + assert out.split(json_file, 1)[1].strip(" :\n"), \ + "the report must say why the file could not be used, not only " \ + "which file it was: {}".format(out) + + +# --------------------------------------------------------------------------- +# T-blocks-14: ConfigBlock.__init__ and update() +# --------------------------------------------------------------------------- + +class TestConfigBlock: + def test_run_button_false(self): + cb = ConfigBlock("test.rst", run_button="False") + # run_button is set dynamically via setattr() in ConfigBlock.__init__, + # so it is looked up with getattr() rather than direct attribute access. + assert getattr(cb, "run_button") is False + + def test_prove_button_true(self): + cb = ConfigBlock("test.rst", prove_button="True") + assert getattr(cb, "prove_button") is True + + def test_accumulate_code_false(self): + cb = ConfigBlock("test.rst", accumulate_code="False") + assert getattr(cb, "accumulate_code") is False + + def test_rst_file_stored(self): + cb = ConfigBlock("my.rst", run_button="True") + assert cb.rst_file == "my.rst" + + def test_opts_stored(self): + cb = ConfigBlock("my.rst", run_button="True", accumulate_code="False") + assert "run_button" in cb._opts + assert "accumulate_code" in cb._opts + + def test_update_replaces_opts(self): + cb1 = ConfigBlock("my.rst", run_button="False", accumulate_code="True") + cb2 = ConfigBlock("my.rst", run_button="True", accumulate_code="False") + cb1.update(cb2) + assert getattr(cb1, "run_button") is True + assert getattr(cb1, "accumulate_code") is False + + def test_no_opts(self): + cb = ConfigBlock("my.rst") + assert cb._opts == {} + + # A configuration value normally arrives as a string, written in a + # code-config directive, and only the string "False" means false. The + # tests above cover that. The ones below cover a caller that hands over a + # real boolean instead, which is what the package's own default + # configuration does -- and which used to be compared against a string it + # could never equal, so that every such value came out true whatever was + # asked for. + # + # Nothing in the package reads these attributes back, so no other + # behavior depends on them and no other test can go red for this. These + # assertions are the whole of what holds it. + + def test_a_real_false_is_kept_false(self): + cb = ConfigBlock("test.rst", run_button=False) + assert getattr(cb, "run_button") is False, \ + "a caller passing a real False means false, and must not be " \ + "given back the opposite of what it asked for" + + def test_a_real_true_is_kept_true(self): + cb = ConfigBlock("test.rst", run_button=True) + assert getattr(cb, "run_button") is True + + def test_real_booleans_that_differ_produce_configurations_that_differ(self): + """Two configurations built from opposite real booleans must not agree. + + The per-value assertions above each name one attribute, so a + coercion that answered true for everything would need all of them to + catch it. This one fails on the collapse itself: the two objects + were once identical and all-true, whatever was asked for. + """ + asked_for_false = ConfigBlock( + "test.rst", run_button=False, prove_button=False, + accumulate_code=False) + asked_for_true = ConfigBlock( + "test.rst", run_button=True, prove_button=True, + accumulate_code=True) + for name in ("run_button", "prove_button", "accumulate_code"): + assert getattr(asked_for_false, name) != \ + getattr(asked_for_true, name), \ + "opposite requests must not produce the same value for " \ + "{}".format(name) + + def test_a_string_that_is_not_False_is_still_true(self): + """The string reading is unchanged: only "False" is false. + + Written down because it is the reading every value coming out of a + directive gets, and because a fix aimed at real booleans could + plausibly have made a string like this one false as well. + """ + cb = ConfigBlock("test.rst", run_button="no") + assert getattr(cb, "run_button") is True + + +# --------------------------------------------------------------------------- +# T-blocks-15: gnatprove_version and gprbuild_version selected attributes +# (covers the "selected" branch of gnatprove= and gprbuild= version parsing) +# --------------------------------------------------------------------------- + +class TestGnatproveVersionSelected: + RST = minimal_rst("""\ +.. code:: ada gnatprove=12.1.0-1 + + procedure P is null; +""") + + def test_gnatprove_version_is_selected(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].gnatprove_version == ["selected", "12.1.0-1"] + + +class TestGprbuildVersionSelected: + RST = minimal_rst("""\ +.. code:: ada gprbuild=22.0.0-1 + + procedure P is null; +""") + + def test_gprbuild_version_is_selected(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].gprbuild_version == ["selected", "22.0.0-1"] + + +# --------------------------------------------------------------------------- +# T-blocks-16: default compiler switch not duplicated when already explicit +# --------------------------------------------------------------------------- + +class TestDefaultSwitchNotDuplicated: + RST = minimal_rst("""\ +.. code:: ada switches=Compiler(-gnata) + + procedure P is null; +""") + + def test_gnata_not_duplicated(self): + """-gnata is both the explicit switch and the default; it must only + appear once in compiler_switches (the default-switches loop must skip + adding it again).""" + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].compiler_switches.count("-gnata") == 1 + + +# --------------------------------------------------------------------------- +# T-blocks-17: switches= value not shaped like Compiler(...) +# --------------------------------------------------------------------------- + +class TestSwitchesValueNotCompilerShaped: + RST = minimal_rst("""\ +.. code:: ada switches=Foo(-gnata) + + procedure P is null; +""") + + def test_parses_without_error(self): + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert len(blocks) == 1 + + def test_no_explicit_switches_beyond_defaults(self): + """switches=Foo(-gnata) is present but not shaped like Compiler(...), + so the captured value is never used; only the default -gnata switch + is present.""" + blocks = Block.get_blocks_from_rst(RST_FILE, self.RST) + assert isinstance(blocks[0], CodeBlock) + assert blocks[0].compiler_switches == ["-gnata"] diff --git a/frontend/python/rst_code_example_pipeline/tests/test_check_code_block.py b/frontend/python/rst_code_example_pipeline/tests/test_check_code_block.py new file mode 100644 index 000000000..6f6854df2 --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_check_code_block.py @@ -0,0 +1,3830 @@ +""" +Unit tests for rst_code_example_pipeline.check_code_block. + +Covers: +- Diag.__repr__: correct "file:line:col: msg" format +- check_block() with block.no_check=True → returns False immediately +- check_block() with prior BlockCheck.status_ok=True in cache + force_checks=False → cache hit +- check_block() with prior BlockCheck.status_ok=False in cache + force_checks=False → cached failure +- check_block() with force_checks=True → a recorded failure is ignored, the block is + checked again, and the record left behind carries this run's own result +- check_block() for a minimal Ada syntax-only block (gcc -gnats) → False +- check_block() for a block with empty buttons list → has_error=True (BUTTONS check fails) +- check_code_block_json() with nonexistent file → returns True (error) +- C compile path (gcc): valid C → False; invalid C → True (requires the Ada toolchain) +- ada-expect-compile-error class: Ada that fails to compile → False (expected failure) +- a failing Ada compile reports its diagnostics against the RST file, with the block's start line added +- C run path: valid C that exits 0 → False (requires the Ada toolchain) +- gnatprove path: C + prove_it → True (requires the Ada toolchain) +- gnatprove path: a pinned, genuinely installed legacy toolchain version still proves cleanly +- each prove button, and each prove class an author writes, selects the gnatprove + switches it names and no others -- read off the recorded command line, since the + fixture block proves cleanly under any switches at all; the plain prove button and + the plain prove class select neither switch, which is what says the others were + selected rather than always present +- verbose cache-skip path: status_ok=True in cache + verbose=True → "already checked" printed +- all_diagnostics flag: a clean Ada compile announces the block, reports SUCCESS and prints no diagnostics +- a corrupt (unparseable) cache file on disk does not crash the check +- an unrecognized language value takes neither the Ada nor the C branch anywhere, + and has neither a build nor a run recorded for it +- the maximum-columns setting reaches the Ada syntax check, and the limit applied + is the one that was asked for +- a toolchain binary missing from PATH falls back to an unknown-version marker instead of aborting the check +- each of the three clean-up commands an Ada compile and run reaches is reported + separately when it fails, the gnatprove --clean one naming the command it ran, + and none of the failures affects the result +- an rm -f clean-up failure after a successful C compile and run is logged without affecting the result +- a run with no executable to run is reported as a failed run and recorded as one, + in both languages, and the run-expect-failure classes do not absorb it +- check_block() driven by the real extraction step rather than by a hand-built block: + the compile, run and prove buttons an author writes in an RST directive, plus the + C run path and the ada-expect-compile-error class, each carry through to the checks + actually performed; a C block asking to be run by class alone, with no button + anywhere, is really built and run -- including one declaring it expects the run to + fail, whose handling was reachable only through a run button before -- and c-norun + takes a run away again; an extracted block that does not build is reported as an error; + and an extracted C block asking only for a compile is compiled without being + linked, while one that is also run is still linked into an executable named + after its main (requires the Ada toolchain). These subsume the hand-built + happy-path compile, run and prove tests that used to sit alongside them +- the other direction of every expect-error declaration: a block that declared a + compile error -- in either language -- or a prove error and then produced none + is reported and fails the check, with the build or the proof recorded as having + succeeded so that the report is known to come from the unmet expectation rather + than from anything going wrong; and a block declaring one of those failures, or + a suppressed run, while asking for no compile, no proof and no run is reported + for that too. The three core cases are covered twice over -- from a hand-built + block and again driven through the real RST directive and the real extraction + step +- a run class that names the language the block is not written in: each of the + six spellings is reported, by name, and fails the check -- including on a + block that a run button separately gets built and run, which is the case a + report read off what the checker decided to do, rather than off what the + block declared, would pass over. The message and the returned value are + asserted by separate tests, so a report that prints and leaves the run at + success reddens the second alone. The controls: the same six classes on the + language they name, the classes that name a language and are deliberately not + paired with one (ada-syntax-only, the two no-check spellings), the class that + names none, and a proof asked for on a C block, which has its own report + already and must not draw a second. The three returns that come before the + declaration checks are pinned as not reporting, since the report sits with + those checks. Driven through the real extraction step as well: a C block + classed ada-run with no button is neither built nor run and fails the check, + and an Ada block classed c-norun keeps the run its button asked for +- the arm of the previous-check lookup that does not read the record: with the + lookup switched off, a recorded failure is neither returned nor announced, and + the block is checked again although the checks were not forced +- Global state: verbose, all_diagnostics, max_columns, force_checks reset before each test + +NOTE: check_block() sets the toolchain up for every block before any early return, so a +test needs the Ada toolchain even when it stops at a no-check block or a cache hit and +never reaches a compiler. Every test that calls check_block() therefore carries the +`toolchain` marker; only the Diag repr tests and the two check_code_block_json() tests +that bail out on a missing file are free of it. +""" +import ast +import json +import os +import re + +import pytest + +import rst_code_example_pipeline.check_code_block as ccb +import rst_code_example_pipeline.extract_projects as ep +from rst_code_example_pipeline import blocks as _blocks_mod +from rst_code_example_pipeline import checks as _checks_mod +import rst_code_example_pipeline.toolchain_info as info + + +def _check_record(directory, block_record): + """The record a check wrote, found as the JSON file beside the block that + is not the one the check was handed. + + A check names that file itself, from the package's own default, so a test + spelling the name out here would restate a choice the package is free to + change -- and would go on passing if the check stopped writing a record + at all, as long as a file of the expected name happened to be lying there + from something else. + """ + handed = os.path.realpath(str(block_record)) + written = sorted(path for path in directory.glob("*.json") + if os.path.realpath(str(path)) != handed) + assert len(written) == 1, \ + "expected the check to write exactly one record beside the block, " \ + "got {}".format([path.name for path in written]) + return written[0] + + +def _reported(block, captured) -> list[str]: + """The messages a check produced for this block, with the location prefix + stripped off. + + Matched on the prefix the checker builds for the block under test, so a + message about some other block could not be mistaken for one of these -- + and so the wording asserted against it is only the part a course author + reads as the explanation. + """ + prefix = "at {}:{} (code block hash: {}): ".format( + block.rst_file, block.line_start, block.text_hash_short) + return [line.split(prefix, 1)[1] + for line in captured.out.splitlines() if prefix in line] + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +# The smallest Ada program that compiles and runs, shared by every test that +# needs a source file but does not care what it contains. +MINIMAL_ADA_SOURCE = """\ +procedure Main is +begin + null; +end Main; +""" + + +def _ada_source_with_a_line_of_width(width: int) -> str: + """A syntactically valid Ada program whose declaration line is exactly + ``width`` characters across. + + For tests that set a column limit to one side of that width and check + what the syntax check makes of it. + """ + head, tail = ' S : constant String := "', '";' + line = head + "x" * (width - len(head) - len(tail)) + tail + assert len(line) == width, \ + "the source line must be exactly the width the test asked for" + return "procedure Main is\n{}\nbegin\n null;\nend Main;\n".format(line) + + +def _installed_version(tool: str) -> str: + """Return a version of ``tool`` declared as installed in the toolchain + configuration, for tests that need to select a version explicitly rather + than take the default one.""" + if not info.TOOLCHAINS: + info.init_toolchain_info() + return info.TOOLCHAINS[tool][0] + + +def _legacy_gnatprove_version() -> str: + """Return the declared GNATprove version that gets the older command line. + + check_block() builds a pre-14 GNATprove command line for any version whose + identifier starts with "12", so a test of that branch needs a declared + version of that generation. Fail with a message naming the branch if none + is declared any more, rather than with an obscure lookup error. + """ + if not info.TOOLCHAINS: + info.init_toolchain_info() + legacy = [v for v in info.TOOLCHAINS["gnatprove"] if v.startswith("12")] + assert legacy, \ + "No GNATprove version of the 12 generation is declared as installed, " \ + "so the older-style command line it needs cannot be exercised" + return legacy[0] + + +def _make_block(project: str = "TestProject", + language: str = "ada", + classes: list[str] | None = None, + buttons: list[str] | None = None, + gnat_version: list[str] | None = None, + gnatprove_version: list[str] | None = None, + gprbuild_version: list[str] | None = None, + no_check: bool | None = None, + syntax_only: bool | None = None, + compile_it: bool | None = None, + run_it: bool | None = None, + source_files: list[str] | None = None, + line_start: int = 1, + text: str = "procedure Main is begin null; end Main;") -> _blocks_mod.CodeBlock: + """Build a minimal CodeBlock for testing. + + NOTE: Pass ``buttons=[]`` explicitly (not ``None``) to produce a block + with an empty buttons list. ``None`` (the default) falls back to + ``["no"]`` so that most tests get a valid button indicator without having + to spell it out each time. + + NOTE: ``line_start`` says where the block sits in its RST file. A test + that checks how a compiler diagnostic is mapped back onto the RST file + should set it higher than any line the compiler could report on its own, + so that an unmapped line cannot be mistaken for a mapped one. + """ + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + classes = classes or [] + # Use explicit None-check so that buttons=[] is preserved as-is. + buttons = ["no"] if buttons is None else buttons + gnat_version = gnat_version or ["default", info.DEFAULT_VERSION["gnat"]] + gnatprove_version = gnatprove_version or ["default", info.DEFAULT_VERSION["gnatprove"]] + gprbuild_version = gprbuild_version or ["default", info.DEFAULT_VERSION["gprbuild"]] + return _blocks_mod.CodeBlock( + rst_file="test.rst", + line_start=line_start, + line_end=line_start + 4, + text=text, + language=language, + project=project, + main_file=None, + gnat_version=gnat_version, + gnatprove_version=gnatprove_version, + gprbuild_version=gprbuild_version, + compiler_switches=["-gnata"], + classes=classes, + manual_chop=False, + buttons=buttons, + no_check=no_check, + syntax_only=syntax_only, + compile_it=compile_it, + run_it=run_it, + source_files=source_files or [], + ) + + +# --------------------------------------------------------------------------- +# T-check_code_block-01: Diag.__repr__ +# --------------------------------------------------------------------------- + +class TestDiagRepr: + def test_format_is_correct(self): + d = ccb.Diag("main.adb", 10, 3, "error: missing semicolon") + assert repr(d) == "main.adb:10:3: error: missing semicolon" + + def test_different_values(self): + d = ccb.Diag("foo.ads", 1, 1, "warning: unused") + assert repr(d) == "foo.ads:1:1: warning: unused" + + def test_zero_line_col(self): + d = ccb.Diag("x.adb", 0, 0, "note") + assert repr(d) == "x.adb:0:0: note" + + +# --------------------------------------------------------------------------- +# T-check_code_block-02: check_block() with no_check=True +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockNoCheck: + def test_returns_false_when_no_check(self, tmp_path): + block = _make_block(classes=["ada-nocheck"], no_check=True) + json_file = str(tmp_path / "block_info.json") + block.to_json_file(json_file) + result = ccb.check_block(block, json_file) + assert result is False + + def test_no_subprocess_called_when_no_check(self, tmp_path, monkeypatch): + """Verify no subprocess is spawned when no_check=True.""" + import subprocess as S + calls = [] + original_check_output = S.check_output + + def mock_check_output(*args, **kwargs): + calls.append(args) + return original_check_output(*args, **kwargs) + + monkeypatch.setattr(S, "check_output", mock_check_output) + + block = _make_block(classes=["ada-nocheck"], no_check=True) + json_file = str(tmp_path / "block_info.json") + block.to_json_file(json_file) + ccb.check_block(block, json_file) + # The only subprocess calls allowed are the toolchain setup calls (set_versions). + # Those run gcc/gnat/gnatprove/gprbuild --version. But no_check returns before + # set_versions is called, so there should be NO subprocess calls at all. + assert calls == [], \ + "check_block() with no_check=True must not call any subprocess" + + +# --------------------------------------------------------------------------- +# T-check_code_block-03: check_block() cache hit (status_ok=True) +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockCacheHitOk: + def test_cache_hit_returns_false(self, work_dir): + """Prior check with status_ok=True and force_checks=False → return False.""" + block = _make_block(buttons=["no"]) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + # Write a fake block_checks.json in the same directory + bc = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + bc.status_ok = True + bc.to_json_file() # writes block_checks.json in cwd + + result = ccb.check_block(block, json_file, force_checks=False) + assert result is False + + +# --------------------------------------------------------------------------- +# T-check_code_block-04: check_block() cache hit (status_ok=False) +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockCacheHitFail: + def test_cached_failure_returns_true(self, work_dir): + """Prior check with status_ok=False and force_checks=False → return True.""" + block = _make_block(buttons=["no"]) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + bc = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + bc.status_ok = False + bc.to_json_file() + + result = ccb.check_block(block, json_file, force_checks=False) + assert result is True + + def test_cached_none_status_ok_reruns(self, work_dir): + """status_ok=None in the cache means previous run was incomplete. + The code does `not ref_block_check.status_ok` which evaluates None as + falsy — so has_error=True and we return True. Verify this edge case.""" + block = _make_block(buttons=["no"]) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + bc = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + bc.status_ok = None # neither True nor False + bc.to_json_file() + + # `not None` is True → has_error = True + result = ccb.check_block(block, json_file, force_checks=False) + assert result is True + + +@pytest.mark.toolchain +class TestCheckBlockCorruptCache: + def test_corrupt_cache_file_is_ignored(self, work_dir): + """A previous-check cache file that is not valid JSON must not crash + check_block(): the read failure is caught, no cached result is used, + and a full check runs and completes normally instead.""" + block = _make_block(buttons=["no"]) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + # Let the package write the cache file, so that the corrupt one lands + # under the name the read is going to look for. Named here instead, + # a rename would leave nothing to be read and this test would pass + # over a check that never met a corrupt file at all. + _checks_mod.BlockCheck(text_hash=block.text_hash, + text_hash_short=block.text_hash_short).to_json_file() + _check_record(work_dir, json_file).write_text("{not valid json") + + result = ccb.check_block(block, json_file) + assert result is False, \ + "An unparseable cache file must be ignored rather than crash the check" + + +# --------------------------------------------------------------------------- +# check_block() with the checks forced against a populated cache +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockForceChecks: + def test_forcing_the_checks_overrides_a_cached_failure(self, work_dir): + """Forcing the checks must ignore what a previous run recorded and + check the block again. + + The block is checkable and clean, but a record of an earlier run + sitting beside it says the block failed. Left alone, that record is + what the caller gets back -- TestCheckBlockCacheHitFail pins that. + Forced, the stale record has to be ignored, the checks have to run for + real, and the answer has to be the one the block earns rather than the + one on disk. + + Both halves are asserted, because the outcome alone cannot tell a + re-check apart from a cache lookup that happened to be dropped: the + record left behind afterwards must carry this run's own result and the + checks it performed. + """ + src = work_dir / "main.adb" + src.write_text(MINIMAL_ADA_SOURCE) + + block = _make_block( + buttons=["no"], + no_check=False, + syntax_only=False, + source_files=["main.adb"], + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + stale = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + stale.status_ok = False + stale.to_json_file() + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "a recorded failure must not be returned when the checks are forced" + + rewritten = json.loads(_check_record(work_dir, json_file).read_text()) + assert rewritten["status_ok"] is True, \ + "the forced run must replace the stale record with its own result" + assert "SYNTAX" in rewritten["checks"], \ + "the forced run must have checked the block, not skipped it" + + +# --------------------------------------------------------------------------- +# check_block() with the previous-check lookup switched off +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockPreviousCheckLookupDisabled: + def test_a_recorded_result_is_ignored_when_the_lookup_is_switched_off( + self, work_dir, monkeypatch): + """With the previous-check lookup switched off, a record beside the + block must not be consulted at all. + + The module carries a switch that decides whether a block already + carrying a record is skipped. It ships on, so every other test in + this file exercises only the arm that reads the record -- and the arm + that does not was never entered by anything. + + The fixture is deliberately the same one TestCheckBlockCacheHitFail + uses: a clean, checkable block with a record beside it saying the + block failed, and the checks *not* forced. That test pins the + recorded failure being handed straight back. Here the answer has to + be the one the block earns instead, and the record left behind has to + carry this run's own result and the checks it performed -- because the + outcome alone cannot tell a re-check apart from a lookup that happened + to find nothing. + """ + monkeypatch.setattr(ccb, "LOOK_FOR_PREVIOUS_CHECKS", False) + + src = work_dir / "main.adb" + src.write_text(MINIMAL_ADA_SOURCE) + + block = _make_block( + buttons=["no"], + no_check=False, + syntax_only=False, + source_files=["main.adb"], + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + stale = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + stale.status_ok = False + stale.to_json_file() + + result = ccb.check_block(block, json_file, force_checks=False) + assert result is False, \ + "with the lookup switched off, a recorded failure must not be " \ + "returned even though the checks were not forced" + + rewritten = json.loads(_check_record(work_dir, json_file).read_text()) + assert rewritten["status_ok"] is True, \ + "the run must replace the stale record with its own result" + assert "SYNTAX" in rewritten["checks"], \ + "the run must have checked the block, not skipped it" + + def test_the_block_is_not_announced_as_already_checked( + self, work_dir, monkeypatch, capsys): + """The message a skipped block gets must not be printed when the + lookup is switched off. + + Asserted separately from the result above because the skip prints + before it returns: a lookup that still ran and still reported the + block as already checked, but whose result was then discarded, would + satisfy the assertions above and be visible only here. Verbose mode + is asked for, since that is the setting under which the message is + produced at all -- and it has to be asked for in the call, because the + module global of that name is only the default the function was + defined with and assigning to it afterwards changes nothing. + """ + monkeypatch.setattr(ccb, "LOOK_FOR_PREVIOUS_CHECKS", False) + + src = work_dir / "main.adb" + src.write_text(MINIMAL_ADA_SOURCE) + + block = _make_block( + buttons=["no"], + no_check=False, + syntax_only=False, + source_files=["main.adb"], + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + recorded = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + recorded.status_ok = True + recorded.to_json_file() + + ccb.check_block(block, json_file, verbose=True, force_checks=False) + captured = capsys.readouterr() + assert "already checked" not in captured.out, \ + "with the lookup switched off, no block may be announced as " \ + "already checked: {}".format(captured.out) + + +# --------------------------------------------------------------------------- +# T-check_code_block-05: check_block() with no buttons (BUTTONS check failure) +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockNoButtons: + def test_empty_buttons_returns_true(self, work_dir): + """A block with empty buttons list must fail the BUTTONS check.""" + # The block asks for nothing but the button validation: it is + # neither no-check nor syntax-only, so the check runs to the end; it + # declares no source files, so the syntax check has nothing to look + # at; and it asks for no compile and no proof. Forcing the checks + # keeps a cached result from short-circuiting all of that. + + block = _make_block(buttons=[], syntax_only=False, no_check=False) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "check_block() must return True (has_error) when buttons list is empty" + + def test_empty_buttons_prints_error(self, work_dir, capsys): + """The diagnostic must name the offending block and say what was + missing, since that text is all a course author gets to act on.""" + block = _make_block(buttons=[], syntax_only=False, no_check=False) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + ccb.check_block(block, json_file, force_checks=True) + captured = capsys.readouterr() + # The "ERROR" prefix and its coloring belong to the message formatter + # and are covered with it; what matters here is the location and the + # wording that follows. + expected = ( + "at {}:{} (code block hash: {}): " + "Expected at least 'no_button' indicator, got none!".format( + block.rst_file, block.line_start, block.text_hash_short)) + assert expected in captured.out + + +# --------------------------------------------------------------------------- +# T-check_code_block-06: check_block() real Ada syntax check +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockRealSyntax: + """Tests that actually invoke gcc -gnats.""" + + ADA_SOURCE = """\ +with Ada.Text_IO; use Ada.Text_IO; +procedure Main is +begin + Put_Line ("Hello, World!"); +end Main; +""" + + def test_valid_ada_syntax_returns_false(self, work_dir): + """A syntactically correct Ada block must pass the syntax check.""" + # Write source file + src = work_dir / "main.adb" + src.write_text(self.ADA_SOURCE) + + block = _make_block( + buttons=["no"], + syntax_only=True, + no_check=False, + source_files=["main.adb"], + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "A syntactically valid Ada block must not produce an error" + + def test_invalid_ada_syntax_returns_true(self, work_dir): + """A syntactically invalid Ada block must fail the syntax check.""" + bad_source = "this is not ada;\n" + src = work_dir / "bad.adb" + src.write_text(bad_source) + + block = _make_block( + buttons=["no"], + syntax_only=True, + no_check=False, + source_files=["bad.adb"], + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "A syntactically invalid Ada block must produce an error" + + +# --------------------------------------------------------------------------- +# T-check_code_block-07: check_code_block_json() with nonexistent file +# --------------------------------------------------------------------------- + +class TestCheckCodeBlockJson: + def test_nonexistent_file_returns_true(self, tmp_path): + """check_code_block_json() on a missing file must return True (error).""" + missing = str(tmp_path / "no_such_file.json") + result = ccb.check_code_block_json(missing) + assert result is True + + def test_nonexistent_file_prints_error(self, tmp_path, capsys): + missing = str(tmp_path / "missing.json") + ccb.check_code_block_json(missing) + captured = capsys.readouterr() + assert "ERROR" in captured.out + + @pytest.mark.toolchain + def test_valid_nocheck_block_json_returns_false(self, work_dir): + """check_code_block_json() on a no-check block must return False.""" + block = _make_block(classes=["ada-nocheck"], no_check=True, buttons=["no"]) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + result = ccb.check_code_block_json(json_file) + assert result is False + + +# --------------------------------------------------------------------------- +# T-check_code_block-08: selected toolchain + non-no button validation +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockSelectedToolchainButtonValidation: + def test_selected_gnat_with_compile_button_fails_buttons_check(self, work_dir): + """When a specific toolchain version is selected, only 'no' button is allowed. + A block with gnat_version=selected and buttons=['compile'] must fail.""" + block = _make_block( + gnat_version=["selected", _installed_version("gnat")], + buttons=["compile"], + syntax_only=False, + no_check=False, + # Suppress compile_it so that we reach the BUTTONS check without + # triggering gprclean/gprbuild (which need a real project file). + compile_it=False, + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "A block with selected toolchain and non-'no' button must fail BUTTONS check" + + +# --------------------------------------------------------------------------- +# T-check_code_block-09: real compile check (gprbuild) +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockRealCompile: + """Tests that actually invoke gprbuild.""" + + BAD_ADA_SOURCE = "procedure Bad is\nbegin\n SYNTAX ERROR HERE!!!\nend Bad;\n" + + @staticmethod + def _compile_failing_block_at(work_dir, capsys, line_start, bad_source): + """Check a block that fails to compile, starting at ``line_start`` in + its RST file. + + Returns the check result, the distinct line numbers the diagnostics + were reported at against the RST file, and the distinct line numbers + the compiler itself used for the extracted source -- the latter read + back from the raw compiler output the check prints alongside them, so + the test never has to know where the compiler places a diagnostic. + + Both are de-duplicated: a failing check reports the same diagnostic + several times over, and how often it does is not what is under test + here. + """ + work_dir.mkdir(parents=True, exist_ok=True) + (work_dir / "bad.adb").write_text(bad_source) + os.chdir(str(work_dir)) + project_filename = ep.write_project_file( + main_file="bad.adb", + compiler_switches=[], + spark_mode=False, + ) + + block = _make_block( + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["bad.adb"], + line_start=line_start, + ) + block.project_filename = project_filename + block.project_main_file = "bad.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + capsys.readouterr() + result = ccb.check_block(block, json_file, force_checks=True) + out = capsys.readouterr().out + + reported = sorted({int(line) for line in re.findall( + r"^{}:(\d+):\d+: ".format(re.escape(block.rst_file)), out, re.M)}) + raw = sorted({int(line) + for line in re.findall(r"bad\.adb:(\d+):\d+: ", out)}) + return result, reported, raw + + def test_compile_error_block_returns_true(self, tmp_path, capsys): + """An Ada block that fails to compile must return True (error) and + report the compiler diagnostics against the RST file, at the lines the + block occupies there. + + The compiler numbers its diagnostics from the top of the extracted + source; check_block has to re-point them at the RST file the reader is + editing and shift them by where the block starts in it. Neither the + compiler's wording nor any particular line is pinned, so a compiler + upgrade that moves or adds a diagnostic does not redden this: + + * against the compiler's own numbering, read back from the raw output + printed alongside the remapped diagnostics, every reported line must + be that number plus the block's start line -- which is what catches a + shift that is missing, doubled, or off by one; + * and compiling the same block a second time from a different start + line must move every reported line by exactly that difference. + """ + first_start, second_start = 100, 250 + + first_result, first_lines, first_raw = self._compile_failing_block_at( + tmp_path / "first", capsys, first_start, self.BAD_ADA_SOURCE) + second_result, second_lines, second_raw = self._compile_failing_block_at( + tmp_path / "second", capsys, second_start, self.BAD_ADA_SOURCE) + + assert first_result is True and second_result is True, \ + "An Ada block that fails to compile must return True (has_error)" + assert first_lines, \ + "no compiler diagnostic was reported against the RST file" + assert first_raw, \ + "the raw compiler output must be shown, or there is nothing to " \ + "compare the remapped line numbers against" + + assert first_lines == [line + first_start for line in first_raw], \ + "each diagnostic must be reported at its compiler line shifted by " \ + "the block's start line; compiler said {}, block starts at {}, " \ + "reported {}".format(first_raw, first_start, first_lines) + assert second_lines == [line + second_start for line in second_raw], \ + "each diagnostic must be reported at its compiler line shifted by " \ + "the block's start line; compiler said {}, block starts at {}, " \ + "reported {}".format(second_raw, second_start, second_lines) + + assert second_lines == [ + line + (second_start - first_start) for line in first_lines], \ + "moving the block down the RST file must move its diagnostics with " \ + "it: {} at line {} became {} at line {}".format( + first_lines, first_start, second_lines, second_start) + + +# --------------------------------------------------------------------------- +# C1 — TestCheckBlockCCompile +# Covers the compile step for a C block. +# Requires gcc in PATH (part of the Ada toolchain). +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockCCompile: + """Tests that actually invoke gcc on C source files.""" + + VALID_C_SOURCE = "int main(void) { return 0; }\n" + INVALID_C_SOURCE = "this is not C at all !@#$\n" + + def test_c_compile_success(self, work_dir): + """A valid C file with compile_it=True and buttons=['compile'] must return False.""" + src = work_dir / "main.c" + src.write_text(self.VALID_C_SOURCE) + + block = _make_block( + language="c", + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "A valid C file must compile without error" + + def test_c_compile_failure(self, work_dir): + """An invalid C file with compile_it=True must return True (has_error).""" + src = work_dir / "main.c" + src.write_text(self.INVALID_C_SOURCE) + + block = _make_block( + language="c", + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "An invalid C file must produce a compile error" + + +# --------------------------------------------------------------------------- +# C2 — TestCheckBlockExpectCompileError + C run path +# Covers ada-expect-compile-error class handling and C run path. +# Requires the Ada toolchain. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockExpectCompileError: + """Tests for ada-expect-compile-error class and C run path.""" + + # This Ada source is syntactically valid (passes gcc -gnats) but fails + # gprbuild compilation because it refers to a non-existent package. + # The nosyntax-check class bypasses the SYNTAX phase so only the BUILD + # phase runs; 'ada-expect-compile-error' suppresses the BUILD failure. + BAD_BUILD_ADA_SOURCE = """\ +with NonExistent_Package; use NonExistent_Package; +procedure Bad is +begin + null; +end Bad; +""" + VALID_C_SOURCE = "int main(void) { return 0; }\n" + + def test_ada_expect_compile_error(self, work_dir): + """A block with classes=['ada-expect-compile-error', 'nosyntax-check'] + and Ada source that fails to compile at the BUILD phase must return False + (the expected compile failure is not treated as an error).""" + src = work_dir / "bad.adb" + src.write_text(self.BAD_BUILD_ADA_SOURCE) + project_filename = ep.write_project_file( + main_file="bad.adb", + compiler_switches=[], + spark_mode=False, + ) + + block = _make_block( + classes=["ada-expect-compile-error", "nosyntax-check"], + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["bad.adb"], + ) + block.project_filename = project_filename + block.project_main_file = "bad.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "An expected compile error must not count as a test failure" + + def test_c_run(self, work_dir): + """A valid C file compiled and run (exits 0) must return False.""" + src = work_dir / "main.c" + src.write_text(self.VALID_C_SOURCE) + + block = _make_block( + language="c", + buttons=["run"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=True, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "A valid C program that exits 0 must not produce a run error" + + +# --------------------------------------------------------------------------- +# C3 — TestCheckBlockGnatprove +# Covers the proof step, which only Ada blocks reach. +# Requires gnatprove in PATH (part of the Ada toolchain). +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockGnatprove: + """Tests that actually invoke gnatprove.""" + + SPARK_SOURCE = """\ +procedure Main with SPARK_Mode is +begin + null; +end Main; +""" + + def test_ada_gnatprove_language_c_else(self, work_dir): + """A block with language="c" and prove_it=True must return True: + proving only supports Ada, so a non-Ada block takes the "wrong + language selected for prove button" error branch instead of + invoking gnatprove.""" + + block = _make_block( + language="c", + buttons=["prove"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + source_files=[], + ) + block.prove_it = True + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "C language with prove_it=True must return True (unsupported)" + + def test_ada_gnatprove_pinned_legacy_version(self, work_dir): + """A prove block pinned to a specific, genuinely installed legacy + GNATprove version must build the older-style command line that + version expects, and a real invocation with it must still prove the + example cleanly.""" + src = work_dir / "main.adb" + src.write_text(self.SPARK_SOURCE) + + spark_project_filename = ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=True, + ) + + block = _make_block( + buttons=["no"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + source_files=["main.adb"], + gnatprove_version=["selected", _legacy_gnatprove_version()], + ) + block.project_filename = None + block.spark_project_filename = spark_project_filename + block.project_main_file = "main.adb" + block.prove_it = True + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "A provable SPARK block must prove cleanly under a pinned legacy GNATprove version" + + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + proved_with = ast.literal_eval(recorded["PROVE"]["cmdline"]) + assert "--no-axiom-guard" in proved_with, \ + "the older command line must ask for the switch only that " \ + "generation understands: {}".format(proved_with) + assert "--checks-as-errors" in proved_with, \ + "the older command line must spell the checks-as-errors switch " \ + "the way that generation accepts it: {}".format(proved_with) + assert "--function-sandboxing=off" not in proved_with, \ + "the older command line must not carry a switch introduced " \ + "after it: {}".format(proved_with) + + +# --------------------------------------------------------------------------- +# Unrecognized-language paths +# Covers cleanup/syntax-check/compile/run all falling through without taking +# either the Ada or the C branch, and without crashing. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockUnrecognizedLanguage: + def test_unrecognized_language_takes_neither_branch(self, tmp_path, + monkeypatch): + """A block whose language is neither 'ada' nor 'c' must fall through + the syntax-check, compile and run steps without taking either + language-specific branch, and must complete without raising. + + The block asks for a compile and a run, and names the main file a + language branch would need, so that a branch wrongly taken would have + enough to proceed rather than tripping over missing state: the check + has to skip it on the language alone. Three things then show it did. + No command but the toolchain version probes is run -- a branch taken + would invoke a compiler -- and the record left behind carries neither + a BUILD nor a RUN phase. A BUILD phase is only added from inside a + language branch. A RUN phase is only added when a run was really + attempted, which is also only decided inside a language branch: a + recorded RUN for a language the checker does not run would claim a + successful run of a command that was never built, and would name a + log file that was never written. + """ + import subprocess as S + + commands = [] + real_check_output = S.check_output + + def recording_check_output(args, *rest, **kwargs): + commands.append(list(args)) + return real_check_output(args, *rest, **kwargs) + + monkeypatch.setattr(S, "check_output", recording_check_output) + + block = _make_block( + language="fortran", + no_check=False, + syntax_only=False, + compile_it=True, + run_it=True, + source_files=["main.f90"], + ) + block.project_main_file = "main.f90" + json_file = str(tmp_path / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + + assert all(command[1:2] == ["--version"] for command in commands), \ + "only the toolchain version probes may run for a language the " \ + "check does not know: {}".format(commands) + + recorded = json.loads( + _check_record(tmp_path, json_file).read_text())["checks"] + assert "BUILD" not in recorded, \ + "a compile was asked for, so a recorded BUILD phase means a " \ + "language branch was taken: {}".format(sorted(recorded)) + + assert "RUN" not in recorded, \ + "a run was asked for, but no language branch could attempt one, " \ + "so a recorded RUN phase describes a run that never happened: " \ + "{}".format(sorted(recorded)) + + assert result is False, \ + "An unrecognized language must not raise and must not report an error" + + +# --------------------------------------------------------------------------- +# Verbose / all_diagnostics paths +# Covers the verbose cache-skip output and the all_diagnostics output path. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockVerbose: + """Tests for verbose and all_diagnostics flag paths.""" + + def test_verbose_cache_skip(self, work_dir, capsys): + """With verbose=True and a cached status_ok=True, check_block must print + 'already checked. Skipping...' (exercises the verbose cache-hit path).""" + block = _make_block(buttons=["no"]) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + bc = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + bc.status_ok = True + bc.to_json_file() + + ccb.verbose = True + result = ccb.check_block(block, json_file, verbose=True, force_checks=False) + assert result is False + out = capsys.readouterr().out + expected = ( + "Code block at {}:{} (code block hash: {}) " + "already checked. Skipping...".format( + block.rst_file, block.line_start, block.text_hash_short)) + assert expected in out + + def test_all_diagnostics_flag(self, work_dir, capsys): + """With all_diagnostics=True and verbose=True, a clean Ada compile must + announce the block it is checking, report success, and print no + diagnostics at all.""" + src = work_dir / "main.adb" + src.write_text(MINIMAL_ADA_SOURCE) + project_filename = ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=False, + ) + + block = _make_block( + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["main.adb"], + ) + block.project_filename = project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + ccb.all_diagnostics = True + ccb.verbose = True + result = ccb.check_block( + block, json_file, all_diagnostics=True, verbose=True, force_checks=True + ) + assert result is False, \ + "A valid Ada compile with all_diagnostics=True and verbose=True must not produce an error" + + out = capsys.readouterr().out + assert "Checking code block at {}:{} (code block hash: {})".format( + block.rst_file, block.line_start, block.text_hash_short) in out + assert "SUCCESS" in out + assert not re.search( + r"^{}:\d+:\d+: ".format(re.escape(block.rst_file)), out, re.M), \ + "a clean compile must not report any diagnostic against the RST file" + + +# --------------------------------------------------------------------------- +# TestCheckBlockMaxColumns +# Covers the maximum-columns setting reaching the Ada syntax check, and the +# limit actually applied being the one that was asked for. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockMaxColumns: + """The maximum-columns setting reaches the Ada syntax check. + + The syntax check already asks for the compiler's own style rules, and + those carry a column limit of their own, narrower than the one either + test below sets. So a block that is wider than the limit it is given + proves nothing on its own -- it would be reported either way -- and only + the block that is *narrower* than the limit it is given can show that the + setting was passed on at all. The two tests together pin both halves: + that the limit is applied, and that it is the one that was asked for. + """ + + #: How wide the one long line of the source below is. Both tests set a + #: limit to one side of it, and both limits are above the compiler's own. + LINE_WIDTH = 90 + + def _check_under_limit(self, work_dir, max_columns: int) -> bool: + """Syntax-check a block holding one LINE_WIDTH-wide line under the + given column limit, and return whether the check reported an error.""" + (work_dir / "main.adb").write_text( + _ada_source_with_a_line_of_width(self.LINE_WIDTH)) + + block = _make_block( + buttons=["no"], + syntax_only=True, + no_check=False, + source_files=["main.adb"], + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + return ccb.check_block(block, json_file, max_columns=max_columns, + force_checks=True) + + def test_line_within_the_column_limit_passes(self, work_dir): + """A line narrower than the limit asked for must pass the syntax + check, even though it is wider than the compiler's own limit. Nothing + but the setting having been passed on can make that happen.""" + assert self._check_under_limit( + work_dir, self.LINE_WIDTH + 10) is False, \ + "a line of {} characters must pass a limit of {}".format( + self.LINE_WIDTH, self.LINE_WIDTH + 10) + + def test_line_beyond_the_column_limit_fails(self, work_dir): + """A line wider than the limit asked for must fail the syntax check, + so that the limit applied is the one that was asked for rather than + some other one that happens to be set.""" + assert self._check_under_limit( + work_dir, self.LINE_WIDTH - 10) is True, \ + "a line of {} characters must not pass a limit of {}".format( + self.LINE_WIDTH, self.LINE_WIDTH - 10) + + +# --------------------------------------------------------------------------- +# TestCheckBlockRunExpectFailure +# Covers the ada-run-expect-failure class: an unexpectedly successful run, +# an expectedly failing run, and an unexpectedly failing run. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockRunExpectFailure: + VALID_ADA_SOURCE = """\ +procedure Main is +begin + null; +end Main; +""" + + FAILING_ADA_SOURCE = """\ +with Ada.Command_Line; +procedure Main is +begin + Ada.Command_Line.Set_Exit_Status (1); +end Main; +""" + + def _setup_project(self, work_dir, source): + src = work_dir / "main.adb" + src.write_text(source) + return ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=False, + ) + + def _make_run_block(self, classes=None): + return _make_block( + classes=classes or [], + buttons=["run"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=True, + source_files=["main.adb"], + ) + + def test_run_success_with_expect_failure_class(self, work_dir): + """A program that exits 0 while marked ada-run-expect-failure must + return True: the run succeeded when a failure was expected.""" + project_filename = self._setup_project(work_dir, self.VALID_ADA_SOURCE) + block = self._make_run_block(classes=["ada-run-expect-failure"]) + block.project_filename = project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True + + def test_ada_run_fail_with_expect_failure_class(self, work_dir, capsys): + """A program that exits non-zero while marked ada-run-expect-failure + must return False: the failure was expected. With verbose enabled, + the expected-failure message is printed.""" + project_filename = self._setup_project(work_dir, self.FAILING_ADA_SOURCE) + block = self._make_run_block(classes=["ada-run-expect-failure"]) + block.project_filename = project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + ccb.verbose = True + result = ccb.check_block(block, json_file, verbose=True, force_checks=True) + assert result is False + out = capsys.readouterr().out + assert "Running of example expectedly failed" in out + + def test_ada_run_fail_with_expect_failure_class_says_nothing_quietly( + self, work_dir, capsys): + """The expected-failure message belongs to the verbose run only. + + The sibling above drives the same path with verbose enabled and + asserts the message; without this one, nothing says the message is + conditional at all, and the quiet run -- the one every real check + makes -- would go unexercised. Its C counterpart is reached by the + extractor-driven expect-failure test further down, which runs quiet. + """ + project_filename = self._setup_project(work_dir, self.FAILING_ADA_SOURCE) + block = self._make_run_block(classes=["ada-run-expect-failure"]) + block.project_filename = project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "a failure the block expects must not be reported as an error" + out = capsys.readouterr().out + assert "Running of example expectedly failed" not in out, \ + "the expected-failure message must be held back on a quiet " \ + "run: {}".format(out) + assert "Running of example failed" not in out, \ + "an expected failure must not be reported as an unexpected one " \ + "either: {}".format(out) + + def test_ada_run_fail_without_expect_failure(self, work_dir): + """A program that exits non-zero without ada-run-expect-failure must + return True: an unexpected run failure.""" + project_filename = self._setup_project(work_dir, self.FAILING_ADA_SOURCE) + block = self._make_run_block() + block.project_filename = project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True + + +# --------------------------------------------------------------------------- +# TestCheckBlockCRunExpectFailure +# Covers the c-run-expect-failure class, symmetric to the Ada case above. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockCRunExpectFailure: + VALID_C_SOURCE = "int main(void) { return 0; }\n" + FAILING_C_SOURCE = "int main(void) { return 1; }\n" + + def _make_c_run_block(self, classes=None): + return _make_block( + language="c", + classes=classes or [], + buttons=["run"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=True, + source_files=["main.c"], + ) + + def test_c_run_fail_with_expect_failure_class(self, work_dir, capsys): + """A C program that exits non-zero while marked c-run-expect-failure + must return False: the failure was expected. With verbose enabled, + the expected-failure message is printed.""" + src = work_dir / "main.c" + src.write_text(self.FAILING_C_SOURCE) + + block = self._make_c_run_block(classes=["c-run-expect-failure"]) + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + ccb.verbose = True + result = ccb.check_block(block, json_file, verbose=True, force_checks=True) + assert result is False + out = capsys.readouterr().out + assert "Running of example expectedly failed" in out + + def test_c_run_success_with_expect_failure_class(self, work_dir): + """A C program that exits 0 while marked c-run-expect-failure must + return True: the run succeeded when a failure was expected.""" + src = work_dir / "main.c" + src.write_text(self.VALID_C_SOURCE) + + block = self._make_c_run_block(classes=["c-run-expect-failure"]) + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True + + def test_c_run_fail_without_expect_failure(self, work_dir): + """A C program that exits non-zero without c-run-expect-failure must + return True: an unexpected run failure.""" + src = work_dir / "main.c" + src.write_text(self.FAILING_C_SOURCE) + + block = self._make_c_run_block() + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True + + +# --------------------------------------------------------------------------- +# TestCheckBlockCExpectCompileError +# Covers the c-expect-compile-error class in the C compile handler. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockCExpectCompileError: + INVALID_C_SOURCE = "this is not C at all !@#$\n" + + def test_c_compile_error_expected(self, work_dir): + """A C file that fails to compile while marked c-expect-compile-error + must return False: the compile failure was expected. + + nosyntax-check is also set: for C, the SYNTAX phase and the BUILD + phase both invoke gcc on the same source, so a genuine syntax error + would already fail (as an unexpected error) during SYNTAX before the + BUILD phase's c-expect-compile-error handling is ever reached -- the + same reason the analogous ada-expect-compile-error test bypasses the + SYNTAX phase.""" + src = work_dir / "main.c" + src.write_text(self.INVALID_C_SOURCE) + + block = _make_block( + language="c", + classes=["c-expect-compile-error", "nosyntax-check"], + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False + + +# --------------------------------------------------------------------------- +# TestCheckBlockProveFailure +# Covers the gnatprove failure handler: the expected (ada-expect-prove-error) +# and unexpected branches. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockProveFailure: + # X is read via Y := X before being initialized: a flow-analysis check + # that reliably fails under --checks-as-errors (mirrors the pattern used + # in the course's own "may not be initialized" SPARK examples). + FAILING_SPARK_SOURCE = """\ +procedure Main with SPARK_Mode is + X, Y : Integer; +begin + Y := X; +end Main; +""" + + def _setup_spark_project(self, work_dir): + src = work_dir / "main.adb" + src.write_text(self.FAILING_SPARK_SOURCE) + return ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=True, + ) + + def _make_prove_block(self, classes=None): + return _make_block( + classes=classes or [], + buttons=["prove"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + source_files=["main.adb"], + ) + + def test_prove_failure_expected(self, work_dir): + """SPARK code that fails to prove while marked ada-expect-prove-error + must return False: the failure was expected.""" + spark_project_filename = self._setup_spark_project(work_dir) + block = self._make_prove_block(classes=["ada-expect-prove-error"]) + block.spark_project_filename = spark_project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False + + def test_prove_failure_unexpected(self, work_dir): + """SPARK code that fails to prove without ada-expect-prove-error must + return True: an unexpected prove failure.""" + spark_project_filename = self._setup_spark_project(work_dir) + block = self._make_prove_block() + block.spark_project_filename = spark_project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True + + +# --------------------------------------------------------------------------- +# TestCheckBlockExpectedErrorThatNeverHappened +# Covers the other direction of every expect-error declaration: the checker +# has to report a block that declared a failure and then did not produce one. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockExpectedErrorThatNeverHappened: + """A block declaring "this must fail" that succeeds instead. + + This is the direction the checker exists for. A course example marked as + expecting a compile error, or a proof failure, is not being checked for + the failure it produces -- it is being watched in case it silently stops + producing one, which is what happens when the example is repaired, the + class is left on it, and nobody notices that the block is now asserting + something untrue about the language. The suite covered only the arm where + the declared failure really occurs, so a checker that dropped these + reports entirely would have stayed green. + + Each test asserts the outcome, the message a course author is given to act + on, and -- where the block reaches a compiler or the prover -- that the + phase itself was recorded as having succeeded. That last one is what + distinguishes the report under test from the block having failed for some + other reason: the build or the proof went through, and it is the button + validation that objected. + """ + + CLEAN_SPARK_SOURCE = """\ +procedure Main with SPARK_Mode is +begin + null; +end Main; +""" + + VALID_C_SOURCE = "int main(void) { return 0; }\n" + + def test_a_compile_error_that_did_not_happen_is_reported( + self, work_dir, capsys): + """Source that compiles cleanly under ada-expect-compile-error must + fail the check. + + The block asks for a compile and gets one; the compiler is happy, so + the declared error never arrives. The build is recorded as having + succeeded, which is what says the report comes from the expectation + being unmet rather than from anything having gone wrong. + """ + src = work_dir / "main.adb" + src.write_text(MINIMAL_ADA_SOURCE) + project_filename = ep.write_project_file( + main_file="main.adb", + compiler_switches=[], + spark_mode=False, + ) + + block = _make_block( + classes=["ada-expect-compile-error"], + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["main.adb"], + ) + block.project_filename = project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a block declaring it expects a compile error must fail the " \ + "check when the source compiles" + + assert "Expected compile error, got none!" in \ + _reported(block, capsys.readouterr()), \ + "the check must say that the declared compile error never arrived" + + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + assert recorded["BUILD"]["status_ok"] is True, \ + "the build must have succeeded, or the failure under test is not " \ + "the missing compile error" + assert recorded["BUTTONS"]["status_ok"] is False, \ + "the unmet expectation must be recorded against the block's " \ + "declarations" + + def test_a_c_compile_error_that_did_not_happen_is_reported( + self, work_dir, capsys): + """C source that compiles cleanly under c-expect-compile-error must + fail the check. + + The C spelling of the test above, and for a long time the only one of + the expect-error declarations that bought the author nothing: a C + block marked "this must not compile" whose code the compiler accepted + was reported as a success, so an example repaired without its class + being taken off went on passing. The compile step raises the same + flag for either language, so the question asked here is the same + question -- and the build is recorded as having succeeded, which is + what says the report comes from the expectation being unmet rather + than from anything having gone wrong. + """ + src = work_dir / "main.c" + src.write_text(self.VALID_C_SOURCE) + + block = _make_block( + language="c", + classes=["c-expect-compile-error"], + buttons=["compile"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=False, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a C block declaring it expects a compile error must fail the " \ + "check when the source compiles" + + assert "Expected compile error, got none!" in \ + _reported(block, capsys.readouterr()), \ + "the check must say that the declared compile error never arrived" + + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + assert recorded["BUILD"]["status_ok"] is True, \ + "the build must have succeeded, or the failure under test is not " \ + "the missing compile error" + assert recorded["BUTTONS"]["status_ok"] is False, \ + "the unmet expectation must be recorded against the block's " \ + "declarations" + + def test_a_prove_error_that_did_not_happen_is_reported( + self, work_dir, capsys): + """SPARK code that proves cleanly under ada-expect-prove-error must + fail the check. + + The mirror of TestCheckBlockProveFailure.test_prove_failure_expected, + which pins the arm where the proof really does fail. Here the prover + is satisfied, so the declared failure never arrives; the proof is + recorded as having succeeded, which is what says the report comes from + the expectation being unmet. + """ + src = work_dir / "main.adb" + src.write_text(self.CLEAN_SPARK_SOURCE) + spark_project_filename = ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=True, + ) + + block = _make_block( + classes=["ada-expect-prove-error"], + buttons=["prove"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + source_files=["main.adb"], + ) + block.spark_project_filename = spark_project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a block declaring it expects a prove error must fail the check " \ + "when the proof succeeds" + + assert "Expected prove error, got none!" in \ + _reported(block, capsys.readouterr()), \ + "the check must say that the declared prove error never arrived" + + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + assert recorded["PROVE"]["status_ok"] is True, \ + "the proof must have succeeded, or the failure under test is not " \ + "the missing prove error" + assert recorded["BUTTONS"]["status_ok"] is False, \ + "the unmet expectation must be recorded against the block's " \ + "declarations" + + def test_expecting_a_compile_error_with_nothing_that_compiles_is_reported( + self, work_dir, capsys): + """A block expecting a compile error while asking for no compile must + be reported. + + Nothing in the block gives the checker a way to produce the error it + declares: there is no compile and no run button, and neither of the + classes that ask for one. Both objections are asserted, because both + are true of this block and each is a separate report -- the missing + button or class, and, unavoidably, the compile error that no compile + could have produced. + """ + block = _make_block( + classes=["ada-expect-compile-error"], + buttons=["no"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a block expecting a compile error with nothing to compile must " \ + "fail the check" + + reported = _reported(block, capsys.readouterr()) + assert "Expected compile or run button/class, got none!" in reported, \ + "the check must say the block asks for no compile: {}".format( + reported) + assert "Expected compile error, got none!" in reported, \ + "the check must also say the declared compile error never " \ + "arrived: {}".format(reported) + + def test_expecting_a_prove_error_without_a_proof_is_reported( + self, work_dir, capsys): + """A block expecting a prove error while asking for no proof must be + reported. + + The class alone does not ask for a proof, so the block declares a + failure the checker is never given the chance to observe. Only the + missing prove button is reported: the arm that reports the missing + failure itself sits behind the proof having been asked for. + """ + block = _make_block( + classes=["ada-expect-prove-error"], + buttons=["no"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + ) + assert block.prove_it is False, \ + "the expect-prove-error class must not by itself ask for a " \ + "proof, or this test is not about a block that asks for none" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a block expecting a prove error without a proof must fail the " \ + "check" + + reported = _reported(block, capsys.readouterr()) + assert "Expected prove button, got none!" in reported, \ + "the check must say the block asks for no proof: {}".format( + reported) + + def test_declaring_no_run_without_a_run_button_is_reported( + self, work_dir, capsys): + """A block classed ada-norun with no run button must be reported. + + Taking a run away is only meaningful for a block that was going to be + run, so the checker requires the run to have been asked for -- and + says so when it was not. + """ + block = _make_block( + classes=["ada-norun"], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + assert (block.run_it, block.compile_it) == (False, False), \ + "the class must have taken the run away, or this block is being " \ + "built and run rather than only validated" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a block classed ada-norun with no run button must fail the check" + + reported = _reported(block, capsys.readouterr()) + assert "Expected run button, got none!" in reported, \ + "the check must say the block asks for no run: {}".format(reported) + + def test_expecting_a_run_failure_without_a_run_button_is_reported( + self, work_dir, capsys): + """The same report is due for a block expecting its run to fail. + + Written separately from the ada-norun block above rather than left to + it: the two class names are read as one set, so a checker that stopped + recognizing this one would still satisfy the other test. The run and + the compile are suppressed so that the block is only validated -- what + is under test is the declaration, not a program. + """ + block = _make_block( + classes=["ada-run-expect-failure"], + buttons=["no"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a block expecting its run to fail with no run button must fail " \ + "the check" + + reported = _reported(block, capsys.readouterr()) + assert "Expected run button, got none!" in reported, \ + "the check must say the block asks for no run: {}".format(reported) + + +# --------------------------------------------------------------------------- +# TestCheckBlockRunClassNamingTheOtherLanguage +# Covers the report for a run class that names a language the block is not +# written in -- the declaration whose consequence is that nothing happens. +# --------------------------------------------------------------------------- + +# The six run classes, each paired with a block of the language it does not +# name. Kept as one table so that the firing cases and the silent controls +# below are driven by the same list and cannot drift apart. +RUN_CLASSES_AND_THEIR_LANGUAGE = [ + ("ada-run", "ada"), + ("ada-norun", "ada"), + ("ada-run-expect-failure", "ada"), + ("c-run", "c"), + ("c-norun", "c"), + ("c-run-expect-failure", "c"), +] + +# The other language, for a table of two. +THE_OTHER_LANGUAGE = {"ada": "c", "c": "ada"} + +WRONG_LANGUAGE_REPORT = "Wrong language selected for run class '{}'" + + +def _no_run_class_report(reported: list[str]) -> bool: + """Whether none of the messages is the wrong-language run-class report. + + Matched on the part of the wording that is common to all six spellings, + so that a report firing for a class this test did not name is caught as + well as one firing for the class it did. + """ + return not any("Wrong language selected for run class" in message + for message in reported) + + +@pytest.mark.toolchain +class TestCheckBlockRunClassNamingTheOtherLanguage: + """A block carrying a run class that names the language it is not written + in. + + The class does nothing for such a block -- that is the point of pairing + each class with its language -- and "does nothing" is exactly the outcome + this checker exists to prevent from passing quietly. A C block tagged + ada-run asks for no run, and therefore for no build, so without a report + it is checked by nothing and recorded as a success. + + The messages and the exit status are asserted by separate tests here, + rather than together, because they are separately losable: this package + already carries a report that prints and leaves the status at zero, so a + new one that did the same is a real possibility rather than a + hypothetical, and it must redden the status tests on its own. + """ + + # A C program that announces itself, for the one test whose block is + # really built and really run. + C_RUN_OUTPUT = "the mis-classed example ran" + + C_SOURCE_THAT_ANNOUNCES_ITSELF = """\ +#include + +int main(void) +{{ + printf("{}\\n"); + return 0; +}} +""".format(C_RUN_OUTPUT) + + @staticmethod + def _checked(block, work_dir, json_file, **kwargs) -> bool: + """Write the block out and check it, with the checks forced. + + Forced because a recorded result would otherwise decide the outcome + on a second run in the same directory, which says nothing about the + declaration under test. + """ + block.to_json_file(json_file) + return ccb.check_block(block, json_file, force_checks=True, **kwargs) + + @pytest.mark.parametrize("code_class,class_language", + RUN_CLASSES_AND_THEIR_LANGUAGE) + def test_a_run_class_naming_the_other_language_is_reported( + self, code_class, class_language, work_dir, capsys): + """Each of the six run classes, on a block of the other language, + must be reported by name. + + One case per class rather than one test over all six: a checker that + recognized five of them would otherwise still pass. The message + names the offending class, so the author is told which word to fix + rather than only that something is wrong with the block. + + Only the message is asserted. That the report also fails the check + is the separate claim held by the tests below, and keeping the two + apart is what makes a report that prints and returns success redden + those and not these. + """ + block = _make_block( + language=THE_OTHER_LANGUAGE[class_language], + classes=[code_class], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + json_file = str(work_dir / "block_info.json") + + self._checked(block, work_dir, json_file) + + reported = _reported(block, capsys.readouterr()) + assert WRONG_LANGUAGE_REPORT.format(code_class) in reported, \ + "the report must name the offending class: {}".format(reported) + + @pytest.mark.parametrize("code_class,class_language", + RUN_CLASSES_AND_THEIR_LANGUAGE) + def test_a_run_class_on_the_language_it_names_is_not_reported( + self, code_class, class_language, work_dir, capsys): + """The control for the six above. + + Without it, a report that fired on every run class whatsoever would + satisfy all six and take every correctly tagged block in the material + down with it. + + Only the absence of this report is asserted, and deliberately not the + block's overall result: two of these six classes separately draw the + pre-existing "Expected run button, got none!" objection, which is a + behavior recorded as it stands rather than one this test should + fasten itself to. + + The compile and the run are switched off rather than derived. On the + language it names, a run class really does ask for a run, and this + block has no project and no source behind it -- what is under test is + the declaration the checker reads, and the derivation it reads it + through is covered where the derivation lives. + """ + block = _make_block( + language=class_language, + classes=[code_class], + buttons=["no"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + ) + json_file = str(work_dir / "block_info.json") + + self._checked(block, work_dir, json_file) + + reported = _reported(block, capsys.readouterr()) + assert _no_run_class_report(reported), \ + "a run class on the language it names must draw no report: " \ + "{}".format(reported) + + def test_a_c_block_declaring_ada_syntax_only_passes_and_stops_there( + self, work_dir): + """The class/language mismatch the material really carries. + + A C block declaring ada-syntax-only sits in the Ada course material + today. The syntax-only class names no language as far as the checker + is concerned, so the block is syntax-checked and stops -- and it must + go on doing that, or a content build fails on an example that is + written the way it is on purpose. + """ + source = work_dir / "main.c" + source.write_text(self.C_SOURCE_THAT_ANNOUNCES_ITSELF) + + block = _make_block( + language="c", + classes=["ada-syntax-only"], + buttons=["no"], + no_check=False, + source_files=["main.c"], + ) + assert block.syntax_only is True, \ + "the class must still make the block syntax-only, or this is not " \ + "the block the material carries" + + json_file = str(work_dir / "block_info.json") + assert self._checked(block, work_dir, json_file) is False, \ + "the C block the material carries must pass the check" + + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + assert sorted(recorded) == ["SYNTAX"], \ + "a syntax-only block must be syntax-checked and nothing else" + + @pytest.mark.parametrize("code_class,language", [ + ("ada-syntax-only", "c"), + ("ada-nocheck", "c"), + ("c-nocheck", "ada"), + ("nosyntax-check", "ada"), + ]) + def test_a_class_that_is_not_a_run_class_is_never_reported( + self, code_class, language, work_dir, capsys): + """The classes that name a language in their spelling and are + deliberately not paired with one, plus the one that names none. + + This is the control that a report written as a scan over class names + beginning with "ada-" or "c-" fails. Two of these are not idle + worries: the syntax-only case is a block the material carries, and + the two no-check spellings are documented as the Ada one and the C + one while being read for either language -- a difference that was + looked at and deliberately left alone, so a report firing here would + quietly take the opposite decision. + + The block is built with the two early returns switched off, so that + it reaches the declaration checks and the report is really consulted. + A block declaring one of these classes would otherwise return before + the report could fire, and this control would hold nothing. + """ + block = _make_block( + language=language, + classes=[code_class], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + json_file = str(work_dir / "block_info.json") + + self._checked(block, work_dir, json_file) + + reported = _reported(block, capsys.readouterr()) + assert _no_run_class_report(reported), \ + "a class that is not a run class must draw no report: {}".format( + reported) + + def test_a_prove_class_on_a_c_block_draws_only_the_prove_report( + self, work_dir, capsys): + """A proof asked for on a C block is already reported, and must not + be reported twice. + + The prove classes name a language in their spelling too, and the + checker has objected to a proof on a non-Ada block all along. A + second report saying the same thing in different words would leave an + author looking for two mistakes where there is one. + """ + block = _make_block( + language="c", + classes=["ada-prove"], + buttons=["no"], + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + ) + assert block.prove_it is True, \ + "the class must ask for a proof, or the existing report is not " \ + "the one being reached" + + json_file = str(work_dir / "block_info.json") + self._checked(block, work_dir, json_file) + + reported = _reported(block, capsys.readouterr()) + assert "Wrong language selected for prove button" in reported, \ + "the existing report must still be made: {}".format(reported) + assert _no_run_class_report(reported), \ + "the same mistake must not be reported a second time: {}".format( + reported) + + def test_a_mis_classed_block_that_a_run_button_rescues_is_still_reported( + self, work_dir, capsys): + """A C block classed ada-run that also carries a run button. + + This is the case that separates a report read off the block's + declarations from one read off what the checker decided to do with + them. The button asks for the run the class failed to ask for, so + the block really is built and really is run, and nothing about the + outcome is wrong -- yet the class still names a language the block is + not written in, and the author still has a word to fix. + + A report derived from "a run class is present and the block is not + being run" passes every other test in this file and fails this one. + + The build and the run are asserted as having happened, with the + program's own output read back out of the run log, so that the report + is known to come from a block the checker fully processed rather than + from one it quietly skipped. + + Like its siblings above this asserts the message and not the returned + value; the status side of this same case is held through the + installed command, where a block with a run button is checked end to + end. + """ + source = work_dir / "main.c" + source.write_text(self.C_SOURCE_THAT_ANNOUNCES_ITSELF) + + block = _make_block( + language="c", + classes=["ada-run"], + buttons=["run"], + syntax_only=False, + no_check=False, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + assert (block.run_it, block.compile_it) == (True, True), \ + "the button must have asked for the run the class did not, or " \ + "this is not the case under test" + + json_file = str(work_dir / "block_info.json") + self._checked(block, work_dir, json_file) + + reported = _reported(block, capsys.readouterr()) + assert WRONG_LANGUAGE_REPORT.format("ada-run") in reported, \ + "the class must be reported although the block was run: " \ + "{}".format(reported) + + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + assert sorted(recorded) == ["BUILD", "BUTTONS", "RUN", "SYNTAX"], \ + "the block must really have been built and run" + assert recorded["RUN"]["status_ok"] is True, \ + "the run itself must have succeeded, or the report cannot be " \ + "attributed to the declaration" + assert (work_dir / recorded["RUN"]["logfile"]).read_text().strip() == \ + self.C_RUN_OUTPUT, \ + "the program the author wrote must be the one that ran" + + # The three returns that used to come before the report, each now a + # decision of its own rather than one rule applied to all three. + # + # The report is read off the declaration, so it is made before the + # recorded result is consulted, and only the block that asked for no + # checking at all escapes it. A block declaring a no-check class asked + # for exactly that and is still skipped in silence. A syntax-only block + # asked for less checking, not for none, and is reported. A block with a + # recorded result asked for nothing less at all -- the record is a cache, + # and it is keyed on a hash of the block's text, so editing only the + # class leaves the key untouched and hands back the success recorded for + # the declaration the block had before the edit. That is the one outcome + # this report exists to prevent, so it is made before the record is read. + # + # Each of the three is pinned below, so that moving the report past one + # of them reddens a test naming the path it was moved past. + + def test_a_mis_classed_block_declaring_no_check_is_not_reported( + self, work_dir, capsys): + """A block declaring a no-check class is skipped before the report. + + Nothing is checked and nothing is recorded, so the mis-classed run + class beside it goes unmentioned. + """ + block = _make_block( + language="c", + classes=["ada-nocheck", "ada-run"], + buttons=["no"], + ) + assert block.no_check is True, \ + "the block must be the one the checker skips outright" + + json_file = str(work_dir / "block_info.json") + assert self._checked(block, work_dir, json_file) is False, \ + "a block declaring no check must still pass" + + assert _no_run_class_report(_reported(block, capsys.readouterr())), \ + "a block the checker never looks at cannot be reported" + + def test_a_mis_classed_block_declaring_syntax_only_is_reported( + self, work_dir, capsys): + """A block declaring itself syntax-only is reported. + + Such a block asked for less checking, not for none: its syntax is + checked and the check stops there. The class it carries is still a + word the author has to fix, and it is knowable from the declaration + without anything being built, so the reduced checking the block asked + for is no reason to withhold it. + + Only the message is asserted; that the report also fails the check is + held separately, as it is for every other message test here. + """ + source = work_dir / "main.c" + source.write_text(self.C_SOURCE_THAT_ANNOUNCES_ITSELF) + + block = _make_block( + language="c", + classes=["ada-syntax-only", "ada-run"], + buttons=["no"], + no_check=False, + source_files=["main.c"], + ) + assert block.syntax_only is True, \ + "the block must be the one the checker stops after the syntax " \ + "check" + + json_file = str(work_dir / "block_info.json") + self._checked(block, work_dir, json_file) + + reported = _reported(block, capsys.readouterr()) + assert WRONG_LANGUAGE_REPORT.format("ada-run") in reported, \ + "a block that stops after the syntax check must still be told " \ + "about the class it carries: {}".format(reported) + + def test_a_mis_classed_block_with_a_recorded_result_is_reported( + self, work_dir, capsys): + """A block whose result is already recorded is reported all the same. + + The record is a cache and its key is a hash of the block's text, so + a class edited without the text being touched hands back the success + recorded for the declaration the block had before the edit -- which + is how a mis-classed block comes into existence in the first place. + The report is read off the declaration and needs nothing the record + holds, so it is made before the record is consulted. + + The recorded result here says the block passed, so nothing but the + declaration can account for the report. Only the message is + asserted; the status side is held separately. + """ + block = _make_block( + language="c", + classes=["ada-run"], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + recorded = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + recorded.status_ok = True + recorded.to_json_file() # beside the block, under the package's name + + ccb.check_block(block, json_file) + + reported = _reported(block, capsys.readouterr()) + assert WRONG_LANGUAGE_REPORT.format("ada-run") in reported, \ + "a recorded success must not absorb the report: {}".format( + reported) + + def test_a_mis_classed_block_is_reported_exactly_once( + self, work_dir, capsys): + """The report is made once, on the path that makes every report. + + The class is read at the top of the check and carried to the end, + where it joins the other declaration objections in the record the + block leaves behind. Carrying it as a second print instead would + tell an author of two mistakes where there is one, and the wording is + the same both times, so nothing in the message would give the + duplication away. + """ + block = _make_block( + language="c", + classes=["ada-run"], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + json_file = str(work_dir / "block_info.json") + + self._checked(block, work_dir, json_file) + + reported = _reported(block, capsys.readouterr()) + assert reported.count(WRONG_LANGUAGE_REPORT.format("ada-run")) == 1, \ + "one mistake must draw one report: {}".format(reported) + + +# --------------------------------------------------------------------------- +# TestCheckBlockRunClassNamingTheOtherLanguageFailsTheRun +# The exit status of the report above, held apart from its wording. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockRunClassNamingTheOtherLanguageFailsTheRun: + """The report has to fail the check, not merely print. + + Kept apart from the wording tests above on purpose. This package already + holds a report that prints and leaves the run at success -- the + wrong-language prove button reported by the extraction step, whose flag + never reaches that function's return value, recorded in this suite as a + known defect. A new report that took the same shape would satisfy every + message test written above while telling a build that the course checked + out. These tests assert the returned value and nothing else, so that + exact defect reddens them alone. + """ + + def test_check_block_returns_an_error_for_a_mis_classed_block( + self, work_dir): + """check_block() itself must return the error, with no reference to + what it printed.""" + block = _make_block( + language="c", + classes=["ada-run"], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + assert ccb.check_block(block, json_file, force_checks=True) is True, \ + "a run class naming the other language must fail the check" + + @staticmethod + def _recording_a_success(block) -> None: + """Leave a record of a successful check beside the block. + + Written through the package's own writer, so that the record is right + in every respect and lands under the name the check looks for without + that name being restated here. + """ + recorded = _checks_mod.BlockCheck( + text_hash=block.text_hash, + text_hash_short=block.text_hash_short, + ) + recorded.status_ok = True + recorded.to_json_file() + + def test_a_recorded_success_does_not_absorb_the_error(self, work_dir): + """A recorded success must not decide the outcome for a block whose + declaration has gone wrong since it was written. + + This is the case a course author actually meets. The per-block + directory is named after a hash of the block's text, the record + beside it is never compared against the declaration, and nothing + removes it -- so editing only the class of an example whose body was + not touched hands back the result of the run before the edit. The + default local driver keeps that directory between runs by design, so + a stale record is the ordinary state there rather than an unusual + one. + + Asserted on the returned value alone, and with no --force: a report + that printed and left the value at success would satisfy the message + test of this same case while telling a build the course checked out. + """ + block = _make_block( + language="c", + classes=["ada-run"], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + self._recording_a_success(block) + + assert ccb.check_block(block, json_file) is True, \ + "a recorded success must not stand in for a check the block " \ + "can no longer pass" + + def test_a_recorded_success_is_still_handed_back_for_a_sound_block( + self, work_dir): + """The control for the test above. + + Reusing a recorded result is what the record is for, and the report + must not cost every block that has one its reuse. The same block + with the class of its own language keeps the recorded success -- and + keeps it without a compiler being reached, which is the whole point + of the record. + """ + block = _make_block( + language="c", + classes=["c-run"], + buttons=["no"], + syntax_only=False, + no_check=False, + ) + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + self._recording_a_success(block) + + assert ccb.check_block(block, json_file) is False, \ + "a block whose declaration is sound must keep its recorded result" + + def test_a_syntax_only_block_fails_the_check_and_the_record( + self, work_dir): + """A syntax-only block carrying the class must fail, and be recorded + as having failed. + + The block stops after the syntax check, so the report is the only + objection it can draw, and the returned value is the only thing + carrying it. The record is asserted beside it because it is the + record the next run reads: one saying the block passed would hand the + failure straight back as a success the moment the check is run again. + """ + source = work_dir / "main.c" + source.write_text( + TestCheckBlockRunClassNamingTheOtherLanguage + .C_SOURCE_THAT_ANNOUNCES_ITSELF) + + block = _make_block( + language="c", + classes=["ada-syntax-only", "ada-run"], + buttons=["no"], + no_check=False, + source_files=["main.c"], + ) + assert block.syntax_only is True, \ + "the block must be the one the checker stops after the syntax " \ + "check" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + assert ccb.check_block(block, json_file, force_checks=True) is True, \ + "a syntax-only block carrying the class must fail the check" + + record = json.loads(_check_record(work_dir, json_file).read_text()) + assert record["status_ok"] is False, \ + "the record the next run reads must say the block failed" + + +# --------------------------------------------------------------------------- +# TestCheckBlockProveExtraArgs +# Covers the gnatprove extra-arguments variants selected via the prove_flow / +# prove_flow_report_all / prove_report_all buttons. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockProveExtraArgs: + SPARK_SOURCE = """\ +procedure Main with SPARK_Mode is +begin + null; +end Main; +""" + + def _setup_spark_project(self, work_dir): + src = work_dir / "main.adb" + src.write_text(self.SPARK_SOURCE) + return ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=True, + ) + + def _prove(self, work_dir, buttons=None, classes=None): + """Prove a SPARK block asking for it the given way, and hand back the + command line the proof phase recorded. + + The switches have to be read off that command line. The fixture + block is trivially valid, so it proves cleanly under any switches at + all, and a passing result therefore says nothing whatever about which + ones were selected. + """ + spark_project_filename = self._setup_spark_project(work_dir) + block = _make_block( + buttons=buttons, + classes=classes, + syntax_only=False, + no_check=False, + compile_it=False, + run_it=False, + source_files=["main.adb"], + ) + block.spark_project_filename = spark_project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + assert ccb.check_block(block, json_file, force_checks=True) is False, \ + "the fixture block must prove cleanly, or what the proof recorded " \ + "is not what this test is about" + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + assert "PROVE" in recorded, \ + "the block must have asked for a proof, or there is no command " \ + "line to look at" + return ast.literal_eval(recorded["PROVE"]["cmdline"]) + + def test_prove_button_selects_neither_switch(self, work_dir): + """A plain prove button asks for neither the flow mode nor the full + report, so the proof runs on the default switches alone.""" + proved_with = self._prove(work_dir, buttons=["prove"]) + assert "--mode=flow" not in proved_with, \ + "a plain prove button must not restrict the proof to flow " \ + "analysis: {}".format(proved_with) + assert "--report=all" not in proved_with, \ + "a plain prove button must not ask for the full report: " \ + "{}".format(proved_with) + + def test_prove_flow_mode(self, work_dir): + """The prove_flow button selects the flow mode and nothing else.""" + proved_with = self._prove(work_dir, buttons=["prove_flow"]) + assert "--mode=flow" in proved_with, \ + "the flow button must restrict the proof to flow analysis: " \ + "{}".format(proved_with) + assert "--report=all" not in proved_with, \ + "the flow button must not also ask for the full report: " \ + "{}".format(proved_with) + + def test_prove_flow_report_all(self, work_dir): + """The prove_flow_report_all button selects both switches.""" + proved_with = self._prove(work_dir, buttons=["prove_flow_report_all"]) + assert "--mode=flow" in proved_with, \ + "the flow report-all button must restrict the proof to flow " \ + "analysis: {}".format(proved_with) + assert "--report=all" in proved_with, \ + "the flow report-all button must ask for the full report: " \ + "{}".format(proved_with) + + def test_prove_report_all(self, work_dir): + """The prove_report_all button selects the full report and nothing + else.""" + proved_with = self._prove(work_dir, buttons=["prove_report_all"]) + assert "--report=all" in proved_with, \ + "the report-all button must ask for the full report: " \ + "{}".format(proved_with) + assert "--mode=flow" not in proved_with, \ + "the report-all button must not also restrict the proof to flow " \ + "analysis: {}".format(proved_with) + + def test_ada_prove_flow_class_selects_the_flow_mode(self, work_dir): + """The class an author writes selects what the matching button does.""" + proved_with = self._prove(work_dir, classes=["ada-prove-flow"]) + assert "--mode=flow" in proved_with, \ + "the flow class must restrict the proof to flow analysis: " \ + "{}".format(proved_with) + assert "--report=all" not in proved_with, \ + "the flow class must not also ask for the full report: " \ + "{}".format(proved_with) + + def test_ada_prove_flow_report_all_class_selects_both(self, work_dir): + """The class an author writes selects what the matching button does.""" + proved_with = self._prove(work_dir, classes=["ada-prove-flow-report-all"]) + assert "--mode=flow" in proved_with, \ + "the flow report-all class must restrict the proof to flow " \ + "analysis: {}".format(proved_with) + assert "--report=all" in proved_with, \ + "the flow report-all class must ask for the full report: " \ + "{}".format(proved_with) + + def test_ada_prove_report_all_class_is_proved(self, work_dir): + """The class alone asks for a proof, with no prove button present. + + Pins the fixture the test below depends on: that test can only + report on the switches of a proof that really happened, so the proof + itself is asserted here, on its own. + """ + assert self._prove(work_dir, classes=["ada-prove-report-all"]) + + def test_ada_prove_report_all_class_asks_for_the_full_report(self, work_dir): + """A block classed ``ada-prove-report-all`` must be proved with + ``--report=all``. + + Each prove button is paired with the class that carries the same + name: prove_flow with ada-prove-flow, prove_flow_report_all with + ada-prove-flow-report-all, and this one with ada-prove-report-all. + That third arm used to test a differently-named class instead, so a + block classed ada-prove-report-all was proved -- it is one of the + classes that select a proof -- and then never reached the switch its + own name asks for. + + This test can only report on the switches of a proof that really + happened, so it depends on the unmarked sibling above, which drives + the same fixture and reddens if the proof stops happening at all. + """ + proved_with = self._prove(work_dir, classes=["ada-prove-report-all"]) + assert "--report=all" in proved_with, \ + "a class that names the full report must select it: {}".format( + proved_with) + assert "--mode=flow" not in proved_with, \ + "the report-all class must not also restrict the proof to flow " \ + "analysis: {}".format(proved_with) + + def test_ada_prove_class_selects_neither_switch(self, work_dir): + """The plain prove class asks for neither the flow mode nor the full + report, so the proof runs on the default switches alone. + + The control for the three class tests above: each of them names a + switch and asserts it was selected, which a proof that always + selected everything would satisfy. This one fails on that. + """ + proved_with = self._prove(work_dir, classes=["ada-prove"]) + assert "--mode=flow" not in proved_with, \ + "a plain prove class must not restrict the proof to flow " \ + "analysis: {}".format(proved_with) + assert "--report=all" not in proved_with, \ + "a plain prove class must not ask for the full report: " \ + "{}".format(proved_with) + + +# --------------------------------------------------------------------------- +# TestCheckCodeBlockJsonInactive +# Covers the inactive-block WARNING printed by check_code_block_json(). +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckCodeBlockJsonInactive: + def test_check_code_block_json_inactive_block(self, work_dir, capsys): + """check_code_block_json() on a block with active=False prints the + deactivation WARNING and still checks it.""" + block = _make_block(classes=["ada-nocheck"], no_check=True, buttons=["no"]) + block.active = False + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_code_block_json(json_file) + assert result is False + assert "WARNING" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# Missing-toolchain path +# Covers the version-lookup fallback when a toolchain binary is missing from +# PATH, in place of monkeypatching the subprocess call. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockMissingToolchain: + def test_missing_toolchain_binary_falls_back_to_unknown_version(self, tmp_path, monkeypatch): + """When none of the toolchain binaries can be found on PATH, the + version lookup must not abort the check: it silently falls back to an + unknown-version marker instead, and check_block() still completes and + returns False. The recorded check result is read back from the raw + written file (not through the round-trip API, which does not restore + the nested per-check dict) to confirm the fallback value was actually + recorded, rather than only asserting the absence of a crash.""" + block = _make_block(buttons=["no"]) + json_file = str(tmp_path / "block_info.json") + block.to_json_file(json_file) + + monkeypatch.setenv("PATH", str(tmp_path)) + + result = ccb.check_block(block, json_file) + assert result is False, \ + "A missing toolchain must not crash the check, only skip real checks" + + written = json.loads(_check_record(tmp_path, json_file).read_text()) + assert written["checks"]["SYNTAX"]["version"] == "", \ + "The version lookup must have failed and recorded the fallback marker" + + +# --------------------------------------------------------------------------- +# Clean-up failure paths +# Covers the gprclean / gnatprove --clean clean-up failures after an Ada +# compile and run, and the rm -f clean-up failure after a C compile and run. +# The clean-up commands are selectively made to fail while every other +# command (the real compile and run) is left untouched. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockCleanupFailures: + """A real Ada compile and run that both succeed, while every clean-up + command invoked along the way is made to fail.""" + + def _setup_project(self, work_dir): + """Write an Ada source file and a .gpr project file into work_dir.""" + src = work_dir / "main.adb" + src.write_text(MINIMAL_ADA_SOURCE) + project_filename = ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=False, + ) + return project_filename + + def test_gprclean_and_gnatprove_clean_failures_do_not_affect_result( + self, work_dir, monkeypatch, capsys): + """Each of the three clean-up commands an Ada block reaches is + reported when it fails, and none of the failures aborts the check or + changes its result. + + The three are a gprclean before the build, and a gprclean and a + gnatprove --clean during the end-of-check clean-up. All three are + made to fail here, so all three have to be reported: a real compile + and run that succeed still make the check pass, but they do so + loudly. + + The counts are exact rather than bounded from below, so that dropping + any one of the three reddens this test. The two gprclean sites print + the same text, so only their number tells that both are still there; + the gnatprove --clean site names the command it ran, so it is + asserted by that name and by being the last of the three to report. + """ + import subprocess as S + + project_filename = self._setup_project(work_dir) + + real_check_output = S.check_output + failed_cleanups = [] + + def fake_check_output(cmd, *args, **kwargs): + if cmd[0] == "gprclean" or (cmd[0] == "gnatprove" and "--clean" in cmd): + failed_cleanups.append(cmd[0]) + raise S.CalledProcessError(1, cmd, output=b"simulated cleanup failure") + return real_check_output(cmd, *args, **kwargs) + + monkeypatch.setattr(S, "check_output", fake_check_output) + + block = _make_block( + buttons=["run"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=True, + source_files=["main.adb"], + ) + block.project_filename = project_filename + block.project_main_file = "main.adb" + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "clean-up failures must not affect the outcome of a successful compile and run" + + # Both clean-up commands must have been reached and must have failed, + # otherwise the test proves nothing about how their failure is handled. + assert "gprclean" in failed_cleanups + assert "gnatprove" in failed_cleanups + + out = capsys.readouterr().out + shared_message = "Failed to clean-up example" + gnatprove_message = shared_message + " (gnatprove --clean)" + + assert out.count(gnatprove_message) == 1, \ + "the gnatprove --clean failure must be reported once, under a " \ + "message that names the command that failed -- three reports " \ + "spelled the same way would say that a clean-up failed and " \ + "never which one: {}".format(out) + + assert out.count(shared_message) - out.count(gnatprove_message) == 2, \ + "both gprclean failures -- the one before the build and the one " \ + "in the end-of-check clean-up -- must be reported, and the two " \ + "print the same text, so only their number tells that neither " \ + "has gone: {}".format(out) + + assert out.rindex(shared_message) == out.index(gnatprove_message), \ + "the gnatprove --clean report belongs to the end-of-check " \ + "clean-up and must therefore come after both gprclean reports: " \ + "{}".format(out) + + assert out.count("simulated cleanup failure") == 3, \ + "each report must carry the output of the command it is about, " \ + "which is the part that says why the clean-up failed: {}".format(out) + + +@pytest.mark.toolchain +class TestCheckBlockCCleanupFailure: + """A real C compile and run that both succeed, while the rm -f clean-up + command is made to fail.""" + + VALID_C_SOURCE = "int main(void) { return 0; }\n" + + def test_rm_cleanup_failure_does_not_affect_result(self, work_dir, monkeypatch, capsys): + """An rm -f clean-up failure after a successful C compile and run is + logged, but it does not abort the check or change its result.""" + import subprocess as S + + src = work_dir / "main.c" + src.write_text(self.VALID_C_SOURCE) + + real_check_output = S.check_output + + def fake_check_output(cmd, *args, **kwargs): + if cmd[0] == "rm": + raise S.CalledProcessError(1, cmd, output=b"simulated rm failure") + return real_check_output(cmd, *args, **kwargs) + + monkeypatch.setattr(S, "check_output", fake_check_output) + + block = _make_block( + language="c", + buttons=["run"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=True, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is False, \ + "an rm -f clean-up failure must not affect the outcome of a successful compile and run" + + assert "Failed to clean-up example" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# A run with no executable to run +# Covers the run step finding nothing to execute, in both languages, with and +# without the class that declares a failing run to be expected. +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockMissingExecutable: + """A run whose executable is gone by the time the run starts. + + The build itself reports success and the executable is removed + afterwards, which is the state the run step has to survive. It used to + escape as a bare FileNotFoundError, which is worse than a failing check + in two separate ways: the run is never recorded at all, and the exception + leaves the whole command, so every block queued behind this one goes + unchecked as well. + + The two languages carry the same handling in two separate places, so both + are exercised: dropping either one has to redden something. + """ + + VALID_C_SOURCE = """\ +#include + +int main(void) +{ + printf("the C example ran\\n"); + return 0; +} +""" + + @staticmethod + def _remove_after(monkeypatch, produced_by, executable): + """Let the build run for real, then take its executable away. + + ``produced_by`` decides which command line is the one that links, so + that neither the toolchain version probes nor the syntax check -- which + invoke the same compiler -- is mistaken for it. + """ + import subprocess as S + + real_check_output = S.check_output + + def fake_check_output(cmd, *args, **kwargs): + output = real_check_output(cmd, *args, **kwargs) + if produced_by(list(cmd)): + assert os.path.isfile(executable), \ + "the build must really have produced {}, or the run has " \ + "nothing to lose".format(executable) + os.remove(executable) + return output + + monkeypatch.setattr(S, "check_output", fake_check_output) + + def _ada_block(self, work_dir, classes): + (work_dir / "main.adb").write_text(MINIMAL_ADA_SOURCE) + project_filename = ep.write_project_file( + main_file="main.adb", + compiler_switches=["-gnata"], + spark_mode=False, + ) + block = _make_block( + classes=classes, + buttons=["run"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=True, + source_files=["main.adb"], + ) + block.project_filename = project_filename + block.project_main_file = "main.adb" + return block + + def _c_block(self, work_dir, classes): + (work_dir / "main.c").write_text(self.VALID_C_SOURCE) + block = _make_block( + language="c", + classes=classes, + buttons=["run"], + syntax_only=False, + no_check=False, + compile_it=True, + run_it=True, + source_files=["main.c"], + ) + block.project_main_file = "main.c" + return block + + @pytest.mark.parametrize("language", ["ada", "c"]) + def test_a_missing_executable_is_reported_and_recorded( + self, language, work_dir, monkeypatch, capsys): + """A run with no executable to run must be reported as a failed run, + and must leave a failed run recorded behind it. + + Both halves matter. Returning rather than raising is what lets the + command go on to the blocks after this one. Recording the run is + what keeps the phase a check writes down honest: the run was + attempted, it failed, and the record has to say so -- a run step that + reported the failure but wrote no RUN phase would leave a block whose + record cannot be told apart from one that was never asked to run. + """ + if language == "ada": + block = self._ada_block(work_dir, []) + self._remove_after(monkeypatch, + lambda cmd: cmd[0] == "gprbuild", "main") + else: + block = self._c_block(work_dir, []) + self._remove_after( + monkeypatch, + lambda cmd: cmd[0] == "gcc" and "-o" in cmd, "main") + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "a run with no executable must be reported as a failure rather " \ + "than leave the check as an exception" + + out = capsys.readouterr().out + assert "no executable to run" in out, \ + "the report must say what was missing, or it is indistinguishable " \ + "from the example itself failing at run time: {}".format(out) + + recorded = json.loads( + _check_record(work_dir, json_file).read_text())["checks"] + assert recorded["RUN"]["status_ok"] is False, \ + "the run was attempted and failed, so it must be recorded as a " \ + "failed run: {}".format(sorted(recorded)) + + @pytest.mark.parametrize("language,expect_failure_class", + [("ada", "ada-run-expect-failure"), + ("c", "c-run-expect-failure")]) + def test_an_expected_run_failure_does_not_absorb_a_missing_executable( + self, language, expect_failure_class, work_dir, monkeypatch, + capsys): + """A block declaring that its run is expected to fail must still be + reported when there is no executable to run. + + The class says the author expects the example to fail when it runs. + Nothing ran here: the checker did not produce the program it was + supposed to run, which is a defect on the checker's side of the line + and not the failure the block declared. Absorbing it would let a + block carrying that class pass over an example that was never built. + """ + if language == "ada": + block = self._ada_block(work_dir, [expect_failure_class]) + self._remove_after(monkeypatch, + lambda cmd: cmd[0] == "gprbuild", "main") + else: + block = self._c_block(work_dir, [expect_failure_class]) + self._remove_after( + monkeypatch, + lambda cmd: cmd[0] == "gcc" and "-o" in cmd, "main") + + json_file = str(work_dir / "block_info.json") + block.to_json_file(json_file) + + result = ccb.check_block(block, json_file, force_checks=True) + assert result is True, \ + "the class expects the example to fail, not the executable to be " \ + "missing, so this must still be reported" + + assert "no executable to run" in capsys.readouterr().out, \ + "the report must name what was missing rather than read as the " \ + "expected run failure the block declared" + + +# --------------------------------------------------------------------------- +# check_block() driven by the real extraction step +# Requires the Ada toolchain (real gnatchop, gprbuild and gnatprove runs). +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockDrivenByTheExtractor: + """check_block() started from what the extraction step really wrote. + + Every other check_block() test in this file assembles a CodeBlock in + memory and then pokes the fields the checker reads -- project_filename, + spark_project_filename, project_main_file, source_files -- into the shape + the path under test needs. That verifies the checker against a state the + extraction step may never produce, so a disagreement between the two + halves about what is written, and about what it contains, stays invisible. + + These tests run the whole chain instead: the RST directive an author types + is parsed, the extraction step chops the block and writes the project + files and the block info beside it, and the check is then started from + that block info exactly as the command line starts it. Nothing is + adjusted in between. + + The trade-off is deliberate: a hand-built block is independent of the + extraction step, and these are not. So the assertions are chosen to fail + when the two halves disagree: + + * the set of phases the check recorded must be exactly the set the + directive's button calls for -- this is the assertion with real + detection power, and it is also the one that pins the phase labels + ("SYNTAX", "BUILD", "RUN", "PROVE", "BUTTONS") as literals. That pin is + a deliberate trade: the labels are the checker's own choice of name, so + renaming one reddens these tests and no others, but the recorded set is + the only observable of *which* checks actually ran, and nothing else in + the suite watches it; + * and the project file the check really used -- read back out of the + command line the check recorded -- must be configured the way that + button requires, which is what catches the extraction step writing a + project for the wrong mode. + + Known limit, so that the messages above are not read as promising more + than they deliver: the extraction step always writes its project files + under the same two names, so a checker that stopped reading the block + info's filename fields and hard-coded those same names instead would + behave identically and go undetected here. What *is* detected is the two + halves being cross-wired (a build driven from the SPARK project, or the + reverse) and a project whose contents do not match the button. + """ + + _RUN_OUTPUT = "extracted example ran" + _C_RUN_OUTPUT = "extracted C example ran" + + # The main file the directives below declare. Kept as one value because + # the tests assert that the generated project names this same file. + _MAIN = "main.adb" + _C_MAIN = "main.c" + + # A name nothing declares, so that a build has to fail on it and the + # compiler has to say so. + _MISSING_NAME = "No_Such_Procedure" + + # What tells a SPARK project apart from an ordinary one: GNATprove only + # treats the unit as SPARK because this pragma is configured in. + _SPARK_CONFIGURATION = "pragma SPARK_Mode (On);" + + # A minimal Ada program that announces itself, so that a test can tell a + # run that really happened from one that was reported as having happened. + _ADA_BODY = """\ +with Ada.Text_IO; use Ada.Text_IO; +procedure Main is +begin + Put_Line ("{}"); +end Main;""".format(_RUN_OUTPUT) + + # Syntactically valid -- so it chops and passes the syntax check -- but it + # calls something that does not exist, so the build must fail. + _BROKEN_ADA_BODY = """\ +procedure Main is +begin + {}; +end Main;""".format(_MISSING_NAME) + + _SPARK_BODY = """\ +procedure Main with SPARK_Mode is +begin + null; +end Main;""" + + # A C block declares its file names inline; the chopper reads them off the + # leading marker lines rather than calling gnatchop. + _C_BODY = """\ +!{} +#include + +int main(void) +{{ + printf("{}\\n"); + return 0; +}}""".format(_C_MAIN, _C_RUN_OUTPUT) + + # The same, for a program that announces itself and then fails. It + # prints before it fails so that a run which really happened can be told + # from one that was reported as having happened: the exit status alone + # would also be produced by no program running at all. + _C_FAIL_OUTPUT = "extracted C example ran and then failed" + + _FAILING_C_BODY = """\ +!{} +#include + +int main(void) +{{ + printf("{}\\n"); + return 1; +}}""".format(_C_MAIN, _C_FAIL_OUTPUT) + + @staticmethod + def _rst(directive: str, body: str, classes: str | None = None) -> str: + """An RST file holding exactly one code block. + + The body is indented the way an author writes it, and the explanatory + paragraph that follows is what tells the parser the block has ended. + """ + indented = "\n".join(" " + line for line in body.splitlines()) + head = directive if classes is None else \ + "{}\n :class: {}".format(directive, classes) + return "{}\n\n{}\n\nExplanatory paragraph.\n".format(head, indented) + + def _extract(self, work_dir, directive: str, body: str, project: str, + classes: str | None = None): + """Run the real extraction step on a one-block RST file. + + Returns the per-block directory it wrote, the block info the checker + will be handed, and the absolute path of that block info file. + + The per-block directory is found by asking the extraction step where + it puts a project, and then by which directory below it holds a block + record -- the staging copy the extraction step keeps alongside holds + none. The record is taken as the one JSON file in that directory + rather than by a name written down here, so that what is read back is + whatever the extraction step wrote. + """ + rst_path = work_dir / "extracted.rst" + rst_path.write_text(self._rst(directive, body, classes)) + + assert ep.analyze_file(str(rst_path)) is False, \ + "the fixture must extract cleanly, or the check that follows is " \ + "not being handed a well-formed block" + + project_dir = work_dir / ep.get_project_dir(project) + block_dirs = sorted(d for d in project_dir.iterdir() + if d.is_dir() and list(d.glob("*.json"))) + assert len(block_dirs) == 1, \ + "expected exactly one per-block directory, got {}".format( + [d.name for d in block_dirs]) + block_dir = block_dirs[0] + records = sorted(block_dir.glob("*.json")) + assert len(records) == 1, \ + "expected exactly one block record, got {}".format( + [record.name for record in records]) + json_file = records[0] + return block_dir, json.loads(json_file.read_text()), str(json_file) + + @staticmethod + def _buttons_asked_for(info) -> tuple[bool, bool, bool]: + """The compile / run / prove decision the checker branches on.""" + return info["compile_it"], info["run_it"], info["prove_it"] + + @staticmethod + def _recorded_checks(block_dir, block_record) -> dict: + """The per-phase results the check wrote beside the block. + + Read straight from the file rather than through + checks.BlockCheck.from_json_file(), which drops the per-phase entries + on the way back in. + """ + return json.loads( + _check_record(block_dir, block_record).read_text())["checks"] + + @staticmethod + def _log_of(block_dir, recorded_check) -> str: + """The log a recorded phase says it wrote.""" + return (block_dir / recorded_check["logfile"]).read_text() + + @staticmethod + def _command_line_of(recorded_check) -> list[str]: + """The argument list a recorded phase really ran. + + Recorded as the printed form of the list, so it reads back as one -- + which is what lets a test assert on the switches the checker chose + rather than on the fact that something was run. + """ + return ast.literal_eval(recorded_check["cmdline"]) + + @staticmethod + def _project_used(recorded_check) -> str: + """The project file a recorded phase really ran against. + + The command line is recorded as the printed form of the argument list, + so it can be read back as one and the project taken from behind the + switch that names it -- rather than by matching a name the test would + otherwise have to know in advance. + + Only for phases that are driven by a project file: the Ada build and + the proof. A C build is a compiler command line with no project on + it, and asking this for one raises rather than returning anything. + """ + args = ast.literal_eval(recorded_check["cmdline"]) + return args[args.index("-P") + 1] + + @staticmethod + def _configuration_pragmas(block_dir, project_filename: str) -> str: + """The configuration pragmas a project file pulls in. + + Followed through the project's own reference to its pragma file, so + that a project generated for the wrong mode is caught by what it + configures rather than by what it happens to be called. + """ + project_text = (block_dir / project_filename).read_text() + named = re.search(r'for Global_Configuration_Pragmas use "([^"]+)"', + project_text) + assert named is not None, \ + "the generated project must name a configuration pragma file" + return (block_dir / named.group(1)).read_text() + + def test_compile_button_block_is_built_as_extracted(self, work_dir): + """A compile button carries from the RST directive through to a real + build with nothing adjusted in between. + + The directive asks for a compile and nothing else, so the block must + reach the checker asking for a compile and nothing else, the checker + must record a build and neither a run nor a proof, and the project it + built against must be an ordinary one naming no main -- a compile + button selects no main to link. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedCompile main={} compile_button".format( + self._MAIN), + self._ADA_BODY, "ExtractedCompile") + + assert info["source_files"] == [self._MAIN], \ + "the chopped source must be recorded, or the syntax check runs " \ + "on nothing and passes vacuously" + + assert self._buttons_asked_for(info) == (True, False, False), \ + "a compile button must reach the checker as a compile and nothing else" + + assert ccb.check_code_block_json(json_file) is False, \ + "the checker must accept the extracted block as it stands" + + recorded = self._recorded_checks(block_dir, json_file) + # Pins the checker's phase labels; see the class docstring for why + # that trade is made deliberately. + assert sorted(recorded) == ["BUILD", "BUTTONS", "SYNTAX"], \ + "a compile button must be syntax-checked and built, and neither " \ + "run nor proved" + assert recorded["BUILD"]["status_ok"] is True + + built_against = self._project_used(recorded["BUILD"]) + assert "for Main use" not in (block_dir / built_against).read_text(), \ + "a compile button selects no main, so the project built against " \ + "must name none" + assert self._SPARK_CONFIGURATION not in \ + self._configuration_pragmas(block_dir, built_against), \ + "a compile button must not be built against a SPARK-configured project" + + def test_run_button_block_is_built_and_run_as_extracted(self, work_dir): + """A run button carries from the RST directive through to the program + actually running. + + A run implies a compile, so both must be asked for and both must be + recorded. The project built against must name the main the directive + declared, or there is nothing for the builder to link. And the output + pinned below is what the author's code prints: it can only reach the + run log if the block was chopped, built from the generated project, + and executed. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedRun main={} run_button".format( + self._MAIN), + self._ADA_BODY, "ExtractedRun") + + assert info["source_files"] == [self._MAIN], \ + "the chopped source must be recorded, or the syntax check runs " \ + "on nothing and passes vacuously" + + assert self._buttons_asked_for(info) == (True, True, False), \ + "a run button must reach the checker as a run, which implies a " \ + "compile, and not as a proof" + + assert ccb.check_code_block_json(json_file) is False, \ + "the checker must accept the extracted block as it stands" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "RUN", "SYNTAX"], \ + "a run button must be syntax-checked, built and run, and not proved" + + built_against = self._project_used(recorded["BUILD"]) + # A localizer, not a detector: the run above cannot happen at all + # unless the project names a main, so this line says which link + # broke rather than being the first to notice. + assert 'for Main use ("{}");'.format(self._MAIN) in \ + (block_dir / built_against).read_text(), \ + "the project built against must name the main the directive declared" + assert self._SPARK_CONFIGURATION not in \ + self._configuration_pragmas(block_dir, built_against), \ + "a run button must not be built against a SPARK-configured project" + + assert self._log_of(block_dir, recorded["RUN"]).strip() == self._RUN_OUTPUT, \ + "the program the author wrote must be the one that ran" + + def test_prove_button_block_is_proved_as_extracted(self, work_dir): + """A prove button carries from the RST directive through to a real + proof. + + Proving needs a project configured for SPARK, which the extraction + step generates separately from the one a build would use. So the + proof must be recorded, the build must not be, and the project the + proof really ran against must be one that turns SPARK mode on -- + asserted through what that project configures, since a project + generated in the wrong mode would still be recorded under the right + field name. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedProve main={} prove_button".format( + self._MAIN), + self._SPARK_BODY, "ExtractedProve") + + assert info["source_files"] == [self._MAIN], \ + "the chopped source must be recorded, or the syntax check runs " \ + "on nothing and passes vacuously" + + assert self._buttons_asked_for(info) == (False, False, True), \ + "a prove button must reach the checker as a proof and nothing else" + + assert ccb.check_code_block_json(json_file) is False, \ + "the checker must accept the extracted block as it stands" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUTTONS", "PROVE", "SYNTAX"], \ + "a prove button must be syntax-checked and proved, and not built" + assert recorded["PROVE"]["status_ok"] is True + + proved_against = self._project_used(recorded["PROVE"]) + assert self._SPARK_CONFIGURATION in \ + self._configuration_pragmas(block_dir, proved_against), \ + "the proof must have run against a project that turns SPARK mode on" + + def test_extracted_block_that_does_not_build_fails_the_check(self, work_dir): + """A block that does not compile must be reported as an error when the + check is driven from the extraction step too. + + Without this the tests above could all pass on a seam that reports + success whatever the compiler said. The block is syntactically valid, + so it chops and passes the syntax check and only the build can fail. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedBadBuild main={} compile_button".format( + self._MAIN), + self._BROKEN_ADA_BODY, "ExtractedBadBuild") + + assert info["source_files"] == [self._MAIN], \ + "the chopped source must be recorded, or the syntax check runs " \ + "on nothing and passes vacuously" + + assert ccb.check_code_block_json(json_file) is True, \ + "an extracted block that does not compile must be reported as an error" + + recorded = self._recorded_checks(block_dir, json_file) + assert recorded["SYNTAX"]["status_ok"] is True, \ + "the block must be syntactically valid, or the build is not what failed" + assert recorded["BUILD"]["status_ok"] is False, \ + "the failure must be recorded against the build" + assert self._MISSING_NAME in self._log_of(block_dir, recorded["BUILD"]), \ + "the build log must name what the compiler could not resolve" + + def test_extracted_block_expecting_a_compile_error_passes(self, work_dir): + """A block declared as expecting a compile error must pass the check + even though the compiler rejects it. + + The class that declares the expectation is written in the RST source, + so it has to survive extraction and reach the checker; if it did not, + this block would be reported as a failure. The build log is checked + as well, because a class that suppressed the build entirely would give + the same answer for the wrong reason. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedExpectError main={} compile_button".format( + self._MAIN), + self._BROKEN_ADA_BODY, "ExtractedExpectError", + classes="ada-expect-compile-error") + + assert info["source_files"] == [self._MAIN], \ + "the chopped source must be recorded, or the syntax check runs " \ + "on nothing and passes vacuously" + + assert "ada-expect-compile-error" in info["classes"], \ + "the class written in the RST source must reach the checker" + + assert ccb.check_code_block_json(json_file) is False, \ + "a compile error the block declared it expects must not fail the check" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "SYNTAX"], \ + "an expected compile error must still be syntax-checked and built" + assert recorded["BUILD"]["status_ok"] is True, \ + "a compile error the block expects must not be recorded as a failure" + assert self._MISSING_NAME in self._log_of(block_dir, recorded["BUILD"]), \ + "the compiler must really have rejected the block, or the " \ + "expectation was satisfied by nothing happening" + + def test_extracted_block_expecting_a_compile_error_that_compiles_fails( + self, work_dir): + """A block declared as expecting a compile error must fail the check + when the compiler accepts it. + + The mirror of the test above, and the one the checker exists for: an + example marked "this must not compile" that quietly starts compiling + is exactly what nobody notices by hand. Driven through the real + directive and the real extraction step, so the class has to survive + both to reach the checker -- and the build is asserted to have + succeeded, since a block that failed to build for some unrelated + reason would also fail the check and would say nothing about the + expectation. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedExpectErrorThatCompiles " + "main={} compile_button".format(self._MAIN), + self._ADA_BODY, "ExtractedExpectErrorThatCompiles", + classes="ada-expect-compile-error") + + assert "ada-expect-compile-error" in info["classes"], \ + "the class written in the RST source must reach the checker" + + assert ccb.check_code_block_json(json_file) is True, \ + "a block declaring a compile error it did not produce must be " \ + "reported as an error" + + recorded = self._recorded_checks(block_dir, json_file) + assert recorded["BUILD"]["status_ok"] is True, \ + "the block must really have compiled, or the failure under test " \ + "is not the missing compile error" + assert recorded["BUTTONS"]["status_ok"] is False, \ + "the unmet expectation must be recorded against the block's " \ + "declarations" + + def test_extracted_c_block_expecting_a_compile_error_that_compiles_fails( + self, work_dir): + """A C block declared as expecting a compile error must fail the check + when the compiler accepts it. + + The C half of the promise the test above pins for Ada, driven the same + way. Only the Ada half was ever enforced, so a C example marked "this + must not compile" that quietly started compiling was reported as a + success -- the one direction of the six expect-error declarations that + nothing watched. The class is written in the RST source, so it has to + survive the directive and the extraction step to reach the checker, + and the build is asserted to have succeeded, since a block that failed + to build for some unrelated reason would also fail the check and would + say nothing about the expectation. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCExpectErrorThatCompiles " + "main={} compile_button".format(self._C_MAIN), + self._C_BODY, "ExtractedCExpectErrorThatCompiles", + classes="c-expect-compile-error") + + assert "c-expect-compile-error" in info["classes"], \ + "the class written in the RST source must reach the checker" + + assert ccb.check_code_block_json(json_file) is True, \ + "a C block declaring a compile error it did not produce must be " \ + "reported as an error" + + recorded = self._recorded_checks(block_dir, json_file) + assert recorded["BUILD"]["status_ok"] is True, \ + "the block must really have compiled, or the failure under test " \ + "is not the missing compile error" + assert recorded["BUTTONS"]["status_ok"] is False, \ + "the unmet expectation must be recorded against the block's " \ + "declarations" + + def test_extracted_block_expecting_a_prove_error_that_proves_fails( + self, work_dir): + """A block declared as expecting a prove error must fail the check + when the prover is satisfied. + + The proof half of the same promise, driven the same way. The proof is + asserted to have succeeded and to have run against a project that + turns SPARK mode on, so a proof that was never really attempted -- or + one attempted against a project the prover treats as ordinary Ada -- + cannot pass for a proof that found nothing to complain about. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedExpectProveErrorThatProves " + "main={} prove_button".format(self._MAIN), + self._SPARK_BODY, "ExtractedExpectProveErrorThatProves", + classes="ada-expect-prove-error") + + assert "ada-expect-prove-error" in info["classes"], \ + "the class written in the RST source must reach the checker" + + assert ccb.check_code_block_json(json_file) is True, \ + "a block declaring a prove error it did not produce must be " \ + "reported as an error" + + recorded = self._recorded_checks(block_dir, json_file) + assert recorded["PROVE"]["status_ok"] is True, \ + "the proof must really have succeeded, or the failure under test " \ + "is not the missing prove error" + assert recorded["BUTTONS"]["status_ok"] is False, \ + "the unmet expectation must be recorded against the block's " \ + "declarations" + + proved_against = self._project_used(recorded["PROVE"]) + assert self._SPARK_CONFIGURATION in \ + self._configuration_pragmas(block_dir, proved_against), \ + "the proof must have run against a project that turns SPARK mode on" + + def test_c_run_button_block_is_built_and_run_as_extracted(self, work_dir): + """A run button on a C block carries through to the program running. + + C blocks take a different route on both sides of the seam: the + extraction step chops them from the file names written into the source + rather than by calling gnatchop, and the checker compiles and links + them with the C compiler instead of the project builder. The output + pinned below is what the author's code prints. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCRun main={} run_button".format( + self._C_MAIN), + self._C_BODY, "ExtractedCRun") + + assert self._buttons_asked_for(info) == (True, True, False), \ + "a run button must reach the checker as a run, which implies a " \ + "compile, and not as a proof" + assert info["source_files"] == [self._C_MAIN], \ + "the C source must have been chopped out under the name the block " \ + "declares for it" + + assert ccb.check_code_block_json(json_file) is False, \ + "the checker must accept the extracted C block as it stands" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "RUN", "SYNTAX"], \ + "a C run button must be syntax-checked, built and run, and not proved" + assert self._log_of(block_dir, recorded["RUN"]).strip() == self._C_RUN_OUTPUT, \ + "the program the author wrote must be the one that ran" + + # A block that is run has a main file resolved for it, and that is the + # arm of the C compile step which links an executable and names it. + # The sibling compile-button test takes the other arm, so both are + # pinned and neither can be made to serve the other's case unnoticed. + built_with = self._command_line_of(recorded["BUILD"]) + assert built_with[:3] == ["gcc", "-o", os.path.splitext(self._C_MAIN)[0]], \ + "a C block with a resolved main must be linked into an executable " \ + "named after that main: {}".format(built_with) + assert "-c" not in built_with, \ + "a C block with a resolved main must be linked, not merely " \ + "compiled: {}".format(built_with) + assert self._C_MAIN in built_with, \ + "the chopped source must be on the command line, or nothing was " \ + "compiled: {}".format(built_with) + + def test_c_compile_button_block_is_built_as_extracted(self, work_dir): + """A compile button on a C block must be compiled. + + Driven by the real extraction step, so the block arrives at the + checker with exactly the fields extraction gives it. The directive + names a main, but extraction resolves a project main file only for + blocks that are also run, so the checker gets none. The C compile + step therefore has to build such a block without an executable to + name: it compiles without linking, which is what a compile button + asks for and the only thing a block holding no main can do at all. + The sibling Ada compile test pins the generated project as naming no + main, so resolving a main for every compiled block is not an + available alternative. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCCompile main={} compile_button".format( + self._C_MAIN), + self._C_BODY, "ExtractedCCompile") + + assert self._buttons_asked_for(info) == (True, False, False), \ + "a compile button must reach the checker as a compile and nothing else" + + assert ccb.check_code_block_json(json_file) is False, \ + "the checker must accept the extracted C block as it stands" + + assert info["project_main_file"] is None, \ + "extraction must leave a compile-only block with no main file " \ + "resolved, or this is not the arm of the compile step under test" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "SYNTAX"], \ + "a C compile button must be syntax-checked and built, and neither " \ + "run nor proved" + assert recorded["BUILD"]["status_ok"] is True + + built_with = self._command_line_of(recorded["BUILD"]) + assert "-c" in built_with, \ + "a compile button asks for a compile and not a link: {}".format( + built_with) + assert "-o" not in built_with, \ + "nothing is being linked, so no executable may be named -- naming " \ + "one is what used to stop the check on an assertion: {}".format( + built_with) + assert self._C_MAIN in built_with, \ + "the chopped source must be on the command line, or nothing was " \ + "compiled: {}".format(built_with) + + # The C run classes an author writes, driven the same way. These are the + # only tests in the file that reach the run path of a C block without a + # run button: every other one either writes the button into the directive + # or hands check_block() a block with run_it already set, and a block that + # arrives with the decision already made cannot show how it was reached. + # That is why a green suite said nothing while a C block asking to be run + # by class alone was never run and the check reported success over it. + + def test_c_run_class_block_is_built_and_run_as_extracted(self, work_dir): + """A C block classed ``c-run`` and carrying no button must be run. + + The class is the whole of what asks for the run here -- the directive + declares ``no_button`` -- so the phase set below is the assertion + with the detection power, and the run log is what says the author's + own program is what executed rather than the run being recorded over + nothing. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCRunClass main={} no_button".format( + self._C_MAIN), + self._C_BODY, "ExtractedCRunClass", classes="c-run") + + assert info["buttons"] == ["no"], \ + "the block must carry no button, or the class is not what asked " \ + "for the run" + assert self._buttons_asked_for(info) == (True, True, False), \ + "a c-run class must reach the checker as a run, which implies a " \ + "compile, and not as a proof" + + assert ccb.check_code_block_json(json_file) is False, \ + "the checker must accept the extracted C block as it stands" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "RUN", "SYNTAX"], \ + "a c-run class must be syntax-checked, built and run, and not proved" + assert recorded["RUN"]["status_ok"] is True + assert self._log_of(block_dir, recorded["RUN"]).strip() == \ + self._C_RUN_OUTPUT, \ + "the program the author wrote must be the one that ran" + + def test_c_run_expect_failure_class_block_is_run_without_a_button( + self, work_dir): + """A C block classed ``c-run-expect-failure`` and carrying no button + must be run, and its failure must be the expected one. + + This is the case a green suite passed over. The checker has long + held a branch that absorbs a failing C run when the block declares it + expects one, but nothing made such a block run from the class alone, + so that branch was reachable only through a run button and the class + on its own bought the block nothing. + + Three things are asserted together, because any two of them are + satisfied by a block that was never run: the run must be recorded, + the program's own output must be in the run log, and the check must + pass even though the program exited non-zero. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCExpectFailure main={} no_button".format( + self._C_MAIN), + self._FAILING_C_BODY, "ExtractedCExpectFailure", + classes="c-run-expect-failure") + + assert info["buttons"] == ["no"], \ + "the block must carry no button, or the class is not what asked " \ + "for the run" + assert self._buttons_asked_for(info) == (True, True, False), \ + "a c-run-expect-failure class must reach the checker as a run, " \ + "which implies a compile, and not as a proof" + + assert ccb.check_code_block_json(json_file) is False, \ + "a run failure the block declared it expects must not fail the check" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "RUN", "SYNTAX"], \ + "the block must really have been run, not merely built and " \ + "reported as passing" + assert recorded["RUN"]["status_ok"] is True, \ + "a failure the block expects must be recorded as a passing run" + assert self._log_of(block_dir, recorded["RUN"]).strip() == \ + self._C_FAIL_OUTPUT, \ + "the program the author wrote must be the one that ran and failed" + + def test_c_norun_class_suppresses_the_run_of_an_extracted_block( + self, work_dir): + """A C block classed ``c-norun`` must not be run, whatever the + directive asks for. + + The mirror of the two above: the class has to be able to take a run + away as well as ask for one, or it is decoration on a block that was + going to be run anyway. The directive carries a compile button + beside the run button, so that the compile survives the suppression + and the block is still built -- which isolates what was suppressed to + the run, and would catch a suppression that quietly stopped the whole + check instead. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCNoRun main={} compile_button " + "run_button".format(self._C_MAIN), + self._C_BODY, "ExtractedCNoRun", classes="c-norun") + + assert "run" in info["buttons"], \ + "the block must carry the run button the class has to suppress" + assert self._buttons_asked_for(info) == (True, False, False), \ + "c-norun must take the run away and leave the compile the " \ + "directive asked for separately" + + assert ccb.check_code_block_json(json_file) is False, \ + "the checker must accept the extracted C block as it stands" + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "SYNTAX"], \ + "a suppressed run must not be recorded as having happened" + assert recorded["BUILD"]["status_ok"] is True, \ + "suppressing the run must not suppress the build as well" + assert not (block_dir / "run.log").exists(), \ + "nothing may have been run, so no run log may have been written" + + def test_a_c_block_classed_ada_run_is_neither_built_nor_run( + self, work_dir): + """A C block classed ``ada-run``, with no button anywhere, must not be + built, must not be run, and must fail the check. + + This is the whole shape of the problem, driven from the directive an + author would really write. The class names Ada, so it asks this + block for nothing; the block carries no button to ask instead; and + the source is never handed to a compiler. + + What the check then does about it is the subject of the test below, + deliberately separated: this one would pass just as well if the block + were quietly accepted, and saying so is the point -- it is about what + was and was not done to the block, and nothing else. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCAdaRunClass main={} no_button".format( + self._C_MAIN), + self._C_BODY, "ExtractedCAdaRunClass", classes="ada-run") + + assert info["buttons"] == ["no"], \ + "the block must carry no button, or something other than the " \ + "class is deciding whether it is run" + assert self._buttons_asked_for(info) == (False, False, False), \ + "a class naming the other language must ask for nothing" + + ccb.check_code_block_json(json_file) + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUTTONS", "SYNTAX"], \ + "the block must have been syntax-checked and nothing more" + assert not (block_dir / "build.log").exists(), \ + "nothing was compiled, so no build log may have been written" + assert not (block_dir / "run.log").exists(), \ + "nothing was run, so no run log may have been written" + + def test_a_c_block_classed_ada_run_fails_the_check_and_the_record( + self, work_dir): + """The same extracted block must fail the check, and must be recorded + as having failed. + + The wrapper level, and the one place the on-disk record is read as + evidence. Neither is covered by asserting what was printed: a report + that printed and handed back success is a defect this package has + already shipped once, in the extraction step's own wrong-language + report, so it is a live possibility rather than a hypothetical. + + The record matters on its own account. It is what the next run reads + to decide the block can be skipped, so a run that fails while + recording success does not merely mislead once -- it tells every + later run that the block was checked and passed. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: c project=ExtractedCAdaRunStatus main={} no_button".format( + self._C_MAIN), + self._C_BODY, "ExtractedCAdaRunStatus", classes="ada-run") + + assert ccb.check_code_block_json(json_file) is True, \ + "a block nothing was done to must not be reported as checked" + + recorded = self._recorded_checks(block_dir, json_file) + assert recorded["BUTTONS"]["status_ok"] is False, \ + "the objection must be recorded against the block's declarations" + record = json.loads( + _check_record(block_dir, json_file).read_text()) + assert record["status_ok"] is False, \ + "the record left beside the block is read back by the next run " \ + "as a result to skip on, so it must not say the block passed" + + def test_an_ada_block_classed_c_norun_is_still_built_and_run( + self, work_dir): + """An Ada block classed ``c-norun`` and carrying a run button must + still be run. + + The mirror direction, and the one where the unpaired reading used to + take something away: a stray C norun canceled the run, and with it + the build, leaving an Ada example that was never compiled. The run + log is what says the author's own program executed rather than a run + being recorded over nothing. + """ + block_dir, info, json_file = self._extract( + work_dir, + ".. code:: ada project=ExtractedAdaCNoRun main={} run_button".format( + self._MAIN), + self._ADA_BODY, "ExtractedAdaCNoRun", classes="c-norun") + + assert "run" in info["buttons"], \ + "the block must carry the run button the class must not suppress" + assert self._buttons_asked_for(info) == (True, True, False), \ + "a norun class naming the other language must take nothing away" + + # The check is run for its effects, and its returned value is + # deliberately not asserted: the stray class is separately reported, + # so the value says something about the report rather than about the + # run this test is here for. + ccb.check_code_block_json(json_file) + + recorded = self._recorded_checks(block_dir, json_file) + assert sorted(recorded) == ["BUILD", "BUTTONS", "RUN", "SYNTAX"], \ + "the block must have been built and run" + assert recorded["RUN"]["status_ok"] is True + assert self._log_of(block_dir, recorded["RUN"]).strip() == \ + self._RUN_OUTPUT, \ + "the program the author wrote must be the one that ran" diff --git a/frontend/python/rst_code_example_pipeline/tests/test_check_projects.py b/frontend/python/rst_code_example_pipeline/tests/test_check_projects.py new file mode 100644 index 000000000..fa4bebdca --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_check_projects.py @@ -0,0 +1,665 @@ +""" +Unit tests for rst_code_example_pipeline.check_projects. + +Covers: +- get_blocks([]) → empty dict +- get_blocks() with a valid block_info.json present → dict with one project entry +- get_blocks() with a block_info.json missing the project field → skips, dict empty +- get_projects(build_dir, projects_list_file=None) with no JSON files → empty dict +- get_projects(build_dir, projects_list_file) with a valid projects-list JSON +- cwd side effect: get_projects calls os.chdir(build_dir) — fixture saves/restores cwd +- check_projects() returns True when a block fails to compile (requires the Ada toolchain) +- a block dropped before it is checked -- its info file unreadable, or naming no + project -- fails the run, each arm pinned separately, and neither costs the + remaining blocks their check +""" +import os + +import pytest + +import rst_code_example_pipeline.check_projects as cp +import rst_code_example_pipeline.extract_projects as ep +from rst_code_example_pipeline import blocks as _blocks_mod +import rst_code_example_pipeline.toolchain_info as info + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +def _write_block_record(block, directory) -> str: + """Write a block's record into ``directory`` under the name the package + chooses for it, and hand back the path it landed at. + + The reader builds its search pattern from its own default name, so a test + that spelled the name out here would be restating a choice the package is + free to change -- and would keep passing if writer and reader ever drifted + apart, since both sides of the seam would have been replaced by the test's + own copy of the name. + """ + directory.mkdir(parents=True, exist_ok=True) + original_cwd = os.getcwd() + os.chdir(str(directory)) + try: + block.to_json_file() + finally: + os.chdir(original_cwd) + written = sorted(directory.glob("*.json")) + assert len(written) == 1, \ + "expected exactly one block record to be written, got {}".format( + [path.name for path in written]) + return str(written[0]) + + +def _make_minimal_block_info(project: str, + tmp_path, + subdir: str = "") -> str: + """ + Write a minimal block_info.json for the given project into tmp_path (or a + subdir of it) and return the absolute path to the JSON file. + """ + # Ensure toolchain_info is initialized + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + + block = _blocks_mod.CodeBlock( + rst_file="test.rst", + line_start=1, + line_end=5, + text="procedure Main is begin null; end Main;", + language="ada", + project=project, + main_file=None, + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=["-gnata"], + classes=["ada-nocheck"], + manual_chop=False, + buttons=["no"], + ) + dest_dir = tmp_path / subdir if subdir else tmp_path + return _write_block_record(block, dest_dir) + + +# --------------------------------------------------------------------------- +# T-check_projects-01: get_blocks() with empty list +# --------------------------------------------------------------------------- + +class TestGetBlocksEmpty: + def test_empty_regex_list_returns_empty_dict(self): + result = cp.get_blocks([]) + assert result == {} + + +# --------------------------------------------------------------------------- +# T-check_projects-02: get_blocks() with a valid block_info.json +# --------------------------------------------------------------------------- + +class TestGetBlocksValid: + def test_one_project_found(self, tmp_path): + json_file = _make_minimal_block_info("MyProject", tmp_path) + result = cp.get_blocks([json_file]) + assert "MyProject" in result + + def test_project_entry_has_one_tuple(self, tmp_path): + json_file = _make_minimal_block_info("MyProject", tmp_path) + result = cp.get_blocks([json_file]) + # A list, not just any sized container: callers append to it as further + # block files for the same project are found. + entry = result["MyProject"] + assert isinstance(entry, list) + assert len(entry) == 1 + + def test_tuple_contains_codeblock_and_path(self, tmp_path): + json_file = _make_minimal_block_info("MyProject", tmp_path) + result = cp.get_blocks([json_file]) + block, path = result["MyProject"][0] + assert isinstance(block, _blocks_mod.CodeBlock) + assert path == json_file + + def test_glob_pattern_finds_file(self, tmp_path): + written = _make_minimal_block_info("GlobProject", tmp_path, subdir="subdir") + pattern = str(tmp_path / "**" / os.path.basename(written)) + result = cp.get_blocks([pattern]) + assert "GlobProject" in result + + def test_two_projects_from_two_files(self, tmp_path): + written = _make_minimal_block_info("Project1", tmp_path, subdir="p1") + _make_minimal_block_info("Project2", tmp_path, subdir="p2") + pattern = str(tmp_path / "**" / os.path.basename(written)) + result = cp.get_blocks([pattern]) + assert "Project1" in result + assert "Project2" in result + + +# --------------------------------------------------------------------------- +# T-check_projects-03: get_blocks() with missing project field +# --------------------------------------------------------------------------- + +class TestGetBlocksMissingProject: + def test_missing_project_field_skipped(self, tmp_path, capsys): + """A block_info.json whose block has project=None must be skipped.""" + # Ensure toolchain_info is initialized + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + + block = _blocks_mod.CodeBlock( + rst_file="test.rst", + line_start=1, + line_end=5, + text="procedure Main is begin null; end Main;", + language="ada", + project=None, # <-- no project + main_file=None, + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=["-gnata"], + classes=["ada-nocheck"], + manual_chop=False, + buttons=["no"], + ) + json_file = str(tmp_path / "block_info.json") + block.to_json_file(json_file) + + result = cp.get_blocks([json_file]) + assert result == {}, "Block with project=None must be skipped" + + def test_missing_project_prints_error(self, tmp_path, capsys): + """When project is None, an ERROR message must be printed.""" + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + + block = _blocks_mod.CodeBlock( + rst_file="test.rst", + line_start=1, + line_end=5, + text="stub", + language="ada", + project=None, + main_file=None, + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=[], + classes=[], + manual_chop=False, + buttons=["no"], + ) + json_file = str(tmp_path / "block_info.json") + block.to_json_file(json_file) + + cp.get_blocks([json_file]) + captured = capsys.readouterr() + assert "ERROR" in captured.out + + +# --------------------------------------------------------------------------- +# T-check_projects-04: get_projects() without projects_list_file +# --------------------------------------------------------------------------- + +class TestGetProjectsNoPrjList: + def test_empty_build_dir_returns_empty_dict(self, tmp_path): + result = cp.get_projects(str(tmp_path), projects_list_file=None) + assert result == {} + + def test_cwd_changed_to_build_dir(self, tmp_path): + cp.get_projects(str(tmp_path), projects_list_file=None) + # After the call, cwd should have been set to tmp_path by get_projects + # (our restore_cwd fixture will reset it after the test, but within the + # test we can verify it was changed) + assert os.getcwd() == str(tmp_path) + + def test_block_info_in_build_dir_found(self, tmp_path): + _make_minimal_block_info("AutoProject", tmp_path, subdir="projects/AutoProject/hash1") + result = cp.get_projects(str(tmp_path), projects_list_file=None) + assert "AutoProject" in result + + +# --------------------------------------------------------------------------- +# T-check_projects-05: get_projects() with projects_list_file +# --------------------------------------------------------------------------- + +class TestGetProjectsWithPrjList: + def test_with_valid_projects_list_returns_project(self, tmp_path): + # Create a project directory and block_info.json + project_name = "ListedProject" + subdir = ep.get_project_dir(project_name) + "/hash123" + _make_minimal_block_info(project_name, tmp_path, subdir=subdir) + + # Create a ProjectsList JSON + pl = ep.ProjectsList() + pl.add(project_name) + prj_list_file = str(tmp_path / "projects.json") + pl.to_json_file(prj_list_file) + + result = cp.get_projects(str(tmp_path), projects_list_file=prj_list_file) + assert project_name in result + + def test_with_empty_projects_list_returns_empty(self, tmp_path): + pl = ep.ProjectsList() + prj_list_file = str(tmp_path / "empty_projects.json") + pl.to_json_file(prj_list_file) + + result = cp.get_projects(str(tmp_path), projects_list_file=prj_list_file) + assert result == {} + + def test_cwd_changed_to_build_dir_with_prj_list(self, tmp_path): + prj_list_file = str(tmp_path / "projects.json") + pl = ep.ProjectsList() + pl.to_json_file(prj_list_file) + + cp.get_projects(str(tmp_path), projects_list_file=prj_list_file) + assert os.getcwd() == str(tmp_path) + + def test_missing_prj_list_file_prints_warning(self, tmp_path, capsys): + """When projects_list_file does not exist, from_json_file returns None + and get_projects must print a WARNING.""" + missing_file = str(tmp_path / "no_such_projects.json") + cp.get_projects(str(tmp_path), projects_list_file=missing_file) + captured = capsys.readouterr() + assert "WARNING" in captured.out + + +# --------------------------------------------------------------------------- +# T-check_projects-06: check_block() thin wrapper +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckBlockWrapper: + def test_no_check_block_returns_false(self, tmp_path): + """check_block() delegates to check_code_block.check_block(); a + no-check block must return False (no error).""" + json_file = _make_minimal_block_info("WrapProject", tmp_path) + # Load the block from JSON (it has no_check=True from the ada-nocheck class) + block = _blocks_mod.CodeBlock.from_json_file(json_file) + assert block is not None + os.chdir(str(tmp_path)) + result = cp.check_block(block, json_file) + assert result is False + + +# --------------------------------------------------------------------------- +# T-check_projects-07: check_projects() integration +# --------------------------------------------------------------------------- + +class TestCheckProjectsIntegration: + @pytest.mark.toolchain + def test_check_projects_with_nocheck_block_returns_false(self, tmp_path): + """check_projects() iterates over all blocks in the build dir and calls + check_block(). A build dir with only no-check blocks must return False.""" + subdir = "projects/MyProj/abc123" + json_file = _make_minimal_block_info("MyProj", tmp_path, subdir=subdir) + result = cp.check_projects(str(tmp_path), projects_list_file=None) + assert result is False + + def test_check_projects_empty_build_dir_returns_false(self, tmp_path): + """check_projects() on an empty build dir (no block_info.json files) + must return False (no errors).""" + result = cp.check_projects(str(tmp_path), projects_list_file=None) + assert result is False + + +# --------------------------------------------------------------------------- +# T-check_projects-08: extended coverage — malformed JSON, verbose, inactive, +# duplicate project +# --------------------------------------------------------------------------- + +class TestCheckProjectsExtended: + def test_get_blocks_from_json_file_returns_none(self, tmp_path, capsys, monkeypatch): + """When from_json_file() returns None, get_blocks() prints ERROR and + skips the entry (exercises the None-block error path in get_blocks).""" + # Write a valid block_info.json so iglob finds the file + json_file = _make_minimal_block_info("NullProject", tmp_path) + + # Patch from_json_file to return None regardless of content + monkeypatch.setattr(_blocks_mod.CodeBlock, "from_json_file", + staticmethod(lambda *args, **kwargs: None)) + + result = cp.get_blocks([json_file]) + assert result == {}, "Expected empty dict when from_json_file returns None" + out = capsys.readouterr().out + assert "ERROR" in out, "Expected ERROR printed when block cannot be loaded" + + def test_get_blocks_duplicate_project(self, tmp_path): + """Two block_info.json files with the same project name: the second block + appends to the existing project entry rather than creating a new key.""" + # Write two files for the same project in different subdirs + written = _make_minimal_block_info("DupProject", tmp_path, subdir="a") + _make_minimal_block_info("DupProject", tmp_path, subdir="b") + pattern = str(tmp_path / "**" / os.path.basename(written)) + result = cp.get_blocks([pattern]) + # Both blocks are in the list under the same project key + assert "DupProject" in result + assert len(result["DupProject"]) == 2, \ + "Expected both blocks accumulated under the same project key" + + @pytest.mark.toolchain + def test_get_projects_verbose(self, tmp_path, capsys): + """check_projects() with verbose=True prints the project header + (exercises the verbose header output path).""" + subdir = "projects/VerbProj/abc123" + _make_minimal_block_info("VerbProj", tmp_path, subdir=subdir) + cp.verbose = True + cp.check_projects(str(tmp_path), projects_list_file=None) + out = capsys.readouterr().out + assert "VerbProj" in out, \ + "Expected verbose project header to contain the project name" + + def test_check_projects_skips_inactive_block(self, tmp_path, monkeypatch): + """A block marked inactive must be skipped by check_projects() + without being checked at all.""" + # Build a block and serialize it with active=False + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + + block = _blocks_mod.CodeBlock( + rst_file="test.rst", + line_start=1, + line_end=5, + text="procedure Main is begin null; end Main;", + language="ada", + project="InactiveProj", + main_file=None, + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=["-gnata"], + classes=["ada-nocheck"], + manual_chop=False, + buttons=["no"], + ) + block.active = False # mark inactive before serializing + + dest_dir = tmp_path / "projects" / "InactiveProj" / "hash000" + json_file = _write_block_record(block, dest_dir) + + # Track calls to check_block + calls = [] + + original_check_block = cp.check_block + + def tracking_check_block(blk, jf): + calls.append(blk) + return original_check_block(blk, jf) + + monkeypatch.setattr(cp, "check_block", tracking_check_block) + + result = cp.check_projects(str(tmp_path), projects_list_file=None) + assert result is False, "Expected no error for inactive block" + assert len(calls) == 0, \ + "check_block must NOT be called for an inactive block" + + +# --------------------------------------------------------------------------- +# C5 — TestCheckProjectsReturnsTrue +# check_projects() must return True when check_block() returns True for a block. +# Requires the Ada toolchain (gprbuild invoked for a failing compile). +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckProjectsReturnsTrue: + """Tests that check_projects() propagates check_error=True.""" + + BAD_ADA_SOURCE = "procedure Bad is\nbegin\n SYNTAX ERROR HERE!!!\nend Bad;\n" + + def test_check_projects_returns_true_on_check_error(self, tmp_path): + """Set up a block_info.json with Ada source that fails to compile. + check_projects() must return True when check_block() reports an error.""" + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + + # Write a bad Ada source file so gprbuild will fail + src = tmp_path / "bad.adb" + src.write_text(self.BAD_ADA_SOURCE) + + # Change to tmp_path so write_project_file creates files there + original_cwd = os.getcwd() + os.chdir(str(tmp_path)) + + project_filename = ep.write_project_file( + main_file="bad.adb", + compiler_switches=[], + spark_mode=False, + ) + + # Build a CodeBlock that will trigger a compile attempt + block = _blocks_mod.CodeBlock( + rst_file="test.rst", + line_start=1, + line_end=5, + text=self.BAD_ADA_SOURCE, + language="ada", + project="FailProject", + main_file="bad.adb", + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=[], + classes=[], + manual_chop=False, + buttons=["compile"], + compile_it=True, + run_it=False, + syntax_only=False, + no_check=False, + source_files=["bad.adb"], + ) + block.project_filename = project_filename + block.project_main_file = "bad.adb" + + # Place the block_info.json in a subdirectory matching check_projects expectations + subdir = tmp_path / "projects" / "FailProject" / "hash001" + subdir.mkdir(parents=True, exist_ok=True) + + # Copy the project files into the subdir (check_block os.chdir's into json_file's dir) + import shutil + shutil.copy(str(tmp_path / project_filename), str(subdir / project_filename)) + shutil.copy(str(tmp_path / "bad.adb"), str(subdir / "bad.adb")) + # Take the configuration pragma files from what write_project_file + # actually wrote, rather than naming one the package chose. + for adc in tmp_path.glob("*.adc"): + shutil.copy(str(adc), str(subdir / adc.name)) + + json_file = _write_block_record(block, subdir) + + os.chdir(original_cwd) + + # Force checks to bypass any cached result + cp.force_checks = True + result = cp.check_projects(str(tmp_path), projects_list_file=None) + assert result is True, \ + "check_projects() must return True when a block fails to compile" + + +# --------------------------------------------------------------------------- +# Blocks that are dropped before they are ever checked +# --------------------------------------------------------------------------- + +class TestABlockThatWasNotCheckedFailsTheRun: + """The two ways a block info file is dropped from the gathering loop. + + One is a file that cannot be turned into a block; the other is a block + that names no project. Each prints an ERROR line of its own and moves on + to the next file, so neither reaches the checking loop and neither can + contribute an error from there. + + What is asserted here is the outcome of the whole run, not the ERROR line + -- the line was already printed while the run still came back clean, and + a run that reports a problem and then says it went fine is the failure + this suite exists to catch. A build gates on the outcome and nothing + else. + + The two arms are pinned separately. They reach the run's outcome through + the same list, so a single test would keep passing with either one of + them disconnected. + """ + + C_SOURCE = "int main(void) { return 0; }\n" + + def _a_readable_block(self, tmp_path, project: str, subdir: str) -> str: + """A block info file that reads back as a block, in its own + directory below the build directory.""" + return _make_minimal_block_info(project, tmp_path, subdir=subdir) + + def _an_unreadable_block(self, tmp_path, subdir: str) -> str: + """A block info file that cannot be turned into a block. + + Written by writing a real one first and then overwriting its text, so + that the file ends up under the name the gathering loop looks for + without that name being spelled out here. + """ + written = _make_minimal_block_info("Unreadable", tmp_path, + subdir=subdir) + with open(written, "w") as f: + f.write("{ this is not a block record") + return written + + def _a_block_naming_no_project(self, tmp_path, subdir: str) -> str: + """A block info file that reads back as a block naming no project. + + The extraction step refuses to write one, so this stands for a file + that was edited or written by hand afterwards. + """ + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + + block = _blocks_mod.CodeBlock( + rst_file="test.rst", + line_start=1, + line_end=5, + text="procedure Main is begin null; end Main;", + language="ada", + project=None, + main_file=None, + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + compiler_switches=["-gnata"], + classes=["ada-nocheck"], + manual_chop=False, + buttons=["no"], + ) + return _write_block_record(block, tmp_path / subdir) + + @staticmethod + def _recording_checker(monkeypatch, fails=()): + """Stand in for the per-block check and record what it was given. + + Which blocks survive the gathering loop and reach the check is what + these tests are about; what a real check would then do to them is + not, and running one would need the toolchain for no gain. Blocks + are recorded by their own file rather than by their project, so that + two blocks of one project can be told apart. ``fails`` names the + block files whose check reports an error. + """ + checked = [] + + def recording_check_block(block, json_file): + checked.append(json_file) + return json_file in fails + + monkeypatch.setattr(cp, "check_block", recording_check_block) + return checked + + def test_an_unreadable_block_info_file_fails_the_run(self, tmp_path): + """A build directory whose one block info file cannot be read must + fail the run. + + Nothing was checked, so a run that came back clean would be reporting + success over an example no one looked at. + """ + self._an_unreadable_block(tmp_path, "projects/Unreadable/hash1") + + assert cp.check_projects(str(tmp_path), projects_list_file=None) \ + is True, \ + "a block info file that could not be read must fail the run" + + def test_a_block_naming_no_project_fails_the_run(self, tmp_path): + """A build directory whose one block info file names no project must + fail the run, for the same reason: that block was never checked.""" + self._a_block_naming_no_project(tmp_path, "projects/NoProject/hash1") + + assert cp.check_projects(str(tmp_path), projects_list_file=None) \ + is True, \ + "a block that names no project must fail the run" + + def test_a_build_directory_of_readable_blocks_still_succeeds( + self, tmp_path, monkeypatch): + """Two blocks that read back and check out must come back clean. + + The control for the two tests above: without it they would still pass + if the run had simply started failing for everything. + """ + checked = self._recording_checker(monkeypatch) + readable = [ + self._a_readable_block(tmp_path, "First", "projects/First/hash1"), + self._a_readable_block(tmp_path, "Second", "projects/Second/hash2"), + ] + + assert cp.check_projects(str(tmp_path), projects_list_file=None) \ + is False, \ + "a build directory whose blocks all read back and check out must " \ + "not fail the run" + assert sorted(checked) == sorted(readable), \ + "both blocks must have been checked: {}".format(sorted(checked)) + + def test_the_other_blocks_are_still_checked(self, tmp_path, monkeypatch): + """A block info file that cannot be read must not cost the other + blocks their check. + + This is the property that decides whether reporting the file instead + of raising on it was an improvement at all. The exception it replaced + left the gathering loop before a single block had been handed to the + checker, so a build directory like this one had none of its examples + checked -- and the run still stopped, which is the only part that was + ever visible. + """ + checked = self._recording_checker(monkeypatch) + self._an_unreadable_block(tmp_path, "projects/Unreadable/hash1") + readable = [ + self._a_readable_block(tmp_path, "First", "projects/First/hash2"), + self._a_readable_block(tmp_path, "Second", "projects/Second/hash3"), + ] + + assert cp.check_projects(str(tmp_path), projects_list_file=None) \ + is True, \ + "the unreadable file must still fail the run" + assert sorted(checked) == sorted(readable), \ + "every block that could be read must still have been checked: " \ + "{}".format(sorted(checked)) + + def test_a_block_that_fails_does_not_stop_the_ones_after_it( + self, tmp_path, monkeypatch): + """A block whose check reports an error must not stop the blocks + after it from being checked. + + The other half of the same property: three of the fixes on this + branch turn an exception raised from inside the check of one block + into a reported failure, and an exception there would have taken the + remaining blocks with it just as surely as one raised while gathering + them. + + Two blocks of one project report an error and a block of another + project does not, so whichever of the two the check reaches first, + both loops it runs -- the one over a project's blocks and the one + over the projects -- still have a block left to visit after a + failure. The order the block files are found in is the file + system's, so this cannot be arranged by putting the failing one + first. + """ + failing = [ + self._a_readable_block(tmp_path, "Shared", "projects/Shared/hash1"), + self._a_readable_block(tmp_path, "Shared", "projects/Shared/hash2"), + ] + passing = self._a_readable_block(tmp_path, "Other", + "projects/Other/hash3") + checked = self._recording_checker(monkeypatch, fails=tuple(failing)) + + assert cp.check_projects(str(tmp_path), projects_list_file=None) \ + is True, \ + "a block whose check reports an error must fail the run" + assert sorted(checked) == sorted(failing + [passing]), \ + "every block must have been checked, whatever the ones before it " \ + "reported: {}".format(sorted(checked)) diff --git a/frontend/python/rst_code_example_pipeline/tests/test_checks.py b/frontend/python/rst_code_example_pipeline/tests/test_checks.py new file mode 100644 index 000000000..5afd9e4ea --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_checks.py @@ -0,0 +1,279 @@ +""" +Unit tests for rst_code_example_pipeline.checks. + +Covers: +- CodeCheck construction and defaults +- BlockCheck.__init__ stores fields; checks dict initially empty +- BlockCheck.add_check() accumulates CodeCheck entries +- BlockCheck.to_json_file() + from_json_file() round-trip +- BlockCheck.from_json_file() with nonexistent file → None +- BlockCheck.from_json_file() with explicit filename +- Adversarial: overwrite, empty JSON {}, TypeError on bad args +""" +import json +import os +import time + +import pytest + +from rst_code_example_pipeline.checks import BlockCheck, CodeCheck + + +# --------------------------------------------------------------------------- +# T-checks-01: CodeCheck defaults +# --------------------------------------------------------------------------- + +class TestCodeCheckDefaults: + def test_default_version_is_none(self): + c = CodeCheck() + assert c.version is None + + def test_default_status_ok_is_none(self): + c = CodeCheck() + assert c.status_ok is None + + def test_default_logfile_is_none(self): + c = CodeCheck() + assert c.logfile is None + + def test_default_cmdline_is_none(self): + c = CodeCheck() + assert c.cmdline is None + + def test_default_timestamp_is_recent_float(self): + before = time.time() + c = CodeCheck() + after = time.time() + assert isinstance(c.timestamp, float) + assert before <= c.timestamp <= after + + def test_explicit_timestamp(self): + c = CodeCheck(timestamp=1234567890.0) + assert c.timestamp == 1234567890.0 + + def test_all_fields_set(self): + c = CodeCheck(timestamp=1.0, version="v1.2", status_ok=True, + logfile="out.log", cmdline="gcc main.c") + assert c.timestamp == 1.0 + assert c.version == "v1.2" + assert c.status_ok is True + assert c.logfile == "out.log" + assert c.cmdline == "gcc main.c" + + +# --------------------------------------------------------------------------- +# T-checks-02: BlockCheck construction +# --------------------------------------------------------------------------- + +class TestBlockCheckInit: + def test_stores_text_hash(self): + bc = BlockCheck(text_hash="abc", text_hash_short="a") + assert bc.text_hash == "abc" + + def test_stores_text_hash_short(self): + bc = BlockCheck(text_hash="abc", text_hash_short="a") + assert bc.text_hash_short == "a" + + def test_checks_initially_empty(self): + bc = BlockCheck(text_hash="h", text_hash_short="s") + assert bc.checks == {} + + def test_checks_empty_even_when_none_passed(self): + bc = BlockCheck(text_hash="h", text_hash_short="s", checks=None) + assert bc.checks == {} + + def test_status_ok_default_none(self): + bc = BlockCheck(text_hash="h", text_hash_short="s") + assert bc.status_ok is None + + def test_timestamp_recent(self): + before = time.time() + bc = BlockCheck(text_hash="h", text_hash_short="s") + after = time.time() + assert before <= bc.timestamp <= after + + def test_explicit_timestamp(self): + bc = BlockCheck(text_hash="h", text_hash_short="s", timestamp=999.0) + assert bc.timestamp == 999.0 + + +# --------------------------------------------------------------------------- +# T-checks-03: add_check() +# --------------------------------------------------------------------------- + +class TestBlockCheckAddCheck: + def test_add_single_check(self): + bc = BlockCheck(text_hash="h", text_hash_short="s") + cc = CodeCheck(status_ok=True) + bc.add_check("syntax", cc) + assert "syntax" in bc.checks + assert bc.checks["syntax"] is cc + + def test_add_multiple_checks(self): + bc = BlockCheck(text_hash="h", text_hash_short="s") + bc.add_check("syntax", CodeCheck(status_ok=True)) + bc.add_check("compile", CodeCheck(status_ok=False)) + assert len(bc.checks) == 2 + assert "syntax" in bc.checks + assert "compile" in bc.checks + + def test_overwrite_check(self): + bc = BlockCheck(text_hash="h", text_hash_short="s") + cc1 = CodeCheck(status_ok=True) + cc2 = CodeCheck(status_ok=False) + bc.add_check("run", cc1) + bc.add_check("run", cc2) + assert bc.checks["run"] is cc2 + + +# --------------------------------------------------------------------------- +# T-checks-04: to_json_file / from_json_file round-trip +# --------------------------------------------------------------------------- + +class TestBlockCheckJsonRoundTrip: + def test_round_trip_top_level_fields(self, tmp_path): + bc = BlockCheck( + text_hash="deadbeef", + text_hash_short="dead", + timestamp=1000.0, + status_ok=True, + ) + f = str(tmp_path / "block_checks.json") + bc.to_json_file(f) + bc2 = BlockCheck.from_json_file(f) + assert bc2 is not None + assert bc2.text_hash == "deadbeef" + assert bc2.text_hash_short == "dead" + assert bc2.timestamp == 1000.0 + assert bc2.status_ok is True + + def test_to_json_file_writes_the_per_phase_checks(self, tmp_path): + """A saved BlockCheck must carry its per-phase checks into the JSON. + + Reloading them is covered by the companion ``xfail`` test below; the + two are kept apart so that losing the written detail fails the suite + on its own.""" + bc = BlockCheck(text_hash="h", text_hash_short="s") + cc = CodeCheck(timestamp=1.0, version="v1", status_ok=True, + logfile="x.log", cmdline="cmd") + bc.add_check("syntax", cc) + assert "syntax" in bc.checks + + f = str(tmp_path / "bc.json") + bc.to_json_file(f) + + with open(f) as json_file: + written = json.load(json_file) + assert "syntax" in written["checks"], \ + "Expected the saved JSON to record the per-phase check" + + fields = written["checks"]["syntax"] + assert fields["timestamp"] == 1.0 + assert fields["version"] == "v1" + assert fields["status_ok"] is True + assert fields["logfile"] == "x.log" + assert fields["cmdline"] == "cmd" + + @pytest.mark.xfail( + strict=True, + reason="BlockCheck.__init__ discards the checks argument, so a JSON " + "round-trip loses every per-phase CodeCheck entry", + ) + def test_round_trip_preserves_the_per_phase_checks(self, tmp_path): + """A saved BlockCheck must come back carrying its per-phase checks. + + What ``to_json_file()`` writes out is covered by the companion test + above; this one covers only what comes back. + + Tracking note — this currently fails. ``BlockCheck.__init__`` accepts a + ``checks`` argument but then unconditionally assigns + ``self.checks = dict()``, so ``from_json_file()`` (which reconstructs + the object with ``BlockCheck(**json_data)``) silently drops every + ``CodeCheck`` entry that ``to_json_file()`` had written out. Nothing + warns: a reloaded block simply looks like one that was never checked, + which defeats the point of persisting the checks at all. A fix would + make ``__init__`` honor the argument and rebuild the ``CodeCheck`` + values from their serialized form; this test then passes and the + ``xfail`` marker must be removed.""" + bc = BlockCheck(text_hash="h", text_hash_short="s") + cc = CodeCheck(timestamp=1.0, version="v1", status_ok=True, + logfile="x.log", cmdline="cmd") + bc.add_check("syntax", cc) + + f = str(tmp_path / "bc.json") + bc.to_json_file(f) + + bc2 = BlockCheck.from_json_file(f) + assert bc2 is not None + assert "syntax" in bc2.checks + + # Accept either a rebuilt CodeCheck or its plain-dict form: the point + # is that the recorded detail survived, not how it is represented. + reloaded = bc2.checks["syntax"] + fields = reloaded if isinstance(reloaded, dict) else vars(reloaded) + assert fields["timestamp"] == 1.0 + assert fields["version"] == "v1" + assert fields["status_ok"] is True + assert fields["logfile"] == "x.log" + assert fields["cmdline"] == "cmd" + + def test_explicit_filename(self, tmp_path): + bc = BlockCheck(text_hash="abc", text_hash_short="a") + f = str(tmp_path / "custom.json") + bc.to_json_file(f) + bc2 = BlockCheck.from_json_file(f) + assert bc2 is not None + assert bc2.text_hash == "abc" + + def test_default_filename(self, tmp_path, monkeypatch): + """to_json_file() and from_json_file() with default filename work when + cwd is set to tmp_path.""" + monkeypatch.chdir(tmp_path) + bc = BlockCheck(text_hash="xyz", text_hash_short="x") + bc.to_json_file() + bc2 = BlockCheck.from_json_file() + assert bc2 is not None + assert bc2.text_hash == "xyz" + + +# --------------------------------------------------------------------------- +# T-checks-05: from_json_file() with nonexistent file +# --------------------------------------------------------------------------- + +class TestBlockCheckFromJsonMissing: + def test_nonexistent_file_returns_none(self, tmp_path): + f = str(tmp_path / "does_not_exist.json") + assert BlockCheck.from_json_file(f) is None + + def test_nonexistent_default_returns_none(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + assert BlockCheck.from_json_file() is None + + +# --------------------------------------------------------------------------- +# T-checks-06: Adversarial +# --------------------------------------------------------------------------- + +class TestBlockCheckAdversarial: + def test_overwrite_existing_file(self, tmp_path): + f = str(tmp_path / "bc.json") + bc1 = BlockCheck(text_hash="first", text_hash_short="f") + bc1.to_json_file(f) + bc2 = BlockCheck(text_hash="second", text_hash_short="s") + bc2.to_json_file(f) + bc_loaded = BlockCheck.from_json_file(f) + assert bc_loaded is not None + assert bc_loaded.text_hash == "second" + + def test_empty_json_raises_type_error(self, tmp_path): + """from_json_file() with '{}' should raise TypeError because __init__ + requires text_hash and text_hash_short.""" + f = tmp_path / "empty.json" + f.write_text("{}") + with pytest.raises(TypeError): + BlockCheck.from_json_file(str(f)) + + def test_from_json_file_none_argument_uses_default(self, tmp_path, monkeypatch): + """Passing None explicitly is equivalent to omitting the argument.""" + monkeypatch.chdir(tmp_path) + assert BlockCheck.from_json_file(None) is None diff --git a/frontend/python/rst_code_example_pipeline/tests/test_chop.py b/frontend/python/rst_code_example_pipeline/tests/test_chop.py new file mode 100644 index 000000000..98814fe3a --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_chop.py @@ -0,0 +1,265 @@ +""" +Unit tests for rst_code_example_pipeline.chop — edge cases and real_gnatchop. + +Covers: +- manual_chop with .ads and .adb extensions +- manual_chop with empty input +- manual_chop with no !filename lines at all (only garbage) +- manual_chop with garbage before first valid file +- cheapo_gnatchop with dotted package name +- cheapo_gnatchop with dotted procedure name +- cheapo_gnatchop with only a spec (package A) +- cheapo_gnatchop with empty input +- cheapo_gnatchop with only garbage (no recognized declaration) +- real_gnatchop: valid Ada, compiler_switches, error handler + (requires the Ada toolchain) +""" +import pytest + +from rst_code_example_pipeline.chop import manual_chop, cheapo_gnatchop, real_gnatchop +from rst_code_example_pipeline.resource import Resource + + +# --------------------------------------------------------------------------- +# T-chop-01: manual_chop — Ada extensions +# --------------------------------------------------------------------------- + +class TestManualChopAdaExtensions: + def test_adb_extension_recognized(self): + lines = ["!main.adb", "procedure Main is", "begin null; end Main;"] + result = manual_chop(lines) + assert len(result) == 1 + assert result[0].basename == "main.adb" + + def test_ads_extension_recognized(self): + lines = ["!pkg.ads", "package Pkg is", "end Pkg;"] + result = manual_chop(lines) + assert len(result) == 1 + assert result[0].basename == "pkg.ads" + + def test_adb_content_correct(self): + lines = ["!main.adb", "procedure Main is", "begin null; end Main;"] + result = manual_chop(lines) + assert result[0].content == "procedure Main is\nbegin null; end Main;" + + def test_ads_content_correct(self): + lines = ["!pkg.ads", "package Pkg is", "end Pkg;"] + result = manual_chop(lines) + assert result[0].content == "package Pkg is\nend Pkg;" + + def test_adb_and_ads_in_same_input(self): + lines = [ + "!spec.ads", + "package Spec is", + "end Spec;", + "!body.adb", + "package body Spec is", + "end Spec;", + ] + result = manual_chop(lines) + assert len(result) == 2 + assert result[0].basename == "spec.ads" + assert result[1].basename == "body.adb" + + +# --------------------------------------------------------------------------- +# T-chop-02: manual_chop — empty and garbage inputs +# --------------------------------------------------------------------------- + +class TestManualChopEdgeCases: + def test_empty_input_returns_empty_list(self): + assert manual_chop([]) == [] + + def test_only_garbage_no_filename_returns_empty_list(self): + lines = ["no file here", "more garbage", "still nothing"] + assert manual_chop(lines) == [] + + def test_garbage_before_first_file_discarded(self): + lines = [ + "garbage line 1", + "garbage line 2", + "!main.adb", + "procedure Main is null;", + ] + result = manual_chop(lines) + assert len(result) == 1 + assert result[0].basename == "main.adb" + assert result[0].content == "procedure Main is null;" + + def test_fake_extension_not_matched(self): + """A line like !fake.txt must not be treated as a valid file.""" + lines = ["!fake.txt", "some content", "!real.adb", "real content"] + result = manual_chop(lines) + assert len(result) == 1 + assert result[0].basename == "real.adb" + + def test_single_filename_no_content(self): + lines = ["!empty.adb"] + result = manual_chop(lines) + assert len(result) == 1 + assert result[0].basename == "empty.adb" + assert result[0].content == "" + + def test_multiple_files_content_correctly_split(self): + lines = [ + "!a.ads", + "package A is", + "end A;", + "!a.adb", + "package body A is", + "end A;", + "!main.adb", + "procedure Main is null;", + ] + result = manual_chop(lines) + assert len(result) == 3 + assert result[0].content == "package A is\nend A;" + assert result[1].content == "package body A is\nend A;" + assert result[2].content == "procedure Main is null;" + + +# --------------------------------------------------------------------------- +# T-chop-03: cheapo_gnatchop — dotted names +# --------------------------------------------------------------------------- + +class TestCheapoGnatchopDottedNames: + def test_dotted_package_body(self): + lines = ["package body Foo.Bar is", "end Foo.Bar;"] + result = cheapo_gnatchop(lines) + assert len(result) == 1 + assert result[0].basename == "foo-bar.adb" + + def test_dotted_procedure(self): + lines = ["procedure Foo.Bar is", "begin null; end Foo.Bar;"] + result = cheapo_gnatchop(lines) + assert len(result) == 1 + assert result[0].basename == "foo-bar.adb" + + def test_triple_dotted_package_body(self): + lines = ["package body A.B.C is", "end A.B.C;"] + result = cheapo_gnatchop(lines) + assert len(result) == 1 + assert result[0].basename == "a-b-c.adb" + + def test_dotted_package_body_content(self): + lines = ["package body Foo.Bar is", "end Foo.Bar;"] + result = cheapo_gnatchop(lines) + assert result[0].content == "package body Foo.Bar is\nend Foo.Bar;" + + +# --------------------------------------------------------------------------- +# T-chop-04: cheapo_gnatchop — spec only +# --------------------------------------------------------------------------- + +class TestCheapoGnatchopSpecOnly: + def test_spec_generates_ads(self): + lines = ["package A is", "end A;"] + result = cheapo_gnatchop(lines) + assert len(result) == 1 + assert result[0].basename == "a.ads" + + def test_spec_content_correct(self): + lines = ["package A is", "end A;"] + result = cheapo_gnatchop(lines) + assert result[0].content == "package A is\nend A;" + + def test_dotted_spec(self): + lines = ["package Foo.Bar is", "end Foo.Bar;"] + result = cheapo_gnatchop(lines) + assert result[0].basename == "foo-bar.ads" + + +# --------------------------------------------------------------------------- +# T-chop-05: cheapo_gnatchop — empty and garbage inputs +# --------------------------------------------------------------------------- + +class TestCheapoGnatchopEdgeCases: + def test_empty_input_returns_empty_list(self): + assert cheapo_gnatchop([]) == [] + + def test_only_garbage_returns_empty_list(self): + lines = ["garbage line", "more garbage", "-- just a comment"] + assert cheapo_gnatchop(lines) == [] + + def test_garbage_before_first_declaration_discarded(self): + lines = [ + "-- header comment", + "with Ada.Text_IO;", + "package body A is", + "end A;", + ] + result = cheapo_gnatchop(lines) + assert len(result) == 1 + assert result[0].basename == "a.adb" + assert "package body A is" in result[0].content + + def test_lowercase_names(self): + lines = ["package body mypackage is", "end mypackage;"] + result = cheapo_gnatchop(lines) + assert result[0].basename == "mypackage.adb" + + def test_procedure_generates_adb(self): + lines = ["procedure Main is", "begin null; end Main;"] + result = cheapo_gnatchop(lines) + assert len(result) == 1 + assert result[0].basename == "main.adb" + + def test_body_before_spec_both_captured(self): + lines = [ + "package body A is", + "end A;", + "package A is", + "end A;", + ] + result = cheapo_gnatchop(lines) + assert len(result) == 2 + assert result[0].basename == "a.adb" + assert result[1].basename == "a.ads" + + +# --------------------------------------------------------------------------- +# T-chop-06: real_gnatchop — Ada toolchain required +# (covers real_gnatchop end to end: success, switch filtering, error handling) +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestRealGnatchop: + """Tests for real_gnatchop; require gnatchop in PATH.""" + + VALID_ADA = ["procedure Main is", "begin null; end Main;"] + + def test_valid_ada_no_switches_returns_resources(self): + """real_gnatchop with no compiler switches returns a non-empty list + of Resource objects.""" + result = real_gnatchop(self.VALID_ADA, compiler_switches=None) + assert len(result) >= 1 + assert all(isinstance(r, Resource) for r in result) + + def test_valid_ada_no_switches_basename(self): + """gnatchop on a minimal procedure Main produces main.adb.""" + result = real_gnatchop(self.VALID_ADA, compiler_switches=None) + basenames = [r.basename for r in result] + assert "main.adb" in basenames + + def test_valid_ada_with_compiler_switches(self): + """real_gnatchop must still chop the source when it is given + compiler switches to pass on.""" + result = real_gnatchop(self.VALID_ADA, compiler_switches=["-gnata"]) + assert len(result) >= 1 + basenames = [r.basename for r in result] + assert "main.adb" in basenames + + def test_invalid_input_raises_exception(self): + """Garbage input causes gnatchop to fail; the CalledProcessError + handler prints the numbered input lines and raises Exception.""" + with pytest.raises(Exception, match="Could not chop files with gnatchop"): + real_gnatchop(["this is not valid Ada at all !@#$"], + compiler_switches=None) + + def test_non_gnat_switch_is_skipped(self): + """A compiler_switches entry that doesn't contain "gnat" (e.g. -Wall) + is silently dropped before invoking gnatchop; gnatchop still succeeds + since gnatchop itself never sees -Wall.""" + result = real_gnatchop(self.VALID_ADA, compiler_switches=["-Wall"]) + assert len(result) == 1 + assert result[0].basename == "main.adb" diff --git a/frontend/python/rst_code_example_pipeline/tests/test_cli.py b/frontend/python/rst_code_example_pipeline/tests/test_cli.py new file mode 100644 index 000000000..19d3ba1d0 --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_cli.py @@ -0,0 +1,958 @@ +""" +End-to-end tests for the command-line entry points. + +Every other test in this suite calls the package's functions directly. These +run the installed commands -- extract-code, check-code and check-block -- as +real processes over a small course directory, and look at what a script +driving them can see: the exit status, and the message that explains it. That +is the contract the package README sets out under "Exit status", and it is +what a build gates on; nothing else in the suite goes near it. + +Covers: +- a course whose one example builds and runs: extract-code and check-code both + succeed, and what the example printed is there in the run log afterwards +- the same course with the example broken: check-code fails, and says which + name the compiler could not resolve +- check-block over a single extracted example: success for one that builds, + failure for one that does not, and failure -- with a message rather than a + crash -- for a block info file that is missing, and for one that is present + and unusable +- check-block over a single extracted example declared as expecting a compile + error whose source compiles: the run fails, says the declared error never + arrived, and the run log shows the example really was built and run. Both + languages, since the same declaration is written in both and each is looked + for separately +- check-block over a single extracted example tagged with a run class naming + the other language: the run fails and names the class the author has to fix. + Both directions, and with the control of the same example tagged with its own + language's class, which checks out. This is the level at which the claim + that the report *fails* the run can be made at all: the exit status is set + outside every function the rest of the suite calls +- check-code over a build directory holding a block info file it has to drop: + one that cannot be read, and one that names no project. Each fails the run + rather than reporting success over an example nothing looked at, and an + unreadable one among several does not cost the others their check +- extract-code over a course whose block names no project: the run fails and + no block info file is written at all, which is why a block naming no project + is only reachable from a file written or edited by hand +- extract-code over a course whose block record was damaged since the last run: + the record is rebuilt, a warning names it as rebuilt, the run still succeeds, + and a check-code run over the rebuilt record still builds and runs the + example +- extract-code over a build directory in which the block record's name is held + by a directory: the run succeeds without a traceback, the block directory is + extracted again, and the record is a readable file once more +- the command lines the README says are rejected: naming neither a build + directory nor a project list fails, and an unknown switch is rejected + outright with the distinct status argument parsing uses + +NOTE: a command that gets as far as checking an example runs the Ada +toolchain over it, so those tests carry the `toolchain` marker. The tests +that stop at argument handling, and the one that stops at an unreadable block +info file, never reach a compiler and carry no marker. + +The commands under test are the console scripts the package installs, so they +must be on PATH -- which they are wherever the package is installed, the same +condition that lets the rest of the suite import it. +""" +import pathlib +import subprocess + +import pytest + +from rst_code_example_pipeline import blocks +from rst_code_example_pipeline import constants +from rst_code_example_pipeline import toolchain_info + + +# A complete Ada example that announces itself when it runs. The course +# below asks for a run, so a check that reports success has to have built the +# example, executed it, and recorded what it printed -- rather than merely not +# failing, which is what a command that checked nothing at all also does. +RUN_OUTPUT = "the example ran" + +WORKING_ADA_BODY = """\ +with Ada.Text_IO; use Ada.Text_IO; +procedure Main is +begin + Put_Line ("{}"); +end Main;""".format(RUN_OUTPUT) + +# A name nothing declares, so the build has to fail on it and the compiler has +# to say so -- which is how a failing run is told apart from one that failed +# for some unrelated reason. +MISSING_NAME = "No_Such_Procedure" + +# Syntactically valid, so it chops and passes the syntax check, but it calls +# something that does not exist. +BROKEN_ADA_BODY = """\ +procedure Main is +begin + {}; +end Main;""".format(MISSING_NAME) + +# The C counterpart, for the one test whose subject is a C example. A C block +# names its own source on a leading marker line rather than having it chopped +# out, so the file name is part of the body here and is also what the +# directive declares as the main. +C_RUN_OUTPUT = "the C example ran" + +C_MAIN = "main.c" + +WORKING_C_BODY = """\ +!{} +#include + +int main(void) +{{ + printf("{}\\n"); + return 0; +}}""".format(C_MAIN, C_RUN_OUTPUT) + + +def _write_course(directory, project: str, body: str, + classes: str | None = None, + language: str = "ada", main: str = "main.adb", + button: str = "run_button"): + """Write a one-block RST file the way a course author would, and return + its name relative to the directory holding it. + + ``classes`` is the ``:class:`` line an author adds to declare what the + example is for -- omitted entirely when there is none, so the common case + stays the directive a course really carries. + + ``language`` and ``main`` are the other two things the directive declares. + They default to the Ada example nearly every test here uses, so that the + call sites reading as a course of Ada say so by not mentioning it. + + ``button`` is the indicator the directive carries. It defaults to the + run button nearly every test here wants; a test whose subject is an + example that nothing builds asks for ``no_button`` instead. + """ + indented = "\n".join(" " + line for line in body.splitlines()) + declared = "" if classes is None else " :class: {}\n".format(classes) + (directory / "course.rst").write_text( + ".. code:: {} project={} main={} {}\n" + "{}" + "\n" + "{}\n" + "\n" + "Explanatory paragraph.\n".format(language, project, main, button, + declared, indented)) + return "course.rst" + + +def _run(command: str, *arguments: str, cwd) -> subprocess.CompletedProcess: + """Run one of the installed commands as a real process.""" + return subprocess.run([command, *arguments], cwd=str(cwd), + capture_output=True, text=True) + + +def _extract(cwd, project: str, body: str, + classes: str | None = None, + language: str = "ada", + main: str = "main.adb", + button: str = "run_button") -> subprocess.CompletedProcess: + """Extract a one-block course into a build directory below ``cwd``.""" + rst_file = _write_course(cwd, project, body, classes, language, main, + button) + return _run("extract-code", "--build-dir", "build", rst_file, cwd=cwd) + + +def _the_extracted_block(cwd) -> str: + """The block info file the extraction step wrote, of which there is one. + + The extraction step keeps a staging copy of the sources alongside the + per-block directory, and only the latter holds a block info file. + """ + written = sorted((cwd / "build").rglob("*.json")) + assert len(written) == 1, \ + "expected the extraction step to write exactly one block info " \ + "file, got {}".format([str(path) for path in written]) + return str(written[0]) + + +def _write_course_of_several_blocks(directory, project: str, + outputs: list[str]) -> str: + """Write a course of several examples, each announcing itself with its + own line, and return its name relative to the directory. + + Each example gets a project of its own, so that one of them being + unreadable cannot be said to have taken its neighbors down with it merely + by sharing a directory. Distinct output then makes each block's run log + identifiable, which is what lets a test say which examples were checked. + """ + blocks_rst = [] + for number, output in enumerate(outputs, start=1): + body = WORKING_ADA_BODY.replace(RUN_OUTPUT, output) + indented = "\n".join(" " + line for line in body.splitlines()) + blocks_rst.append( + ".. code:: ada project={}{} main=main.adb run_button\n" + "\n" + "{}\n" + "\n" + "Explanatory paragraph.\n".format(project, number, indented)) + (directory / "course.rst").write_text("\n".join(blocks_rst)) + return "course.rst" + + +def _the_extracted_blocks(cwd) -> list: + """Every block info file the extraction step wrote, in a stable order.""" + return sorted((cwd / "build").rglob(constants.BLOCK_INFO_FILENAME)) + + +def _block_info_files_written(cwd) -> list: + """Every block info file below the build directory, or none if the + extraction step did not get as far as making one.""" + build = cwd / "build" + return _the_extracted_blocks(cwd) if build.is_dir() else [] + + +def _a_block_record_naming_no_project(directory) -> str: + """Write a well-formed block record that names no project. + + The extraction step refuses to write one -- it stops the whole run on a + code block with no project name before writing anything -- so this state + only exists in a file written or edited by hand. It is produced through + the package's own writer so that the record is right in every respect + except the one under test, and lands under the name the check looks for + without that name being restated here. + """ + if not toolchain_info.DEFAULT_VERSION: + toolchain_info.init_toolchain_info() + + versions = toolchain_info.DEFAULT_VERSION + block = blocks.CodeBlock( + rst_file="course.rst", + line_start=1, + line_end=5, + text="procedure Main is begin null; end Main;", + language="ada", + project=None, + main_file=None, + gnat_version=["default", versions["gnat"]], + gnatprove_version=["default", versions["gnatprove"]], + gprbuild_version=["default", versions["gprbuild"]], + compiler_switches=[], + classes=["ada-nocheck"], + manual_chop=False, + buttons=["no"], + ) + directory.mkdir(parents=True, exist_ok=True) + written = str(directory / constants.BLOCK_INFO_FILENAME) + block.to_json_file(written) + return written + + +def _the_run_log(cwd) -> str: + """What the example printed when it was run, of which there is one. + + A run writes its output beside the example rather than to the command's + own output, so reading it back is the only way to tell a course that + really ran something from one that reported success over nothing. + """ + written = sorted((cwd / "build").rglob("run.log")) + assert len(written) == 1, \ + "expected the check to write exactly one run log, got {}".format( + [str(path) for path in written]) + return written[0].read_text() + + +# --------------------------------------------------------------------------- +# A course whose examples all build +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCourseThatChecksOut: + def test_extract_and_check_both_succeed(self, tmp_path): + """A course whose one example builds and runs must be extracted and + checked without either command reporting a failure. + + The status alone cannot tell success apart from having checked + nothing, which the package README warns is possible, so the output the + example printed is asserted as well: it can only be there if the block + was extracted, built and executed. + """ + extracted = _extract(tmp_path, "CliCourseGood", WORKING_ADA_BODY) + assert extracted.returncode == 0, \ + "extracting a well-formed course must succeed: {}".format( + extracted.stdout) + + checked = _run("check-code", "--build-dir", "build", cwd=tmp_path) + assert checked.returncode == 0, \ + "checking a course whose example builds must succeed: {}".format( + checked.stdout) + + assert RUN_OUTPUT in _the_run_log(tmp_path), \ + "a course reported as checked must have run its example, and the "\ + "run log is where what it printed ends up" + + +# --------------------------------------------------------------------------- +# A course with one example that does not build +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCourseWithABrokenExample: + def test_check_code_fails_and_says_why(self, tmp_path): + """A course with one example that does not build must be extracted + without complaint -- the block is well-formed, it just does not + compile -- and then fail the check. + + The message is asserted as well as the status, so that a run which + fails because the course fixture itself is wrong cannot be mistaken + for the failure the test is about. + """ + extracted = _extract(tmp_path, "CliCourseBroken", BROKEN_ADA_BODY) + assert extracted.returncode == 0, \ + "the course must extract cleanly, or the check that follows is " \ + "not failing on the example: {}".format(extracted.stdout) + + checked = _run("check-code", "--build-dir", "build", cwd=tmp_path) + assert checked.returncode == 1, \ + "checking a course with an example that does not build must " \ + "fail: {}".format(checked.stdout) + assert MISSING_NAME in checked.stdout, \ + "the failure must name what the compiler could not resolve: " \ + "{}".format(checked.stdout) + + +# --------------------------------------------------------------------------- +# A single extracted example +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestCheckingASingleBlock: + def test_a_block_that_builds_succeeds(self, tmp_path): + """check-block on one previously extracted example that builds must + succeed.""" + assert _extract(tmp_path, "CliBlockGood", + WORKING_ADA_BODY).returncode == 0 + checked = _run("check-block", "--force", _the_extracted_block(tmp_path), + cwd=tmp_path) + assert checked.returncode == 0, \ + "checking one example that builds must succeed: {}".format( + checked.stdout) + + def test_a_block_that_does_not_build_fails(self, tmp_path): + """check-block on one previously extracted example that does not + build must fail, and name what the compiler could not resolve.""" + assert _extract(tmp_path, "CliBlockBroken", + BROKEN_ADA_BODY).returncode == 0 + checked = _run("check-block", "--force", _the_extracted_block(tmp_path), + cwd=tmp_path) + assert checked.returncode == 1, \ + "checking one example that does not build must fail: {}".format( + checked.stdout) + assert MISSING_NAME in checked.stdout, \ + "the failure must name what the compiler could not resolve: " \ + "{}".format(checked.stdout) + + def test_a_block_expecting_a_compile_error_that_compiles_fails( + self, tmp_path): + """check-block on an example declared as expecting a compile error, + whose source compiles, must fail and say the error never arrived. + + This is the check the package exists to perform, seen from where a + build sees it: the example is marked "this must not compile", the + compiler accepts it anyway, and the only thing standing between that + and a green build is this command's exit status. Driven through the + installed command rather than in process, because an author's class + has to survive the RST source, the extraction step and the exit-status + contract to have any effect at all. + """ + assert _extract(tmp_path, "CliBlockExpectErrorThatCompiles", + WORKING_ADA_BODY, + "ada-expect-compile-error").returncode == 0 + checked = _run("check-block", "--force", _the_extracted_block(tmp_path), + cwd=tmp_path) + assert checked.returncode == 1, \ + "checking an example that declares a compile error it does not " \ + "produce must fail: {}".format(checked.stdout) + assert "Expected compile error, got none!" in checked.stdout, \ + "the failure must say that the declared compile error never " \ + "arrived: {}".format(checked.stdout) + assert RUN_OUTPUT in _the_run_log(tmp_path), \ + "the example must really have been built and run, or the " \ + "expectation was left unmet by nothing having happened" + + def test_a_c_block_expecting_a_compile_error_that_compiles_fails( + self, tmp_path): + """check-block on a C example declared as expecting a compile error, + whose source compiles, must fail and say the error never arrived. + + The C spelling of the test above, seen from the same place: the two + languages end at the same report, and only the Ada one used to be + made. A C example marked "this must not compile" that the compiler + accepted left the command at status zero with nothing printed, so a + build gating on the status was told the course checked out over an + example asserting something untrue about the language. Driven + through the installed command for the same reason its Ada twin is: + the author's class has to survive the RST source, the extraction step + and the exit-status contract to have any effect at all. + """ + assert _extract(tmp_path, "CliCBlockExpectErrorThatCompiles", + WORKING_C_BODY, "c-expect-compile-error", + language="c", main=C_MAIN).returncode == 0 + checked = _run("check-block", "--force", _the_extracted_block(tmp_path), + cwd=tmp_path) + assert checked.returncode == 1, \ + "checking a C example that declares a compile error it does not " \ + "produce must fail: {}".format(checked.stdout) + assert "Expected compile error, got none!" in checked.stdout, \ + "the failure must say that the declared compile error never " \ + "arrived: {}".format(checked.stdout) + assert C_RUN_OUTPUT in _the_run_log(tmp_path), \ + "the example must really have been built and run, or the " \ + "expectation was left unmet by nothing having happened" + + def test_a_c_block_classed_for_ada_fails_and_names_the_class( + self, tmp_path): + """check-block on a C example tagged with an Ada run class must fail + and name the class. + + A run class names a language and buys a block of the other language + nothing at all. Seen from where a build sees it, that is the worst + shape a mistake can take: without this failure the command exits zero + over an example whose author asked for something that did not happen. + + The example carries a run button as well, so it really is built and + run and the outcome is fine -- which is what makes the failure + attributable to the class the author wrote rather than to anything + that went wrong. Extraction is asserted to succeed first, so that + the failure is localized to the check. + """ + assert _extract(tmp_path, "CliCBlockClassedForAda", WORKING_C_BODY, + "ada-run", language="c", main=C_MAIN).returncode == 0, \ + "the extraction step must accept the example, or the failure " \ + "below is not the check's" + checked = _run("check-block", "--force", _the_extracted_block(tmp_path), + cwd=tmp_path) + assert checked.returncode == 1, \ + "checking an example tagged with the other language's run class " \ + "must fail: {}".format(checked.stdout) + assert "Wrong language selected for run class 'ada-run'" \ + in checked.stdout, \ + "the failure must name the class the author has to fix: " \ + "{}".format(checked.stdout) + assert C_RUN_OUTPUT in _the_run_log(tmp_path), \ + "the example must really have been built and run, or the " \ + "failure cannot be attributed to the class" + + def test_an_ada_block_classed_for_c_fails_and_names_the_class( + self, tmp_path): + """The mirror, so that the command-level claim is not held by a + single direction. + + Written out rather than left to the C case above: the two languages' + class names are separate words in the source, so a command that had + stopped recognizing one of them would still fail the other test. + """ + assert _extract(tmp_path, "CliAdaBlockClassedForC", WORKING_ADA_BODY, + "c-norun").returncode == 0 + checked = _run("check-block", "--force", _the_extracted_block(tmp_path), + cwd=tmp_path) + assert checked.returncode == 1, \ + "checking an Ada example tagged with a C run class must fail: " \ + "{}".format(checked.stdout) + assert "Wrong language selected for run class 'c-norun'" \ + in checked.stdout, \ + "the failure must name the class the author has to fix: " \ + "{}".format(checked.stdout) + + def test_a_class_only_edit_is_not_absorbed_by_the_recorded_result( + self, tmp_path): + """The whole defect, and the whole fix, through the installed + command and over a build directory that was not thrown away. + + A course author writes an example, checks it, and it passes. Later + they change only its ``:class:`` line -- the source text of the + example is not touched -- and check again without deleting anything. + The per-block directory is named after a hash of the example's text, + so the same directory is reused, and the record of the earlier + successful check is still sitting in it. + + Without --force, that record is what the second run would otherwise + hand back. The example is now tagged with the other language's run + class, so it asks for no run and therefore for no build, and a run + reporting success over it would be reporting success over an example + nothing compiled. This is the shape the continuous-integration run + is protected from only by deleting the build directory first, and the + shape the documented local loop meets, because the local driver keeps + that directory between runs on purpose. + + The example carries no run button, so nothing else can ask for the + build the class stopped asking for. + """ + assert _extract(tmp_path, "CliStaleRecord", WORKING_C_BODY, + "c-run", language="c", main=C_MAIN, + button="no_button").returncode == 0 + first = _run("check-code", "--build-dir", "build", cwd=tmp_path) + assert first.returncode == 0, \ + "the example must check out before its class is edited: " \ + "{}".format(first.stdout) + + extracted = _the_extracted_blocks(tmp_path) + assert _extract(tmp_path, "CliStaleRecord", WORKING_C_BODY, + "ada-run", language="c", main=C_MAIN, + button="no_button").returncode == 0, \ + "the extraction step must accept the edited example, or the " \ + "failure below is not the check's" + assert _the_extracted_blocks(tmp_path) == extracted, \ + "the edit must land in the same block directory, or the stale " \ + "record this test is about was never reached" + + checked = _run("check-code", "--build-dir", "build", cwd=tmp_path) + assert checked.returncode == 1, \ + "an example whose class now names the other language must fail, " \ + "although a successful check of it is on record: {}".format( + checked.stdout) + assert "Wrong language selected for run class 'ada-run'" \ + in checked.stdout, \ + "the failure must name the class the author has to fix: " \ + "{}".format(checked.stdout) + + def test_an_unchanged_example_keeps_its_recorded_result(self, tmp_path): + """The control for the test above. + + Reusing the record of an earlier successful check is what the record + is for, and the report above must not cost every example that has one + its reuse. The same example checked twice, with nothing edited in + between, checks out both times. + """ + assert _extract(tmp_path, "CliUnchangedRecord", WORKING_C_BODY, + "c-run", language="c", main=C_MAIN, + button="no_button").returncode == 0 + for attempt in ("first", "second"): + checked = _run("check-code", "--build-dir", "build", cwd=tmp_path) + assert checked.returncode == 0, \ + "the {} check of an unedited example must succeed: " \ + "{}".format(attempt, checked.stdout) + + def test_a_c_block_classed_for_c_succeeds(self, tmp_path): + """The control for the two above. + + The same C example, tagged with the run class of its own language, + must go through both commands at status zero -- so the failures above + are attributable to the class naming the wrong language and not to + anything about the example, the directive or the fixture. + """ + assert _extract(tmp_path, "CliCBlockClassedForC", WORKING_C_BODY, + "c-run", language="c", main=C_MAIN).returncode == 0 + checked = _run("check-block", "--force", _the_extracted_block(tmp_path), + cwd=tmp_path) + assert checked.returncode == 0, \ + "an example tagged with its own language's run class must " \ + "check out: {}".format(checked.stdout) + assert C_RUN_OUTPUT in _the_run_log(tmp_path), \ + "the example must really have been built and run" + + +class TestBlockInfoThatCannotBeRead: + def test_a_missing_block_info_file_fails_with_a_message(self, tmp_path): + """A block info file that cannot be loaded counts as a failure, so a + script gating on the status is not told the example checked out when + nothing was checked at all. + + The command must say which file it could not read; a crash would also + end in a failing status and would tell the reader nothing. + """ + missing = str(tmp_path / "no_such_block.json") + result = _run("check-block", missing, cwd=tmp_path) + + assert result.returncode == 1, \ + "a block info file that cannot be loaded must count as a failure" + assert missing in result.stdout, \ + "the message must name the file that could not be read: " \ + "{}".format(result.stdout) + assert "Traceback" not in result.stderr, \ + "the file must be reported, not crashed on: {}".format( + result.stderr) + + def test_an_unusable_block_info_file_fails_with_a_message(self, tmp_path): + """A block info file that is there but cannot be turned into a block + must be reported the same way a missing one is. + + This is the case a file damaged after it was written falls into -- + truncated, edited, half-copied. It used to leave the command as a + traceback: the status was 1 all the same, but only because that is + what Python gives an uncaught exception, and nothing in the output + told the reader which file was at fault or why. + """ + unusable = tmp_path / constants.BLOCK_INFO_FILENAME + unusable.write_text("{ this is not a block record") + + result = _run("check-block", str(unusable), cwd=tmp_path) + + assert result.returncode == 1, \ + "a block info file that cannot be loaded must count as a failure" + assert str(unusable) in result.stdout, \ + "the message must name the file that could not be read: " \ + "{}".format(result.stdout) + assert "Traceback" not in result.stderr, \ + "the file must be reported, not crashed on: {}".format( + result.stderr) + + +# --------------------------------------------------------------------------- +# A block the check dropped instead of checking +# --------------------------------------------------------------------------- + +class TestABlockTheCheckNeverLookedAt: + """check-code over a build directory holding a block info file it drops. + + Each of the two ways in prints an ERROR line and moves on to the next + file, so neither block reaches the check and neither can report an error + from there. What is asserted here is the status of the command, because + that is what a build gates on -- and both of these have printed their + ERROR line while the command still exited 0, which is a run reporting + success over an example nothing looked at. + + Both files are made here rather than extracted. One stands for a record + damaged after the extraction step wrote it; the other for a record + written or edited by hand, since the extraction step refuses to write a + block that names no project. + """ + + def test_an_unreadable_block_info_file_fails_the_run(self, tmp_path): + """A build directory whose one block info file cannot be read must + fail the run.""" + block_dir = tmp_path / "build" / "projects" / "Damaged" / "hash1" + block_dir.mkdir(parents=True) + unreadable = block_dir / constants.BLOCK_INFO_FILENAME + unreadable.write_text("{ this is not a block record") + + result = _run("check-code", "--build-dir", "build", cwd=tmp_path) + + assert result.returncode == 1, \ + "a block info file that could not be read means an example was " \ + "never checked, and the run must say so: {}".format(result.stdout) + assert str(unreadable) in result.stdout, \ + "the run must name the file it could not read: {}".format( + result.stdout) + assert "Traceback" not in result.stderr, \ + "the file must be reported, not crashed on: {}".format( + result.stderr) + + def test_a_block_naming_no_project_fails_the_run(self, tmp_path): + """A build directory whose one block info file names no project must + fail the run, for the same reason: that block was never checked.""" + written = _a_block_record_naming_no_project( + tmp_path / "build" / "projects" / "NoProject" / "hash1") + + result = _run("check-code", "--build-dir", "build", cwd=tmp_path) + + assert result.returncode == 1, \ + "a block that names no project is a block that was not checked, " \ + "and the run must say so: {}".format(result.stdout) + assert written in result.stdout, \ + "the run must name the file whose block it dropped: {}".format( + result.stdout) + + def test_an_empty_build_directory_still_succeeds(self, tmp_path): + """A build directory with nothing in it must not fail the run. + + The control for the two tests above: without it they would go on + passing if check-code had simply started failing for everything. + """ + (tmp_path / "build").mkdir() + + result = _run("check-code", "--build-dir", "build", cwd=tmp_path) + + assert result.returncode == 0, \ + "a build directory holding no blocks has nothing to report: " \ + "{}".format(result.stdout) + + +@pytest.mark.toolchain +class TestOneBadBlockAmongSeveral: + """A course whose block info files are not all readable. + + The one property that decides whether reporting an unreadable file + instead of raising on it was an improvement: the exception it replaced + left the command while the block info files were still being gathered, so + not one example in the course was checked, whatever else was wrong with + it. + """ + + OUTPUTS = ["the first example ran", + "the second example ran", + "the third example ran"] + + def test_the_other_examples_are_still_checked(self, tmp_path): + """One unreadable block info file must fail the run and must not cost + the other examples in the course their check. + + The run logs are what shows they were checked: an example's output + can only reach one by being extracted, built and executed. + """ + rst_file = _write_course_of_several_blocks( + tmp_path, "CliCourseMixed", self.OUTPUTS) + extracted = _run("extract-code", "--build-dir", "build", rst_file, + cwd=tmp_path) + assert extracted.returncode == 0, \ + "the course must extract cleanly, or the check that follows is " \ + "not failing on the damaged file: {}".format(extracted.stdout) + + written = _the_extracted_blocks(tmp_path) + assert len(written) == len(self.OUTPUTS), \ + "expected one block info file per example, got {}".format( + [str(path) for path in written]) + + damaged = written[0] + damaged_output = [output for output in self.OUTPUTS + if output in damaged.read_text()] + assert len(damaged_output) == 1, \ + "the file about to be damaged must belong to exactly one of the " \ + "examples, got {}".format(damaged_output) + damaged.write_text("{ this is not a block record") + + checked = _run("check-code", "--build-dir", "build", cwd=tmp_path) + + assert checked.returncode == 1, \ + "the damaged file means an example was never checked, and the " \ + "run must say so: {}".format(checked.stdout) + + ran = "\n".join(path.read_text() + for path in (tmp_path / "build").rglob("run.log")) + + for output in self.OUTPUTS: + if output == damaged_output[0]: + assert output not in ran, \ + "the example whose block info file was damaged cannot " \ + "have been run: {}".format(ran) + else: + assert output in ran, \ + "an example whose block info file was untouched must " \ + "still have been checked and run: {} is missing from " \ + "{}".format(output, ran) + + +# --------------------------------------------------------------------------- +# A course the extraction step refuses +# --------------------------------------------------------------------------- + +class TestACourseWhoseBlockNamesNoProject: + def test_nothing_is_extracted_and_the_run_fails(self, tmp_path): + """A course with a code block that names no project must fail the + extraction, and must leave no block info file behind. + + That is what makes the check's "block has no project" arm reachable + only from a file written or edited by hand: the extraction step stops + the whole run on such a block before it writes anything, so no file + it produced can carry one. The good block is written first so that a + step which wrote as it went would be caught leaving the first one on + disk. + """ + indented = "\n".join(" " + line + for line in WORKING_ADA_BODY.splitlines()) + (tmp_path / "course.rst").write_text( + ".. code:: ada project=CliCourseNamed main=main.adb run_button\n" + "\n" + "{}\n" + "\n" + "Explanatory paragraph.\n" + "\n" + ".. code:: ada main=main.adb run_button\n" + "\n" + "{}\n" + "\n" + "Another paragraph.\n".format(indented, indented)) + + result = _run("extract-code", "--build-dir", "build", "course.rst", + cwd=tmp_path) + + assert result.returncode == 1, \ + "a code block with no project name must fail the extraction: " \ + "{}".format(result.stdout) + assert _block_info_files_written(tmp_path) == [], \ + "the extraction step must write no block info file at all when " \ + "it refuses a course: {}".format( + [str(path) for path in _block_info_files_written(tmp_path)]) + + +# --------------------------------------------------------------------------- +# A course whose block record was damaged between runs +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestACourseWhoseBlockRecordWasDamaged: + """extract-code finding a record it wrote earlier and cannot read now. + + A build directory is reused between runs, so a record damaged by an + interrupted run survives into the next one. Extraction rewrites it and + carries on, which is the right outcome and used to be a traceback -- and + because the outcome is a success, the message is the only thing that says + the file was ever damaged. + + Asserted through the commands rather than in process, because what makes + the repair honest is the pair of statuses: the extraction succeeds, and + the example it repaired is then really checked. + """ + + DAMAGED_RECORD = "{ this is not a block record" + + def test_the_record_is_rebuilt_and_the_example_is_still_checked( + self, tmp_path): + """A damaged block record must be rebuilt with a warning naming it, + the extraction must still succeed, and the example must still be + checked afterwards. + + The last clause is not something the warning claims -- it says only + that the example is still extracted and the run was not cut short -- + which is exactly why it is the one most likely to rot: a repair that + printed the line and left the record unusable would satisfy the + status and the message and still leave the example unchecked. The + run log is what settles it -- the output below can only get there by + the example being built and executed. + """ + assert _extract(tmp_path, "CliCourseRebuilt", + WORKING_ADA_BODY).returncode == 0, \ + "the course must extract cleanly first, or there is no record to " \ + "damage" + + written = _the_extracted_blocks(tmp_path) + assert len(written) == 1, \ + "expected one block record after the first extraction, got " \ + "{}".format([str(path) for path in written]) + record = written[0] + record.write_text(self.DAMAGED_RECORD) + + again = _run("extract-code", "--build-dir", "build", "course.rst", + cwd=tmp_path) + + assert again.returncode == 0, \ + "rebuilding a damaged record is a recovery, so the extraction " \ + "must still succeed: {}".format(again.stdout) + assert "WARNING" in again.stdout, \ + "a rebuilt record must be announced as a warning: {}".format( + again.stdout) + # The repair runs from inside the project directory, so the record is + # named relative to it. Derived from the real path rather than + # written out here. + named_as = "{}/{}".format(record.parent.name, record.name) + assert named_as in again.stdout, \ + "the warning must name the record it rebuilt: {}".format( + again.stdout) + assert "course.rst" in again.stdout, \ + "the warning must say which block it is about, or the record it " \ + "names cannot be located from the message alone: {}".format( + again.stdout) + assert "extracted and the run was not cut short" in again.stdout, \ + "the warning must say the run was not cut short: {}".format( + again.stdout) + assert "Traceback" not in again.stderr, \ + "the record must be rebuilt, not crashed on: {}".format( + again.stderr) + + assert record.read_text() != self.DAMAGED_RECORD, \ + "the damaged record must have been rewritten, not merely reported" + + checked = _run("check-code", "--build-dir", "build", cwd=tmp_path) + assert checked.returncode == 0, \ + "the example whose record was rebuilt must still check out: " \ + "{}".format(checked.stdout) + assert RUN_OUTPUT in _the_run_log(tmp_path), \ + "the example must really have been built and run after its " \ + "record was rebuilt" + + +# --------------------------------------------------------------------------- +# A build directory in which the record's name is held by a directory +# --------------------------------------------------------------------------- + + +@pytest.mark.toolchain +class TestACourseWhoseBlockRecordIsADirectory: + """extract-code finding a directory where a record it wrote earlier stood. + + An interrupted copy into a kept build directory leaves this behind. It is + not a damaged record -- it is no record at all, because nothing can open + it -- and the two cases end differently: a block directory with no record + is removed and extracted again, while a record reported as rebuilt is one + that was read and found unusable. + + Asserted through the command because the cost of getting it wrong is paid + there: the run reports a repair it did not make and then ends in a + traceback, which is what a build driving these commands sees. + """ + + def test_the_block_directory_is_extracted_again_without_a_traceback( + self, tmp_path): + """A record name held by a directory must leave the run succeeding, + with the record a readable file again and no rebuild announced.""" + assert _extract(tmp_path, "CliCourseRecordIsADirectory", + WORKING_ADA_BODY).returncode == 0, \ + "the course must extract cleanly first, or there is no record " \ + "for a directory to stand in place of" + + record = pathlib.Path(_the_extracted_block(tmp_path)) + record.unlink() + record.mkdir() + + again = _run("extract-code", "--build-dir", "build", "course.rst", + cwd=tmp_path) + + assert "Traceback" not in again.stderr, \ + "a record name held by a directory must be extracted again, not " \ + "crashed on: {}".format(again.stderr) + assert again.returncode == 0, \ + "extracting the block again is a recovery, so the run must still " \ + "succeed: {}".format(again.stdout) + assert "no JSON info file" in again.stdout, \ + "nothing could be read, so the run must report a block directory " \ + "with no record rather than a record it rebuilt: {}".format( + again.stdout) + assert "being rebuilt" not in again.stdout, \ + "no record was read, so none may be announced as rebuilt: " \ + "{}".format(again.stdout) + + assert record.is_file(), \ + "the block directory was extracted again, so its record must be " \ + "a file once more" + assert blocks.CodeBlock.from_json_file(str(record)) is not None, \ + "the record written in place of the directory must read back as " \ + "a block: {}".format(record.read_text()) + + checked = _run("check-code", "--build-dir", "build", cwd=tmp_path) + assert checked.returncode == 0, \ + "the example extracted again must check out: {}".format( + checked.stdout) + assert RUN_OUTPUT in _the_run_log(tmp_path), \ + "the example must really have been built and run after its block " \ + "directory was extracted again" + + +# --------------------------------------------------------------------------- +# Command lines that are rejected before any example is looked at +# --------------------------------------------------------------------------- + +class TestRejectedCommandLines: + def test_check_code_needs_somewhere_to_look(self, tmp_path): + """check-code with neither a build directory nor a project list has + nothing to check and must fail rather than report success over + nothing.""" + result = _run("check-code", cwd=tmp_path) + assert result.returncode == 1, \ + "check-code must fail when it is told nowhere to look: " \ + "{}".format(result.stdout) + + def test_extract_code_needs_somewhere_to_write(self, tmp_path): + """extract-code with neither a build directory nor a project list has + nowhere to put what it extracts and must fail.""" + rst_file = _write_course(tmp_path, "CliNoDestination", WORKING_ADA_BODY) + result = _run("extract-code", rst_file, cwd=tmp_path) + assert result.returncode == 1, \ + "extract-code must fail when it is told nowhere to write: " \ + "{}".format(result.stdout) + + @pytest.mark.parametrize("command", + ["extract-code", "check-code", "check-block"]) + def test_an_unknown_switch_is_rejected_outright(self, command, tmp_path): + """A command line that cannot be parsed is rejected with a status of + its own, so that a script can tell a mistyped invocation apart from an + example that failed its check.""" + result = _run(command, "--no-such-switch", cwd=tmp_path) + assert result.returncode == 2, \ + "{} must reject an unknown switch with the argument-parsing " \ + "status: {}".format(command, result.stderr) diff --git a/frontend/python/rst_code_example_pipeline/tests/test_colors.py b/frontend/python/rst_code_example_pipeline/tests/test_colors.py new file mode 100644 index 000000000..528e11bdd --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_colors.py @@ -0,0 +1,305 @@ +""" +Unit tests for rst_code_example_pipeline.colors. + +Covers: +- col() with colors enabled and disabled +- printcol() output captured via capsys +- no_colors() context manager (disable inside, restore outside) +- Colors.disable_colors() and state restore +- Adversarial: direct __enter__/__exit__ use on no_colors(), and restoring the + previous setting when the guarded block raises +- both answers to the terminal test the module makes when it is imported: + colors survive an import under a terminal and are switched off as soon as + either of the two output streams is not one +""" +import importlib +import sys + +import pytest + +from rst_code_example_pipeline import colors as colors_module +from rst_code_example_pipeline.colors import Colors, col, no_colors, printcol + + +# --------------------------------------------------------------------------- +# T-colors-01: col() enabled +# --------------------------------------------------------------------------- + +class TestColEnabled: + def test_col_wraps_with_prefix_and_endc(self): + Colors._enabled = True + result = col("hello", Colors.RED) + assert result == f"{Colors.RED}hello{Colors.ENDC}" + + def test_col_endc_does_not_double_wrap(self): + """Passing Colors.ENDC as color should still wrap correctly.""" + Colors._enabled = True + result = col("msg", Colors.ENDC) + assert result == f"{Colors.ENDC}msg{Colors.ENDC}" + + +# --------------------------------------------------------------------------- +# T-colors-02: col() disabled +# --------------------------------------------------------------------------- + +class TestColDisabled: + def test_col_returns_bare_string_when_disabled(self): + Colors._enabled = False + assert col("hello", Colors.RED) == "hello" + + def test_col_empty_string_disabled(self): + Colors._enabled = False + assert col("", Colors.BLUE) == "" + + +# --------------------------------------------------------------------------- +# T-colors-03: printcol() output +# --------------------------------------------------------------------------- + +class TestPrintcol: + def test_printcol_prints_the_bare_message_when_disabled(self, capsys): + Colors._enabled = False + printcol("hello output", Colors.GREEN) + captured = capsys.readouterr() + assert captured.out == "hello output\n" + assert captured.err == "" + + def test_printcol_prints_the_wrapped_message_when_enabled(self, capsys): + Colors._enabled = True + printcol("msg", Colors.RED) + captured = capsys.readouterr() + assert captured.out == f"{Colors.RED}msg{Colors.ENDC}\n" + assert captured.err == "" + + +# --------------------------------------------------------------------------- +# T-colors-04: no_colors() context manager +# --------------------------------------------------------------------------- + +class TestNoColors: + def test_no_colors_disables_inside(self): + Colors._enabled = True + with no_colors(): + assert Colors._enabled is False + + def test_no_colors_restores_outside_when_was_true(self): + Colors._enabled = True + with no_colors(): + pass + assert Colors._enabled is True + + def test_no_colors_restores_outside_when_was_false(self): + Colors._enabled = False + with no_colors(): + pass + assert Colors._enabled is False + + def test_no_colors_col_returns_bare_inside(self): + Colors._enabled = True + with no_colors(): + result = col("bare", Colors.RED) + assert result == "bare" + + def test_no_colors_col_colored_outside(self): + Colors._enabled = True + with no_colors(): + pass + result = col("colored", Colors.RED) + assert Colors.RED in result + + def test_no_colors_nested(self): + """Nested no_colors() context managers must each restore correctly.""" + Colors._enabled = True + with no_colors(): + assert Colors._enabled is False + with no_colors(): + assert Colors._enabled is False + assert Colors._enabled is False + assert Colors._enabled is True + + +# --------------------------------------------------------------------------- +# T-colors-05: disable_colors() +# --------------------------------------------------------------------------- + +class TestDisableColors: + def test_disable_colors_sets_enabled_false(self): + Colors._enabled = True + Colors.disable_colors() + assert Colors._enabled is False + + def test_col_after_disable_colors(self): + Colors._enabled = True + Colors.disable_colors() + assert col("test", Colors.GREEN) == "test" + + +# --------------------------------------------------------------------------- +# T-colors-06: Adversarial — direct __enter__/__exit__ on no_colors() +# --------------------------------------------------------------------------- + +class TestNoColorsAdversarial: + def test_direct_enter_exit(self): + """Using __enter__/__exit__ directly (without `with`) must still restore state.""" + Colors._enabled = True + ctx = no_colors() + ctx.__enter__() + assert Colors._enabled is False + ctx.__exit__(None, None, None) + assert Colors._enabled is True + + def test_direct_enter_exit_when_was_false(self): + Colors._enabled = False + ctx = no_colors() + ctx.__enter__() + assert Colors._enabled is False + ctx.__exit__(None, None, None) + assert Colors._enabled is False + + def test_no_colors_restores_state_when_the_block_raises(self): + """An exception escaping the 'with' block must still restore the + previous color setting: no_colors() only narrows the scope it was + given, so a caller that lets an exception through must not be left + with colors silently disabled for the rest of the process.""" + Colors._enabled = True + with pytest.raises(ValueError): + with no_colors(): + assert Colors._enabled is False + raise ValueError("oops") + assert Colors._enabled is True + + +# --------------------------------------------------------------------------- +# T-colors-07: the terminal test the module makes when it is imported +# --------------------------------------------------------------------------- + +class _StreamAnsweringIsatty: + """An output stream that answers the terminal question a given way. + + Everything else is handed to the real stream, so a wrapped stream stays + usable -- which matters because the one being wrapped is the one pytest + has put in place to capture output. + """ + + def __init__(self, stream, is_a_tty: bool): + self._stream = stream + self._is_a_tty = is_a_tty + + def isatty(self) -> bool: + return self._is_a_tty + + def __getattr__(self, name): + return getattr(self._stream, name) + + +@pytest.fixture() +def imported_under_streams(monkeypatch): + """Import the module again with the two output streams answering the + terminal question a given way, and hand back what it decided. + + The decision is made once, at import, from ``sys.stdout`` and + ``sys.stderr`` -- so the only way to drive it is to import the module + again with those streams replaced. Under pytest neither is a terminal, + which is why the arm that keeps colors on was never entered by anything. + + Re-importing rebinds every name in the module, including the class the + rest of this file and the shared color-state fixture hold directly. The + module's contents are therefore put back afterwards, so that the class + those two are holding is the class the module goes on exposing -- and so + that the two modules importing this one are not left looking at a + different class from everybody else. + """ + saved = dict(colors_module.__dict__) + saved_enabled = Colors._enabled + + def reimport(stdout_is_a_tty: bool, stderr_is_a_tty: bool): + monkeypatch.setattr( + sys, "stdout", + _StreamAnsweringIsatty(sys.stdout, stdout_is_a_tty)) + monkeypatch.setattr( + sys, "stderr", + _StreamAnsweringIsatty(sys.stderr, stderr_is_a_tty)) + importlib.reload(colors_module) + return colors_module + + yield reimport + + colors_module.__dict__.clear() + colors_module.__dict__.update(saved) + Colors._enabled = saved_enabled + + +class TestTheTerminalTestMadeAtImport: + def test_colors_are_kept_when_both_streams_are_terminals( + self, imported_under_streams): + """Under a terminal on both streams the module must leave colors on. + + This is the arm the whole class exists for: nothing had ever imported + the module with a terminal on both streams, so a module that switched + colors off unconditionally would have looked identical. Asserted + through what col() produces as well as through the setting, since the + setting is only interesting for what it makes the output do. + """ + reimported = imported_under_streams(True, True) + assert reimported.Colors._enabled is True, \ + "an import under a terminal must leave colors enabled" + assert reimported.col("msg", reimported.Colors.RED) == \ + "{}msg{}".format(reimported.Colors.RED, reimported.Colors.ENDC), \ + "colors left enabled must actually color the output" + + def test_colors_are_switched_off_when_stdout_is_not_a_terminal( + self, imported_under_streams): + """Standard output not being a terminal must switch colors off. + + The case that matters in practice: the command's output is being piped + into a log or a build report, where escape sequences are noise. + """ + reimported = imported_under_streams(False, True) + assert reimported.Colors._enabled is False, \ + "colors must be switched off when standard output is not a terminal" + assert reimported.col("msg", reimported.Colors.RED) == "msg", \ + "colors switched off must leave the output bare" + + def test_colors_are_switched_off_when_only_stderr_is_not_a_terminal( + self, imported_under_streams): + """Standard error alone not being a terminal must switch colors off + too. + + Written separately rather than left to the test above: the module asks + the question of both streams, and one that stopped asking it of + standard error would still satisfy every other test here. + """ + reimported = imported_under_streams(True, False) + assert reimported.Colors._enabled is False, \ + "colors must be switched off when standard error is not a terminal" + assert reimported.col("msg", reimported.Colors.RED) == "msg", \ + "colors switched off must leave the output bare" + + def test_reimporting_really_produces_a_new_class( + self, imported_under_streams): + """The re-import must really replace the class, not hand back the one + already in place. + + Without this, the three tests above could all be reading the setting + of the class this file imported at collection time -- which pytest + leaves switched off -- and would go on passing whatever the module + decided under the streams they set up. + """ + assert imported_under_streams(True, True).Colors is not Colors, \ + "re-importing must produce a new class, or the tests above are " \ + "asserting against the class that was already there" + + def test_the_module_still_exposes_the_class_this_file_imported(self): + """After the re-imports, the module must expose the same class again. + + Collected after them, so it sees what they left behind. Everything + else in this file, the shared color-state fixture, and the two modules + that import this one all hold names bound before any re-import: if the + module were left exposing the replacement class, they would be setting + and restoring a flag nothing reads, and every one of those tests would + pass over an output nobody colored. + """ + assert colors_module.Colors is Colors, \ + "the module must expose the class this file imported" + assert colors_module.col is col, \ + "the module must expose the function this file imported" diff --git a/frontend/python/rst_code_example_pipeline/tests/test_extract_projects.py b/frontend/python/rst_code_example_pipeline/tests/test_extract_projects.py new file mode 100644 index 000000000..3d3f225e5 --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_extract_projects.py @@ -0,0 +1,1480 @@ +""" +Unit tests for rst_code_example_pipeline.extract_projects. + +Covers: +- the configuration the module starts every run with: built from real booleans, + and holding what it declares rather than the opposite +- get_project_dir(): simple and dotted project names +- write_project_file(): all four combinations of spark_mode × main_file × compiler_switches +- write_project_file(): the generated project points at the configuration pragma + file the same call wrote, in both plain and SPARK mode +- write_project_file(): the plain and SPARK modes write separate project and + pragma files, so a block that is both proved and run keeps both +- ProjectsList: init, add(), to_json_file(), from_json_file() round-trip, missing file +- analyze_file(): minimal no-check / syntax-only Ada block +- analyze_file(): a block directory left over from a prior run whose info JSON file was + deleted is detected as stale, logged, and removed rather than reused +- analyze_file(): a block directory left over from a prior run whose info JSON file's name + is held by a directory is treated as having no record at all -- removed and extracted + again rather than announced as a rebuilt record and then written over +- analyze_file(): a block record left over from a prior run that is present but cannot be + read is rebuilt, the run still succeeds, and a warning names the file as rebuilt -- while + a record that reads back is repaired silently, because it was never damaged +- analyze_file() integration: compile_button / run_button / prove_button Ada blocks -- + the extracted source, the per-block directory name and the generated project files + (requires the Ada toolchain — real gnatchop and write_project_file calls) +- analyze_file(): a block whose source text chops into zero source files is logged and + skipped rather than crashing the whole analysis +- Global state (verbose, code_block_at, current_config) reset before each test + +NOTE: a no-check block does not spare analyze_file() the toolchain. The chop step runs +before the no-check test, and every block reaching it goes through the toolchain setup, +which writes into the toolchain installation tree. Every analyze_file() test that +reaches the block loop therefore carries the `toolchain` marker; the only unmarked +analyze_file() tests are the two that return before that loop (a block without a +project, and a file whose blocks are all inactive). The get_project_dir(), +write_project_file(), ProjectsList and Diag tests never call analyze_file() at all and +need no marker. +""" +import json +import os +import re +import shutil + +import pytest + +import rst_code_example_pipeline.extract_projects as ep +from rst_code_example_pipeline import blocks as _blocks_mod +from rst_code_example_pipeline import constants as _constants + + +def _pragma_file(directory, project_filename: str): + """The configuration pragma file a generated project points at. + + Located through the project's own reference rather than through a name + written down here, so that a test says what a build against that project + would pick up instead of restating a name the package was free to choose. + """ + project_text = (directory / project_filename).read_text() + named = re.search(r'for Global_Configuration_Pragmas use "([^"]+)"', + project_text) + assert named is not None, \ + "the generated project must name a configuration pragma file" + return directory / named.group(1) + + +def _configuration_pragmas(directory, project_filename: str) -> str: + """The configuration pragmas a generated project pulls in. + + A project naming a file nobody wrote is caught here, by name, instead of + surfacing much later as a build that quietly used none of them. + """ + pragma_file = _pragma_file(directory, project_filename) + assert pragma_file.is_file(), \ + "{} names {}, which was never written".format( + project_filename, pragma_file.name) + return pragma_file.read_text() + + +# --------------------------------------------------------------------------- +# TestDefaultConfiguration: the module's own starting configuration +# --------------------------------------------------------------------------- + +class TestDefaultConfiguration: + """The configuration the module starts every run with. + + It is the one place in the package that builds a configuration out of + real booleans rather than out of the strings a code-config directive + produces, and those were once read by comparing them against a string + they could never equal -- so all three came out true and the module + began every run with the opposite of two of the values it declares. + + Asserted against the declared call rather than against a list of values + repeated here, so that changing what the module declares changes what + this test expects, and only the reading of it is pinned. + """ + + @staticmethod + def _declared() -> dict: + """What the module asked for, taken from the configuration itself. + + ConfigBlock keeps the arguments it was constructed with, so the + request and the answer can be compared without either being written + down in this file. + """ + return ep.current_config._opts + + def test_the_module_declares_its_configuration_with_real_booleans(self): + """The precondition for the test below: if these stopped being real + booleans the reading under test would not be the one exercised.""" + declared = self._declared() + assert declared, \ + "the module must start from a configuration that asks for something" + assert all(isinstance(value, bool) for value in declared.values()), \ + "the module's own configuration is the real-boolean caller this " \ + "reading exists for: {}".format(declared) + + def test_the_starting_configuration_holds_what_the_module_asked_for(self): + for name, requested in self._declared().items(): + assert getattr(ep.current_config, name) is requested, \ + "the starting configuration must hold the value the module " \ + "declared for {}, not its opposite".format(name) + + def test_the_starting_configuration_is_not_uniformly_true(self): + """The control for the test above. + + A reading that answered true for everything satisfied the values the + module happens to ask for as true, so a request that is all-true + would not distinguish the two readings at all. + """ + assert not all(self._declared().values()), \ + "the module's own configuration must ask for at least one false " \ + "value, or it cannot tell a correct reading from one that " \ + "answers true for everything" + + +# --------------------------------------------------------------------------- +# T-extract_projects-01: get_project_dir() +# --------------------------------------------------------------------------- + +class TestGetProjectDir: + def test_simple_name(self): + assert ep.get_project_dir("Simple") == "projects/Simple" + + def test_dotted_name_two_parts(self): + assert ep.get_project_dir("Foo.Bar") == "projects/Foo/Bar" + + def test_dotted_name_three_parts(self): + assert ep.get_project_dir("A.B.C") == "projects/A/B/C" + + def test_base_prefix_always_present(self): + result = ep.get_project_dir("X") + assert result.startswith("projects/") + + def test_no_trailing_slash(self): + result = ep.get_project_dir("Foo") + assert not result.endswith("/") + + +# --------------------------------------------------------------------------- +# T-extract_projects-02: write_project_file() +# --------------------------------------------------------------------------- + +class TestWriteProjectFile: + def test_no_main_no_switches_not_spark_creates_gpr(self, work_dir): + ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=False) + assert list(work_dir.glob("*.gpr")), "a project file must be written" + + def test_no_main_no_switches_not_spark_creates_adc(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=False) + assert _pragma_file(work_dir, result).is_file(), \ + "the pragma file the project points at must be written" + + def test_returns_gpr_filename_not_spark(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=False) + assert (work_dir / result).is_file(), \ + "the name returned must be the project file that was written" + + def test_no_main_placeholder_absent_when_none(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=False) + content = (work_dir / result).read_text() + assert "for Main use" not in content + + def test_with_main_file_gpr_contains_main_use(self, work_dir): + result = ep.write_project_file(main_file="main.adb", compiler_switches=[], + spark_mode=False) + content = (work_dir / result).read_text() + assert 'for Main use ("main.adb")' in content + + def test_with_compiler_switch_gpr_contains_switch(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=["-gnatwa"], + spark_mode=False) + content = (work_dir / result).read_text() + assert '"-gnatwa"' in content + + def test_multiple_switches_all_present(self, work_dir): + result = ep.write_project_file( + main_file=None, compiler_switches=["-gnatwa", "-gnatwe"], spark_mode=False + ) + content = (work_dir / result).read_text() + assert '"-gnatwa"' in content + assert '"-gnatwe"' in content + + def test_spark_mode_creates_main_spark_gpr(self, work_dir): + ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=True) + assert list(work_dir.glob("*.gpr")), \ + "a project file must be written in SPARK mode too" + + def test_spark_mode_creates_main_spark_adc(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=True) + assert _pragma_file(work_dir, result).is_file(), \ + "the pragma file the SPARK project points at must be written" + + def test_spark_mode_returns_spark_gpr_filename(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=True) + assert (work_dir / result).is_file(), \ + "the name returned must be the project file that was written" + + def test_the_two_modes_write_separate_projects(self, work_dir): + """A SPARK project and a plain one can sit side by side. + + The two modes are asked for one after the other for the same block -- + a block that is both proved and run gets both -- so they have to write + to different places. Were they to share a name the second call would + overwrite the first, and the block would be built against whichever + project happened to be written last. + """ + plain = ep.write_project_file(main_file=None, compiler_switches=[], + spark_mode=False) + spark = ep.write_project_file(main_file=None, compiler_switches=[], + spark_mode=True) + assert plain != spark, \ + "the two modes must not write to the same project file" + assert (work_dir / plain).is_file() and (work_dir / spark).is_file(), \ + "both project files must survive the other being written" + assert _pragma_file(work_dir, plain) != _pragma_file(work_dir, spark), \ + "the two projects must not share a configuration pragma file" + + def test_spark_adc_contains_spark_mode_pragma(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=True) + assert "pragma SPARK_Mode (On);" in _configuration_pragmas(work_dir, result) + + def test_non_spark_adc_does_not_contain_spark_pragma(self, work_dir): + result = ep.write_project_file(main_file=None, compiler_switches=[], spark_mode=False) + assert "pragma SPARK_Mode" not in _configuration_pragmas(work_dir, result) + + @pytest.mark.parametrize("spark_mode", [False, True], ids=["plain", "spark"]) + def test_project_names_the_pragma_file_the_same_call_wrote( + self, work_dir, spark_mode): + """The pragma file a generated project points at is the one written + beside it. + + The project text and the pragma file are produced by two separate + parts of one call, and nothing in the generator checks that the two + agree on the name. A disagreement leaves both files on disk and is + invisible here; only a later build against the project would meet it. + """ + result = ep.write_project_file( + main_file=None, compiler_switches=[], spark_mode=spark_mode + ) + assert _configuration_pragmas(work_dir, result).strip(), \ + "the pragma file the project names must have something in it" + + def test_full_combo_main_switches_spark(self, work_dir): + result = ep.write_project_file( + main_file="main.adb", compiler_switches=["-gnatwa"], spark_mode=True + ) + gpr = (work_dir / result).read_text() + assert 'for Main use ("main.adb")' in gpr + assert '"-gnatwa"' in gpr + # Says the project really is the SPARK one, by what it configures + # rather than by what it is called. + assert "pragma SPARK_Mode (On);" in _configuration_pragmas(work_dir, result) + + +# --------------------------------------------------------------------------- +# T-extract_projects-03: ProjectsList +# --------------------------------------------------------------------------- + +class TestProjectsList: + def test_init_no_args_empty_projects(self): + pl = ep.ProjectsList() + assert pl.projects == {} + + def test_init_with_projects_arg(self): + pl = ep.ProjectsList(projects={"Foo": True}) + assert pl.projects == {"Foo": True} + + def test_add_project_appears_in_dict(self): + pl = ep.ProjectsList() + pl.add("MyProject") + assert "MyProject" in pl.projects + assert pl.projects["MyProject"] is True + + def test_add_multiple_projects(self): + pl = ep.ProjectsList() + pl.add("A") + pl.add("B") + assert set(pl.projects.keys()) == {"A", "B"} + + def test_to_json_file_creates_file(self, tmp_path): + pl = ep.ProjectsList() + pl.add("Foo") + dest = str(tmp_path / "projects.json") + pl.to_json_file(dest) + assert os.path.isfile(dest) + + def test_to_json_file_content_is_valid_json(self, tmp_path): + pl = ep.ProjectsList() + pl.add("Bar") + dest = str(tmp_path / "projects.json") + pl.to_json_file(dest) + with open(dest) as f: + data = json.load(f) + assert "projects" in data + assert data["projects"]["Bar"] is True + + def test_round_trip_preserves_projects(self, tmp_path): + pl = ep.ProjectsList() + pl.add("Alpha") + pl.add("Beta") + dest = str(tmp_path / "roundtrip.json") + pl.to_json_file(dest) + pl2 = ep.ProjectsList.from_json_file(dest) + assert pl2 is not None + assert set(pl2.projects.keys()) == {"Alpha", "Beta"} + + def test_from_json_file_nonexistent_returns_none(self, tmp_path): + result = ep.ProjectsList.from_json_file(str(tmp_path / "no_such.json")) + assert result is None + + def test_to_json_file_overwrites_silently(self, tmp_path): + pl1 = ep.ProjectsList() + pl1.add("First") + dest = str(tmp_path / "over.json") + pl1.to_json_file(dest) + + pl2 = ep.ProjectsList() + pl2.add("Second") + pl2.to_json_file(dest) + + pl_loaded = ep.ProjectsList.from_json_file(dest) + assert pl_loaded is not None + assert "Second" in pl_loaded.projects + assert "First" not in pl_loaded.projects + + +# --------------------------------------------------------------------------- +# T-extract_projects-04: analyze_file() — minimal no-check block +# --------------------------------------------------------------------------- + +class TestAnalyzeFile: + # A minimal RST file with a single Ada block marked as no-check. + # The no-check class keeps analyze_file() from compiling or running the + # block, but it is still chopped and still goes through the toolchain + # setup, so these tests need the Ada toolchain all the same. + # NOTE: analyze_file() requires every code block to have a project attribute; + # blocks without one cause exit(1). Always include project=... here. + NOCHECK_RST = """\ +.. code:: ada project=NoCheckProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + + # A single Ada block whose chopping is made to yield nothing, so no source + # file is ever written out for it. + EMPTY_CHOP_RST = """\ +.. code:: ada project=EmptyChopProject main=main.adb compile_button + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + + def _write_rst(self, tmp_path, content: str) -> str: + rst_path = tmp_path / "test_nocheck.rst" + rst_path.write_text(content) + return str(rst_path) + + @pytest.mark.toolchain + def test_no_crash_on_nocheck_block(self, work_dir): + rst_file = self._write_rst(work_dir, self.NOCHECK_RST) + # analyze_file() must return without raising + result = ep.analyze_file(rst_file) + assert result is False + + @pytest.mark.toolchain + def test_no_crash_on_nocheck_block_with_project(self, work_dir): + rst_content = """\ +.. code:: ada project=TestProj + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False + + @pytest.mark.toolchain + def test_analyze_file_creates_project_dirs(self, work_dir): + rst_content = """\ +.. code:: ada project=MyProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + ep.analyze_file(rst_file) + project_dir = work_dir / "projects" / "MyProject" + assert project_dir.exists(), \ + f"Expected project directory {project_dir} to be created" + + @pytest.mark.toolchain + def test_analyze_file_with_projects_list_file(self, work_dir): + rst_content = """\ +.. code:: ada project=ListedProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + prj_list_file = str(work_dir / "projects.json") + ep.analyze_file(rst_file, prj_list_file) + # The projects list JSON file must have been created + assert os.path.isfile(prj_list_file), \ + "analyze_file() must write the projects list JSON file" + with open(prj_list_file) as f: + data = json.load(f) + assert "projects" in data + assert "ListedProject" in data["projects"] + + @pytest.mark.toolchain + def test_analyze_file_verbose_existing_projects_list_file(self, work_dir, capsys): + """verbose=True + extracted_projects_list_file pointing at a file that + already exists prints the 'Extracted list of projects...' message.""" + prj_list = work_dir / "projects.json" + prj_list.write_text('{"projects": {}}') + ep.verbose = True + rst_file = self._write_rst(work_dir, self.NOCHECK_RST) + result = ep.analyze_file(rst_file, str(prj_list)) + assert result is False + assert "Extracted list" in capsys.readouterr().out + + @pytest.mark.toolchain + def test_analyze_file_verbose_missing_projects_list_file(self, work_dir, capsys): + """verbose=True + extracted_projects_list_file pointing at a file that + does not exist yet prints the 'will be created' message.""" + prj_list = work_dir / "new_projects.json" + ep.verbose = True + rst_file = self._write_rst(work_dir, self.NOCHECK_RST) + result = ep.analyze_file(rst_file, str(prj_list)) + assert result is False + assert "will be created" in capsys.readouterr().out + + @pytest.mark.toolchain + def test_analyze_file_existing_projects_list_loaded(self, work_dir): + # Pre-create a projects list JSON with an existing entry + prj_list_file = str(work_dir / "projects.json") + existing = ep.ProjectsList() + existing.add("ExistingProject") + existing.to_json_file(prj_list_file) + + rst_content = """\ +.. code:: ada project=NewProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + ep.analyze_file(rst_file, prj_list_file) + + with open(prj_list_file) as f: + data = json.load(f) + # Both the pre-existing and the new project must be in the file + assert "NewProject" in data["projects"], \ + "New project must be added to the existing projects list" + + @pytest.mark.toolchain + def test_analyze_file_syntax_only_block(self, work_dir): + rst_content = """\ +.. code:: ada project=SyntaxProject + :class: ada-syntax-only + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + # syntax_only blocks are still processed (no toolchain invocation needed + # inside analyze_file for the project extraction phase) + assert result is False + + def test_analyze_file_no_project_raises_system_exit(self, work_dir): + """analyze_file() calls exit(1) when a block has no project attribute.""" + rst_content = """\ +.. code:: ada + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + with pytest.raises(SystemExit): + ep.analyze_file(rst_file) + + @pytest.mark.toolchain + def test_analyze_file_no_button_block(self, work_dir): + """A non-no-check, non-syntax-only block with buttons=["no"] reaches + the project extraction path and writes block_info.json without error.""" + rst_content = """\ +.. code:: ada project=NoBtnProject no_button + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False + + @pytest.mark.toolchain + def test_analyze_file_config_block(self, work_dir): + """A :code-config: line produces a ConfigBlock; analyze_file() must handle + it (via isinstance check) without crashing.""" + rst_content = """\ +:code-config:`run_button=False;prove_button=True;accumulate_code=False` + +.. code:: ada project=CfgProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False + + @pytest.mark.toolchain + def test_analyze_file_manual_chop_block(self, work_dir): + """A C block uses manual_chop=True; analyze_file() must call manual_chop + (not real_gnatchop) and succeed.""" + rst_content = """\ +.. code:: c project=CProject no_button + + !main.c + int main(void) { return 0; } + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False + + @pytest.mark.toolchain + def test_code_block_at_matches_one_block(self, work_dir): + """A block whose line range contains the requested line must stay + active and be extracted.""" + ep.code_block_at = 4 + rst_file = self._write_rst(work_dir, self.NOCHECK_RST) + result = ep.analyze_file(rst_file) + assert result is False + # The block stayed active, so its project directory must exist. + assert (work_dir / "projects" / "NoCheckProject").exists() + + def test_code_block_at_sets_inactive(self, work_dir, capsys): + """A requested line that falls inside no block must leave every block + inactive, so that nothing is extracted.""" + # code_block_at=9999 is far beyond any line in the small RST fixture + ep.code_block_at = 9999 + rst_file = self._write_rst(work_dir, self.NOCHECK_RST) + result = ep.analyze_file(rst_file) + assert result is False + # No project directory should have been created (all blocks inactive) + assert not (work_dir / "projects" / "NoCheckProject").exists(), \ + "No project dir expected when all blocks are inactive" + + @pytest.mark.toolchain + def test_verbose_prints_headers(self, work_dir, capsys): + """Set verbose=True and confirm that project header lines are printed.""" + ep.verbose = True + rst_content = """\ +.. code:: ada project=VerboseProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + ep.analyze_file(rst_file) + out = capsys.readouterr().out + # The verbose header and block count line should appear + assert "VerboseProject" in out, \ + "Expected project name in verbose output" + + @pytest.mark.toolchain + def test_second_call_same_project_logs_exists(self, work_dir, capsys): + """Call analyze_file() twice with the same project; the second call + must print 'already exists' when verbose=True.""" + ep.verbose = True + rst_content = """\ +.. code:: ada project=RepeatedProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + ep.analyze_file(rst_file) # first call: creates the project dir + # reset verbose (it gets cleared by the autouse fixture between tests, + # but we are in one test so set it again for the second call) + ep.verbose = True + capsys.readouterr() # discard first-call output + ep.analyze_file(rst_file) # second call: dir already exists + out = capsys.readouterr().out + assert "already exists" in out, \ + "Expected 'already exists' in verbose output on second call" + + @pytest.mark.toolchain + def test_stale_block_dir_missing_json_is_removed_and_recreated(self, work_dir, capsys): + """If a code block's per-block directory already exists from a prior + run but its info JSON file has since been deleted, the directory must + be treated as stale: logged and removed rather than reused, and the + analysis must complete without crashing.""" + rst_content = """\ +.. code:: ada project=StaleProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + rst_file = self._write_rst(work_dir, rst_content) + ep.analyze_file(rst_file) # first call: creates the block's info JSON + + block_jsons = list(work_dir.rglob("*.json")) + assert len(block_jsons) == 1, \ + f"Expected exactly 1 block record after the first call; found {len(block_jsons)}" + block_jsons[0].unlink() + + capsys.readouterr() # discard first-call output + result = ep.analyze_file(rst_file) # second call: block dir is stale + assert result is False + + out = capsys.readouterr().out + assert "no JSON info file" in out, \ + "Expected the stale-directory message when the info JSON is missing" + + # A block directory left over from an earlier run in which the record's + # name is taken by a directory rather than a file. An interrupted copy + # leaves this behind, and it is the state that tells the caller's guard + # apart from a looser one: a record that is not a file is one the reader + # will not open, so the only honest reading of it is that there is no + # record here at all. + RECORD_IS_A_DIRECTORY_RST = """\ +.. code:: ada project=RecordIsADirectoryProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + + @pytest.mark.toolchain + def test_block_record_whose_name_is_taken_by_a_directory_is_no_record( + self, work_dir, capsys): + """A block directory whose record name is held by a directory must be + treated as holding no record: removed, extracted again, and left with + a readable record in its place. + + The two repairs this code makes are told apart by whether a record is + there to be read. Only a *file* can be: the reader opens the record + through a guard of its own that asks for one, and hands back nothing + for anything else without saying why. So a directory standing where + the record belongs has to take the branch for a block directory with + no record -- the one that removes the directory and extracts the block + again -- and not the branch that announces a record it rebuilt. + + Taking the wrong branch here is not a cosmetic mislabeling. That + branch keeps the block directory, so the run goes on to write the + block's record into the name the directory holds, and ends in a + traceback about writing to a directory -- after having reported that + it repaired a file nothing ever read. + """ + rst_file = self._write_rst(work_dir, self.RECORD_IS_A_DIRECTORY_RST) + ep.analyze_file(rst_file) + + written = list(work_dir.rglob(_constants.BLOCK_INFO_FILENAME)) + assert len(written) == 1, \ + "expected exactly one block record after the first run, got " \ + "{}".format([str(path) for path in written]) + record = written[0] + + # The name the record stood under, taken over by a directory: what an + # interrupted copy leaves behind, and what the record must be again + # once the block directory has been rebuilt. + record.unlink() + record.mkdir() + + capsys.readouterr() # discard the first run's output + result = ep.analyze_file(rst_file) + out = capsys.readouterr().out + + assert result is False, \ + "removing a block directory that holds no readable record and " \ + "extracting the block again is a recovery, not a failure of the run" + + assert "no JSON info file" in out, \ + "a name held by a directory is no record, so the branch that " \ + "removes the block directory and extracts it again must be the " \ + "one that ran: {}".format(out) + assert "being rebuilt" not in out, \ + "nothing was read, so nothing may be reported as rebuilt -- that " \ + "message promises a record was read back and found damaged: " \ + "{}".format(out) + + assert record.is_file(), \ + "the block directory was rebuilt, so the record must be a file " \ + "again rather than the directory that stood in its place: " \ + "{}".format([str(path) for path in + self._project_dir( + work_dir, "RecordIsADirectoryProject").rglob("*")]) + assert _blocks_mod.CodeBlock.from_json_file(str(record)) is not None, \ + "the record written in place of the directory must read back as " \ + "a block: {}".format(record.read_text()) + + # The block record left over from an earlier run, in the two states the + # repair path tells apart. The reader refuses the first and accepts the + # second unchanged. + DAMAGED_RECORD = "{ this is not a block record" + + # The one project the file below extracts, and the two blocks it holds, + # in the order the extraction walks them. The first block owns the record + # that gets damaged. The second one exists only so that "the run was not + # cut short" can be settled by work that only a run continuing past the + # repair could have done, rather than by the wording of the message that + # claims it -- and it is put in the *same* project deliberately: blocks are + # walked in one loop per project, so a second block of the same project can + # only be reached by that loop carrying on past the repair, while a block + # of another project would be reached by an outer loop starting afresh and + # would prove nothing about the repaired block's own run. + REPAIRED_PROJECT = "RebuiltProject" + REPAIRED_UNIT = "Main" + UNIT_AFTER_THE_REPAIR = "Later" + + # Both blocks carry a no-check class, which is what makes this file the + # right fixture rather than a convenient one: the example is extracted and + # then deliberately skipped, so a warning promising it is *checked* would + # be false here. The test below asserts that promise is not made. + REBUILT_RST = """\ +.. code:: ada project=RebuiltProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. + +.. code:: ada project=RebuiltProject + :class: ada-nocheck + + procedure Later is + begin + null; + end Later; + +Another paragraph. +""" + + # The single source file gnatchop writes for the damaged block's example, + # named after the compilation unit its text declares. + EXTRACTED_SOURCE = "main.adb" + + def _project_dir(self, work_dir, project: str): + """The directory the extraction keeps one project's blocks under.""" + return work_dir / ep.get_project_dir(project) + + def _block_records(self, work_dir, project: str) -> dict: + """The block records below one project's directory, keyed by the name + of the compilation unit each one describes. + + Keyed by what the record says rather than by where it sits, because + the directory holding it is named after a hash of the block's text and + says nothing a test could read. A record that does not read back as a + block is left out: this reports what a run wrote, and a damaged record + describes no block at all. Looking one up therefore *answers* rather + than asserting, so that a missing one is reported by the assertion + that names the property it was looked up for. + """ + records = dict() + for path in sorted(self._project_dir(work_dir, project).rglob( + _constants.BLOCK_INFO_FILENAME)): + block = _blocks_mod.CodeBlock.from_json_file(str(path)) + if block is None: + continue + for unit in (self.REPAIRED_UNIT, self.UNIT_AFTER_THE_REPAIR): + if "procedure {}".format(unit) in block.text: + records[unit] = path + return records + + @pytest.mark.toolchain + def test_damaged_block_record_is_rebuilt_and_the_rebuild_is_announced( + self, work_dir, capsys): + """A block record that is present but cannot be read must be rebuilt, + must not fail the run, and must say so. + + This is the repair a kept build directory makes necessary: the record + of a block extracted earlier is damaged -- by an interrupted run, an + edit, a half-finished copy -- and the next extraction finds it there + and unreadable. Rewriting it and carrying on is the right outcome, + and it used to end the run with a traceback instead. + + The warning makes two claims, and both are checked against what the + run actually did rather than against its own wording: that the example + is still extracted -- the chopped source file is back on disk with the + block's code in it -- and that the run was not cut short -- the block + that follows the repaired one in the same project, whose output is + deleted before the repair run, is extracted again, which only a run + carrying on through that project's blocks past the repair can do. + + The wording is pinned on top of that, in both directions. The clause + must be present, so that a run that repaired silently cannot pass; and + the older, wider promise that the example is still *checked* must be + absent, because both blocks here carry a no-check class and are + extracted and then deliberately skipped, which would make that promise + false for exactly this input. + + The neighboring repair -- a block directory whose record has gone + missing entirely -- takes a different branch with a different message + and removes the directory. Its message is asserted absent, so this + test cannot pass by having taken that path instead. + + That the record is genuinely rebuilt is asserted last and matters + most: it is what the message promises, and a repair that printed the + line without rewriting the file would satisfy everything above it. + """ + rst_file = self._write_rst(work_dir, self.REBUILT_RST) + ep.analyze_file(rst_file) + + written = self._block_records(work_dir, self.REPAIRED_PROJECT) + assert set(written) == {self.REPAIRED_UNIT, + self.UNIT_AFTER_THE_REPAIR}, \ + "the first run must write one readable record per block of the " \ + "project, or there is nothing to damage and nothing to look for " \ + "afterwards: {}".format( + {unit: str(path) for unit, path in written.items()}) + + record = written[self.REPAIRED_UNIT] + original = record.read_text() + record.write_text(self.DAMAGED_RECORD) + + # Everything the second block produced is taken away again -- its + # whole directory, record and extracted source alike -- so that + # finding it back after the repair run can only mean that run reached + # it. Left in place, the first run's leftovers would satisfy the + # not-cut-short check for free. The staging directory the two blocks + # share is not a leftover either: the run empties it before the first + # block is extracted. + shutil.rmtree(written[self.UNIT_AFTER_THE_REPAIR].parent) + + capsys.readouterr() # discard the first run's output + result = ep.analyze_file(rst_file) + out = capsys.readouterr().out + + assert result is False, \ + "repairing the record is a recovery, not a failure of the run" + + assert "WARNING" in out, \ + "a rebuilt record must be announced as a warning, not left to be " \ + "inferred from the reader's error line: {}".format(out) + assert "Block info file could not be read and is being rebuilt" in out, \ + "the warning must say what was done to the file: {}".format(out) + # The repair runs from inside the project directory, so the record is + # named relative to it -- the block directory and the file within it. + # Taken from the real path rather than written out, and paired with + # the location prefix below, which is what makes a relative path + # enough to find the block again. + named_as = os.path.join(record.parent.name, record.name) + assert named_as in out, \ + "the warning must name the record it rebuilt: {}".format(out) + assert rst_file in out, \ + "the warning must say which block it is about, or the record it " \ + "names cannot be located from the message alone: {}".format(out) + assert "extracted and the run was not cut short" in out, \ + "the warning must say the run was not cut short, or a reader " \ + "cannot tell it apart from the fatal case: {}".format(out) + assert "still extracted and checked" not in out, \ + "the block carries a no-check class, so it is extracted and then " \ + "deliberately skipped -- the warning must not promise it is " \ + "checked: {}".format(out) + + assert "no JSON info file" not in out, \ + "the record was present, so the branch that removes a directory " \ + "with no record at all must not be the one that ran: {}".format(out) + + # The first claim the warning makes, taken from disk rather than from + # the message: the example really was extracted again. + extracted = (self._project_dir(work_dir, self.REPAIRED_PROJECT) + / "latest" / self.EXTRACTED_SOURCE) + assert extracted.is_file(), \ + "the warning says the example is still extracted, so its source " \ + "file must be on disk: {}".format( + [str(path) for path in + self._project_dir(work_dir, + self.REPAIRED_PROJECT).rglob("*")]) + assert "procedure Main" in extracted.read_text(), \ + "the extracted source must hold the block's code, not an empty " \ + "file left behind by a chop that wrote nothing: {}".format( + extracted.read_text()) + + # The second claim, likewise: the run carried on past the repair, + # through the rest of the same project's blocks, and extracted the one + # that follows it -- whose output was removed before this run started. + # Looked up rather than asserted for, so that a run cut short at the + # repair is reported by the assertion below, which names the property, + # rather than by a helper counting records. + rebuilt_project = self._block_records(work_dir, self.REPAIRED_PROJECT) + assert self.UNIT_AFTER_THE_REPAIR in rebuilt_project, \ + "the block after the repaired one, in the same project, must " \ + "have been extracted again, or the run was cut short at the " \ + "repair after all: {}".format( + {unit: str(path) for unit, path in rebuilt_project.items()}) + + # The record the repair rewrote is the one that was damaged, read back + # from where it stood: a repair that wrote a fresh record somewhere + # else would leave this one exactly as it was damaged. + rebuilt = record + assert rebuilt.read_text() != self.DAMAGED_RECORD, \ + "the damaged record must have been rewritten, not merely reported" + assert _blocks_mod.CodeBlock.from_json_file(str(rebuilt)) is not None, \ + "the rebuilt record must read back as a block, or the repair " \ + "left behind a record no more usable than the damaged one" + assert json.loads(rebuilt.read_text()) == json.loads(original), \ + "the rebuilt record must describe the same block the undamaged " \ + "run wrote" + + @pytest.mark.toolchain + def test_a_block_record_that_reads_back_is_not_announced_as_rebuilt( + self, work_dir, capsys): + """A second extraction over an undamaged record must say nothing + about rebuilding it. + + The control for the test above. A warning that fires whenever a + block directory is reused would satisfy every assertion there and + would tell a reader that a healthy build directory is damaged, which + is worse than saying nothing at all. + """ + rst_file = self._write_rst(work_dir, self.REBUILT_RST) + ep.analyze_file(rst_file) + + capsys.readouterr() # discard the first run's output + ep.analyze_file(rst_file) # the record is reused exactly as written + out = capsys.readouterr().out + + assert "being rebuilt" not in out, \ + "nothing was damaged, so nothing may be reported as rebuilt: " \ + "{}".format(out) + assert "WARNING" not in out, \ + "a reused build directory in good order must produce no warning " \ + "at all: {}".format(out) + + @pytest.mark.toolchain + def test_no_check_verbose_skip(self, work_dir, capsys): + """With verbose=True a no-check block must print a 'Skipping' message.""" + ep.verbose = True + rst_file = self._write_rst(work_dir, self.NOCHECK_RST) + ep.analyze_file(rst_file) + out = capsys.readouterr().out + assert "Skipping" in out, \ + "Expected 'Skipping' message for no-check block in verbose mode" + + @pytest.mark.toolchain + def test_chopper_returning_no_source_files_is_reported( + self, work_dir, monkeypatch, capsys): + """A block whose source text chops to nothing must be reported. + + Two distinct messages are printed, one from the immediate failure site + and one from the surrounding handler that moves on to the next block, + and the block itself is still logged so the remaining blocks get their + turn. + + The overall result the same run must report is covered by the + companion ``xfail`` test below; the two are kept apart so that losing + these messages fails the suite on its own.""" + monkeypatch.setattr(ep, "real_gnatchop", lambda *a, **kw: []) + + rst_file = self._write_rst(work_dir, self.EMPTY_CHOP_RST) + ep.analyze_file(rst_file) + + out = capsys.readouterr().out + assert "Failed to chop example" in out, \ + "Expected the immediate failure message when chopping yields nothing" + assert "Error while updating code for the block, continuing with next one!" in out, \ + "Expected the surrounding handler to report that it moves on" + assert list(work_dir.rglob("*.json")), \ + "Expected the failing block to still be logged before moving on" + + @pytest.mark.toolchain + @pytest.mark.xfail( + strict=True, + reason="the error flag raised when a block cannot be chopped is set on " + "a nested function's local, so analyze_file() still reports success", + ) + def test_chopper_returning_no_source_files_fails_the_run( + self, work_dir, monkeypatch): + """A block whose source text chops to nothing must fail the analysis. + + Chopping producing no source files at all means the block's code was + never written out, so the run cannot be called successful. The block + itself is still logged and skipped so the remaining blocks get their + turn, and the companion test above covers the diagnostics printed + along the way; the overall result, though, must report an error. + + Tracking note — this currently fails. The failure site assigns the + analysis-error flag inside a nested helper function, which makes it a + fresh local of that helper instead of updating the flag + ``analyze_file()`` eventually returns, so the run reports success and + the caller's exit code stays zero. The same site also re-raises with no + exception in flight, which turns the real diagnostic into Python's + ``No active exception to reraise`` message. A fix would declare the + flag ``nonlocal`` (and raise a real exception carrying the reason); + this test then passes and the ``xfail`` marker must be removed.""" + monkeypatch.setattr(ep, "real_gnatchop", lambda *a, **kw: []) + + rst_file = self._write_rst(work_dir, self.EMPTY_CHOP_RST) + assert ep.analyze_file(rst_file) is True, \ + "a per-block chopping failure must surface as an overall error" + + +# --------------------------------------------------------------------------- +# T-extract_projects-05: Diag class +# --------------------------------------------------------------------------- + +class TestDiag: + def test_fields_stored(self): + d = ep.Diag("f.adb", 3, 7, "error message") + assert d.file == "f.adb" + assert d.line == 3 + assert d.col == 7 + assert d.msg == "error message" + + def test_repr_format(self): + d = ep.Diag("f.adb", 3, 7, "error message") + assert repr(d) == "f.adb:3:7: error message" + + def test_repr_edge_case_zero_and_empty(self): + d = ep.Diag("", 0, 0, "") + assert repr(d) == ":0:0: " + + +# --------------------------------------------------------------------------- +# T-extract_projects-06: same-project second block +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestAnalyzeFileSameProjectTwoBlocks: + TWO_BLOCKS_RST = """\ +.. code:: ada project=SameProject + :class: ada-nocheck + + procedure Main is + begin + null; + end Main; + +First explanatory paragraph. + +.. code:: ada project=SameProject + :class: ada-nocheck + + procedure Helper is + begin + null; + end Helper; + +Second explanatory paragraph. +""" + + def _write_rst(self, tmp_path, content: str) -> str: + rst_path = tmp_path / "two_blocks.rst" + rst_path.write_text(content) + return str(rst_path) + + def test_two_blocks_same_project(self, work_dir): + """Two no-check Ada blocks declaring the same project= attribute must + both be extracted under that one project.""" + rst_file = self._write_rst(work_dir, self.TWO_BLOCKS_RST) + result = ep.analyze_file(rst_file) + assert result is False + # The project directory must have been created + assert (work_dir / "projects" / "SameProject").exists() + # Two separate block records must exist (each block has its own + # hash-named subdirectory) + block_jsons = list((work_dir / "projects" / "SameProject").rglob("*.json")) + assert len(block_jsons) == 2, \ + f"Expected 2 block records; found {len(block_jsons)}" + + +# --------------------------------------------------------------------------- +# C4 — TestAnalyzeFileIntegration +# analyze_file() with compile_button / run_button / prove_button Ada blocks. +# Requires the Ada toolchain (real gnatchop called for non-no-check blocks). +# --------------------------------------------------------------------------- + +@pytest.mark.toolchain +class TestAnalyzeFileIntegration: + """Integration tests for analyze_file() with real Ada compilation paths. + + Each RST fixture uses a valid Ada ``procedure Main`` body so that + real_gnatchop can parse it into exactly one source file. The block + attributes (compile_button / run_button / prove_button) set compile_it / + run_it / prove_it on the parsed CodeBlock. + """ + + # A minimal but valid Ada procedure that gnatchop can chop into one file. + _ADA_BODY = """\ +procedure Main is +begin + null; +end Main;""" + + # A C block asking for a prove button: proving is Ada-only, so this is a + # malformed example. + _C_PROVE_RST = ( + ".. code:: c project=TestCProve prove_button\n\n" + " !main.c\n" + " int main(void) { return 0; }\n\n" + "Explanatory paragraph.\n" + ) + + # A compile/run-eligible Ada block declaring no button indicator at all, + # not even no_button. + _NO_BUTTONS_RST = """\ +.. code:: ada project=TestNoBtns main=main.adb + + procedure Main is + begin + null; + end Main; + +Explanatory paragraph. +""" + + @staticmethod + def _write_rst(work_dir, content: str, name: str = "test_integration.rst") -> str: + rst_path = work_dir / name + rst_path.write_text(content) + return str(rst_path) + + @staticmethod + def _block_dir(work_dir, project: str): + """Return the single per-block directory written for ``project``. + + Every block gets its own directory below the project, named after the + short hash of its text so that two blocks cannot collide; ``latest`` + is the staging copy and is not one of them.""" + project_dir = work_dir / "projects" / project + block_dirs = sorted(d for d in project_dir.iterdir() + if d.is_dir() and d.name != "latest") + assert len(block_dirs) == 1, \ + "expected exactly one per-block directory, got {}".format( + [d.name for d in block_dirs]) + return block_dirs[0] + + @staticmethod + def _block_info(block_dir) -> dict: + """The record the extraction step wrote for a block, of which there is + one. + + Taken as the JSON file that is there rather than by a name written + down here: the extraction step chooses that name from the package's + own default, and the check step goes looking for the same default. + """ + written = sorted(block_dir.glob("*.json")) + assert len(written) == 1, \ + "expected exactly one block record, got {}".format( + [path.name for path in written]) + return json.loads(written[0].read_text()) + + def test_analyze_file_compile_button(self, work_dir): + """RST with a compile_button Ada block: analyze_file() must call + real_gnatchop, write the project file, write block_info.json, and + return False (no error).""" + rst_content = ( + ".. code:: ada project=TestCompile main=main.adb compile_button\n" + "\n" + + "\n".join(" " + line for line in self._ADA_BODY.splitlines()) + + "\n\nExplanatory paragraph.\n" + ) + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False, \ + "analyze_file() must return False for a valid compile_button block" + + block_dir = self._block_dir(work_dir, "TestCompile") + info = self._block_info(block_dir) + assert block_dir.name == info["text_hash_short"], \ + "the block directory must be named after the block's short hash" + # The chopped source is what the compiler will see, so it must be the + # author's code, unchanged and un-reindented. + assert (block_dir / "main.adb").read_text() == self._ADA_BODY + assert info["source_files"] == ["main.adb"] + # The project file the record names must be the one on disk, or the + # check step goes looking for a project that is not there. + assert info["project_filename"] is not None and \ + (block_dir / info["project_filename"]).is_file(), \ + "the recorded project file must be the one that was written" + assert info["spark_project_filename"] is None, \ + "no SPARK project may be written for a block that is not proved" + # A compile button alone is not runnable, so no main is selected and + # the generated project must not name one. + assert info["project_main_file"] is None + assert "for Main use" not in \ + (block_dir / info["project_filename"]).read_text() + + def test_analyze_file_run_button(self, work_dir): + """RST with a run_button Ada block: analyze_file() must call + real_gnatchop, write the project file, write block_info.json, and + return False (no error).""" + rst_content = ( + ".. code:: ada project=TestRun main=main.adb run_button\n" + "\n" + + "\n".join(" " + line for line in self._ADA_BODY.splitlines()) + + "\n\nExplanatory paragraph.\n" + ) + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False, \ + "analyze_file() must return False for a valid run_button block" + + block_dir = self._block_dir(work_dir, "TestRun") + info = self._block_info(block_dir) + assert (block_dir / "main.adb").read_text() == self._ADA_BODY + assert info["source_files"] == ["main.adb"] + assert info["project_filename"] is not None and \ + (block_dir / info["project_filename"]).is_file(), \ + "the recorded project file must be the one that was written" + assert info["spark_project_filename"] is None + # A runnable block selects a main, and the project must name it or + # there is nothing for the builder to link. + assert info["project_main_file"] == "main.adb" + assert 'for Main use ("main.adb");' in \ + (block_dir / info["project_filename"]).read_text() + + def test_analyze_file_prove_button(self, work_dir): + """RST with a prove_button SPARK Ada block: analyze_file() must call + real_gnatchop, write the SPARK project file, write block_info.json, and + return False (no error).""" + spark_body = """\ +procedure Main with SPARK_Mode is +begin + null; +end Main;""" + rst_content = ( + ".. code:: ada project=TestProve main=main.adb prove_button\n" + "\n" + + "\n".join(" " + line for line in spark_body.splitlines()) + + "\n\nExplanatory paragraph.\n" + ) + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False, \ + "analyze_file() must return False for a valid prove_button block" + + block_dir = self._block_dir(work_dir, "TestProve") + info = self._block_info(block_dir) + assert (block_dir / "main.adb").read_text() == spark_body + assert info["source_files"] == ["main.adb"] + # A prove button alone builds only the SPARK project. + assert info["spark_project_filename"] is not None and \ + (block_dir / info["spark_project_filename"]).is_file(), \ + "the recorded SPARK project file must be the one that was written" + assert info["project_filename"] is None + assert [p.name for p in block_dir.glob("*.gpr")] == \ + [info["spark_project_filename"]], \ + "the SPARK project must be the only project file written" + # GNATprove only treats the unit as SPARK because of this pragma. + assert "pragma SPARK_Mode (On);" in \ + _configuration_pragmas(block_dir, info["spark_project_filename"]) + + def test_analyze_file_run_button_no_main(self, work_dir): + """RST with run_button and no main= attribute: get_main_filename() + falls back to using the chopped source file as the main file.""" + rst_content = ( + ".. code:: ada project=TestRunNoMain run_button\n" + "\n" + + "\n".join(" " + line for line in self._ADA_BODY.splitlines()) + + "\n\nExplanatory paragraph.\n" + ) + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False, \ + "analyze_file() must return False for a run_button block with no main=" + + block_dir = self._block_dir(work_dir, "TestRunNoMain") + info = self._block_info(block_dir) + assert info["main_file"] is None, \ + "the fixture must not declare a main= attribute, or the fallback " \ + "this test exists for is never taken" + # With nothing declared, the last chopped source becomes the main file. + assert info["source_files"] == ["main.adb"] + assert info["project_main_file"] == "main.adb" + assert info["project_filename"] is not None and \ + (block_dir / info["project_filename"]).is_file(), \ + "the recorded project file must be the one that was written" + assert 'for Main use ("main.adb");' in \ + (block_dir / info["project_filename"]).read_text() + + def test_analyze_file_prove_and_run_button(self, work_dir): + """RST with both prove_button and run_button: the main file is + resolved via get_main_filename() inside the prove_it handling as well + as the compile_it handling, and both project files are written.""" + spark_body = """\ +procedure Main with SPARK_Mode is +begin + null; +end Main;""" + rst_content = ( + ".. code:: ada project=TestProveRun prove_button run_button\n\n" + + "\n".join(" " + line for line in spark_body.splitlines()) + + "\n\nExplanatory paragraph.\n" + ) + rst_file = self._write_rst(work_dir, rst_content) + result = ep.analyze_file(rst_file) + assert result is False, \ + "analyze_file() must return False for a valid prove_button+run_button block" + + block_dir = self._block_dir(work_dir, "TestProveRun") + info = self._block_info(block_dir) + assert (block_dir / "main.adb").read_text() == spark_body + # Both projects are written, and both must name the resolved main file. + assert info["main_file"] is None + assert info["project_main_file"] == "main.adb" + for gpr in (info["project_filename"], info["spark_project_filename"]): + assert gpr is not None and (block_dir / gpr).is_file(), \ + "both recorded project files must be the ones that were written" + assert 'for Main use ("main.adb");' in (block_dir / gpr).read_text(), \ + "{} must name the main file".format(gpr) + assert "pragma SPARK_Mode (On);" in \ + _configuration_pragmas(block_dir, info["spark_project_filename"]) + + def test_analyze_file_c_prove_button_reports_the_wrong_language( + self, work_dir, capsys): + """A prove button on a C block must be reported as a wrong language. + + Proving is Ada-only, so a C block asking for a prove button is a + malformed example, and the run must name the problem. + + The overall result the same run must report is covered by the + companion ``xfail`` test below; the two are kept apart so that losing + this message fails the suite on its own.""" + rst_file = self._write_rst(work_dir, self._C_PROVE_RST) + ep.analyze_file(rst_file) + assert "Wrong language selected for prove button" in capsys.readouterr().out, \ + "Expected the wrong-language message for a prove button on a C block" + + @pytest.mark.xfail( + strict=True, + reason="the per-block error flag is never merged into analyze_file()'s " + "return value, so a prove button on a non-Ada block reports success", + ) + def test_analyze_file_c_prove_button_fails_the_run(self, work_dir): + """A prove button on a C block must fail the analysis. + + Proving is Ada-only, so a C block asking for a prove button is a + malformed example: the message is printed — the companion test above + covers that — and the run must report an error so the caller's exit + code reflects it. + + Tracking note — this currently fails, and so does the sibling + ``xfail`` test covering a block that carries no button indicator at + all: both paths set the same per-block error flag, which is written + but never read. Nothing merges it into the value ``analyze_file()`` + returns, so the run reports success and a broken example passes + unnoticed. One fix — folding the per-block flag into the overall + analysis result — closes both; when it lands, both tests pass and + both ``xfail`` markers must be removed.""" + rst_file = self._write_rst(work_dir, self._C_PROVE_RST) + assert ep.analyze_file(rst_file) is True, \ + "a prove button on a non-Ada block must surface as an overall error" + + def test_analyze_file_no_buttons_block_is_reported(self, work_dir, capsys): + """A compile/run-eligible block with no button indicator must be + reported. + + Every such block is expected to declare at least a no_button + indicator, so a block declaring none is a malformed example, and the + run must name the problem. + + The overall result the same run must report is covered by the + companion ``xfail`` test below; the two are kept apart so that losing + this message fails the suite on its own.""" + rst_file = self._write_rst(work_dir, self._NO_BUTTONS_RST) + ep.analyze_file(rst_file) + assert "Expected at least" in capsys.readouterr().out, \ + "Expected the missing-indicator message for a block with no buttons" + + @pytest.mark.xfail( + strict=True, + reason="the per-block error flag is never merged into analyze_file()'s " + "return value, so a block carrying no button indicator reports success", + ) + def test_analyze_file_no_buttons_block_fails_the_run(self, work_dir): + """A compile/run-eligible block with no button indicator must fail the + analysis. + + Every such block is expected to declare at least a no_button + indicator, so a block declaring none is a malformed example: the + message is printed — the companion test above covers that — and the + run must report an error so the caller's exit code reflects it. + + Tracking note — this currently fails, for the same reason as the + sibling ``xfail`` test covering a prove button on a C block. Both + paths set the same per-block error flag, which is written but never + read: nothing merges it into the value ``analyze_file()`` returns, so + the run reports success and a broken example passes unnoticed. One + fix — folding the per-block flag into the overall analysis result — + closes both; when it lands, both tests pass and both ``xfail`` + markers must be removed.""" + rst_file = self._write_rst(work_dir, self._NO_BUTTONS_RST) + assert ep.analyze_file(rst_file) is True, \ + "a block with no button indicator must surface as an overall error" diff --git a/frontend/python/rst_code_example_pipeline/tests/test_fmt_utils.py b/frontend/python/rst_code_example_pipeline/tests/test_fmt_utils.py new file mode 100644 index 000000000..d43189154 --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_fmt_utils.py @@ -0,0 +1,169 @@ +""" +Unit tests for rst_code_example_pipeline.fmt_utils. + +Covers: +- header(): the message followed by a '*' underline of matching length +- error(): "ERROR : " written to stdout +- warning(): "WARNING : " written to stdout, and colored differently + from error() when colors are on +- simple_error() and simple_success(): the message written to stdout +- Adversarial: empty string, Unicode string with non-ASCII characters + +Each function gets one exact-output assertion rather than several partial +ones, plus the edge cases that exercise a different input shape. The +underline lengths below are spelled out as literals on purpose: recomputing +them with the same '*' * len(...) expression the source uses would hide a +character-versus-byte length bug instead of catching it. +""" +import pytest + +from rst_code_example_pipeline import fmt_utils +from rst_code_example_pipeline.colors import Colors + + +@pytest.fixture(autouse=True) +def disable_colors_for_tests(): + """Disable ANSI codes so assertions on plain text are predictable. + + The shared fixture in conftest.py puts the previous setting back, so this + one only has to establish the setting these tests need. + """ + Colors._enabled = False + + +# --------------------------------------------------------------------------- +# T-fmt_utils-01: header() +# --------------------------------------------------------------------------- + +class TestHeader: + def test_header_exact_output(self): + assert fmt_utils.header("Hello") == "Hello\n*****\n" + + def test_header_empty_string(self): + """An empty message underlines nothing, so both lines are empty.""" + assert fmt_utils.header("") == "\n\n" + + def test_header_unicode(self): + """The underline is as long as the message in characters, not bytes: + the seven letters below occupy more than seven bytes in UTF-8.""" + assert fmt_utils.header("Ünïcödé") == "Ünïcödé\n*******\n" + + +# --------------------------------------------------------------------------- +# T-fmt_utils-02: error() +# --------------------------------------------------------------------------- + +class TestError: + def test_error_exact_output(self, capsys): + fmt_utils.error("src/foo.rst:42", "something went wrong") + captured = capsys.readouterr() + assert captured.out == "ERROR src/foo.rst:42: something went wrong\n" + assert captured.err == "" + + def test_error_empty_loc_and_msg(self, capsys): + fmt_utils.error("", "") + captured = capsys.readouterr() + assert captured.out == "ERROR : \n" + + def test_error_unicode(self, capsys): + fmt_utils.error("über.rst:1", "Ünïcödé error") + captured = capsys.readouterr() + assert captured.out == "ERROR über.rst:1: Ünïcödé error\n" + + +# --------------------------------------------------------------------------- +# T-fmt_utils-03: warning() +# --------------------------------------------------------------------------- + +class TestWarning: + def test_warning_exact_output(self, capsys): + fmt_utils.warning("src/foo.rst:42", "something was repaired") + captured = capsys.readouterr() + assert captured.out == \ + "WARNING src/foo.rst:42: something was repaired\n" + assert captured.err == "" + + def test_warning_empty_loc_and_msg(self, capsys): + fmt_utils.warning("", "") + captured = capsys.readouterr() + assert captured.out == "WARNING : \n" + + def test_warning_unicode(self, capsys): + fmt_utils.warning("über.rst:1", "Ünïcödé warning") + captured = capsys.readouterr() + assert captured.out == "WARNING über.rst:1: Ünïcödé warning\n" + + def test_warning_is_not_colored_like_an_error(self, capsys): + """With colors on, a warning must not come out in the color an error + does. + + This is the one property the plain-text assertions above cannot see, + and it is what stops a reader skimming a build log from taking a + recovery for a failure. Both lines are produced here rather than one, + so the test says the two differ instead of restating whichever escape + sequence each happens to use. + """ + Colors._enabled = True + + fmt_utils.warning("src/foo.rst:42", "something was repaired") + warned = capsys.readouterr().out + fmt_utils.error("src/foo.rst:42", "something went wrong") + errored = capsys.readouterr().out + + assert warned != errored, \ + "a warning that reads exactly like an error tells the reader " \ + "nothing: {!r}".format(warned) + assert Colors.YELLOW in warned, \ + "a warning must be colored as one: {!r}".format(warned) + assert Colors.RED not in warned, \ + "a warning must not be colored as an error: {!r}".format(warned) + assert warned.endswith( + "WARNING{} src/foo.rst:42: something was repaired\n".format( + Colors.ENDC)), \ + "only the level marker is colored; the rest of the line is " \ + "plain: {!r}".format(warned) + + +# --------------------------------------------------------------------------- +# T-fmt_utils-04: simple_error() +# --------------------------------------------------------------------------- + +class TestSimpleError: + def test_simple_error_exact_output(self, capsys): + fmt_utils.simple_error("bad stuff") + captured = capsys.readouterr() + assert captured.out == "bad stuff\n" + assert captured.err == "" + + def test_simple_error_empty(self, capsys): + fmt_utils.simple_error("") + captured = capsys.readouterr() + # print("") still emits a newline + assert captured.out == "\n" + + def test_simple_error_unicode(self, capsys): + fmt_utils.simple_error("erreur: Ünïcödé") + captured = capsys.readouterr() + assert captured.out == "erreur: Ünïcödé\n" + + +# --------------------------------------------------------------------------- +# T-fmt_utils-05: simple_success() +# --------------------------------------------------------------------------- + +class TestSimpleSuccess: + def test_simple_success_exact_output(self, capsys): + fmt_utils.simple_success("all good") + captured = capsys.readouterr() + assert captured.out == "all good\n" + assert captured.err == "" + + def test_simple_success_empty(self, capsys): + fmt_utils.simple_success("") + captured = capsys.readouterr() + assert captured.out == "\n" + + def test_simple_success_unicode(self, capsys): + fmt_utils.simple_success("Ünïcödé success") + captured = capsys.readouterr() + assert captured.out == "Ünïcödé success\n" diff --git a/frontend/python/rst_code_example_pipeline/tests/test_resource.py b/frontend/python/rst_code_example_pipeline/tests/test_resource.py new file mode 100644 index 000000000..649cd46ae --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_resource.py @@ -0,0 +1,104 @@ +""" +Unit tests for rst_code_example_pipeline.resource. + +Covers: +- Resource constructor: basename stored, content=None → empty, content=[] → empty, + single-element list, multi-element list joined with newline +- append() adds a line; empty resource then append +- Adversarial: append empty string; append line with embedded newline +""" +from rst_code_example_pipeline.resource import Resource + + +# --------------------------------------------------------------------------- +# T-resource-01: constructor +# --------------------------------------------------------------------------- + +class TestResourceConstructor: + def test_basename_stored(self): + r = Resource("foo.adb") + assert r.basename == "foo.adb" + + def test_content_none_is_empty(self): + r = Resource("f.adb", content=None) + assert r.content == "" + + def test_content_default_is_empty(self): + r = Resource("f.adb") + assert r.content == "" + + def test_content_empty_list_is_empty(self): + r = Resource("f.ads", content=[]) + assert r.content == "" + + def test_content_single_element(self): + r = Resource("f.adb", content=["line one"]) + assert r.content == "line one" + + def test_content_two_elements_joined_with_newline(self): + r = Resource("f.adb", content=["a", "b"]) + assert r.content == "a\nb" + + def test_content_multi_element(self): + r = Resource("f.adb", content=["a", "b", "c"]) + assert r.content == "a\nb\nc" + + +# --------------------------------------------------------------------------- +# T-resource-02: append() +# --------------------------------------------------------------------------- + +class TestResourceAppend: + def test_append_to_empty(self): + r = Resource("f.adb") + r.append("first line") + assert r.content == "first line" + + def test_append_adds_line(self): + r = Resource("f.adb", content=["existing"]) + r.append("new line") + assert r.content == "existing\nnew line" + + def test_multiple_appends(self): + r = Resource("f.adb") + r.append("a") + r.append("b") + r.append("c") + assert r.content == "a\nb\nc" + + def test_append_empty_string(self): + r = Resource("f.adb", content=["line"]) + r.append("") + # Join adds a newline between the two elements + assert r.content == "line\n" + + +# --------------------------------------------------------------------------- +# T-resource-03: Adversarial +# --------------------------------------------------------------------------- + +class TestResourceAdversarial: + def test_append_line_with_embedded_newline(self): + """A line with an embedded newline is stored as a single element. + The content join must use \\n between list elements, not within them, + so the embedded newline is preserved literally.""" + r = Resource("f.adb", content=["a"]) + r.append("b\nc") + # The list is ["a", "b\nc"]; joined by "\n" → "a\nb\nc" + assert r.content == "a\nb\nc" + + def test_initial_content_with_embedded_newlines(self): + """If content list elements themselves contain newlines, join still + inserts exactly one \\n between each element.""" + r = Resource("f.adb", content=["x\ny", "z"]) + assert r.content == "x\ny\nz" + + def test_basename_with_path_separators(self): + """basename is stored verbatim even if it contains slashes.""" + r = Resource("dir/file.adb") + assert r.basename == "dir/file.adb" + + def test_large_content_list(self): + lines = [str(i) for i in range(1000)] + r = Resource("big.adb", content=lines) + assert r.content == "\n".join(lines) diff --git a/frontend/python/rst_code_example_pipeline/tests/test_smoke.py b/frontend/python/rst_code_example_pipeline/tests/test_smoke.py index c0d727c3b..7e9c6140e 100644 --- a/frontend/python/rst_code_example_pipeline/tests/test_smoke.py +++ b/frontend/python/rst_code_example_pipeline/tests/test_smoke.py @@ -1,78 +1,116 @@ """ Smoke tests for rst_code_example_pipeline. -Run with: - python -m unittest discover -s tests/ -or (from the project root): - python -m unittest rst_code_example_pipeline.tests.test_smoke +Covers: +- package metadata: the declared version is the one the distribution was + installed under, and the declared title is the name the package is imported + under +- every module of the package is importable without side effects +- every command-line entry point accepts --help and exits successfully """ -import unittest -from unittest.mock import patch - -import rst_code_example_pipeline +from importlib import import_module, metadata +import sys +import pytest -class TestPackageMetadata(unittest.TestCase): - def test_title(self) -> None: - self.assertEqual(rst_code_example_pipeline.__title__, - 'rst_code_example_pipeline') - - def test_version(self) -> None: - self.assertRegex(rst_code_example_pipeline.__version__, - r'^\d+\.\d+\.\d+$') +import rst_code_example_pipeline -class TestModuleImports(unittest.TestCase): +def _distribution_name() -> str: + """The name the package is installed under. + + Read back from the installed metadata rather than written down here: the + distribution is named with hyphens where the import package uses + underscores, and only the metadata knows which distribution provides + which import package. + + The same distribution is reported once per metadata directory on the + import path. An editable install routinely has two -- the one written + beside the interpreter and the build residue left in the source tree -- + so repeats of a single name are expected and are collapsed here. Two + *different* names is the case worth failing on: the lookup would then be + ambiguous and the version compared below could come from either. + """ + provided_by = set(metadata.packages_distributions()[ + rst_code_example_pipeline.__name__]) + assert len(provided_by) == 1, \ + "expected exactly one distribution to provide the package, got " \ + "{}".format(sorted(provided_by)) + return provided_by.pop() + + +class TestPackageMetadata: + def test_version_matches_the_installed_distribution(self): + """The version the package declares must be the one it was installed + under. + + The version is written down twice -- in the package and in the + packaging metadata -- and nothing ties the two together, so a release + that bumps one and forgets the other would otherwise pass unnoticed + and ship a package that misreports its own version. + """ + installed = metadata.version(_distribution_name()) + assert rst_code_example_pipeline.__version__ == installed, \ + "the package declares version {} but was installed as {}".format( + rst_code_example_pipeline.__version__, installed) + + def test_title_is_the_name_the_package_is_imported_under(self): + """The declared title must be the name the package is imported under. + + It is not the distribution name, which is spelled with hyphens: the + title has tracked the import package since before the package was + distributed at all. Checking it against the name the import machinery + supplies catches a package that was renamed without the title + following it. + """ + assert rst_code_example_pipeline.__title__ == \ + rst_code_example_pipeline.__name__, \ + "the package declares the title {} but is imported as {}".format( + rst_code_example_pipeline.__title__, + rst_code_example_pipeline.__name__) + + +class TestModuleImports: """Each module must be importable without side-effects.""" - def test_import_colors(self) -> None: + def test_import_colors(self): from rst_code_example_pipeline import colors # noqa: F401 - def test_import_fmt_utils(self) -> None: + def test_import_fmt_utils(self): from rst_code_example_pipeline import fmt_utils # noqa: F401 - def test_import_checks(self) -> None: + def test_import_checks(self): from rst_code_example_pipeline import checks # noqa: F401 - def test_import_blocks(self) -> None: + def test_import_blocks(self): from rst_code_example_pipeline import blocks # noqa: F401 - def test_import_toolchain_info(self) -> None: + def test_import_toolchain_info(self): from rst_code_example_pipeline import toolchain_info # noqa: F401 - def test_import_toolchain_setup(self) -> None: + def test_import_toolchain_setup(self): from rst_code_example_pipeline import toolchain_setup # noqa: F401 - def test_import_check_code_block(self) -> None: + def test_import_check_code_block(self): from rst_code_example_pipeline import check_code_block # noqa: F401 - def test_import_extract_projects(self) -> None: + def test_import_extract_projects(self): from rst_code_example_pipeline import extract_projects # noqa: F401 - def test_import_check_projects(self) -> None: + def test_import_check_projects(self): from rst_code_example_pipeline import check_projects # noqa: F401 -class TestEntryPoints(unittest.TestCase): +class TestEntryPoints: """Entry-point main() functions must accept --help (exit 0).""" - def _assert_help_exits_zero(self, entry: str) -> None: - from importlib import import_module - mod = import_module(f'rst_code_example_pipeline.cli.{entry}') - with patch('sys.argv', [entry, '--help']): - with self.assertRaises(SystemExit) as ctx: - mod.main() - self.assertEqual(ctx.exception.code, 0) - - def test_check_block_help(self) -> None: - self._assert_help_exits_zero('check_block') - - def test_extract_help(self) -> None: - self._assert_help_exits_zero('extract') - - def test_check_help(self) -> None: - self._assert_help_exits_zero('check') + @pytest.mark.parametrize("entry", ["check_block", "extract", "check"]) + def test_help_exits_zero(self, entry, monkeypatch): + module = import_module("rst_code_example_pipeline.cli.{}".format(entry)) + monkeypatch.setattr(sys, "argv", [entry, "--help"]) + with pytest.raises(SystemExit) as raised: + module.main() -if __name__ == '__main__': - unittest.main() + assert raised.value.code == 0, \ + "{} --help must exit successfully".format(entry) diff --git a/frontend/python/rst_code_example_pipeline/tests/test_toolchain_info.py b/frontend/python/rst_code_example_pipeline/tests/test_toolchain_info.py new file mode 100644 index 000000000..8b43def02 --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_toolchain_info.py @@ -0,0 +1,185 @@ +""" +Unit tests for rst_code_example_pipeline.toolchain_info. + +Covers: +- init_toolchain_info() populates DEFAULT_VERSION, TOOLCHAINS, TOOLCHAIN_PATH +- every declared version has the release shape a toolchain download URL needs +- get_toolchain_default_version() for gnat, gnatprove, gprbuild +- the default version of each tool is one of the versions declared for it +- Re-initialization idempotency +- get_toolchain_default_version() for unknown tool raises KeyError +- State isolation: each test that mutates module-level dicts resets them + +The assertions below are deliberately invariants rather than snapshots of the +versions currently configured: a toolchain upgrade must not redden this file. + +NOTE: These tests require the Ada toolchain .ini file to be present +""" +import re + +import pytest + +import rst_code_example_pipeline.toolchain_info as info + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def reset_module_state(): + """Reset module-level dicts before and after every test.""" + info.DEFAULT_VERSION.clear() + info.TOOLCHAINS.clear() + info.TOOLCHAIN_PATH.clear() + yield + info.DEFAULT_VERSION.clear() + info.TOOLCHAINS.clear() + info.TOOLCHAIN_PATH.clear() + + +# --------------------------------------------------------------------------- +# T-toolchain_info-01: init_toolchain_info() populates the module dicts +# --------------------------------------------------------------------------- + +class TestInitToolchainInfo: + def test_default_version_keys_after_init(self): + info.init_toolchain_info() + assert set(info.DEFAULT_VERSION.keys()) == {"gnat", "gnatprove", "gprbuild"} + + def test_toolchains_keys_after_init(self): + info.init_toolchain_info() + assert set(info.TOOLCHAINS.keys()) == {"gnat", "gnatprove", "gprbuild"} + + def test_toolchain_path_keys_after_init(self): + info.init_toolchain_info() + assert set(info.TOOLCHAIN_PATH.keys()) == {"root", "selected", "default"} + + def test_default_version_values_nonempty(self): + info.init_toolchain_info() + for tool in ("gnat", "gnatprove", "gprbuild"): + assert info.DEFAULT_VERSION[tool], \ + f"DEFAULT_VERSION[{tool!r}] must be a non-empty string" + + def test_toolchains_entries_are_release_versions(self): + """Every declared version must be a non-empty release identifier of the + form ..-. + + That shape is not a matter of taste: the download URL of each + toolchain is built by interpolating this exact token, so a malformed or + missing entry produces a download failure far away from its cause. It + is also stronger than merely checking the value is a list: splitting an + empty configuration entry on whitespace yields a one-element list + holding an empty string, which no other test rejects. + """ + info.init_toolchain_info() + for tool in ("gnat", "gnatprove", "gprbuild"): + versions = info.TOOLCHAINS[tool] + assert versions, \ + f"TOOLCHAINS[{tool!r}] must declare at least one version" + for ver in versions: + assert re.fullmatch(r"\d+\.\d+\.\d+-\d+", ver), \ + f"TOOLCHAINS[{tool!r}] entry {ver!r} is not a release version" + + def test_toolchain_path_values_nonempty_strings(self): + info.init_toolchain_info() + for key in ("root", "selected", "default"): + val = info.TOOLCHAIN_PATH[key] + assert isinstance(val, str) and val, \ + f"TOOLCHAIN_PATH[{key!r}] must be a non-empty string" + + +# --------------------------------------------------------------------------- +# T-toolchain_info-02: get_toolchain_default_version() auto-initializes +# --------------------------------------------------------------------------- + +class TestGetToolchainDefaultVersion: + def test_gnat_returns_string(self): + # Dicts are empty; the function must initialize and return a value + result = info.get_toolchain_default_version("gnat") + assert isinstance(result, str) and result + + def test_gnatprove_returns_string(self): + result = info.get_toolchain_default_version("gnatprove") + assert isinstance(result, str) and result + + def test_gprbuild_returns_string(self): + result = info.get_toolchain_default_version("gprbuild") + assert isinstance(result, str) and result + + def test_default_version_is_one_of_the_declared_versions(self): + """The default version of each tool must be one of the versions + declared as installed for that tool. + + Only the declared versions are downloaded and installed, and the + default is then pointed at one of them, so a default that is not in + the list leaves a dangling symlink where the toolchain is expected. + """ + for tool in ("gnat", "gnatprove", "gprbuild"): + result = info.get_toolchain_default_version(tool) + assert result in info.TOOLCHAINS[tool], \ + f"Default {tool} version {result!r} is not declared as installed: " \ + f"{info.TOOLCHAINS[tool]}" + + def test_auto_init_populates_default_version_dict(self): + # Before the call the dict is empty (fixture cleared it) + assert len(info.DEFAULT_VERSION) == 0 + info.get_toolchain_default_version("gnat") + # After the call the dict must have been populated + assert len(info.DEFAULT_VERSION) > 0 + + def test_unknown_tool_raises_key_error(self): + # init_toolchain_info() is called internally because dict is empty; + # the key "unknown_tool" was never set so KeyError must propagate. + with pytest.raises(KeyError): + info.get_toolchain_default_version("unknown_tool") + + +# --------------------------------------------------------------------------- +# T-toolchain_info-03: re-initialization idempotency +# --------------------------------------------------------------------------- + +class TestReInitIdempotency: + def test_second_init_gnat_default_unchanged(self): + info.init_toolchain_info() + first = info.DEFAULT_VERSION["gnat"] + info.init_toolchain_info() + second = info.DEFAULT_VERSION["gnat"] + assert first == second + + def test_second_init_toolchain_path_unchanged(self): + info.init_toolchain_info() + first = dict(info.TOOLCHAIN_PATH) + info.init_toolchain_info() + assert dict(info.TOOLCHAIN_PATH) == first + + def test_second_init_toolchains_unchanged(self): + info.init_toolchain_info() + first_gnat = list(info.TOOLCHAINS["gnat"]) + info.init_toolchain_info() + assert list(info.TOOLCHAINS["gnat"]) == first_gnat + + def test_many_inits_stable(self): + for _ in range(5): + info.init_toolchain_info() + # All keys must still be present + assert "gnat" in info.DEFAULT_VERSION + assert "root" in info.TOOLCHAIN_PATH + assert "gprbuild" in info.TOOLCHAINS + + +# --------------------------------------------------------------------------- +# T-toolchain_info-04: state isolation — verify the fixture works correctly +# --------------------------------------------------------------------------- + +class TestStateIsolation: + def test_dicts_empty_at_test_start(self): + # The autouse fixture clears dicts before every test; verify that here. + assert len(info.DEFAULT_VERSION) == 0 + assert len(info.TOOLCHAINS) == 0 + assert len(info.TOOLCHAIN_PATH) == 0 + + def test_manual_mutation_does_not_bleed_across(self): + info.DEFAULT_VERSION["gnat"] = "fake-version" + assert info.DEFAULT_VERSION["gnat"] == "fake-version" + # The fixture teardown clears it; the next test will see an empty dict. diff --git a/frontend/python/rst_code_example_pipeline/tests/test_toolchain_setup.py b/frontend/python/rst_code_example_pipeline/tests/test_toolchain_setup.py new file mode 100644 index 000000000..2c5edf05e --- /dev/null +++ b/frontend/python/rst_code_example_pipeline/tests/test_toolchain_setup.py @@ -0,0 +1,307 @@ +""" +Unit tests for rst_code_example_pipeline.toolchain_setup. + +Covers: +- reset_toolchain() when no symlinks exist → no exception +- reset_toolchain() when symlinks exist → symlinks removed +- set_toolchain(block) with gnat_version=["default", …] → no symlink created +- set_toolchain(block) with a non-default version selector → symlink created +- set_toolchain() followed by reset_toolchain() → symlinks removed +- Adversarial: set_toolchain() called twice without reset → must not fail +- State isolation: teardown_function resets toolchain after every test + +NOTE: nearly every test here is toolchain-free. The isolated_toolchain_path fixture +redirects TOOLCHAIN_PATH into a tmp_path-based directory, so the symlinks are created +and removed there and the real toolchain installation tree is never touched. The one +exception is the test that deletes the 'root' key: set_toolchain() then re-reads the +toolchain configuration, which restores the real installation paths over the redirect, +so the call writes into the real tree. That single test carries the `toolchain` marker. +""" +import os + +import pytest + +import rst_code_example_pipeline.toolchain_info as info +import rst_code_example_pipeline.toolchain_setup as setup +from rst_code_example_pipeline.blocks import CodeBlock + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +def _installed_version(tool: str) -> str: + """Return a version of ``tool`` declared as installed in the toolchain + configuration. + + Tests needing a non-default toolchain selector read the version from the + configuration instead of spelling one out, so that changing the installed + set cannot leave them selecting a version that no longer exists. + """ + if not info.TOOLCHAINS: + info.init_toolchain_info() + return info.TOOLCHAINS[tool][0] + + +def _make_block(gnat_version: list[str], + gnatprove_version: list[str] | None = None, + gprbuild_version: list[str] | None = None) -> CodeBlock: + """Build a minimal CodeBlock with the given toolchain version selectors.""" + # Ensure toolchain_info is initialized so default version strings exist + if not info.DEFAULT_VERSION: + info.init_toolchain_info() + gnatprove_version = gnatprove_version or ["default", info.DEFAULT_VERSION["gnatprove"]] + gprbuild_version = gprbuild_version or ["default", info.DEFAULT_VERSION["gprbuild"]] + return CodeBlock( + rst_file="test.rst", + line_start=1, + line_end=5, + text="procedure Main is begin null; end Main;", + language="ada", + project="TestProject", + main_file=None, + gnat_version=gnat_version, + gnatprove_version=gnatprove_version, + gprbuild_version=gprbuild_version, + compiler_switches=["-gnata"], + classes=[], + manual_chop=False, + buttons=["no"], + ) + + +@pytest.fixture() +def isolated_toolchain_path(tmp_path, monkeypatch): + """ + Redirect TOOLCHAIN_PATH so symlinks are created in tmp_path instead of + the selected directory of the real toolchain installation tree. Also + creates stub target directories matching the installed toolchain versions + so os.symlink targets exist. + """ + # Ensure toolchain_info is initialized + if not info.TOOLCHAINS: + info.init_toolchain_info() + + root = tmp_path / "ada" + selected = root / "selected" + default_dir = root / "default" + selected.mkdir(parents=True) + default_dir.mkdir(parents=True) + + # Create a stub version directory for every version declared as installed, + # so that a symlink to any of them has an existing target + for tool, versions in info.TOOLCHAINS.items(): + for ver in versions: + tool_dir = root / tool / ver + tool_dir.mkdir(parents=True, exist_ok=True) + + # Patch the module-level dict values + monkeypatch.setitem(info.TOOLCHAIN_PATH, "root", str(root)) + monkeypatch.setitem(info.TOOLCHAIN_PATH, "selected", str(selected)) + monkeypatch.setitem(info.TOOLCHAIN_PATH, "default", str(default_dir)) + + yield { + "root": str(root), + "selected": str(selected), + "default": str(default_dir), + } + + # Teardown: call reset_toolchain() so no symlinks survive across tests + try: + setup.reset_toolchain() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# T-toolchain_setup-01: reset_toolchain() without prior symlinks +# --------------------------------------------------------------------------- + +class TestResetToolchainNoSymlinks: + def test_no_exception_when_symlinks_absent(self, isolated_toolchain_path): + # No symlinks have been created; reset must silently succeed + setup.reset_toolchain() # must not raise + + def test_selected_dir_still_exists_after_reset(self, isolated_toolchain_path): + setup.reset_toolchain() + assert os.path.isdir(isolated_toolchain_path["selected"]) + + +# --------------------------------------------------------------------------- +# T-toolchain_setup-02: reset_toolchain() removes existing symlinks +# --------------------------------------------------------------------------- + +class TestResetToolchainRemovesSymlinks: + def test_symlinks_removed_after_reset(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + root = isolated_toolchain_path["root"] + + # Manually create symlinks to simulate a prior set_toolchain call + for tool in ("gnat", "gnatprove", "gprbuild"): + link = os.path.join(selected, tool) + target_ver = list(os.listdir(os.path.join(root, tool)))[0] + target = os.path.join(root, tool, target_ver) + os.symlink(target, link) + + # Verify they were created + for tool in ("gnat", "gnatprove", "gprbuild"): + assert os.path.exists(os.path.join(selected, tool)) + + setup.reset_toolchain() + + for tool in ("gnat", "gnatprove", "gprbuild"): + assert not os.path.exists(os.path.join(selected, tool)), \ + f"Symlink for {tool!r} was not removed by reset_toolchain()" + + def test_reset_idempotent_after_removal(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + root = isolated_toolchain_path["root"] + + for tool in ("gnat",): + link = os.path.join(selected, tool) + target_ver = list(os.listdir(os.path.join(root, tool)))[0] + target = os.path.join(root, tool, target_ver) + os.symlink(target, link) + + setup.reset_toolchain() + # Second reset must not raise even though symlinks are already gone + setup.reset_toolchain() + + +# --------------------------------------------------------------------------- +# T-toolchain_setup-03: set_toolchain() with all "default" versions +# --------------------------------------------------------------------------- + +class TestSetToolchainDefaultVersion: + def test_no_symlink_created_for_default_gnat(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + block = _make_block(gnat_version=["default", info.DEFAULT_VERSION["gnat"]]) + setup.set_toolchain(block) + assert not os.path.exists(os.path.join(selected, "gnat")), \ + "No symlink should be created when gnat_version is 'default'" + + def test_no_symlink_created_for_any_default(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + block = _make_block( + gnat_version=["default", info.DEFAULT_VERSION["gnat"]], + gnatprove_version=["default", info.DEFAULT_VERSION["gnatprove"]], + gprbuild_version=["default", info.DEFAULT_VERSION["gprbuild"]], + ) + setup.set_toolchain(block) + for tool in ("gnat", "gnatprove", "gprbuild"): + assert not os.path.exists(os.path.join(selected, tool)), \ + f"No symlink should be created for tool {tool!r} in default mode" + + +# --------------------------------------------------------------------------- +# T-toolchain_setup-04: set_toolchain() with "selected" gnat version +# --------------------------------------------------------------------------- + +class TestSetToolchainSelectedVersion: + def test_gnat_symlink_created(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + block = _make_block(gnat_version=["selected", _installed_version("gnat")]) + setup.set_toolchain(block) + link_path = os.path.join(selected, "gnat") + assert os.path.exists(link_path), \ + "Symlink selected/gnat must exist after set_toolchain() with 'selected'" + + def test_gnat_symlink_points_to_correct_version(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + root = isolated_toolchain_path["root"] + version = _installed_version("gnat") + block = _make_block(gnat_version=["selected", version]) + setup.set_toolchain(block) + link_path = os.path.join(selected, "gnat") + expected_target = os.path.join(root, "gnat", version) + assert os.readlink(link_path) == expected_target, \ + f"Symlink must point to {expected_target!r}" + + def test_no_gnatprove_symlink_when_only_gnat_selected(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + block = _make_block(gnat_version=["selected", _installed_version("gnat")]) + setup.set_toolchain(block) + assert not os.path.exists(os.path.join(selected, "gnatprove")), \ + "gnatprove symlink must not be created when only gnat is 'selected'" + + def test_all_three_selected(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + block = _make_block( + gnat_version=["selected", _installed_version("gnat")], + gnatprove_version=["selected", _installed_version("gnatprove")], + gprbuild_version=["selected", _installed_version("gprbuild")], + ) + setup.set_toolchain(block) + for tool in ("gnat", "gnatprove", "gprbuild"): + assert os.path.exists(os.path.join(selected, tool)), \ + f"Symlink for {tool!r} must be created when version is 'selected'" + + +# --------------------------------------------------------------------------- +# T-toolchain_setup-05: set_toolchain() followed by reset_toolchain() +# --------------------------------------------------------------------------- + +class TestSetThenReset: + def test_symlinks_removed_after_reset(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + block = _make_block(gnat_version=["selected", _installed_version("gnat")]) + setup.set_toolchain(block) + assert os.path.exists(os.path.join(selected, "gnat")) + setup.reset_toolchain() + assert not os.path.exists(os.path.join(selected, "gnat")), \ + "Symlink must be gone after reset_toolchain()" + + def test_set_then_reset_is_idempotent(self, isolated_toolchain_path): + block = _make_block(gnat_version=["selected", _installed_version("gnat")]) + setup.set_toolchain(block) + setup.reset_toolchain() + # A second reset must not raise + setup.reset_toolchain() + + +# --------------------------------------------------------------------------- +# T-toolchain_setup-06: adversarial — double set_toolchain() without reset +# --------------------------------------------------------------------------- + +class TestAdversarialDoubleSet: + def test_double_set_does_not_fail(self, isolated_toolchain_path): + """set_toolchain() calls reset_toolchain() internally, so calling it + twice without an explicit reset in between must not raise.""" + block = _make_block(gnat_version=["selected", _installed_version("gnat")]) + setup.set_toolchain(block) + # Second call must not raise (reset is called inside set_toolchain) + setup.set_toolchain(block) + + def test_after_double_set_symlink_still_present(self, isolated_toolchain_path): + selected = isolated_toolchain_path["selected"] + block = _make_block(gnat_version=["selected", _installed_version("gnat")]) + setup.set_toolchain(block) + setup.set_toolchain(block) + assert os.path.exists(os.path.join(selected, "gnat")), \ + "Symlink must still be present after two consecutive set_toolchain() calls" + + +# --------------------------------------------------------------------------- +# T-toolchain_setup-07: set_toolchain() with uninitialized TOOLCHAIN_PATH +# (exercises the lazy init_toolchain_info() guard at the start of set_toolchain()) +# --------------------------------------------------------------------------- + +class TestSetToolchain: + @pytest.mark.toolchain + def test_set_toolchain_reinitialises_toolchain_path( + self, isolated_toolchain_path, monkeypatch): + """When TOOLCHAIN_PATH has no 'root' key, set_toolchain() calls + init_toolchain_info() to populate it.""" + # Remove 'root' so the guard 'if not "root" in info.TOOLCHAIN_PATH:' + # evaluates to True + monkeypatch.delitem(info.TOOLCHAIN_PATH, "root") + assert "root" not in info.TOOLCHAIN_PATH, \ + "Precondition: 'root' must be absent before the call" + + block = _make_block(gnat_version=["default", info.DEFAULT_VERSION["gnat"]]) + # set_toolchain() must call init_toolchain_info() internally and succeed + setup.set_toolchain(block) + + # After the call, 'root' must be back (init_toolchain_info() re-populated it) + assert "root" in info.TOOLCHAIN_PATH, \ + "Expected 'root' to be present after set_toolchain() triggers init"