diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17a74fc..422e07f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# Lints, tests every supported Python version on Linux and macOS, and checks +# the built distributions and documentation. # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python name: CI @@ -6,74 +7,145 @@ name: CI on: push: branches: [ "main", "dev" ] + # Pull requests into any branch, including stacked feature branches. pull_request: - branches: [ "main", "dev" ] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - build: + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Install ruff + run: python -m pip install ruff + - name: Lint with ruff + run: ruff check . - runs-on: ubuntu-latest + # Every job runs the whole suite, including the loopback OpenSSH tests, so + # each Python version and both operating systems exercise every module. + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 25 strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10","3.11","3.12","3.13","3.14"] - + # Pinned because Python 3.9 builds may not exist for newer images. + os: [ubuntu-24.04] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + include: + - os: macos-latest + python-version: "3.9" + - os: macos-latest + python-version: "3.14" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install the OpenSSH server + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends openssh-server + - name: Install Hallmark and test tools run: | python -m pip install --upgrade pip - python -m pip install ruff pytest - python -m pip install -e . - - name: Lint with ruff - run: | - ruff check --select E,F --exclude ./demo + python -m pip install -e . --group dev - name: Test with pytest + env: + HALLMARK_RUN_SSH_TESTS: "1" + COVERAGE_FILE: .coverage.${{ runner.os }}-py${{ matrix.python-version }} + run: python -m pytest --cov --cov-report= --timeout=600 -ra --durations=15 + - name: Upload coverage data + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-${{ runner.os }}-py${{ matrix.python-version }} + path: .coverage.${{ runner.os }}-py${{ matrix.python-version }} + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + + coverage: + needs: test + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Install coverage + run: python -m pip install coverage + - uses: actions/download-artifact@v8 + with: + pattern: coverage-* + merge-multiple: true + - name: Combine and report coverage run: | - pytest + coverage combine + coverage report + coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY" + coverage xml + coverage html + - name: Upload the combined report + uses: actions/upload-artifact@v7 + with: + name: coverage-report + path: | + coverage.xml + htmlcov/ + - name: Upload coverage to Codecov + continue-on-error: true + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + files: coverage.xml + disable_search: true + fail_ci_if_error: true + use_oidc: ${{ !github.event.pull_request.head.repo.fork }} + use_pypi: true + - name: Enforce the coverage floor + run: coverage report --fail-under=89 distribution-and-docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + persist-credentials: false + - uses: actions/setup-python@v7 with: - python-version: "3.12" - - name: Build source and wheel distributions + python-version: "3.13" + - name: Build and check source and wheel distributions run: | - python -m pip install build + python -m pip install build twine python -m build + python -m twine check --strict dist/* - name: Install the wheel and documentation tools run: python -m pip install dist/*.whl sphinx - - name: Check the installed CLI - run: hm --help + - name: Check both installed commands + run: | + hm --help + hallmark --help working-directory: ${{ runner.temp }} - name: Build documentation with warnings as errors run: python -m sphinx -b html -W --keep-going doc "${{ runner.temp }}/hallmark-docs" - - ssh-transport: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v3 - with: - python-version: "3.9" - - name: Install Linux SSH server - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y openssh-server - - name: Install Hallmark and tests - run: python -m pip install -e . pytest - - name: Exercise disposable OpenSSH server and client - env: - HALLMARK_RUN_SSH_TESTS: "1" - run: python -m pytest test/test_ssh_integration.py test/test_transport.py -q diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 0000000..f2dfe12 --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,84 @@ +# Tests the oldest dependency versions pyproject.toml allows, and the newest +# releases including pre-releases. + +name: Dependencies + +on: + schedule: + - cron: "17 7 * * 1" + workflow_dispatch: + pull_request: + paths: + - pyproject.toml + - .github/workflows/dependencies.yml + +permissions: + contents: read + +concurrency: + group: dependencies-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + lowest: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.9" + - name: Install the OpenSSH server + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends openssh-server + # Resolved separately so the test tools are not held at their oldest versions. + - name: Install the oldest allowed dependencies + run: | + python -m pip install --upgrade pip uv + uv pip compile --python-version 3.9 --resolution lowest-direct --no-header pyproject.toml -o "$RUNNER_TEMP/minimum.txt" + python -m pip install -r "$RUNNER_TEMP/minimum.txt" -e . --group dev + python -m pip check + { + echo "### Oldest allowed dependencies" + echo '```' + grep -v '^ *#' "$RUNNER_TEMP/minimum.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Test with pytest + env: + HALLMARK_RUN_SSH_TESTS: "1" + run: python -m pytest --timeout=600 -ra + + pre-release: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + python-version: ["3.14", "3.15"] + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + - name: Install the OpenSSH server + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends openssh-server + - name: Install the newest dependencies, including pre-releases + env: + PYTHON_VERSION: ${{ matrix.python-version }} + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade --pre -e . --group dev + { + echo "### Installed on Python $PYTHON_VERSION" + echo '```' + python -m pip list --format=freeze --exclude-editable + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Test with pytest + env: + HALLMARK_RUN_SSH_TESTS: "1" + run: python -m pytest --timeout=600 -ra diff --git a/.gitignore b/.gitignore index 3a2b4eb..7e64860 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ venv/ .pytest_cache/ .ruff_cache/ .coverage +.coverage.* +coverage.xml htmlcov/ build/ dist/ diff --git a/README.rst b/README.rst index 04b526b..f2e118e 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,10 @@ Reproducibility is the |hallmark|_ of the scientific method. +.. image:: https://codecov.io/gh/lanisrc/hallmark/graph/badge.svg?branch=main + :target: https://codecov.io/gh/lanisrc/hallmark + :alt: Coverage + Modern science has become so complex that many science projects rely on multiple software packages to work in unison, resulting in networks of data products along the analyses. @@ -135,8 +139,8 @@ version, operating system, and a minimal example without credentials or private From a development checkout, install the validation tools and run:: - python -m pip install -e . pytest ruff sphinx - python -m pytest + python -m pip install -e . --group dev --group lint --group doc + python -m pytest --cov ruff check --select E,F mod test python -m sphinx -b html -W doc doc/_build diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..7671050 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,25 @@ +# Codecov reports are informational; CI enforces the coverage floor in +# .github/workflows/ci.yml from the combined report it uploads. +codecov: + notify: + after_n_builds: 1 +coverage: + precision: 1 + round: down + status: + project: + default: + target: auto + threshold: 1% + informational: true + patch: + default: + target: 80% + informational: true +comment: + layout: "condensed_header, diff, flags, files" + after_n_builds: 1 +ignore: + - "test/**" + - "demo/**" + - "doc/**" diff --git a/doc/CONTRIBUTING.md b/doc/CONTRIBUTING.md index dce1cbf..5378cb2 100644 --- a/doc/CONTRIBUTING.md +++ b/doc/CONTRIBUTING.md @@ -28,6 +28,8 @@ * If the implementation of a feature or fix is less clear than the desired outcome, write unit or end-to-end tests first to define the expected functionality. Then write the code to make those tests pass. * If the implementation is as clear as (or clearer than) the goal, implement the solution first. Then write tests to ensure the solution covers edge cases and realistic user scenarios. The tests may reveal places where the implementation needs to be fixed, expanded, or simplified. +* Run `pytest --cov` to see which lines the tests miss, or `pytest --cov --cov-report=html` for a browsable report in `htmlcov/`. CI runs the whole suite, including the loopback SSH tests, on every supported Python version, combines coverage from every job, and fails if the total drops below the floor set in `.github/workflows/ci.yml`. Codecov comments on each pull request with the coverage of its changed lines; those comments are informational. +* Each week, and on pull requests that change `pyproject.toml`, `.github/workflows/dependencies.yml` runs the suite against the oldest dependency versions `pyproject.toml` allows (on Python 3.9) and against the newest releases including pre-releases (on the two newest Python versions). If you start using a newer feature of a dependency, raise its lower bound in `pyproject.toml`. ## Data Management diff --git a/doc/usecase.rst b/doc/usecase.rst index c6fd20e..be22692 100644 --- a/doc/usecase.rst +++ b/doc/usecase.rst @@ -357,5 +357,5 @@ SSH configuration and known-hosts files. They do not connect to a lab server or change the user's SSH trust files. Enabling the tests requires the OpenSSH tools to be installed; missing tools cause a test failure. -CI installs the server and runs these tests separately from the -Python-version matrix. +CI installs the server and runs the complete suite, these tests included, +in every Linux and macOS test job. diff --git a/pyproject.toml b/pyproject.toml index b5ab0d7..751b474 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,14 +10,14 @@ readme = "README.rst" requires-python = ">=3.9" dependencies =[ - "click", - "GitPython", - "numpy", - "pandas", - "parse", - "pyyaml", - "requests", - "tqdm", + "click>=8.1", + "GitPython>=3.1.30", + "numpy>=1.21", + "pandas>=1.5", + "parse>=1.19", + "pyyaml>=5.4", + "requests>=2.26", + "tqdm>=4.60", ] authors = [ @@ -28,7 +28,9 @@ authors = [ {name = "Ram Adithya"}, {name = "Hayden Foote"}, {name = "Samuel Jacobs"}, - {name = "Yaqing Su"} + {name = "Yaqing Su"}, + {name = "Tianrui (Dennis) Wu"}, + {name = "Arshia Akhtarkavan"} ] [project.scripts] @@ -41,7 +43,7 @@ Repository = "https://github.com/l6a/hallmark" Issues = "https://github.com/l6a/hallmark/issues" [dependency-groups] -dev = ["pytest"] +dev = ["pytest", "pytest-cov", "pytest-timeout"] doc = ["sphinx"] lint = ["ruff"] @@ -50,10 +52,11 @@ where = ["mod"] [tool.ruff] target-version = "py39" +# Excluded at the top level so the shell cells in demo notebooks are not parsed. +extend-exclude = ["doc", "demo"] [tool.ruff.lint] select = ["E", "F"] -exclude = ["doc", "demo"] [tool.pytest.ini_options] testpaths = ["test"] @@ -62,3 +65,16 @@ markers = [ "ssh_client: exercise the actual local OpenSSH SFTP command parser", "ssh_integration: disposable loopback sshd (HALLMARK_RUN_SSH_TESTS=1 required)", ] + +[tool.coverage.run] +source_pkgs = ["hallmark"] +branch = true +# Store repository-relative paths so data from Linux and macOS jobs combines. +relative_files = true + +[tool.coverage.paths] +hallmark = ["mod/hallmark", "*/mod/hallmark", "*/site-packages/hallmark"] + +[tool.coverage.report] +show_missing = true +skip_covered = true diff --git a/test/test_cli.py b/test/test_cli.py index 52bd799..c4a141e 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -15,7 +15,7 @@ import os import requests -import importlib +import importlib.metadata import pytest import pandas as pd from pathlib import Path @@ -30,6 +30,10 @@ from hallmark.remote.plan import DownloadItem, DownloadPlan from hallmark.utils import use_working_directory +_CLICK_RUNNER_SIGNALS_EOF = tuple( + int(part) for part in importlib.metadata.version("click").split(".")[:2] +) >= (8, 2) + cli_module = importlib.import_module("hallmark.cli") # files to create for testing, with a variety of a and i values to test regex encoding @@ -1187,6 +1191,10 @@ def capture_get(url, **kwargs): assert "1 file(s); 4 bytes" in result.output assert "Download these files? [y/N]" in result.output assert "Source: https://example.test/data/" in result.output + # Click 8.1 (the newest for Python 3.9) reads exhausted CliRunner input + # as an empty line instead of end of input, so "" acts like "\n" there. + if answer == "" and not _CLICK_RUNNER_SIGNALS_EOF: + answer = "\n" if answer == "y\n": assert result.exit_code == 0, result.output assert requests_made == ["https://example.test/data/tiny.fits"]