Add coverage, Codecov and dependency-bounds CI to the codex branch

.github/workflows/ci.yml

@@ -1,79 +1,151 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# Lints, tests every supported Python version on Linux and macOS, and checks
+# the built distributions and documentation.
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
 name: CI
 
 on:
   push:
     branches: [ "main", "dev" ]
+  # Pull requests into any branch, including stacked feature branches.
   pull_request:
-    branches: [ "main", "dev" ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  build:
+  lint:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        persist-credentials: false
+    - uses: actions/setup-python@v7
+      with:
+        python-version: "3.13"
+    - name: Install ruff
+      run: python -m pip install ruff
+    - name: Lint with ruff
+      run: ruff check .
 
-    runs-on: ubuntu-latest
+  # Every job runs the whole suite, including the loopback OpenSSH tests, so
+  # each Python version and both operating systems exercise every module.
+  test:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10","3.11","3.12","3.13","3.14"]
-
+        # Pinned because Python 3.9 builds may not exist for newer images.
+        os: [ubuntu-24.04]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        include:
+        - os: macos-latest
+          python-version: "3.9"
+        - os: macos-latest
+          python-version: "3.14"
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
+      with:
+        persist-credentials: false
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Install the OpenSSH server
+      if: runner.os == 'Linux'
+      run: sudo apt-get update && sudo apt-get install -y --no-install-recommends openssh-server
+    - name: Install Hallmark and test tools
       run: |
         python -m pip install --upgrade pip
-        python -m pip install ruff pytest
-        python -m pip install -e .
-    - name: Lint with ruff
-      run: |
-        ruff check --select E,F --exclude ./demo
+        python -m pip install -e . --group dev
     - name: Test with pytest
+      env:
+        HALLMARK_RUN_SSH_TESTS: "1"
+        COVERAGE_FILE: .coverage.${{ runner.os }}-py${{ matrix.python-version }}
+      run: python -m pytest --cov --cov-report= --timeout=600 -ra --durations=15
+    - name: Upload coverage data
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@v7
+      with:
+        name: coverage-${{ runner.os }}-py${{ matrix.python-version }}
+        path: .coverage.${{ runner.os }}-py${{ matrix.python-version }}
+        include-hidden-files: true
+        if-no-files-found: error
+        retention-days: 7
+
+  coverage:
+    needs: test
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        persist-credentials: false
+    - uses: actions/setup-python@v7
+      with:
+        python-version: "3.13"
+    - name: Install coverage
+      run: python -m pip install coverage
+    - uses: actions/download-artifact@v8
+      with:
+        pattern: coverage-*
+        merge-multiple: true
+    - name: Combine and report coverage
       run: |
-        pytest
+        coverage combine
+        coverage report
+        coverage report --format=markdown >> "$GITHUB_STEP_SUMMARY"
+        coverage xml
+        coverage html
+    - name: Upload the combined report
+      uses: actions/upload-artifact@v7
+      with:
+        name: coverage-report
+        path: |
+          coverage.xml
+          htmlcov/
+    - name: Upload coverage to Codecov
+      continue-on-error: true
+      uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+      with:
+        files: coverage.xml
+        disable_search: true
+        fail_ci_if_error: true
+        use_oidc: ${{ !github.event.pull_request.head.repo.fork }}
+        use_pypi: true
+    - name: Enforce the coverage floor
+      run: coverage report --fail-under=89
 
   distribution-and-docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
       with:
         fetch-depth: 0
-    - uses: actions/setup-python@v5
+        persist-credentials: false
+    - uses: actions/setup-python@v7
       with:
-        python-version: "3.12"
-    - name: Build source and wheel distributions
+        python-version: "3.13"
+    - name: Build and check source and wheel distributions
       run: |
-        python -m pip install build
+        python -m pip install build twine
         python -m build
+        python -m twine check --strict dist/*
     - name: Install the wheel and documentation tools
       run: python -m pip install dist/*.whl sphinx
-    - name: Check the installed CLI
-      run: hm --help
+    - name: Check both installed commands
+      run: |
+        hm --help
+        hallmark --help
       working-directory: ${{ runner.temp }}
     - name: Build documentation with warnings as errors
       run: python -m sphinx -b html -W --keep-going doc "${{ runner.temp }}/hallmark-docs"
-
-  ssh-transport:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v3
-      with:
-        python-version: "3.9"
-    - name: Install Linux SSH server
-      if: runner.os == 'Linux'
-      run: sudo apt-get update && sudo apt-get install -y openssh-server
-    - name: Install Hallmark and tests
-      run: python -m pip install -e . pytest
-    - name: Exercise disposable OpenSSH server and client
-      env:
-        HALLMARK_RUN_SSH_TESTS: "1"
-      run: python -m pytest test/test_ssh_integration.py test/test_transport.py -q

.github/workflows/dependencies.yml

@@ -0,0 +1,84 @@
+# Tests the oldest dependency versions pyproject.toml allows, and the newest
+# releases including pre-releases.
+
+name: Dependencies
+
+on:
+  schedule:
+  - cron: "17 7 * * 1"
+  workflow_dispatch:
+  pull_request:
+    paths:
+    - pyproject.toml
+    - .github/workflows/dependencies.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: dependencies-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lowest:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        persist-credentials: false
+    - uses: actions/setup-python@v7
+      with:
+        python-version: "3.9"
+    - name: Install the OpenSSH server
+      run: sudo apt-get update && sudo apt-get install -y --no-install-recommends openssh-server
+    # Resolved separately so the test tools are not held at their oldest versions.
+    - name: Install the oldest allowed dependencies
+      run: |
+        python -m pip install --upgrade pip uv
+        uv pip compile --python-version 3.9 --resolution lowest-direct --no-header pyproject.toml -o "$RUNNER_TEMP/minimum.txt"
+        python -m pip install -r "$RUNNER_TEMP/minimum.txt" -e . --group dev
+        python -m pip check
+        {
+          echo "### Oldest allowed dependencies"
+          echo '```'
+          grep -v '^ *#' "$RUNNER_TEMP/minimum.txt"
+          echo '```'
+        } >> "$GITHUB_STEP_SUMMARY"
+    - name: Test with pytest
+      env:
+        HALLMARK_RUN_SSH_TESTS: "1"
+      run: python -m pytest --timeout=600 -ra
+
+  pre-release:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.14", "3.15"]
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        persist-credentials: false
+    - uses: actions/setup-python@v7
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install the OpenSSH server
+      run: sudo apt-get update && sudo apt-get install -y --no-install-recommends openssh-server
+    - name: Install the newest dependencies, including pre-releases
+      env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade --pre -e . --group dev
+        {
+          echo "### Installed on Python $PYTHON_VERSION"
+          echo '```'
+          python -m pip list --format=freeze --exclude-editable
+          echo '```'
+        } >> "$GITHUB_STEP_SUMMARY"
+    - name: Test with pytest
+      env:
+        HALLMARK_RUN_SSH_TESTS: "1"
+      run: python -m pytest --timeout=600 -ra

.gitignore

@@ -7,6 +7,8 @@ venv/
 .pytest_cache/
 .ruff_cache/
 .coverage
+.coverage.*
+coverage.xml
 htmlcov/
 build/
 dist/

README.rst

@@ -3,6 +3,10 @@
 
     Reproducibility is the |hallmark|_ of the scientific method.
 
+..  image:: https://codecov.io/gh/lanisrc/hallmark/graph/badge.svg?branch=main
+    :target: https://codecov.io/gh/lanisrc/hallmark
+    :alt: Coverage
+
 Modern science has become so complex that many science projects rely
 on multiple software packages to work in unison, resulting in networks of
 data products along the analyses.
@@ -135,8 +139,8 @@ version, operating system, and a minimal example without credentials or private
 
 From a development checkout, install the validation tools and run::
 
-    python -m pip install -e . pytest ruff sphinx
-    python -m pytest
+    python -m pip install -e . --group dev --group lint --group doc
+    python -m pytest --cov
     ruff check --select E,F mod test
     python -m sphinx -b html -W doc doc/_build
 

codecov.yml

@@ -0,0 +1,25 @@
+# Codecov reports are informational; CI enforces the coverage floor in
+# .github/workflows/ci.yml from the combined report it uploads.
+codecov:
+  notify:
+    after_n_builds: 1
+coverage:
+  precision: 1
+  round: down
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+        informational: true
+    patch:
+      default:
+        target: 80%
+        informational: true
+comment:
+  layout: "condensed_header, diff, flags, files"
+  after_n_builds: 1
+ignore:
+  - "test/**"
+  - "demo/**"
+  - "doc/**"

doc/CONTRIBUTING.md

@@ -28,6 +28,8 @@
 
 * If the implementation of a feature or fix is less clear than the desired outcome, write unit or end-to-end tests first to define the expected functionality. Then write the code to make those tests pass.
 * If the implementation is as clear as (or clearer than) the goal, implement the solution first. Then write tests to ensure the solution covers edge cases and realistic user scenarios. The tests may reveal places where the implementation needs to be fixed, expanded, or simplified.
+* Run `pytest --cov` to see which lines the tests miss, or `pytest --cov --cov-report=html` for a browsable report in `htmlcov/`. CI runs the whole suite, including the loopback SSH tests, on every supported Python version, combines coverage from every job, and fails if the total drops below the floor set in `.github/workflows/ci.yml`. Codecov comments on each pull request with the coverage of its changed lines; those comments are informational.
+* Each week, and on pull requests that change `pyproject.toml`, `.github/workflows/dependencies.yml` runs the suite against the oldest dependency versions `pyproject.toml` allows (on Python 3.9) and against the newest releases including pre-releases (on the two newest Python versions). If you start using a newer feature of a dependency, raise its lower bound in `pyproject.toml`.
 
 ## Data Management
 

doc/usecase.rst

@@ -357,5 +357,5 @@ SSH configuration and known-hosts files.
 They do not connect to a lab server or change the user's SSH trust files.
 Enabling the tests requires the OpenSSH tools to be installed; missing
 tools cause a test failure.
-CI installs the server and runs these tests separately from the
-Python-version matrix.
+CI installs the server and runs the complete suite, these tests included,
+in every Linux and macOS test job.