cdm-data-loaders/.github/workflows/tests.yml at develop · kbase/cdm-data-loaders · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
name: Run tests
permissions:
  contents: read

on:
  workflow_call:

  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review

jobs:
  code_format_and_lint:
    name: Check code formatting
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

      - name: Install uv
        uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884
        with:
          enable-cache: true

      - name: Install dependencies
        run: uv sync

      - name: Run code formatting checks
        shell: bash
        run: uv run ruff format --check

      - name: Run code linting checks
        continue-on-error: true
        run: uv run ruff check --output-format=github .

  container_tests:
    name: Run container tests (${{ matrix.suite.name }})
    needs: code_format_and_lint
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        suite:
          - name: pipelines
            args: tests/cdm_data_loaders/pipelines/
          - name: integration
            args: tests/integration/
          - name: everything else
            args: --ignore=tests/integration --ignore=tests/cdm_data_loaders/pipelines

    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

      - name: Run container tests
        shell: bash
        continue-on-error: false
        run: |
          docker build -t cdm-data-loaders-test:latest .
          echo "Running tests inside container..."
          # slightly nasty hack
          chmod 777 .
          docker run --rm -v .:/out cdm-data-loaders-test:latest test --cov-report=xml:/out/coverage.xml --junitxml=/out/junit.xml -o junit_family=legacy ${{ matrix.suite.args }}

      - name: Send coverage to Codecov
        id: upload_coverage
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5
        with:
          files: coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Send test reports to Codecov
        id: upload_test_reports
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5
        if: ${{ !cancelled() }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          report_type: test_results

  docker_tests:
    name: Run docker pipeline tests
    runs-on: ubuntu-24.04
    needs: code_format_and_lint
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

      - name: Install uv
        uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: "3.14"

      - name: Install dependencies
        run: uv sync --dev

      - name: Run docker pipeline tests
        shell: bash
        continue-on-error: false
        run: uv run pytest --run-docker -m "docker and not requires_spark and not requires_ceph" --cov --junitxml=junit.xml -o junit_family=legacy tests/cdm_data_loaders/pipelines/

      - name: Send test reports to Codecov
        id: upload_test_reports
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5
        if: ${{ !cancelled() }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          report_type: test_results

  tests:
    name: Run local tests (${{ matrix.suite.name }}, ${{ matrix.python-version }}, ${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    needs: code_format_and_lint
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.13", "3.14"]
        os: ["ubuntu-24.04"]
        suite:
          - name: pipelines
            args: tests/cdm_data_loaders/pipelines/
          - name: integration
            args: tests/integration/
          - name: everything else
            args: --ignore=tests/integration --ignore=tests/cdm_data_loaders/pipelines

    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

      - name: Install uv
        uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: uv sync --dev

      - name: Run local tests
        shell: bash
        continue-on-error: false
        run: uv run pytest -m "not requires_spark and not requires_ceph" --cov --junitxml=junit.xml -o junit_family=legacy ${{matrix.suite.args}}

      - name: Send test reports to Codecov
        id: upload_test_reports
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5
        if: ${{ !cancelled() }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          report_type: test_results