From b94a7df018c48dce99bd0158b75b3fbee2fbb849 Mon Sep 17 00:00:00 2001 From: Gaurav Jadhav Date: Thu, 5 Nov 2026 16:05:07 +0530 Subject: [PATCH] CI: run the backend tests on 4 workers so the job finishes inside its timeout The backend job ran 4,120 tests one at a time and needed about 45 minutes, so it hit its 15 minute limit on every run and no image was ever pushed. With pytest-xdist on 4 workers (each with its own PostgreSQL, Redis and object store containers) the full suite with coverage takes 7.5 minutes on a 4-CPU machine. The job limit goes to 30 minutes for a slower runner. The workflow also runs when infra/compose/postgres/initdb changes, because the tests build their database from those role scripts. --- .github/workflows/backend.yml | 8 +- backend/check.sh | 5 +- backend/pyproject.toml | 1 + backend/uv.lock | 206 +++++++++++++++++++--------------- 4 files changed, 125 insertions(+), 95 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 7304c3d8..c9bef58d 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -3,9 +3,10 @@ name: backend on: push: branches: [main] - paths: ["backend/**", ".github/workflows/backend.yml"] + # The tests build their database from the role scripts in infra/compose/postgres/initdb (tests/conftest.py) + paths: ["backend/**", "infra/compose/postgres/initdb/**", ".github/workflows/backend.yml"] pull_request: - paths: ["backend/**", ".github/workflows/backend.yml"] + paths: ["backend/**", "infra/compose/postgres/initdb/**", ".github/workflows/backend.yml"] concurrency: group: backend-${{ github.ref }} @@ -17,7 +18,8 @@ permissions: jobs: checks: runs-on: ubuntu-24.04 - timeout-minutes: 15 + # 4,120 tests on 4 workers take about 7.5 minutes on a 4-CPU machine; this leaves room for a slower runner + timeout-minutes: 30 defaults: run: working-directory: backend diff --git a/backend/check.sh b/backend/check.sh index ff8a639a..246ecfde 100755 --- a/backend/check.sh +++ b/backend/check.sh @@ -6,4 +6,7 @@ uv run ruff check . uv run ruff format --check . uv run mypy uv run lint-imports -uv run pytest "$@" +# Four workers, each with its own PostgreSQL, Redis and object store containers (the session fixtures run +# once per worker). The suite waits on the databases far more than on the CPU, so this fits the 4-vCPU CI +# runner and a 4-CPU Colima. Pass -n 0 to run serially. +uv run pytest -n 4 "$@" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b20d391f..921e0955 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -162,6 +162,7 @@ dev = [ "pytest>=9.1.1", "pytest-asyncio>=1.4.0", "pytest-cov>=7.1.0", + "pytest-xdist>=3.8", "ruff>=0.16.9", "testcontainers[postgres,redis]>=4.15.0", "types-aiobotocore[s3]>=3.9.1", diff --git a/backend/uv.lock b/backend/uv.lock index 51b56b11..3128e698 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -459,6 +459,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "fastapi" version = "0.142.1" @@ -692,6 +701,99 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, ] +[[package]] +name = "idraft" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "aiobotocore" }, + { name = "aiohttp" }, + { name = "alembic" }, + { name = "argon2-cffi" }, + { name = "asyncpg" }, + { name = "cryptography" }, + { name = "fastapi" }, + { name = "google-re2" }, + { name = "httpx" }, + { name = "idna" }, + { name = "publicsuffixlist" }, + { name = "pyahocorasick" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyotp" }, + { name = "python-multipart" }, + { name = "redis" }, + { name = "sentry-sdk", extra = ["fastapi"] }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "structlog" }, + { name = "taskiq" }, + { name = "taskiq-redis" }, + { name = "tzdata" }, + { name = "uvicorn", extra = ["standard"] }, + { name = "zstandard" }, +] + +[package.dev-dependencies] +dev = [ + { name = "asgi-lifespan" }, + { name = "asyncpg-stubs" }, + { name = "hypothesis" }, + { name = "import-linter" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "pytest-xdist" }, + { name = "ruff" }, + { name = "testcontainers", extra = ["redis"] }, + { name = "types-aiobotocore", extra = ["s3"] }, +] + +[package.metadata] +requires-dist = [ + { name = "aiobotocore", specifier = ">=3.9.1" }, + { name = "aiohttp", specifier = ">=3.14.3" }, + { name = "alembic", specifier = ">=1.20.0" }, + { name = "argon2-cffi", specifier = ">=25.1.0" }, + { name = "asyncpg", specifier = ">=0.31.0" }, + { name = "cryptography", specifier = ">=50.0.1" }, + { name = "fastapi", specifier = ">=0.142.1" }, + { name = "google-re2", specifier = "==1.1.20251105" }, + { name = "httpx", specifier = ">=0.28.1" }, + { name = "idna", specifier = "==3.20" }, + { name = "publicsuffixlist", specifier = "==1.0.2.20260925" }, + { name = "pyahocorasick", specifier = "==2.3.1" }, + { name = "pydantic", specifier = ">=2.13.5" }, + { name = "pydantic-settings", specifier = ">=2.15.0" }, + { name = "pyotp", specifier = ">=2.10.0" }, + { name = "python-multipart", specifier = ">=0.0.32" }, + { name = "redis", specifier = ">=8.1.0" }, + { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.71.0" }, + { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.1.1" }, + { name = "structlog", specifier = ">=26.1.0" }, + { name = "taskiq", specifier = ">=0.13.0" }, + { name = "taskiq-redis", specifier = ">=1.2.4" }, + { name = "tzdata", specifier = ">=2026.4" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.54.0" }, + { name = "zstandard", specifier = ">=0.25.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "asgi-lifespan", specifier = ">=2.1.0" }, + { name = "asyncpg-stubs", specifier = ">=0.31.3" }, + { name = "hypothesis", specifier = ">=6.168.3" }, + { name = "import-linter", specifier = ">=2.15" }, + { name = "mypy", specifier = ">=2.3.1" }, + { name = "pytest", specifier = ">=9.1.1" }, + { name = "pytest-asyncio", specifier = ">=1.4.0" }, + { name = "pytest-cov", specifier = ">=7.1.0" }, + { name = "pytest-xdist", specifier = ">=3.8" }, + { name = "ruff", specifier = ">=0.16.9" }, + { name = "testcontainers", extras = ["postgres", "redis"], specifier = ">=4.15.0" }, + { name = "types-aiobotocore", extras = ["s3"], specifier = ">=3.9.1" }, +] + [[package]] name = "import-linter" version = "2.15" @@ -1097,6 +1199,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1235,97 +1350,6 @@ fastapi = [ { name = "fastapi" }, ] -[[package]] -name = "idraft" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "aiobotocore" }, - { name = "aiohttp" }, - { name = "alembic" }, - { name = "argon2-cffi" }, - { name = "asyncpg" }, - { name = "cryptography" }, - { name = "fastapi" }, - { name = "google-re2" }, - { name = "httpx" }, - { name = "idna" }, - { name = "publicsuffixlist" }, - { name = "pyahocorasick" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pyotp" }, - { name = "python-multipart" }, - { name = "redis" }, - { name = "sentry-sdk", extra = ["fastapi"] }, - { name = "sqlalchemy", extra = ["asyncio"] }, - { name = "structlog" }, - { name = "taskiq" }, - { name = "taskiq-redis" }, - { name = "tzdata" }, - { name = "uvicorn", extra = ["standard"] }, - { name = "zstandard" }, -] - -[package.dev-dependencies] -dev = [ - { name = "asgi-lifespan" }, - { name = "asyncpg-stubs" }, - { name = "hypothesis" }, - { name = "import-linter" }, - { name = "mypy" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "pytest-cov" }, - { name = "ruff" }, - { name = "testcontainers", extra = ["redis"] }, - { name = "types-aiobotocore", extra = ["s3"] }, -] - -[package.metadata] -requires-dist = [ - { name = "aiobotocore", specifier = ">=3.9.1" }, - { name = "aiohttp", specifier = ">=3.14.3" }, - { name = "alembic", specifier = ">=1.20.0" }, - { name = "argon2-cffi", specifier = ">=25.1.0" }, - { name = "asyncpg", specifier = ">=0.31.0" }, - { name = "cryptography", specifier = ">=50.0.1" }, - { name = "fastapi", specifier = ">=0.142.1" }, - { name = "google-re2", specifier = "==1.1.20251105" }, - { name = "httpx", specifier = ">=0.28.1" }, - { name = "idna", specifier = "==3.20" }, - { name = "publicsuffixlist", specifier = "==1.0.2.20260925" }, - { name = "pyahocorasick", specifier = "==2.3.1" }, - { name = "pydantic", specifier = ">=2.13.5" }, - { name = "pydantic-settings", specifier = ">=2.15.0" }, - { name = "pyotp", specifier = ">=2.10.0" }, - { name = "python-multipart", specifier = ">=0.0.32" }, - { name = "redis", specifier = ">=8.1.0" }, - { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.71.0" }, - { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.1.1" }, - { name = "structlog", specifier = ">=26.1.0" }, - { name = "taskiq", specifier = ">=0.13.0" }, - { name = "taskiq-redis", specifier = ">=1.2.4" }, - { name = "tzdata", specifier = ">=2026.4" }, - { name = "uvicorn", extras = ["standard"], specifier = ">=0.54.0" }, - { name = "zstandard", specifier = ">=0.25.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "asgi-lifespan", specifier = ">=2.1.0" }, - { name = "asyncpg-stubs", specifier = ">=0.31.3" }, - { name = "hypothesis", specifier = ">=6.168.3" }, - { name = "import-linter", specifier = ">=2.15" }, - { name = "mypy", specifier = ">=2.3.1" }, - { name = "pytest", specifier = ">=9.1.1" }, - { name = "pytest-asyncio", specifier = ">=1.4.0" }, - { name = "pytest-cov", specifier = ">=7.1.0" }, - { name = "ruff", specifier = ">=0.16.9" }, - { name = "testcontainers", extras = ["postgres", "redis"], specifier = ">=4.15.0" }, - { name = "types-aiobotocore", extras = ["s3"], specifier = ">=3.9.1" }, -] - [[package]] name = "six" version = "1.17.0"