From b1872b7bfe2aa8be251bef04b92cfd69cff50d1f Mon Sep 17 00:00:00 2001 From: Valtora Date: Wed, 21 Oct 2026 21:24:49 +0000 Subject: [PATCH] fix(deps): move Pillow to 13.0.0 past fastembed's cap fastembed 0.9.0 declares pillow<13.0, which held every image on 12.3.0 under 20 advisories. Nojoin uses only its TextEmbedding, so fastembed now installs with --no-deps from requirements/worker-no-deps.txt on every surface, worker.txt pins the rest of what it declares, and a test checks those pins against its metadata in place of pip. Fixes: #526 --- .github/CONTRIBUTING.md | 2 +- .github/dependabot.yml | 10 +- .github/workflows/ci.yml | 3 + backend/tests/test_no_deps_requirements.py | 143 +++++++++++++++++++++ docker/Dockerfile.worker | 19 ++- docs/DEVELOPMENT.md | 9 +- requirements/base.txt | 10 +- requirements/dev.txt | 4 + requirements/local.txt | 2 + requirements/test.txt | 2 + requirements/worker-no-deps.txt | 16 +++ requirements/worker.txt | 19 ++- 12 files changed, 220 insertions(+), 19 deletions(-) create mode 100644 backend/tests/test_no_deps_requirements.py create mode 100644 requirements/worker-no-deps.txt diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 86a120453..cd0c9ccdc 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -30,7 +30,7 @@ Install only what your change needs: - **Documentation:** nothing. `python3 scripts/validate_docs.py` uses only the standard library. - **Frontend:** Node and `npm install` in `frontend/`. No Python or GPU. -- **Backend without a GPU:** a virtual environment with `requirements/dev.txt`, the test stack plus Ruff, mypy and pre-commit. `requirements/local.txt` adds pyannote.audio for running the processing pipeline on a GPU host. +- **Backend without a GPU:** a virtual environment with `requirements/dev.txt`, the test stack plus Ruff, mypy and pre-commit, followed by `requirements/worker-no-deps.txt` installed with `--no-deps` ([Fresh Checkout Setup](../docs/DEVELOPMENT.md#fresh-checkout-setup)). `requirements/local.txt` adds pyannote.audio for running the processing pipeline on a GPU host. Before opening a pull request, run the checks for every area you touched, as listed in [Verification By Change Scope](../docs/DEVELOPMENT.md#verification-by-change-scope). `python scripts/check.py` runs every Python-side check, pytest included. CI runs the same checks and the aggregate `CI gate` must pass to merge ([Required Pull Request Checks](../docs/DEVELOPMENT.md#required-pull-request-checks)). diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1637919ad..cb44469bb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -46,14 +46,12 @@ updates: # numpy 2.5 forces an ancient numba that breaks the openai-whisper resolve. - dependency-name: "numpy" versions: [">=2.5.0"] - # fastembed 0.9.0, the newest release, declares huggingface-hub<2.0, so a - # 2.x pin cannot resolve in the worker. Remove when fastembed allows 2.x. + # fastembed 0.9.0, the newest release, declares huggingface-hub<2.0. It is + # installed with --no-deps (requirements/worker-no-deps.txt), so pip no + # longer enforces that, but backend/tests/test_no_deps_requirements.py + # fails a 2.x pin. Remove when fastembed allows 2.x. - dependency-name: "huggingface-hub" versions: [">=2.0.0"] - # fastembed 0.9.0 also declares pillow<13.0, so a 13.x pin cannot resolve - # in the worker. Remove when fastembed allows 13.x. - - dependency-name: "pillow" - versions: [">=13.0.0"] # --- Frontend npm dependencies --- - package-ecosystem: npm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70ffee62a..e6defb4e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,10 +130,13 @@ jobs: cache: pip cache-dependency-path: requirements/test.txt + # fastembed goes in without its declared dependencies, as in the worker + # image; requirements/worker-no-deps.txt explains. - name: Install backend test dependencies run: | python -m pip install --upgrade pip python -m pip install -r requirements/test.txt + python -m pip install --no-deps -r requirements/worker-no-deps.txt # The images ship ffmpeg, and the tests that run real media through it # skip where it is missing. Ubuntu's package is the 6.1 series, older diff --git a/backend/tests/test_no_deps_requirements.py b/backend/tests/test_no_deps_requirements.py new file mode 100644 index 000000000..2ddb4f235 --- /dev/null +++ b/backend/tests/test_no_deps_requirements.py @@ -0,0 +1,143 @@ +"""Packages installed with --no-deps still get what they declare (#526). + +requirements/worker-no-deps.txt holds the packages every install surface puts +in with ``pip install --no-deps`` after worker.txt. fastembed is the only one: +it caps pillow below 13.0, the first release with the fixes for 20 advisories, +and Nojoin loads none of the image models that cap serves. pip no longer checks +what these packages declare, so this module does. Every declared requirement +must be pinned in worker.txt or a file it includes, at a version the package +accepts, unless OVERRIDES names it. An override must still be needed: once the +package accepts the pinned version, it belongs back in worker.txt. +""" + +from __future__ import annotations + +import importlib.metadata as metadata +import re +from pathlib import Path + +import pytest +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +REQUIREMENTS = Path(__file__).resolve().parents[2] / "requirements" +NO_DEPS_FILE = REQUIREMENTS / "worker-no-deps.txt" + +# Declared requirements Nojoin overrides on purpose, by package. +OVERRIDES: dict[str, set[str]] = { + # pillow<13.0 blocks the release that fixes 20 advisories (#526). + "fastembed": {"pillow"}, +} + +# Pinned only in worker.txt, so an install that has it installed worker.txt. +WORKER_SENTINEL = "onnx-asr" + +_PIN = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?==([^\s;#]+)") +_INCLUDE = re.compile(r"^-r\s+(\S+)") + + +def _pins(path: Path) -> dict[str, str]: + """Exact pins in ``path`` and in every file it includes with ``-r``.""" + pins: dict[str, str] = {} + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + include = _INCLUDE.match(line) + if include: + pins.update(_pins(path.parent / include.group(1))) + elif pin := _PIN.match(line): + pins[canonicalize_name(pin.group(1))] = pin.group(2) + return pins + + +def _entries(path: Path) -> list[str]: + lines = (raw.strip() for raw in path.read_text(encoding="utf-8").splitlines()) + return [line for line in lines if line and not line.startswith("#")] + + +def _installed(package: str) -> str | None: + try: + return metadata.version(package) + except metadata.PackageNotFoundError: + return None + + +def _declared(package: str) -> list[Requirement]: + """What ``package`` requires on this interpreter, extras left out.""" + requirements = [Requirement(raw) for raw in metadata.requires(package) or []] + return [ + requirement + for requirement in requirements + if requirement.marker is None or requirement.marker.evaluate({"extra": ""}) + ] + + +def _problems(package: str, pins: dict[str, str]) -> list[str]: + declared = _declared(package) + overrides = OVERRIDES.get(package, set()) + problems = [ + f"OVERRIDES names {name} for {package}, which no longer declares it." + for name in sorted(overrides - {canonicalize_name(r.name) for r in declared}) + ] + for requirement in declared: + name = canonicalize_name(requirement.name) + pinned = pins.get(name) + accepted = pinned is not None and requirement.specifier.contains( + pinned, prereleases=True + ) + if name in overrides: + if accepted: + problems.append( + f"{package} now accepts {name}=={pinned}, so it no longer needs " + f"--no-deps. Move it back into worker.txt, delete " + f"worker-no-deps.txt and its install lines, and drop it here." + ) + elif pinned is None: + problems.append( + f"{package} requires {requirement}, which worker.txt does not pin." + ) + elif not accepted: + problems.append( + f"{package} requires {requirement}, but worker.txt pins " + f"{name}=={pinned}." + ) + return problems + + +def test_no_deps_file_holds_only_exact_pins() -> None: + entries = _entries(NO_DEPS_FILE) + assert entries + for entry in entries: + assert _PIN.match(entry), f"{entry!r} is not an exact pin" + assert set(OVERRIDES) <= set(_pins(NO_DEPS_FILE)) + + +def test_no_other_requirements_file_resolves_a_no_deps_package() -> None: + no_deps = set(_pins(NO_DEPS_FILE)) + for path in sorted(REQUIREMENTS.glob("*.txt")): + if path == NO_DEPS_FILE: + continue + clash = no_deps & set(_pins(path)) + assert not clash, ( + f"{path.name} resolves {sorted(clash)} with its dependencies, which " + f"worker-no-deps.txt exists to skip." + ) + + +@pytest.mark.parametrize("package", sorted(_pins(NO_DEPS_FILE))) +def test_worker_pins_what_each_no_deps_package_requires(package: str) -> None: + installed = _installed(package) + if installed is None: + if _installed(WORKER_SENTINEL) is not None: + pytest.fail( + f"worker.txt is installed but {package} is not. Run: python -m pip " + f"install --no-deps -r requirements/worker-no-deps.txt" + ) + pytest.skip("the worker requirements are not installed") + + pinned = _pins(NO_DEPS_FILE)[package] + assert installed == pinned, ( + f"{package} {installed} is installed but worker-no-deps.txt pins {pinned}; " + f"reinstall it to check the pinned release's requirements." + ) + problems = _problems(package, _pins(REQUIREMENTS / "worker.txt")) + assert not problems, "\n".join(problems) diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 8256b8f8a..ec811a456 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -32,6 +32,11 @@ RUN python -c "import torch; print(f'torch=={torch.__version__}')" > constraints # Install dependencies using the constraints to force usage of system torch RUN pip install --no-cache-dir -r requirements/worker.txt -c constraints.txt +# fastembed caps pillow below the release that fixes its advisories, so it goes +# in without its declared dependencies; worker.txt pins the ones it needs. +# requirements/worker-no-deps.txt explains. +RUN pip install --no-cache-dir --no-deps -r requirements/worker-no-deps.txt + # Install pyannote.audio without dependencies to use system torch. The pin is # read from local.txt so the image and a GPU host install cannot drift apart. RUN pip install --no-cache-dir --no-deps \ @@ -46,8 +51,8 @@ RUN torchcodec_version="$(python -c 'import importlib.metadata as m; print(m.ver pip install --no-cache-dir --no-deps --force-reinstall \ --index-url https://download.pytorch.org/whl/cpu "torchcodec==${torchcodec_version}+cpu" -# Parakeet ASR (onnx-asr) requires the CUDA build of ONNX Runtime. fastembed -# pulls the CPU-only onnxruntime; replace it with onnxruntime-gpu. Its PyPI +# Parakeet ASR (onnx-asr) requires the CUDA build of ONNX Runtime. worker.txt +# pins the CPU-only onnxruntime; replace it with onnxruntime-gpu. Its PyPI # builds target CUDA 13 from 1.27.0 and CUDA 12 before that, so this pin must # move with the CUDA major of the base image above. RUN pip uninstall -y onnxruntime onnxruntime-gpu && \ @@ -112,8 +117,7 @@ RUN apt-get update && \ # remain on disk and are flagged by the image scan. Upgrade the system copies in # place (as root, before the USER switch) to the same patched versions. Keep these # pins in step with requirements/base.txt and requirements/worker.txt: -# pillow==12.3.0 closes the batch of HIGH pillow CVEs disclosed after v1.5.0 -# (CVE-2026-54058/54059/54060, 55379/55380, 59197/59199/59200/59204/59205); +# pillow==13.0.0 closes the 20 advisories fixed only in that release (#526); # urllib3==2.8.0 closes CVE-2026-44431/44432 and CVE-2026-97687/97689; # fsspec==2026.9.0 keeps the CVE-2026-104851 fix. # @@ -121,7 +125,7 @@ RUN apt-get update && \ # and from 26.2 it ships an SBOM of its vendored urllib3, msgpack and setuptools, # whose fixable HIGH CVEs the image scan reports like installed packages. RUN /usr/bin/python3 -m pip install --no-cache-dir --break-system-packages \ - --root-user-action=ignore --upgrade pillow==12.3.0 urllib3==2.8.0 fsspec==2026.9.0 && \ + --root-user-action=ignore --upgrade pillow==13.0.0 urllib3==2.8.0 fsspec==2026.9.0 && \ /usr/bin/python3 -m pip uninstall -y --break-system-packages \ --root-user-action=ignore pip @@ -157,6 +161,11 @@ COPY --from=builder /opt/venv /opt/venv # this image's torch, CUDA runtime and FFmpeg. RUN python -c "from torchcodec.decoders import AudioDecoder" +# Likewise for fastembed, installed without its declared dependencies: its +# import loads every one of them, so a dependency worker.txt stopped pinning +# fails the build here. +RUN python -c "from fastembed import TextEmbedding" + WORKDIR /app COPY . . diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 94c805089..c91f8e21e 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -45,13 +45,14 @@ python3 -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip python -m pip install -r requirements/dev.txt +python -m pip install --no-deps -r requirements/worker-no-deps.txt pre-commit install # optional: Ruff, whitespace and the doc and Alembic validators on commit cd frontend npm install ``` -`requirements/dev.txt` is the test stack (with PyPI's torch, which runs on the CPU) plus the lint, format and type tooling. On a GPU host that runs the live processing stack, install `requirements/local.txt` instead. `requirements/backend.txt` and `requirements/worker.txt` are the API and worker runtime sets. +`requirements/dev.txt` is the test stack (with PyPI's torch, which runs on the CPU) plus the lint, format and type tooling. On a GPU host that runs the live processing stack, install `requirements/local.txt` instead. `requirements/backend.txt` and `requirements/worker.txt` are the API and worker runtime sets. Whichever of these includes `worker.txt`, follow it with the `--no-deps` line, which installs fastembed without the dependencies it declares ([fastembed's Pillow cap](#fastembeds-pillow-cap)). ## Required Pull Request Checks @@ -500,6 +501,12 @@ The base image also fixes the worker's Python minor. The API image, CI, mypy and To move torch, change test.txt, local.txt, both `FROM` lines in Dockerfile.worker (tag and digest) and `MATCHED_STACKS` together, and move torchvision to its pair. Keep the base on the CUDA build PyPI's torch uses (CUDA 13.0 for 2.14.1), so that local.txt installs the same build without an index URL; pip ignores an `--index-url` written on a requirement line. Then run diarisation, Whisper, an ONNX engine and text embeddings on a GPU in the rebuilt worker image, because CI has no GPU and installs torch from PyPI. Dependabot raises the pip and Docker halves as separate pull requests, each of which fails the drift check alone, so combine them. +#### fastembed's Pillow cap + +fastembed 0.9.0, the newest release, declares `pillow<13.0`. Pillow 13.0.0 is the first release with the fixes for 20 advisories, two of them critical (#526), and the cap serves fastembed's image models, which Nojoin never loads: it uses fastembed only for `TextEmbedding`. So fastembed is the one package in [requirements/worker-no-deps.txt](../requirements/worker-no-deps.txt), and every surface installs that file with `pip install --no-deps` after `worker.txt`: Dockerfile.worker, CI's `Backend tests` job and the [Fresh Checkout Setup](#fresh-checkout-setup). `worker.txt` pins everything else fastembed declares, the CPU `onnxruntime` included, which Dockerfile.worker then swaps for `onnxruntime-gpu`. + +pip no longer checks fastembed's requirements, so [backend/tests/test_no_deps_requirements.py](../backend/tests/test_no_deps_requirements.py) does. It fails when a requirement is not pinned in `worker.txt` or `base.txt`, or is pinned outside the range fastembed accepts. The `huggingface-hub` hold in [dependabot.yml](../.github/dependabot.yml) now rests on that check. The test also fails once fastembed accepts the pinned Pillow, normally on Dependabot's fastembed bump. Then move fastembed back into `worker.txt`, delete `worker-no-deps.txt` and its install lines (this guide, CONTRIBUTING.md, CI, Dockerfile.worker and the comments in the requirements files), and delete the test. The worker image build also imports fastembed, and the import loads every dependency it declares. + #### Holding a pin `HOLDS` is empty: no pin is held under an open advisory. Dependabot's `ignore:` stops version-update pull requests but not security alerts, and an advisory raises a fresh alert each time its range is revised, once per manifest that resolves the package (three for a package in test.txt and local.txt, which [requirements/dev.txt](../requirements/dev.txt) includes). To hold a pin: diff --git a/requirements/base.txt b/requirements/base.txt index e7750f39b..7a23e8242 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -19,10 +19,12 @@ httpx==0.28.1 # Security pins: force patched transitive dependencies across both images. # urllib3 (via requests/httpx): CVE-2026-44431 (sensitive header leak on # cross-origin redirect), CVE-2026-44432 (DoS via response decompression). -# pillow (export chart images and reportlab in the api image; matplotlib/fastembed -# in the worker): -# CVE-2026-40192 (FITS decompression-bomb DoS), CVE-2026-42311. +# pillow (export chart images and reportlab in the api image; matplotlib, +# python-pptx, pytesseract and fastembed in the worker): 13.0.0 is the first +# release with the fixes for 20 advisories, two of them critical (#526). +# fastembed caps pillow below 13, so it is installed without its declared +# dependencies: see requirements/worker-no-deps.txt. urllib3==2.8.0 -pillow==12.3.0 +pillow==13.0.0 alembic==1.20.0 pgvector==0.5.1 diff --git a/requirements/dev.txt b/requirements/dev.txt index 1b6cf545c..c1c4267e5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,6 +6,10 @@ # contributor needs to run the same fast checks as CI: # # python -m pip install -r requirements/dev.txt +# python -m pip install --no-deps -r requirements/worker-no-deps.txt +# +# The second line installs fastembed, which must skip its declared +# dependencies; worker-no-deps.txt says why. # # For a GPU host with the live processing stack, use local.txt instead. -r test.txt diff --git a/requirements/local.txt b/requirements/local.txt index bcbf76560..213342063 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -1,3 +1,5 @@ +# Follow this file with: +# python -m pip install --no-deps -r requirements/worker-no-deps.txt -r backend.txt -r worker.txt aiosqlite==0.22.1 diff --git a/requirements/test.txt b/requirements/test.txt index d560674cf..bea1bd0d8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,3 +1,5 @@ +# Follow this file with: +# python -m pip install --no-deps -r requirements/worker-no-deps.txt -r backend.txt -r worker.txt aiosqlite==0.22.1 diff --git a/requirements/worker-no-deps.txt b/requirements/worker-no-deps.txt new file mode 100644 index 000000000..908530dde --- /dev/null +++ b/requirements/worker-no-deps.txt @@ -0,0 +1,16 @@ +# Packages installed without their declared dependencies, after worker.txt or +# any file that includes it (test.txt, dev.txt, local.txt): +# +# python -m pip install --no-deps -r requirements/worker-no-deps.txt +# +# Never include this file from another one: pip would then resolve the +# dependencies this file exists to skip. +# +# fastembed 0.9.0 declares pillow<13.0, which blocks Pillow 13.0.0 and its fixes +# for 20 advisories (#526). Nojoin uses fastembed only for TextEmbedding; the +# cap serves its image models, which Nojoin never loads. worker.txt pins the +# rest of what it declares, and backend/tests/test_no_deps_requirements.py +# checks those pins against fastembed's metadata, since pip no longer does. +# When a fastembed release accepts the pinned pillow, that test fails: move +# fastembed back into worker.txt and delete this file and its install lines. +fastembed==0.9.0 diff --git a/requirements/worker.txt b/requirements/worker.txt index c8b2dbe7b..d0db92862 100644 --- a/requirements/worker.txt +++ b/requirements/worker.txt @@ -1,10 +1,12 @@ +# Follow this file with: +# python -m pip install --no-deps -r requirements/worker-no-deps.txt -r base.txt celery[redis]==5.6.3 openai-whisper==20250625 # Parakeet ASR engine (pluggable transcription backend) onnx-asr==0.12.0 # onnxruntime-gpu (CUDA build) is installed in docker/Dockerfile.worker: it must -# replace the CPU-only onnxruntime that fastembed pulls in, not coexist with it. +# replace the CPU-only onnxruntime pinned below, not coexist with it. silero-vad==6.2.3 numpy==2.4.6 pandas==3.0.6 @@ -61,4 +63,17 @@ pyannoteai-sdk==0.4.0 # PyPI builds torchcodec against the CUDA of PyPI's torch. docker/Dockerfile.worker # swaps it for the CPU build of this version, which links no CUDA library. torchcodec==0.17.0 -fastembed==0.9.0 + +# fastembed's own dependencies. fastembed itself is in worker-no-deps.txt and is +# installed with --no-deps, because it caps pillow below the release that fixes +# its advisories (#526), so everything else it declares is pinned here. +# backend/tests/test_no_deps_requirements.py fails if a pin is missing or falls +# outside fastembed's range. huggingface_hub, numpy and requests are above and +# in base.txt. Keep onnxruntime in step with onnxruntime-gpu in +# docker/Dockerfile.worker, which replaces it in the image. +onnxruntime==1.31.0 +tokenizers==0.23.3 +loguru==0.7.3 +mmh3==5.3.1 +py-rust-stemmers==0.1.8 +tqdm==4.70.1