From aec381b0d1d776dd6b47cfffb5922b48894c03cf Mon Sep 17 00:00:00 2001 From: Ryan McLaughlin Date: Fri, 9 Oct 2026 21:16:42 -0400 Subject: [PATCH 1/2] Add licensed nested-container checks and document validation limits --- docs/containers.md | 32 +++++++++++++ scripts/check_nested_ptools.py | 84 ++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 scripts/check_nested_ptools.py diff --git a/docs/containers.md b/docs/containers.md index 799793a..6973af5 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -69,3 +69,35 @@ The public MP image does not include Pathway Tools or MetaCyc. Licensed users sh ```{include} includes/cami-references.md ``` + +## Nested Pathway Tools containers + +Running MP inside a SIF and asking it to build or execute a second Pathway Tools +SIF is not currently validated as a supported installation route. Use the Mamba +installation of MP with host Apptainer for licensed Pathway Tools workflows. + +A local check of the 4.0.0 release SIF found that nested BLAST creation/search +worked, but Pathway Tools startup timed out. A direct-host comparison also timed +out; nested runtime support therefore remains inconclusive. A separate nested +build probe failed because inherited outer bind mounts targeted paths absent +from the writable build root. Nextflow startup inside the MP SIF also stalled +on a FUSE filesystem request on this host. These results are host-specific and +do not establish that nesting is impossible on every installation. + +Maintainers can repeat the licensed checks on a target host from a source checkout: + +```bash +python scripts/check_nested_ptools.py \ + --mp-image /absolute/path/metapathways.sif \ + --pt-image /absolute/path/pathway-tools.sif \ + --installer /absolute/path/pathway-tools-29.5-linux-64-tier1-install \ + --output /absolute/path/new-nested-test +``` + +Activate the host Apptainer environment first so its helper tools are on `PATH`. +Omit `--installer` to check only execution of an existing licensed image. The +output directory must be new; it contains private logs and a machine-readable +result. Each check has a configurable timeout (`--timeout`, default 900 seconds). +The test does not upload licensed files or change the normal MP image registry. +A passed startup check verifies Pathway Tools readiness and BLAST integration; +it does not replace a complete PGDB inference test. diff --git a/scripts/check_nested_ptools.py b/scripts/check_nested_ptools.py new file mode 100644 index 0000000..305da70 --- /dev/null +++ b/scripts/check_nested_ptools.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Opt-in licensed integration check: MP SIF -> Pathway Tools SIF/build. + +Run on the target host. Nothing is uploaded; installer and SIF remain private. +""" +import argparse +import json +import os +import signal +from pathlib import Path +import shutil +import subprocess +import sys + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--mp-image', required=True, type=Path) + p.add_argument('--pt-image', required=True, type=Path) + p.add_argument('--installer', type=Path, help='also test full nested build_pt') + p.add_argument('--output', required=True, type=Path, help='new diagnostic directory') + p.add_argument('--apptainer', default=shutil.which('apptainer')) + p.add_argument('--timeout', type=int, default=900, help='seconds allowed per check [900]') + a = p.parse_args() + if a.timeout <= 0: + p.error('--timeout must be positive') + if not a.apptainer: + p.error('Host Apptainer is required') + images = [a.mp_image.resolve(), a.pt_image.resolve()] + if a.installer: + images.append(a.installer.resolve()) + for path in images: + if not path.is_file() or any(c in str(path) for c in ':,\n'): + p.error(f'Invalid or missing input: {path}') + output = a.output.resolve() + if any(c in str(output) for c in ':,\n'): + p.error('Output path cannot contain colons, commas or newlines') + output.mkdir(parents=True, exist_ok=False) + command = [a.apptainer, 'exec', '--cleanenv', '--bind', f'{output}:{output}', + '--pwd', str(output)] + for path in images[1:]: + command += ['--bind', f'{path}:{path}:ro'] + for name, value in [('XDG_DATA_HOME', output / 'config'), + ('NXF_HOME', output / 'nextflow'), + ('APPTAINER_CACHEDIR', output / 'cache'), + ('APPTAINER_TMPDIR', output / 'tmp')]: + Path(value).mkdir(exist_ok=True) + command += ['--env', f'{name}={value}'] + command += [str(images[0])] + tests = [('runtime', ['python', '-c', + 'import sys; from metapathways.pt_container import validate; ' + 'print(validate(sys.argv[1], sys.argv[2]))', + str(images[1]), str(output / 'runtime')])] + if a.installer: + tests.append(('build', ['metapathways', 'build_pt', '-i', str(images[2]), + '-o', str(output / 'built'), '--threads', '2', + '--max_cpus', '2', '--memory', '4 GB', '--max_memory', '4 GB'])) + results = {'mp_image': str(images[0]), 'pt_image': str(images[1]), 'checks': {}} + for name, args in tests: + print(f'Testing nested {name}; log: {output / (name + ".log")}', flush=True) + with (output / (name + '.log')).open('w') as log: + process = subprocess.Popen(command + args, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True) + timed_out = False + try: + code = process.wait(timeout=a.timeout) + except (subprocess.TimeoutExpired, KeyboardInterrupt) as error: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + if isinstance(error, KeyboardInterrupt): + raise + code, timed_out = 124, True + results['checks'][name] = {'exit_code': code, 'passed': code == 0, + 'timed_out': timed_out} + (output / 'result.json').write_text(json.dumps(results, indent=2) + '\n') + print(f'{name}: {"PASS" if code == 0 else "FAIL"}', flush=True) + return int(any(not r['passed'] for r in results['checks'].values())) + + +if __name__ == '__main__': + sys.exit(main()) From 111940c8e8b04082929548666b5b8c626451ad21 Mon Sep 17 00:00:00 2001 From: Ryan McLaughlin Date: Fri, 9 Oct 2026 22:34:27 -0400 Subject: [PATCH 2/2] Prepare 4.0.1 with bundled dependencies for nested Pathway Tools builds --- CITATION.cff | 2 +- MANIFEST.in | 1 + README.md | 2 +- changelog.md | 8 ++++ docker/Dockerfile.ptools-base | 20 ++++++++++ docker/Dockerfile.release | 1 + docs/containers.md | 70 ++++++++++++++++++++++++++-------- docs/installation.md | 6 +-- docs/pathway-tools.md | 9 +++++ docs/releasing.md | 10 ++--- metapathways/_version.py | 2 +- metapathways/pt_container.py | 60 +++++++++++++++++++++++++---- scripts/check_nested_ptools.py | 7 +++- scripts/containers.py | 5 ++- tests/test_pt_container.py | 44 +++++++++++++++++++++ 15 files changed, 210 insertions(+), 37 deletions(-) create mode 100644 docker/Dockerfile.ptools-base diff --git a/CITATION.cff b/CITATION.cff index 87af383..ca18491 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -41,4 +41,4 @@ keywords: - functional annotation - pathways - Nextflow -version: "4.0.0" +version: "4.0.1" diff --git a/MANIFEST.in b/MANIFEST.in index ca9f217..f56e86f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,6 +17,7 @@ include tests/test_read_mapping.py include tests/test_pt_container.py include tests/test_nextflow.py include docker/Dockerfile.release +include docker/Dockerfile.ptools-base include docker/README.quay.md recursive-include metapathways/report_assets *.html *.js *.css diff --git a/README.md b/README.md index 7f8b892..b7625cf 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ On Linux x86-64, install MetaPathways and its workflow dependencies with Mamba: ```bash mamba create -n metapathways --override-channels --strict-channel-priority \ - -c hallamlab -c conda-forge -c bioconda metapathways=4.0.0 + -c hallamlab -c conda-forge -c bioconda metapathways=4.0.1 conda activate metapathways ``` diff --git a/changelog.md b/changelog.md index 7955167..a3d7854 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 4.0.1 + +- Bundle an unlicensed Pathway Tools dependency rootfs in release containers so nested licensed builds do not need APT user-switching privileges. +- Clear inherited outer binds only for the inner image build; retain package provenance and blocking image validation. +- Retry transient official-patch HTTP failures with bounded delays. +- Add optional outer-image extraction and isolated configuration to the licensed nested test helper. + + ## 4.0.0 - Nextflow controllers for local and Slurm workflows, multi-sample manifests, resource budgets, database builds and licensed Pathway Tools container preparation. diff --git a/docker/Dockerfile.ptools-base b/docker/Dockerfile.ptools-base new file mode 100644 index 0000000..72a0024 --- /dev/null +++ b/docker/Dockerfile.ptools-base @@ -0,0 +1,20 @@ +# Public system dependencies only. No Pathway Tools installer, software, +# MetaCyc data, credentials, or vendor patches belong in this image. +# Keep this package list aligned with metapathways/pt_container.py until the +# prepared-base build path has passed licensed integration testing. +FROM ubuntu:22.04 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates xterm openssl libxml2 xvfb xauth libxm4 libssl-dev \ + procps bzip2 ncbi-blast+ \ + && mkdir -p /data /opt/bin \ + && printf '[ncbi]\nData=/usr/share/ncbi/data\n' > /opt/mp-ncbirc \ + && test -d /usr/share/ncbi/data \ + && blastp -version \ + && makeblastdb -version \ + && dpkg-query -W ncbi-blast+ > /opt/mp-blast-version.txt \ + && dpkg-query -W > /opt/mp-base-packages.tsv \ + && rm -rf /var/lib/apt/lists/* + +LABEL org.metapathways.purpose="Pathway Tools system dependencies only" diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release index 8928fb4..5e47eaa 100644 --- a/docker/Dockerfile.release +++ b/docker/Dockerfile.release @@ -4,6 +4,7 @@ USER root RUN apt-get update && apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends procps && \ rm -rf /var/lib/apt/lists/* +COPY --from=ptools-base / /opt/metapathways/ptools-base/ USER $MAMBA_USER ARG VERSION ARG REVISION diff --git a/docs/containers.md b/docs/containers.md index 6973af5..ce22bf6 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -2,7 +2,7 @@ [Home and quick start](installation.md) · [Test dataset](test.md) · [Licensed Pathway Tools](pathway-tools.md) -Use the versioned `quay.io/hallamlab/metapathways:4.0.0` image on Linux x86-64. It includes MP, its workflow dependencies (including MAGSplitter and Camelot), and the three-sample test dataset derived from CAMI II ([Meyer et al., 2022](#cami-references)). Production references and licensed Pathway Tools are supplied separately. +Use the versioned `quay.io/hallamlab/metapathways:4.0.1` image on Linux x86-64. It includes MP, its workflow dependencies (including MAGSplitter and Camelot), and the three-sample test dataset derived from CAMI II ([Meyer et al., 2022](#cami-references)). Production references and licensed Pathway Tools are supplied separately. ## Docker three-sample test @@ -11,9 +11,9 @@ Create a working directory and open a shell in the image. The mount keeps inputs ```bash mkdir -p ~/mp-test-docker cd ~/mp-test-docker -docker pull quay.io/hallamlab/metapathways:4.0.0 +docker pull quay.io/hallamlab/metapathways:4.0.1 docker run --rm -it --network host --user "$(id -u):$(id -g)" \ - -v "$PWD:/work" -w /work quay.io/hallamlab/metapathways:4.0.0 bash + -v "$PWD:/work" -w /work quay.io/hallamlab/metapathways:4.0.1 bash ``` Inside the container, run: @@ -40,7 +40,7 @@ Create a working directory and pull the image on an internet-connected host: ```bash mkdir -p ~/mp-test-apptainer cd ~/mp-test-apptainer -apptainer pull metapathways.sif docker://quay.io/hallamlab/metapathways:4.0.0 +apptainer pull metapathways.sif docker://quay.io/hallamlab/metapathways:4.0.1 apptainer exec --bind "$PWD:/work" --pwd /work metapathways.sif bash ``` @@ -72,17 +72,52 @@ The public MP image does not include Pathway Tools or MetaCyc. Licensed users sh ## Nested Pathway Tools containers -Running MP inside a SIF and asking it to build or execute a second Pathway Tools -SIF is not currently validated as a supported installation route. Use the Mamba -installation of MP with host Apptainer for licensed Pathway Tools workflows. - -A local check of the 4.0.0 release SIF found that nested BLAST creation/search -worked, but Pathway Tools startup timed out. A direct-host comparison also timed -out; nested runtime support therefore remains inconclusive. A separate nested -build probe failed because inherited outer bind mounts targeted paths absent -from the writable build root. Nextflow startup inside the MP SIF also stalled -on a FUSE filesystem request on this host. These results are host-specific and -do not establish that nesting is impossible on every installation. +MP's release container includes an unlicensed Ubuntu dependency base for +`build_pt`. When present, MP uses it automatically instead of installing system +packages inside the nested build. Conda installations without the bundled base +continue to build from Ubuntu using host Apptainer. + +A local nested build passed installation, official patch startup, SIF creation, +BLAST validation and Pathway Tools readiness. Complete nested PGDB inference and +portability across HPC installations require separate validation. Use the Mamba +route when a host restricts nested user namespaces or container execution. + +### What the nested checks established + +- Java/Nextflow stalled on a FUSE filesystem request when executing from the + outer SIF on the test host. Launching the outer image with `--unsquash` + resolved this; extraction requires additional temporary space and startup time. +- Outer bind mounts were inherited by the inner build and targeted directories + absent from its writable root. Clearing inherited bind variables inside the + outer container let the build proceed. Explicit inner runtime mounts are still + required; removing all mounts indiscriminately is not a general solution. +- The inner build fell back to a root-mapped namespace without full UID mappings + or a fakeroot helper. Package installation failed when APT tried to switch to + its `_apt` user. Preparing dependencies outside this restricted namespace + removed that step; the licensed installer then completed successfully. +- The prepared-base build completed in approximately six minutes, including + official patch startup, SIF packaging, BLAST database/search validation and + the Pathway Tools readiness marker. This duration excludes building the base. +- SRI returned HTTP 503 twice during the experiment. The successful test reused + 107 unmodified official patch files from an existing private SIF, checking + their hashes against its retained manifest. A fresh vendor download was + therefore not validated in that successful attempt. + +### Build behavior and provenance + +The inner build subprocess clears inherited bind variables, while leaving the +parent environment unchanged. The built image metadata records its installer, +recipe, image and official-patch hashes, plus a package-manifest hash for the +bundled dependency base. Validation remains blocking: a failed build is never +registered. Transient vendor HTTP errors receive at most three attempts; missing +patches still stop the build. + +Use `--unsquash` on the outer Apptainer invocation if Java stalls on its FUSE +mount. This is a host launch setting; code inside an already-mounted container +cannot remount that outer image. Keep configuration inside the mounted project +when building from a container, and pass the container-visible SIF path with +`--image` when running it. A `/work` registration is meaningful inside that mount, +not from an unrelated host shell. Maintainers can repeat the licensed checks on a target host from a source checkout: @@ -98,6 +133,9 @@ Activate the host Apptainer environment first so its helper tools are on `PATH`. Omit `--installer` to check only execution of an existing licensed image. The output directory must be new; it contains private logs and a machine-readable result. Each check has a configurable timeout (`--timeout`, default 900 seconds). -The test does not upload licensed files or change the normal MP image registry. +The test does not upload licensed files. Its configuration directory is isolated +under the requested output, so it does not replace the normal MP image registry. +Use `--unsquash` when testing the extraction route described above; the default +continues to test the mounted SIF. A passed startup check verifies Pathway Tools readiness and BLAST integration; it does not replace a complete PGDB inference test. diff --git a/docs/installation.md b/docs/installation.md index a2848bf..5fd0014 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -6,7 +6,7 @@ Install MetaPathways, try the included three-sample dataset, then use your own d ```bash mamba create -n metapathways --override-channels --strict-channel-priority \ - -c hallamlab -c conda-forge -c bioconda metapathways=4.0.0 + -c hallamlab -c conda-forge -c bioconda metapathways=4.0.1 conda activate metapathways ``` @@ -30,10 +30,10 @@ Open the URL printed by the report server. On a remote server, use an [SSH tunne ```bash # Docker -docker pull quay.io/hallamlab/metapathways:4.0.0 +docker pull quay.io/hallamlab/metapathways:4.0.1 # Or Apptainer -apptainer pull metapathways.sif docker://quay.io/hallamlab/metapathways:4.0.0 +apptainer pull metapathways.sif docker://quay.io/hallamlab/metapathways:4.0.1 ``` The image includes the same workflow dependencies and test data. Follow the [Docker three-sample test](containers.md#docker-three-sample-test) or [Apptainer three-sample test](containers.md#apptainer-three-sample-test) to run the commands with your working directory mounted for persistent results. Licensed Pathway Tools is a separate image. diff --git a/docs/pathway-tools.md b/docs/pathway-tools.md index 2737723..5701fad 100644 --- a/docs/pathway-tools.md +++ b/docs/pathway-tools.md @@ -18,6 +18,15 @@ The compatibility list removes only unsafe **explicit reaction assignments**, not genes, annotations or sequences; the same reaction may still be inferred later. The sections below explain the evidence, limits and diagnostic records. +## Building from the MP Apptainer container + +The release container includes an unlicensed Ubuntu dependency base. `build_pt` +selects it automatically, avoiding package installation inside the restricted +nested build. Your licensed installer and official patches are still required +and remain private. See [nested containers](containers.md#nested-pathway-tools-containers) +for tested behavior and host limitations. On hosts affected by Java/FUSE stalls, +launch the outer MP SIF with `--unsquash`; this needs temporary extraction space. + ## Understand the three different databases | Name | What it contains | How you use it | diff --git a/docs/releasing.md b/docs/releasing.md index c10979a..827da9c 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -6,8 +6,8 @@ This chapter is for maintainers preparing and publishing packages. Every release must pass package and workflow validation before publication. The source version in `metapathways/_version.py` is authoritative. The current -release series is **4.0**, with package version **4.0.0** and tag **v4.0.0**. -Use subsequent patch versions such as **4.0.1**, or **4.0.1rc1** for a preview. +release series is **4.0**, with package version **4.0.1** and tag **v4.0.1**. +Use subsequent patch versions such as **4.0.2**, or **4.0.2rc1** for a preview. Tags have a leading `v`; package versions do not. Updating a repository does not upgrade an already installed environment. @@ -25,10 +25,10 @@ Open feature work as a draft PR targeting `dev` while benchmarks or test checks Prepare any version changes on the feature branch before final PR review: ```bash -python scripts/release.py prepare 4.0.0 +python scripts/release.py prepare 4.0.1 git diff git add README.md CITATION.cff metapathways/_version.py conda_recipe/meta_template.yaml -git commit -m "Prepare MetaPathways 4.0.0" +git commit -m "Prepare MetaPathways 4.0.1" ``` Commit the release script, workflow, tests, and other implementation files as well @@ -281,7 +281,7 @@ The intended sequence is **feature branch → single-server and HPC benchmarks p Use the [PR tester checklist](pr-testing.md) for independent installation, the single- and two-sample CAMI workflows, restart behavior, optional licensed Pathway Tools, and report exports. Record the tested Git commit. Record the source commit as well as the package version when testing a candidate. -The release version is **4.0.0**. Preparing this version does not publish it; branch and PR testing still happen before a release tag is pushed. Do not reuse 3.5.1 or its existing tags. Change the version with `prepare` before final release validation; it also updates the citation version and resets the Conda build number for a new version. Avoid editing runtime code/version files while a benchmark is executing from an editable checkout. +The release version is **4.0.1**. Preparing this version does not publish it; branch and PR testing still happen before a release tag is pushed. Do not reuse 3.5.1 or its existing tags. Change the version with `prepare` before final release validation; it also updates the citation version and resets the Conda build number for a new version. Avoid editing runtime code/version files while a benchmark is executing from an editable checkout. Before merging, manually dispatch the Release workflow on the feature branch with blank publication inputs. Select `test_containers` to test the exact Conda artifact inside Docker and its SIF conversion. Download `release-assets`, `container-assets`, and the security reports from Actions for review. Core integration uses explicit 2 GB task reservations, a 4 GB memory budget, and two CPUs so the small fixture fits CI runners. These limits are not production metagenome sizing recommendations. diff --git a/metapathways/_version.py b/metapathways/_version.py index 740e66c..d6d0e00 100644 --- a/metapathways/_version.py +++ b/metapathways/_version.py @@ -1,5 +1,5 @@ __author__ = "Ryan J. McLaughlin, Tony X. Liu, Tomer Altman, Aditi N. Nallan, Aria S. Hahn, Julia Anstett, Connor Morgan-Lang, Kishori M. Konwar, Steven J. Hallam" -__version__ = "4.0.0" +__version__ = "4.0.1" __maintainer__ = "Ryan J. McLaughlin" __contact__ = "mcglock@student.ubc.ca" __status__ = "Release" diff --git a/metapathways/pt_container.py b/metapathways/pt_container.py index 5e0027f..bcd8824 100644 --- a/metapathways/pt_container.py +++ b/metapathways/pt_container.py @@ -17,6 +17,7 @@ from collections import deque import uuid from urllib.request import build_opener, HTTPRedirectHandler +from urllib.error import HTTPError, URLError from metapathways import nextflow @@ -76,6 +77,32 @@ ''' +PREPARED_BASE = Path('/opt/metapathways/ptools-base') + + +def build_recipe(): + """Use the bundled unlicensed rootfs when running the release container.""" + if not PREPARED_BASE.is_dir(): + return RECIPE, None + manifest = PREPARED_BASE / 'opt/mp-base-packages.tsv' + if not manifest.is_file(): + raise RuntimeError('Bundled Pathway Tools dependency base lacks provenance') + recipe = RECIPE.replace('Bootstrap: docker\nFrom: ubuntu:22.04', + 'Bootstrap: localimage\nFrom: ' + str(PREPARED_BASE)) + recipe = recipe.replace(' apt-get update\n', '') + recipe = recipe.replace(' apt-get install -y --no-install-recommends ca-certificates xterm openssl libxml2 xvfb xauth libxm4 libssl-dev procps bzip2 ncbi-blast+\n', '') + return recipe, dict(kind='bundled-rootfs', packages_sha256=digest(manifest)) + + +def inner_build_environment(): + """Outer binds must not become mounts inside the writable installation root.""" + env = os.environ.copy() + for name in ('APPTAINER_BIND', 'APPTAINER_BINDPATH', + 'SINGULARITY_BIND', 'SINGULARITY_BINDPATH'): + env.pop(name, None) + return env + + def digest(path): h = hashlib.sha256() with Path(path).open('rb') as f: @@ -112,6 +139,21 @@ def installer_version(installer, explicit=None): return version +def open_vendor_url(opener, url): + """Retry transient vendor errors without accepting redirects or partial files.""" + for attempt in range(3): + try: + return opener.open(url, timeout=60) + except (HTTPError, URLError) as exc: + if isinstance(exc, HTTPError) and exc.code not in (429, 500, 502, 503, 504): + raise + if attempt == 2: + raise + delay = (5, 15)[attempt] + print(f'Official patch download temporarily unavailable; retrying in {delay}s', flush=True) + time.sleep(delay) + + def download_patches(version, destination): """Snapshot SRI's official release feed; never silently use partial downloads.""" if not re.fullmatch(r'\d+\.\d+', version): @@ -122,7 +164,7 @@ def download_patches(version, destination): files.mkdir(parents=True) opener = build_opener(_NoRedirect()) try: - with opener.open(url, timeout=60) as response: + with open_vendor_url(opener, url) as response: listing = response.read() links = _PatchLinks() links.feed(listing.decode('utf-8')) @@ -133,7 +175,7 @@ def download_patches(version, destination): for name in sorted(links.names): print('Downloading official Pathway Tools patch: ' + name, flush=True) target = files / name - with opener.open(url + name, timeout=60) as response, target.open('wb') as stream: + with open_vendor_url(opener, url + name) as response, target.open('wb') as stream: shutil.copyfileobj(response, stream) if target.stat().st_size == 0: raise ValueError('Empty patch: ' + name) @@ -360,18 +402,19 @@ def build(installer, image, installer_sha256, threads, version=None): (work / 'installer').symlink_to(installer) patches = download_patches(installer_version(installer, version), work / 'official-patches') definition = work / 'pathway-tools.def' - definition.write_text(RECIPE) + recipe, base = build_recipe() + definition.write_text(recipe) partial = work / 'pathway-tools.sif' subprocess.run([executable, 'build', '--fakeroot', '--mksquashfs-args', f'-processors {threads}', str(partial), str(definition)], - cwd=work, check=True) + cwd=work, check=True, env=inner_build_environment()) validation_log = validate(partial, work / 'validation') metadata = dict(installer_sha256=installer_sha256, image=str(image), - image_sha256=digest(partial), recipe_sha256=hashlib.sha256(RECIPE.encode()).hexdigest(), - official_patches=patches, validation_log=validation_log) + image_sha256=digest(partial), recipe_sha256=hashlib.sha256(recipe.encode()).hexdigest(), + dependency_base=base, official_patches=patches, validation_log=validation_log) partial.replace(image) save_json(str(image) + '.json', metadata) - image.with_suffix('.def').write_text(RECIPE) + image.with_suffix('.def').write_text(recipe) def parser(): @@ -403,7 +446,8 @@ def main(argv=None): if not shutil.which(executable): p.error(f'{executable} must be installed and on PATH') checksum = digest(installer) - recipe_hash = hashlib.sha256(RECIPE.encode()).hexdigest() + recipe, _ = build_recipe() + recipe_hash = hashlib.sha256(recipe.encode()).hexdigest() try: version = installer_version(installer, args.ptools_version) except ValueError as exc: diff --git a/scripts/check_nested_ptools.py b/scripts/check_nested_ptools.py index 305da70..e0cf4cc 100644 --- a/scripts/check_nested_ptools.py +++ b/scripts/check_nested_ptools.py @@ -21,6 +21,8 @@ def main(): p.add_argument('--output', required=True, type=Path, help='new diagnostic directory') p.add_argument('--apptainer', default=shutil.which('apptainer')) p.add_argument('--timeout', type=int, default=900, help='seconds allowed per check [900]') + p.add_argument('--unsquash', action='store_true', + help='extract the outer MP image before execution to avoid FUSE mounts') a = p.parse_args() if a.timeout <= 0: p.error('--timeout must be positive') @@ -38,9 +40,12 @@ def main(): output.mkdir(parents=True, exist_ok=False) command = [a.apptainer, 'exec', '--cleanenv', '--bind', f'{output}:{output}', '--pwd', str(output)] + if a.unsquash: + command += ['--unsquash'] for path in images[1:]: command += ['--bind', f'{path}:{path}:ro'] - for name, value in [('XDG_DATA_HOME', output / 'config'), + for name, value in [('XDG_CONFIG_HOME', output / 'config'), + ('XDG_DATA_HOME', output / 'data'), ('NXF_HOME', output / 'nextflow'), ('APPTAINER_CACHEDIR', output / 'cache'), ('APPTAINER_TMPDIR', output / 'tmp')]: diff --git a/scripts/containers.py b/scripts/containers.py index 83da7c0..5eaf218 100644 --- a/scripts/containers.py +++ b/scripts/containers.py @@ -52,7 +52,10 @@ def build(args): work = Path(temp) context = work / "context" (context / "package").mkdir(parents=True) - shutil.copy2(release.ROOT / "docker/Dockerfile.release", context / "Dockerfile") + base = (release.ROOT / "docker/Dockerfile.ptools-base").read_text() + base = base.replace('FROM ubuntu:22.04', 'FROM ubuntu:22.04 AS ptools-base', 1) + runtime = (release.ROOT / "docker/Dockerfile.release").read_text() + (context / "Dockerfile").write_text(base + "\n" + runtime) shutil.copy2(source / package, context / "package" / package) (context / "package/explicit.txt").write_text( explicit_lock((source / "conda-explicit.txt").read_text(), package)) diff --git a/tests/test_pt_container.py b/tests/test_pt_container.py index 2c39936..513032c 100644 --- a/tests/test_pt_container.py +++ b/tests/test_pt_container.py @@ -58,6 +58,32 @@ def test_changed_installer_rejected_before_execution(self): pt.build(self.installer, self.image, 'wrong-checksum', 2) run.assert_not_called() + def test_bundled_base_avoids_package_installation_and_records_provenance(self): + base = self.root / 'base' + (base / 'opt').mkdir(parents=True) + manifest = base / 'opt/mp-base-packages.tsv' + manifest.write_text('ncbi-blast+\t2.12.0\n') + with patch.object(pt, 'PREPARED_BASE', base): + recipe, provenance = pt.build_recipe() + self.assertIn('Bootstrap: localimage', recipe) + self.assertNotIn('apt-get', recipe) + self.assertEqual(provenance['packages_sha256'], pt.digest(manifest)) + + def test_bundled_base_without_provenance_is_rejected(self): + with patch.object(pt, 'PREPARED_BASE', self.root): + with self.assertRaisesRegex(RuntimeError, 'lacks provenance'): + pt.build_recipe() + + def test_inner_build_drops_inherited_binds_without_mutating_parent(self): + with patch.dict(os.environ, {'APPTAINER_BIND': '/work:/work', + 'SINGULARITY_BINDPATH': '/installer', + 'APPTAINER_CACHEDIR': '/cache'}): + env = pt.inner_build_environment() + self.assertNotIn('APPTAINER_BIND', env) + self.assertNotIn('SINGULARITY_BINDPATH', env) + self.assertEqual(env['APPTAINER_CACHEDIR'], '/cache') + self.assertEqual(os.environ['APPTAINER_BIND'], '/work:/work') + @patch.object(pt.shutil, 'which', return_value='/bin/apptainer') def test_runtime_isolation(self, _): cmd = pt.exec_command(self.image, self.root / 'private state', ['sh', '-c', 'echo hello']) @@ -153,6 +179,24 @@ def test_patch_snapshot_only_accepts_official_listing_filenames(self): self.assertEqual(manifest['files'][0]['sha256'], pt.digest(self.root/'patches/files/p123.fasl')) self.assertTrue(all(c.args[0].startswith(manifest['source']) for c in opener.return_value.open.call_args_list)) + def test_vendor_retry_is_bounded_and_does_not_retry_client_errors(self): + from urllib.error import HTTPError + from unittest.mock import Mock + opener = Mock() + opener.open.side_effect = HTTPError('https://vendor/', 503, 'unavailable', {}, None) + with patch.object(pt.time, 'sleep') as sleep: + with self.assertRaises(HTTPError): + pt.open_vendor_url(opener, 'https://vendor/') + self.assertEqual(opener.open.call_count, 3) + self.assertEqual([c.args[0] for c in sleep.call_args_list], [5, 15]) + opener.reset_mock() + opener.open.side_effect = HTTPError('https://vendor/', 403, 'forbidden', {}, None) + with patch.object(pt.time, 'sleep') as sleep: + with self.assertRaises(HTTPError): + pt.open_vendor_url(opener, 'https://vendor/') + self.assertEqual(opener.open.call_count, 1) + sleep.assert_not_called() + def test_failed_patch_download_does_not_build_or_publish(self): self.download_patches.side_effect = RuntimeError('official feed unavailable') with patch.object(pt.shutil, 'which', return_value='/bin/apptainer'), patch.object(pt.subprocess, 'run') as run: