diff --git a/.dockerignore b/.dockerignore index 92f55c7..cd038aa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,7 @@ ROADMAP.md *-workspace work test_run +# accelerator build artifacts generated by bin/build_consensus_accel.py +bin/consensus_accel*.so +bin/consensus_accel.c +bin/__pycache__ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..585550a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,129 @@ +name: CI + +on: [push, pull_request] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -el {0} + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up ChoCallate environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-file: environment.yaml + environment-name: ChoCallate + cache-environment: true + cache-downloads: true + init-shell: bash + + - name: Ruff + run: ruff check . + + - name: Nextflow lint + run: nextflow lint . + + - name: Actionlint + run: actionlint + + pytest: + name: pytest (${{ matrix.accelerator }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + accelerator: [pure, accel] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up ChoCallate environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-file: environment.yaml + environment-name: ChoCallate + cache-environment: true + cache-downloads: true + init-shell: bash + + - name: Build consensus accelerator + if: matrix.accelerator == 'accel' + run: python bin/build_consensus_accel.py + + - name: Verify accelerator import + if: matrix.accelerator == 'accel' + run: >- + python -c "import sys; sys.path.insert(0, 'bin'); + import consensus_accel" + + - name: Run pytest + run: pytest -q tests/python + + docker: + name: Docker build + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build image + uses: docker/build-push-action@v7 + with: + context: . + push: false + load: true + tags: chocallate:ci + cache-from: type=gha + cache-to: type=gha + + - name: Verify consensus accelerator + run: >- + docker run --rm --entrypoint micromamba chocallate:ci + run -n ChoCallate python -c + "import sys; sys.path.insert(0, '/opt/chocallate/bin'); + import consensus_accel" + + - name: Verify offline plugins with arbitrary UID + run: >- + docker run --rm --network none -u 12345:12345 -w /tmp + --entrypoint micromamba chocallate:ci + run -n ChoCallate nextflow inspect /opt/chocallate/main.nf + + nf-test: + name: nf-test + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up ChoCallate environment + uses: mamba-org/setup-micromamba@v3 + with: + environment-file: environment.yaml + environment-name: ChoCallate + cache-environment: true + cache-downloads: true + init-shell: bash + + - name: Run nf-test + run: nf-test test --ci diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 7e07113..2f5b050 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -14,9 +14,6 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.gitignore b/.gitignore index 22ce215..eba1aef 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,26 @@ test_data/*.bt2 test_data/chocallate_test_* .vscode/ ChoCallate.code-workspace +bin/consensus_accel.c +bin/consensus_accel*.so +bin/.build_accel/ +test_run/ChoCallate_output* +test_run/merge_* +test_run/consensus_* +test_run/bcfs +test_run/ad_perf +test_run/accs_list.txt +test_run/ERR* +test_run/*.bt2 +AGENTS.md +test_run/samples.tsv +test_run/arth_chr1.fasta.gz.* +test_run/benchmark_* +test_run/test_merge* +test_run/test_config* +test_run/run_test_2.sh +test_run/include.bed +.pytest_cache +.nf-test/ +.nf-test.log +.ruff_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9404b04..03910a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,91 @@ All notable changes to this project will be documented in this file. -## [Unreleased] +## [4.0.0] - 2026-10-20 + +### Added + +- `nextflow_schema.json` and the `nf-schema` plugin validate every parameter before the first task: enum values, numeric minimums, input-file existence and unknown keys abort with the offending key and the accepted values. Every parameter now has a default in `nextflow.config` and documentation in `assets/templates/config.yaml`. +- Samplesheet validation: blank lines and comments are skipped, only the first non-comment line can be a header, sample ids must be unique and shell-safe, every row must have the exact column count for `input.format`/`input.reads_type`, and relative read paths are resolved from the samplesheet directory and must exist. +- `calling..min_snp_qual` and `calling..min_indel_qual` provide per-caller, per-variant-type quality filters and are required for every configured caller. +- `calling.allow_caller_failure` (default `false`) enables degraded mode: a failed caller is skipped with a warning, `CallersFailed` is recorded, the run continues while `consensus.threshold` is satisfiable, and samples with no surviving caller are listed in `dropped_samples.txt` (and in `SamplesDropped` in a merged output). +- Caller `extra_args` that request non-diploid ploidy, a copy-number map or custom ploidy regions are rejected, because consensus genotypes are diploid. +- `consensus.absent_vote` (`ref`/`abstain`), `consensus.filtered_vote` (`abstain`/`ref`) and `consensus.tie_policy` (`nocall`/`first`) configure the voting model. +- `output.compress_reference_blocks` (default `true`) collapses runs of hom-reference positions into multi-base gVCF blocks with `INFO/END` when reference records are written, in both per-sample and merged output. +- `output.publish_mode` (`copy` or `link`, default `copy`) copies or links results instead of moving them out of the work directory; `copy` keeps results independent of work-directory cleanup, while `link` requires the work directory to be retained. +- `input.reference_index_prefix` sets the mapper index prefix when it differs from the reference file name; it must be a plain file name without path separators. +- `mapping.platform` (default `ILLUMINA`) and `mapping.library` (default the sample id) set the read-group `PL` and `LB` values for mapping and BAM input; values are restricted to letters, digits, `_`, `.`, `:` and `-`. +- `mapping.aligner_reserve_mb` reserves memory for the aligner index while `samtools sort` runs in the same task; `0` estimates the reserve from the staged index files plus 1 GiB. +- `resources.low`, `resources.medium` and `resources.high` define per-label CPU, memory and time limits; `resources.limits` caps those values after retry escalation, and JVM heaps are derived from task memory. +- `resources.max_retries` controls retries for transient failures (exit status 137 or 140), with memory and time multiplied by the attempt number. +- `boost.cleanup` is now a params-file option (`"v1"`, `"v2"` or `false`) instead of a hard-coded `nextflow.config` setting. +- `SCATTER_INTERVALS` computes the coverage BED chunks once per sample and shares them between every caller and consensus. +- `EXPORT_OUTPUT` applies `output.format`, `output.remove_invariant` and `output.split_multiallelic` once at the end and publishes per-sample or merged files according to `output.type`; internal consensus and merge files are always BCF, including for `output.format: "vcf"`. +- Mapping statistics are published under `mapping_stats/`: raw `samtools flagstat` plus final `samtools flagstat`, `samtools stats` and `samtools idxstats`, and `mapping_summary.tsv` with per-sample alignment, duplicate, MAPQ, insert-size and error-rate metrics. +- `bin/postprocess_bam.sh` provides shared mapping/BAM post-processing: mate fixing, `samtools sort`, optional `samtools markdup`, MAPQ filtering, optional GATK `LeftAlignIndels` and one CSI index. +- BAM input read groups are normalised with `samtools addreplacerg` (`ID`/`SM` set to the sample id, `PL`/`LB` from `mapping.platform`/`mapping.library`), so GATK and `bcftools reheader` see consistent read groups. +- BAM inputs and mapping output are checked against the reference: `@SQ` names and lengths must match the `.fai`, and when the BAM header carries `M5` checksums they are compared with the sequence dictionary; mismatches abort before calling. +- The generated `.fai` is checked for contigs longer than 2,147,483,647 bp (2^31-1), because BAM/BCF positions and GATK use 32-bit coordinates. +- `bin/filter_bed.py` strips `track`, `browser` and `#` lines before `sort-bed` and rejects contigs absent from the reference `.fai`; consensus rejects coverage BED intervals with negative or reversed coordinates or ends past the contig. +- `coverage.on_empty` (`fail` by default, or `skip`) controls what happens when no base of a sample reaches `coverage.min_coverage`; skipped samples are listed in `dropped_samples.txt` (and in `SamplesDropped` in a merged output). +- `bin/merge_outputs.sh` is the production merge chain used by `MERGE_OUTPUTS` and the pytest merge tests; `bin/drop_refonly_duplicates.py` removes the redundant `<*>` record created by `bcftools merge --gvcf`. +- `bin/merge_cho_headers.py` builds the merged `##ChoCallate=` line from the union of per-sample `Callers` and `CallersFailed`, adds `SamplesDropped` when samples were dropped, and writes `##ChoCallateSamples=` when caller sets differ or any caller failed. +- The merge step checks the process open-file limit before the first task for `output.type: "single"` and aborts with the required `ulimit -n` value when the sample count and `consensus.cpu` would exceed it. +- `bin/build_consensus_accel.py` builds an optional Cython accelerator for `generate_consensus.py`; the Docker image builds it automatically, and an equivalent pure-pysam implementation remains as a fallback. +- One structured `##ChoCallate=` header line per consensus file records the version, threshold, callers, rule, tie policy, absence policy, filtered-record policy and reference-block flag. +- The `tests/python` pytest suite covers consensus (pure and accelerated), merge, BAM post-processing, validation, resources, retries, resume/cache and cleanup; the `tests/nf` nf-test suite contains module tests and end-to-end pipeline cases on the bundled `test_run` data. +- GitHub Actions CI runs `ruff`, `nextflow lint`, `actionlint`, pytest with and without the Cython accelerator, nf-test, and Docker image smoke tests including offline plugin resolution. + +### Changed + +- `output.remove_invariant` now defaults to `false`, so a default run keeps monomorphic/reference sites in both per-sample and merged BCF/VCF output. +- Hom-reference positions are written as gVCF-style `ALT="<*>"` records with `INFO/END` instead of `ALT="."`; single-base records carry `AD=(reference depth, other-allele depth)`, while multi-base blocks omit `AD` and carry the block minimum in `FORMAT/DP`. `MERGE_OUTPUTS` uses `bcftools merge --gvcf`, so merged `FORMAT/AD` reports honest ALT depth for hom-reference samples. +- `coverage.min_coverage` now defaults to `5` (was `2`), must be at least `1`, and `bam_filter.min_map_qual` now defaults to `10` (was `0`). +- `rmdup.mode` (`none`, `mark` or `remove`; default `mark`) replaces `rmdup.enabled` and `picard MarkDuplicates`; `samtools markdup` flags duplicates by default, and callers, coverage and consensus skip duplicate-flagged reads. +- Reference compression is detected from the BGZF header, so `ref_genome.bgzip` is no longer needed; plain gzip references are rejected while plain and BGZF-compressed FASTA are accepted. +- `input.reference_index_dir` is required only for `format: "fastq"`, so BAM-only runs no longer need mapper indexes. +- Mapper indexes are resolved from `input.reference_index_dir` and staged as task inputs instead of being read from that host path at task runtime, so mapping works in containers, on cloud executors and with `scratch`; an incomplete or missing index aborts before the first task. +- Consensus inputs are staged as `tmp/callers/.bcf` and passed as `--input caller=path`, so caller identity and voting order no longer depend on positional file order when a caller fails. +- Callers are normalised with `bcftools norm -f ref -m -any --atomize`; multi-allelic and complex records are split, low-QUAL records are kept with `FILTER=LowQual`, and the old `--min-alleles 2 --max-alleles 2` filter is no longer applied. +- Consensus is now two-stage and caller-order independent by default: an ALT becomes a candidate once `consensus.threshold` callers support it, then the majority zygosity among its supporters decides the genotype; ties are a no-call unless `consensus.tie_policy: "first"`, and multiple supported ALT alleles produce ploidy-consistent split records. +- Filtered caller records abstain from the vote by default (`consensus.filtered_vote: "abstain"`), no-calls no longer count as hom-reference votes, and positions with an IUPAC-ambiguous reference base or caller allele are omitted from consensus output. +- `FORMAT/DP` counts collapsed observations (one per read name) instead of every covering read, matching the overlap-corrected coverage BED; a deletion is not an allele, so an overlapping mate's base is preferred when tallying reference `AD`. +- `calling.bcftools.mpileup.max_depth` and `max_idepth` (default `100000`) override the bcftools default of 250; FreeBayes receives `--use-best-n-alleles` (default `4`) and `--limit-coverage` (default `100000`); GATK receives `--standard-min-confidence-threshold-for-calling min(min_snp_qual, min_indel_qual)`. +- GATK calling intervals are padded by `calling.gatk.interval_padding` (default `150`), and each padded chunk's calls are restricted to its own unpadded BED interval before concatenation. +- Coverage is computed with `mosdepth --no-per-base --quantize` and sorted with `sort-bed`/`bedops` instead of a per-base `samtools depth | awk` stream. +- Mapping and BAM preparation share `bin/postprocess_bam.sh`; in mapping tasks `mapping.cpu` is split between mapper and post-processing, sort memory is derived from task memory after the aligner reserve, and each BAM is indexed once. +- Every process now uses `process_low`, `process_medium` or `process_high`; every JVM receives an `-Xmx` derived from task memory, `Runtime.runtime.availableProcessors()` is no longer used, `maxForks` directives are gone, and caller failures are no longer ignored unconditionally. +- All process scripts run under `bash -euo pipefail`, process-local `$TMPDIR` is used for temporary files, and cleanup removes `tmp/`, `index/` and `TMPDIR` only after a successful task. +- The bundled `test_run` dataset is reduced to a 200-kb slice and 20,000 read pairs, so the end-to-end test takes about 3 minutes. +- `environment.yaml` adds `mosdepth`, `cython`, `setuptools` and `c-compiler` and updates the tool stack; `htslib` provides `bgzip`/`tabix`. +- The Docker image installs the pipeline under `/opt/chocallate`, pins the `nf-schema` and `nf-boost` plugins at build time for offline runs, and provides a world-writable `HOME`, so it works with Docker, Apptainer and Singularity under an arbitrary UID. +- `nextflow.config` now declares defaults for every parameter plus `manifest.name`, `nextflowVersion`, `homePage` and `mainScript`. +- `generate_consensus.py` streams one cursor per caller BCF instead of materialising all records of a coverage span and is about 20x faster on a fragmented 2.1 Mbp coverage BED through span grouping, a fast path for positions without caller records and bulk pileup accessors; an optional Cython accelerator speeds up the pileup and record writing further, with an equivalent pure-pysam fallback. +- `merge_bcf_tree.py` handles a single input file and aborts pending FIFO readers/writers when the final merge fails. + +### Removed + +- `ref_genome.bgzip`; compression is detected automatically. +- `rmdup.enabled`; use `rmdup.mode` instead. +- The shared `calling.min_snp_qual` key; configure `calling..min_snp_qual` and `calling..min_indel_qual` instead. +- `bin/fill_missing_ad.py`; the gVCF-style `<*>` representation makes post-merge AD filling unnecessary. +- The standalone `tabix` and unused `bedtools` conda dependencies. +- The `--remove_invariant` and `--split_multiallelic` options of `generate_consensus.py`; `EXPORT_OUTPUT` now applies both transformations. + +### Fixed + +- `generate_consensus.py` no longer drops the last base of every coverage BED interval, so 1-bp intervals produce output and variants at interval ends are retained. +- Deletion handling no longer skips the deletion REF span or loses overlapping caller records: the walk advances one base at a time, caller records inside a deletion are still voted on, and reference records are not written over deleted bases. +- ALT support is keyed by `(REF, ALT)` instead of ALT alone, so deletions of different lengths and atomised records with different REF alleles at the same position remain distinct and caller-order independent. +- `generate_consensus.py` no longer crashes with `TypeError` on a caller record with a missing genotype (`./.`, `0/.`). +- `MERGE_OUTPUTS` keeps monomorphic sites when `output.remove_invariant: false`, matching the documented meaning; the merged output no longer drops every reference-only record. +- `output.type: "single"` with `output.format: "vcf"` no longer fails by passing per-sample `.vcf.gz` files to a merge stage that expects BCF; internal files are always BCF and the VCF is created only by the export stage. +- `PREPARE_BAM` declares the BAM it actually writes (`mapping.bam`), so `input.format: "bam"` no longer fails with "Missing output file(s)". +- `merge_bcf_tree.py` no longer fails with "Expected two or more files to merge" when it runs with a single chunk (`--cpus 1`), and a failed final merge no longer hangs on FIFOs. +- `bin/stage_cleanup.sh` preserves staged inputs and temporary files when a task fails and never deletes a `TMPDIR` outside the task work directory. +- `GENERATE_COVERAGE` stages `include_bed` and `exclude_bed` in separate directories, so BED inputs with the same file name no longer overwrite each other. +- `test_run/run_test.sh` sets `NXF_OFFLINE` (the variable was misspelled `NFX_OFFLINE`), so the offline setting takes effect. + ## [3.2.3] - 2026-10-01 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 166a28b..19bd6c0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to the ChoCallate -We welcome you contributions to the ChoCallate project +We welcome your contributions to the ChoCallate project ## How to report a bug @@ -14,10 +14,10 @@ In general, we follow the "fork-and-pull" Git workflow. 2. **Clone** the project to your own machine 3. **Commit** changes to your own branch 4. **Push** your work back up to your fork - 5. Submit a **Pull request** so that we can review your changes + 5. Submit a **Pull request** with your changes -Note: Be sure to merge the latest changes before making the pull request. You can increase your chances of a successfull pull request by indicating clearly what branch you are contributing to and making a detailed commit message. It also helps if you open an issue first to let us know you are working on this project. +Note: Be sure to merge the latest changes before making the pull request. You can increase your chances of a successful pull request by indicating clearly what branch you are contributing to and making a detailed commit message. It also helps if you open an issue first to let us know you are working on this project. ## License -ChoCallate is licensed undert [MIT License](https://github.com/alermol/ChoCallate/blob/main/LICENSE) \ No newline at end of file +ChoCallate is licensed under [MIT License](https://github.com/alermol/ChoCallate/blob/main/LICENSE) diff --git a/Dockerfile b/Dockerfile index ccecf56..0ca2f4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,37 @@ COPY environment.yaml . RUN micromamba create -y -f environment.yaml && \ micromamba clean --all --yes -COPY . . +# Install the pipeline outside /tmp. Apptainer and Singularity bind the host +# /tmp into the container by default, which would hide a pipeline installed +# there. +# +# The COPY destination is created with the ownership given by --chown, so the +# image user can create NXF_HOME below it. The image runs as MAMBA_USER and +# the accelerator build writes into bin/, so the copied project must be owned +# by that user as well. +COPY --chown=${MAMBA_USER}:${MAMBA_USER} . /opt/chocallate/ +WORKDIR /opt/chocallate + +# Build the optional Cython accelerator for generate_consensus.py in place. +# generate_consensus.py falls back to an equivalent pure-pysam implementation +# if the module is missing, so this is an optimisation, not a requirement. +RUN micromamba run -n ChoCallate python bin/build_consensus_accel.py + +# Pin the plugins in the image so a run does not need network access. +ENV NXF_HOME=/opt/chocallate/.nextflow +RUN micromamba run -n ChoCallate nextflow plugin install \ + nf-schema@3.0.0,nf-boost@0.8.0 && \ + chmod -R a+rwX "${NXF_HOME}" + +# Both micromamba run and Nextflow need a writable HOME. With docker run -u +# $(id -u) the UID has no passwd entry, so ~ is not writable. Apptainer and +# Singularity bind the host /tmp, which is why HOME is not set to /tmp either. +ENV HOME=/opt/chocallate/.home +RUN mkdir -p "${HOME}" && chmod 0777 "${HOME}" ENV ENV_NAME=ChoCallate WORKDIR /workspace -ENTRYPOINT ["micromamba", "run", "-n", "ChoCallate", "/tmp/main.nf"] +ENTRYPOINT ["micromamba", "run", "-n", "ChoCallate", \ + "nextflow", "run", "/opt/chocallate/main.nf"] diff --git a/README.md b/README.md index 55bad90..5159ec6 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,53 @@ # ChoCallate 🍫 -![GitHub Release](https://img.shields.io/github/v/release/alermol/ChoCallate) ![Static Badge](https://img.shields.io/badge/Changelog-orange) ![GitHub License](https://img.shields.io/github/license/alermol/chocallate) ![Static Badge](https://img.shields.io/badge/Wiki-red?link=https%3A%2F%2Fgithub.com%2Falermol%2FChoCallate%2Fwiki) +[![GitHub Release](https://img.shields.io/github/v/release/alermol/ChoCallate)](https://github.com/alermol/ChoCallate/releases) [![Changelog](https://img.shields.io/badge/Changelog-orange)](CHANGELOG.md) [![GitHub License](https://img.shields.io/github/license/alermol/chocallate)](LICENSE) [![Wiki](https://img.shields.io/badge/Wiki-red)](https://github.com/alermol/ChoCallate/wiki) -**ChoCallate** (**Cho**rus of **Call**ers) - a **Nextflow** pipeline for **consensus-based variant calling**. +**ChoCallate** (**Cho**rus of **Call**ers) is a Nextflow pipeline for **consensus-based variant calling**. It maps FASTQ reads or accepts aligned BAMs, calls variants with bcftools, FreeBayes and GATK, and applies a configurable consensus rule to produce high-confidence SNVs and indels. -ChoCallate runs several variant callers and applies configurable consensus rules to produce high-confidence **SNVs** and **INDELs**. It addresses a critical challenge in variant calling: individual variant callers can produce different results for the same genomic data, leading to uncertainty in variant identification. By implementing a consensus-driven approach, ChoCallate combines results from multiple state-of-the-art variant callers and applies configurable consensus rules to generate reliable, high-quality variant calls. +## Quick start -## Requirements - -- **Linux** (tested). macOS/Windows are not currently tested. -- **Conda** (Miniconda/Anaconda) or **Mamba** -- **Git** -- **Nextflow** - - - -## Installation +Requirements: Linux, Conda/Mamba and Git. Nextflow and all analysis tools are installed by the environment. The first run resolves the `nf-schema` and `nf-boost` plugins; an offline Conda setup must install the pinned versions once, while the Docker image bundles them. ```bash git clone --depth 1 https://github.com/alermol/ChoCallate.git cd ChoCallate conda env create -y -f environment.yaml --no-default-packages conda activate ChoCallate +cp assets/templates/config.yaml my_run.yaml ``` -Optional verification and cleanup: +Minimal `my_run.yaml`: -```bash -cd test_run -bash run_test.sh -bash cleanup.sh +```yaml +input: + samples_tsv: /absolute/path/samples.tsv + reference_genome: /absolute/path/reference.fa.gz + reference_index_dir: /absolute/path/indexes + +output: + directory: ChoCallate_output ``` +`reference_genome` must be plain FASTA or BGZF-compressed FASTA; plain gzip is rejected. For FASTQ input, `reference_index_dir` must contain the index files for the selected `mapping.mapper` (Bowtie2 by default). +`samples.tsv` is tab-separated: -## Docker +```text +sample1 /path/sample1_R1.fq.gz /path/sample1_R2.fq.gz +``` + +Supported layouts are paired-end (`sample_id`, R1, R2), single-end (`sample_id`, R1), mixed (`sample_id`, R1, R2, unpaired; Bowtie2 only) and BAM (`sample_id`, BAM). `reference_index_dir` is required for FASTQ input and unused for BAM input. Index files must match the selected mapper. -ChoCallate is available as a Docker image on DockerHub. For a fuller walkthrough, see the Wiki: [Installing ChoCallate](https://github.com/alermol/ChoCallate/wiki/Installing-ChoCallate#use-the-docker-container). +Run: ```bash -docker pull alermol/chocallate:latest +nextflow run main.nf -params-file my_run.yaml ``` -Mount your run directory to `/workspace` and run: +## Docker ```bash +docker pull alermol/chocallate:latest docker run --rm \ -u "$(id -u):$(id -g)" \ -v "${PWD}/input_data:/workspace" \ @@ -53,81 +56,36 @@ docker run --rm \ -params-file config.yaml ``` -Outputs will be written to the configured `outdir` (default: `ChoCallate_output`) inside `input_data`. - -## Usage - -ChoCallate is configured via a Nextflow params YAML file. Start from the template. - -```bash -cp assets/templates/config.yaml my_run.yaml -``` - -Minimum set of parameters in `my_run.yaml`: - -- `samples_tsv`: input samples TSV (formats below) -- `reference_genome`: reference FASTA (plain or bgzip-compressed) -- `reference_index_dir`: path to directory with index files for reference genome - -After configuration is complete you can run the ChoCallate - -```bash -nextflow run main.nf -params-file my_run.yaml -``` - - - -## Inputs - -- **Reads**: FASTQs (`format: "fastq"`) or a pre-aligned BAM (`format: "bam"`). If you provide a BAM, mapping is skipped. -- **Reference genome**: Plain or bgzipped FASTA file. -- **Indexes of reference genome**: All indexes required by the selected mapper/callers. -- **Tip**: Use **absolute paths** for inputs. - -`samples_tsv` formats: - -- **FASTQ + paired-end** (`reads_type: "pe"`): `sample_idR1R2` -- **FASTQ + single-end** (`reads_type: "se"`): `sample_idR1` -- **FASTQ + mixed** (`reads_type: "mx"`): `sample_idR1R2U` (Bowtie2 mapping only) -- **BAM** (`format: "bam"`): `sample_idbam_path` - - +The image bundles the Nextflow plugins and works offline and under an arbitrary UID. Paths in `config.yaml` and `samples.tsv` must refer to files inside the mounted `/workspace`. ## Outputs -Published outputs are written to `outdir` (default: `ChoCallate_output`), including standard Nextflow reports: - -- `pipeline_report.html` -- `timeline_report.html` -- `trace.txt` - -Consensus outputs depend on `output.type` and `output.format`: - -- **Single merged** (default): `/consensus.bcf` or `/consensus.vcf.gz` -- **Per-sample**: `/per_sample/.bcf` or `.vcf.gz` - -Consensus VCF/BCF records include `FORMAT/GT`, `AD`, and `DP` (AD/DP from BAM), plus site-level `INFO` tags (`AN`, `AC`, `AF`, `NS`, total `DP`, etc.). Details: [Output Structure](https://github.com/alermol/ChoCallate/wiki/Output-Structure). - -When `output.type` is `"single"` and `N samples ≥ 2`, single consensus BCF is merged using chunked merge through named pipes. No intermediate BCF files are written to disk. Details: [Merge Strategies and Testing](https://github.com/alermol/ChoCallate/wiki/Merge-Strategies-and-Testing). +By default, a run publishes one merged multi-sample `consensus.bcf`. Set `output.type: "sample"` for one file per sample, and `output.format: "vcf"` for bgzip-compressed VCF. All covered sites are kept by default, including monomorphic positions; set `output.remove_invariant: true` to drop them. Mapping statistics, `dropped_samples.txt` and standard Nextflow reports are also published. -## Additional documentation +## Documentation -- **Wiki home**: [ChoCallate Wiki](https://github.com/alermol/ChoCallate/wiki) -- **Install**: [Installing ChoCallate](https://github.com/alermol/ChoCallate/wiki/Installing-ChoCallate) -- **Quick start / config & CLI**: [CLI Reference](https://github.com/alermol/ChoCallate/wiki/CLI-Reference) -- **Multi-sample merge**: [Merge Strategies and Testing](https://github.com/alermol/ChoCallate/wiki/Merge-Strategies-and-Testing) -- **Outputs**: [Output Structure](https://github.com/alermol/ChoCallate/wiki/Output-Structure) +Detailed documentation is in the [Wiki](https://github.com/alermol/ChoCallate/wiki): +- [Installation](https://github.com/alermol/ChoCallate/wiki/Installing-ChoCallate), including Docker, Apptainer and offline use. +- [Pipeline overview](https://github.com/alermol/ChoCallate/wiki/Pipeline-Overview) and [inputs and reference](https://github.com/alermol/ChoCallate/wiki/Inputs-and-Reference) requirements. +- [CLI reference](https://github.com/alermol/ChoCallate/wiki/CLI-Reference): every parameter, including resources, callers and consensus options. +- [Consensus model](https://github.com/alermol/ChoCallate/wiki/Consensus-Model): voting, caller normalization and degraded mode. +- [Output structure](https://github.com/alermol/ChoCallate/wiki/Output-Structure): files, header metadata and FORMAT/INFO annotations. +- [Merge strategies and testing](https://github.com/alermol/ChoCallate/wiki/Merge-Strategies-and-Testing): multi-sample merge and open-file limits. +- [Operations and troubleshooting](https://github.com/alermol/ChoCallate/wiki/Operations-and-Troubleshooting): resources, retries and common errors. +- [Testing and CI](https://github.com/alermol/ChoCallate/wiki/Testing): pytest, nf-test, snapshots and lint commands. +- [Developer guide](https://github.com/alermol/ChoCallate/wiki/Developer-Guide): repository layout and process conventions. +The full configuration template is [`assets/templates/config.yaml`](assets/templates/config.yaml). -## Contribution +## Contributing -See [CONTRIBUTING.md](https://github.com/alermol/ChoCallate/blob/main/CONTRIBUTING.md) +See [CONTRIBUTING.md](CONTRIBUTING.md). ## License -[MIT](https://github.com/alermol/ChoCallate/blob/main/LICENSE) +[MIT](LICENSE) ## Development roadmap -See [Development Roadmap](https://github.com/alermol/ChoCallate/blob/main/ROADMAP.md) for planned container support and additional callers/mappers. \ No newline at end of file +See [ROADMAP.md](ROADMAP.md). diff --git a/ROADMAP.md b/ROADMAP.md index 36f45a4..e3bb11c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -7,7 +7,7 @@ This document outlines the future direction of our project. It's a living docume - [x] Docker image - [ ] Singularity image -## Ployploid calling +## Polyploid calling - [ ] Integration of [Octopus](https://github.com/luntergroup/octopus) into the pipeline diff --git a/assets/templates/config.yaml b/assets/templates/config.yaml index 5e11bfa..ab2a0d0 100644 --- a/assets/templates/config.yaml +++ b/assets/templates/config.yaml @@ -1,53 +1,247 @@ +# ChoCallate configuration template. +# +# Copy this file, edit it, and run: +# nextflow run main.nf -params-file my_run.yaml +# +# The values below are the pipeline defaults. Only override what differs. +# Every parameter is documented here; the pipeline has no --help output. + input: + # Tab-separated samplesheet. Columns depend on input.format/reads_type: + # fastq + pe: sample_id, R1, R2 + # fastq + se: sample_id, R1 + # fastq + mx: sample_id, R1, R2, unpaired + # bam: sample_id, BAM + # Relative read paths are resolved from the samplesheet directory. samples_tsv: null + + # Reference FASTA. Plain and bgzip-compressed files are detected + # automatically. Contigs longer than 2^31-1 bp are rejected. reference_genome: null + + # Directory containing the mapper and caller indexes. Required for + # format: fastq; unused for format: bam. reference_index_dir: null + + # Index prefix inside reference_index_dir. Must be a plain file name + # without path separators. null uses the file name of reference_genome. + reference_index_prefix: null + + # Optional BED of regions to call. null calls every covered region. include_bed: null + + # Optional BED of regions to exclude. null excludes no region. exclude_bed: null + + # Input molecule type: "fastq" or "bam". format: "fastq" + + # FASTQ layout: "se" (single-end), "pe" (paired-end) or "mx" (mixed + # paired/unpaired; bowtie2 only). reads_type: "pe" output: + # Directory for published results and Nextflow reports. directory: "ChoCallate_output" + + # "single": one merged multi-sample file; "sample": one file per sample. + # The pipeline always builds internal per-sample BCFs (and a merged BCF for + # "single") before this layout is applied to the final export. type: "single" + + # Final published format: "bcf" or "vcf" (VCF is bgzip-compressed). + # Internals are always BCF; this option only affects EXPORT_OUTPUT. format: "bcf" - remove_invariant: true + + # Drop monomorphic/reference-only records when exporting the final output. + # false (default) keeps monomorphic/reference records. + remove_invariant: false + + # Keep one record per ALT allele in the final output instead of re-joining + # multi-allelic sites. split_multiallelic: false -ref_genome: - bgzip: true + # Compress consecutive hom-reference positions into multi-base gVCF + # blocks with INFO/END in the final output. Reference records are always + # gVCF-style internally (ALT="<*>") so that a multi-sample merge assigns + # the ALT depth of hom-reference samples correctly; this option only + # controls whether runs of them are additionally collapsed into blocks. + # Multi-base blocks carry the minimum DP and omit AD, so merged ALT depth + # for a hom-reference sample is missing rather than a fabricated zero. + compress_reference_blocks: true + + # Publishing mode for result files: "copy" writes independent copies, + # "link" creates symbolic links into the Nextflow work directory. Linked + # results depend on the work directory and must not be combined with + # work-directory cleanup. + publish_mode: "copy" + +resources: + # Resource limits for the three process labels. Memory and time are + # multiplied by the task attempt number, so retries after an out-of-memory + # exit (137) or a walltime exit (140) run with more memory and time; JVM + # heap sizes are derived from the resulting task memory. + # Maximum number of retries for exit status 137 or 140. + max_retries: 3 + + # Hard ceilings applied to every task after the per-label values and the + # attempt multiplier. Set these to what the executor can provide so a + # late retry cannot request resources that will never become available. + limits: + cpus: 64 + memory_mb: 262144 + time: "7d" + + low: + # Index creation, decompression and coverage tasks. + cpus: 1 + memory_mb: 4096 + time: "4h" + + medium: + # Mapping, variant calling, scattering and merging tasks. + cpus: 6 + memory_mb: 16384 + time: "24h" + + high: + # GATK HaplotypeCaller tasks, which start one JVM per assigned CPU. + cpus: 8 + memory_mb: 32768 + time: "48h" mapping: + # Read mapper: "bowtie2", "bwa" or "minimap2". A minimap2 index must be + # built with `minimap2 -x sr -d`; the k/w settings stored in the .mmi + # override -x sr at mapping time. mapper: "bowtie2" + + # Extra arguments passed verbatim to the mapper. extra_args: "" + + # CPUs requested per mapping task. The mapper and BAM post-processing run + # concurrently, so this is split between them. Post-processing further + # divides its share between its concurrent samtools stages. cpu: 10 + # Memory (MiB) reserved in each aligner mapping task for the index and + # mapper working set, which stay resident while samtools sort runs in the + # same pipe. Sort receives half of the task memory that remains after this + # reserve. 0 estimates it from the staged index files plus 1 GiB; set it + # explicitly for large genomes when the aligner needs more than the index + # files suggest. The reserve must be smaller than the task memory. + aligner_reserve_mb: 0 + + # Read-group PL value. Accepted characters: letters, digits, "_", ".", + # ":" and "-". + platform: "ILLUMINA" + + # Read-group LB value. null uses the sample id. Accepted characters: + # letters, digits, "_", ".", ":" and "-". + library: null + bam_filter: - min_map_qual: 0 + # Minimum mapping quality; alignments below it are discarded. + min_map_qual: 10 rmdup: - enabled: false + # Duplicate handling with samtools markdup: + # "none" leave duplicate reads unflagged + # "mark" flag duplicates without removing them (default) + # "remove" flag and remove duplicate reads from the BAM + # Callers, mosdepth coverage and the consensus pileup skip reads flagged + # as duplicates, so "mark" is enough for calling while keeping the BAM. + mode: "mark" left_align_indels: + # Run GATK LeftAlignIndels on the final BAM. enabled: true coverage: - min_coverage: 2 + # Minimum depth for a base to enter the coverage BED and consensus. + # Must be at least 1; zero-depth positions are not included. + min_coverage: 5 + + # Action when no base of a sample reaches min_coverage (empty coverage BED): + # "fail" aborts with the sample id and threshold; "skip" drops the sample, + # logs a warning and records it in dropped_samples.txt. + on_empty: "fail" calling: + # Comma-separated non-empty subset of bcftools, freebayes and gatk. callers: "bcftools,gatk,freebayes" - min_snp_qual: 20 + + # CPUs requested per calling task. cpu: 4 + + # Continue with the surviving callers when one fails. false aborts the + # run. + allow_caller_failure: false + bcftools: + # Minimum QUAL for SNPs and for indels/other records. + min_snp_qual: 20 + min_indel_qual: 20 call: + # Extra arguments for bcftools call. extra_args: "" mpileup: + # Maximum raw depth per input file. bcftools defaults to 250, which + # silently down-samples high-coverage libraries. + max_depth: 100000 + # Maximum per-file depth for indel calling. bcftools also defaults this + # separately to 250. + max_idepth: 100000 + # Extra arguments for bcftools mpileup. extra_args: "" + freebayes: + # Minimum QUAL for SNPs and for indels/other records. + min_snp_qual: 20 + min_indel_qual: 20 + # Evaluate only the N best alleles per site, ranked by supporting + # quality; 0 evaluates all and can blow up in repeat regions. + use_best_n_alleles: 4 + # Cap per-sample coverage; freebayes down-samples above this depth. + limit_coverage: 100000 + # Extra arguments for freebayes. extra_args: "" + gatk: + # Minimum QUAL for SNPs and for indels/other records. + min_snp_qual: 20 + min_indel_qual: 20 + # Padding added to each calling interval so indels at covered-region + # edges are assembled. + interval_padding: 150 + # Extra arguments for GATK HaplotypeCaller. extra_args: "" consensus: + # Minimum number of callers supporting an ALT allele (or the reference). threshold: 2 + + # CPUs requested for consensus and merge tasks. cpu: 4 + + # Vote of a caller without a record: "ref" or "abstain". "abstain" + # requires at least coverage.min_coverage reads at the position. + absent_vote: "ref" + + # Vote of a caller record whose FILTER is not PASS/".": "abstain" or + # "ref". "abstain" keeps a soft-filtered record out of the vote instead of + # counting it as reference evidence. + filtered_vote: "abstain" + + # Zygosity tie-breaker: "nocall" or "first". + tie_policy: "nocall" + +boost: + # nf-boost work-directory cleanup observer. "v1" and "v2" delete the files + # of a completed task once no downstream process needs them; "v2" is the + # current observer. + # Deleted files cannot be reused, so cleanup and -resume are mutually + # exclusive: a resumed run re-executes every task whose files were removed. + # Set this to false to keep the work directory and let -resume reuse + # completed tasks. + cleanup: "v2" diff --git a/bin/build_consensus_accel.py b/bin/build_consensus_accel.py new file mode 100644 index 0000000..ff0f059 --- /dev/null +++ b/bin/build_consensus_accel.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Build the optional Cython accelerator for generate_consensus.py in place. + +Usage: + python bin/build_consensus_accel.py + +Produces ``bin/consensus_accel..so``. generate_consensus.py imports it if +present and otherwise falls back to an equivalent pure-pysam implementation. +""" + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def main(): + try: + import pysam + except ImportError: + sys.exit("pysam is required to build the consensus accelerator") + try: + from Cython.Build import cythonize + except ImportError: + sys.exit("Cython is required to build the consensus accelerator") + + try: + from setuptools import Distribution, Extension + except ImportError: + sys.exit("setuptools is required to build the consensus accelerator") + + ext = Extension( + "consensus_accel", + [os.path.join(HERE, "consensus_accel.pyx")], + include_dirs=pysam.get_include(), + define_macros=pysam.get_defines(), + # resolve htslib symbols against the copies bundled in pysam + extra_objects=pysam.get_libraries(), + ) + dist = Distribution({ + "name": "consensus_accel", + "ext_modules": cythonize([ext], language_level=3, quiet=True), + }) + build_ext = dist.get_command_obj("build_ext") + build_ext.inplace = True + build_ext.build_lib = HERE + build_ext.build_temp = os.path.join(HERE, ".build_accel") + build_ext.ensure_finalized() + # build from bin/ so the in-place copy lands next to the .pyx + old_cwd = os.getcwd() + os.chdir(HERE) + try: + build_ext.run() + finally: + os.chdir(old_cwd) + built = [f for f in os.listdir(HERE) if f.startswith("consensus_accel") + and f.endswith(".so")] + print("built:", ", ".join(built)) + + +if __name__ == "__main__": + main() diff --git a/bin/check_bam_reference.py b/bin/check_bam_reference.py new file mode 100755 index 0000000..faf98f7 --- /dev/null +++ b/bin/check_bam_reference.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Check that a BAM header matches a reference .fai index. + +A BAM aligned to a different assembly, or to a reference whose contigs have +different lengths, would silently produce calls at wrong coordinates. +``PREPARE_BAM`` and ``postprocess_bam.sh`` therefore compare the ``@SQ`` +names and lengths of the BAM with the generated ``.fai`` and report every +difference as a diff. When a sequence dictionary is supplied, matching +``M5`` checksums are compared as well, which catches a different assembly +with identically named and sized contigs. +""" + +import argparse +import difflib +import sys + +import pysam + + +def read_fai(path): + """Return the ``(contig, length)`` pairs of a ``.fai`` file.""" + entries = [] + with open(path, encoding='utf-8') as handle: + for line in handle: + fields = line.rstrip('\n').split('\t') + if len(fields) < 2 or not fields[0]: + continue + try: + length = int(fields[1]) + except ValueError: + continue + entries.append((fields[0], length)) + return entries + + +def read_bam_sq(path): + """Return the ``@SQ`` ``(name, length)`` pairs of a BAM header.""" + with pysam.AlignmentFile(path, 'rb') as bam: + return list(zip(bam.references, bam.lengths)) + + +def read_bam_m5(path): + """Return the ``@SQ`` ``M5`` checksums of a BAM, keyed by contig.""" + with pysam.AlignmentFile(path, 'rb') as bam: + entries = bam.header.to_dict().get('SQ', []) + return { + entry['SN']: entry['M5'].lower() + for entry in entries + if 'SN' in entry and 'M5' in entry + } + + +def read_dict_m5(path): + """Return the ``@SQ`` ``M5`` checksums of a .dict, keyed by contig.""" + checksums = {} + with open(path, encoding='utf-8') as handle: + for line in handle: + if not line.startswith('@SQ'): + continue + fields = {} + for field in line.rstrip('\n').split('\t')[1:]: + key, separator, value = field.partition(':') + if separator: + fields[key] = value + name = fields.get('SN') + checksum = fields.get('M5') + if name and checksum: + checksums[name] = checksum.lower() + return checksums + + +def format_entries(entries): + """Format ``(name, length)`` pairs for a line-based diff.""" + return ['{}\t{}'.format(name, length) for name, length in entries] + + +def diff_entries(bam_entries, fai_entries): + """Return a unified diff between the BAM and .fai contig lists.""" + return list(difflib.unified_diff( + format_entries(bam_entries), + format_entries(fai_entries), + fromfile='BAM @SQ', + tofile='reference .fai', + lineterm='', + )) + + +def diff_m5(bam_m5, dict_m5): + """Return ``M5`` mismatches for contigs present in both headers.""" + return [ + '{}\tBAM {} != reference {}'.format( + name, bam_m5[name], dict_m5[name]) + for name in sorted(bam_m5.keys() & dict_m5.keys()) + if bam_m5[name] != dict_m5[name] + ] + + +def main(): + parser = argparse.ArgumentParser( + description='Fail when the BAM @SQ entries differ from the .fai.') + parser.add_argument('bam', help='Input BAM') + parser.add_argument('fai', help='Reference FASTA .fai index') + parser.add_argument('--dict', dest='dict_path', + help='Reference sequence dictionary (.dict)') + args = parser.parse_args() + + errors = [] + bam_entries = read_bam_sq(args.bam) + fai_entries = read_fai(args.fai) + if bam_entries != fai_entries: + diff = diff_entries(bam_entries, fai_entries) + errors.append( + ("ERROR: BAM header @SQ entries do not match reference " + "'{}':\n{}\nThe BAM must be aligned to the reference passed " + 'as input.reference_genome.').format( + args.fai, '\n'.join(diff))) + + if args.dict_path: + mismatches = diff_m5(read_bam_m5(args.bam), + read_dict_m5(args.dict_path)) + if mismatches: + errors.append( + ("ERROR: BAM @SQ M5 checksums do not match reference " + "dictionary '{}':\n{}\nThe BAM must be aligned to the " + 'reference passed as input.reference_genome.').format( + args.dict_path, '\n'.join(mismatches))) + + if errors: + sys.exit('\n'.join(errors)) + + +if __name__ == '__main__': + main() diff --git a/bin/check_contig_lengths.py b/bin/check_contig_lengths.py new file mode 100755 index 0000000..41064b9 --- /dev/null +++ b/bin/check_contig_lengths.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Reject reference contigs that exceed the 32-bit coordinate range. + +BAM ``pos``, BCF ``POS`` and GATK/Picard use signed 32-bit positions, so a +contig longer than ``2**31 - 1`` bp cannot be processed by this pipeline. +Checking the generated ``.fai`` before any mapping task gives an immediate, +actionable error instead of a failure deep inside a caller. +""" + +import argparse +import sys + +MAX_CONTIG_LENGTH = (1 << 31) - 1 + + +def read_fai_lengths(path): + """Yield ``(contig, length)`` pairs from a ``.fai`` file.""" + with open(path, encoding='utf-8') as handle: + for line in handle: + fields = line.rstrip('\n').split('\t') + if len(fields) < 2 or not fields[0]: + continue + try: + length = int(fields[1]) + except ValueError: + continue + yield fields[0], length + + +def oversized_contigs(path, maximum=MAX_CONTIG_LENGTH): + """Return the ``(contig, length)`` pairs longer than ``maximum``.""" + return [(contig, length) + for contig, length in read_fai_lengths(path) + if length > maximum] + + +def main(): + parser = argparse.ArgumentParser( + description='Fail when a reference contig exceeds the 32-bit limit.') + parser.add_argument('fai', help='Reference FASTA .fai index') + args = parser.parse_args() + + oversized = oversized_contigs(args.fai) + if not oversized: + return + details = ', '.join( + "'{}' ({} bp)".format(contig, length) + for contig, length in oversized) + sys.exit( + 'ERROR: reference contig(s) exceed the maximum supported length of ' + '{} bp (2^31-1): {}. BAM/BCF positions and GATK use 32-bit ' + 'coordinates; split these contigs or use a long-contig-capable ' + 'workflow.'.format(MAX_CONTIG_LENGTH, details)) + + +if __name__ == '__main__': + main() diff --git a/bin/consensus_accel.pyx b/bin/consensus_accel.pyx new file mode 100644 index 0000000..1ce7ce8 --- /dev/null +++ b/bin/consensus_accel.pyx @@ -0,0 +1,459 @@ +# cython: language_level=3, boundscheck=False, wraparound=False, cdivision=True +"""Optional Cython accelerators for ``generate_consensus.py``. + +The module is compiled in place (see ``bin/build_consensus_accel.py``). When it +is not available, ``generate_consensus.py`` falls back to an equivalent +pure-pysam implementation, so the pipeline keeps working from a plain checkout. + +``PileupTally`` + Computes FORMAT/DP and the number of reads supporting the reference base for + the current column of a pysam pileup iterator directly from the htslib + pileup buffer. This mirrors ``build_pileup_site``'s overlap handling: one + observation per query name, highest base quality wins (ties keep the first + observation), and a deletion is an observation without an allele, so DP + matches the overlap-corrected mosdepth coverage BED. + +``BcfLineWriter`` + Writes records from VCF text lines through htslib's ``vcf_parse`` + + ``bcf_write``. Constructing records with ``VariantFile.new_record`` plus + per-sample assignments costs ~20us/record, which dominates the runtime at + one record per position; text lines cost a fraction of that. +""" + +from libc.stdint cimport int32_t, uint8_t, uint32_t, uint64_t +from libc.stdlib cimport free, malloc +from libc.string cimport memcmp, memset + +from pysam.libcalignmentfile cimport IteratorColumn +from pysam.libcbcf cimport VariantHeader +from pysam.libchtslib cimport ( + bam_get_qname, + bam_get_qual, + bam_get_seq, + bam_pileup1_t, + bam_seqi, + bcf1_t, + bcf_clear1, + bcf_destroy1, + bcf_hdr_t, + bcf_hdr_write, + bcf_init1, + bcf_update_alleles_str, + bcf_update_format_int32, + bcf_update_genotypes, + bcf_update_info_int32, + bcf_write, + hts_close, + hts_open, + htsFile, + seq_nt16_str, +) + +DEF MISSING_QUAL = 0xff +# Collapsed-observation state for a deletion: counted in DP, never an allele. +DEF NO_ALLELE = 2 +# Table slots above this are refused; the caller falls back to the Python path. +DEF MAX_TABLE_SLOTS = 1 << 21 + + +cdef inline uint64_t _hash_qname(const char *name, size_t length) noexcept: + """FNV-1a hash of a query name (bytes in ``name``).""" + cdef uint64_t h = 14695981039346656037ULL + cdef size_t i + for i in range(length): + h ^= name[i] + h *= 1099511628211ULL + return h + + +cdef class PileupTally: + """Overlap-corrected DP/reference-base counts and full column entries of a + pysam pileup. + + The object drives the pileup itself (through the iterator's C ``cnext``), + so no PileupColumn object is created per reference position. ``sync`` + advances to a target position, ``simple`` tallies the current column and + ``entries`` materialises it for the rare full-tally path. + """ + + cdef IteratorColumn it + cdef size_t cap + cdef uint64_t *hashes + cdef char **qnames + cdef uint32_t *name_lens + cdef uint32_t *best_q + cdef uint8_t *is_ref + cdef uint32_t *epoch + cdef uint32_t cur_epoch + cdef bint started + cdef bint done + + def __cinit__(self, object iterator): + self.it = iterator + self.cap = 0 + self.hashes = NULL + self.qnames = NULL + self.name_lens = NULL + self.best_q = NULL + self.is_ref = NULL + self.epoch = NULL + self.cur_epoch = 0 + self.started = False + self.done = False + + def __dealloc__(self): + free(self.hashes) + free(self.qnames) + free(self.name_lens) + free(self.best_q) + free(self.is_ref) + free(self.epoch) + + cdef int _reserve(self, int n) except -1: + cdef size_t need = n * 2 + 8 + cdef size_t new_cap = 64 + if need <= self.cap: + return 0 + while new_cap < need: + new_cap <<= 1 + free(self.hashes) + free(self.qnames) + free(self.name_lens) + free(self.best_q) + free(self.is_ref) + free(self.epoch) + self.hashes = malloc(new_cap * sizeof(uint64_t)) + self.qnames = malloc(new_cap * sizeof(char *)) + self.name_lens = malloc(new_cap * sizeof(uint32_t)) + self.best_q = malloc(new_cap * sizeof(uint32_t)) + self.is_ref = malloc(new_cap * sizeof(uint8_t)) + self.epoch = malloc(new_cap * sizeof(uint32_t)) + self.cap = 0 + if (self.hashes is NULL or self.qnames is NULL + or self.name_lens is NULL or self.best_q is NULL + or self.is_ref is NULL or self.epoch is NULL): + raise MemoryError() + memset(self.epoch, 0, new_cap * sizeof(uint32_t)) + self.cap = new_cap + self.cur_epoch = 0 + return 0 + + cpdef bint sync(self, int pos1): + """Advance the pileup to the first column at or after ``pos1``. + + Returns True if that column is exactly at ``pos1`` (1-based), False if + it is further ahead or the pileup is exhausted. + """ + cdef int p + if self.done: + return False + while True: + if self.started and self.it.pos + 1 >= pos1: + break + # bam_mplp_auto returns 1 while columns are available, 0 once the + # region is exhausted and < 0 on error. + p = self.it.cnext() + if p == 0: + self.done = True + return False + if p < 0: + raise OSError("error while reading the pileup") + self.started = True + return self.it.pos + 1 == pos1 + + def entries(self): + """Materialise the current column like pysam's bulk accessors. + + Returns ``(names, sequences, qualities)`` where sequences carry + samtools-style indel annotations ('A+2TT', 'A-2', '*', '>', '<') and + qualities use htslib's 255 for missing values, so the shared Python + full-tally code can consume them unchanged. + """ + cdef int n = self.it.n_plp + cdef const bam_pileup1_t *start = self.it.plp + cdef const bam_pileup1_t *p + cdef char *qname + cdef char *buf + cdef int i, j, l_qname, qpos, indel + cdef uint32_t qual + cdef str base, inserted, seq + names = [] + seqs = [] + quals = [] + for i in range(n): + p = start + i + qname = bam_get_qname(p.b) + l_qname = p.b.core.l_qname + names.append(qname[:l_qname - 1].decode('utf-8')) + if p.is_refskip: + seq = '<' if (p.b.core.flag & 16) else '>' + qual = 255 + elif p.is_del or p.qpos < 0: + seq = '*' + qual = 255 + else: + qpos = p.qpos + indel = p.indel + base = chr(seq_nt16_str[bam_seqi(bam_get_seq(p.b), qpos)]) + if indel > 0: + buf = malloc(indel) + if buf is NULL: + raise MemoryError() + for j in range(indel): + buf[j] = seq_nt16_str[bam_seqi(bam_get_seq(p.b), + qpos + 1 + j)] + inserted = buf[:indel].decode('ascii') + free(buf) + seq = base + '+' + str(indel) + inserted + elif indel < 0: + seq = base + '-' + str(-indel) + else: + seq = base + qual = (bam_get_qual(p.b))[qpos] + seqs.append(seq) + quals.append(qual) + return names, seqs, quals + + def simple(self, str ref_base, int max_slots=MAX_TABLE_SLOTS): + """Return ``(dp, n_ref, n_obs)`` for the current column, or None if the + column is too deep for the scratch table (caller falls back to Python). + + ``max_slots`` is the table capacity used for that decision; tests + lower it to exercise the fallback without a million-read column. + + ``dp`` is the number of collapsed observations and equals ``n_obs``: + one per distinct query name that is not a reference skip. This + matches the overlap-corrected mosdepth coverage BED. A deletion is an + observation without an allele, so an overlapping mate's allele always + wins over it. ``n_ref`` counts observations whose collapsed allele is + the plain reference base, so ``dp - n_ref`` is the ``<*>`` depth. + """ + cdef int n = self.it.n_plp + cdef const bam_pileup1_t *start = self.it.plp + cdef const bam_pileup1_t *p + cdef char ref_char + cdef uint32_t epoch, qual, prev_q + cdef uint64_t h + cdef size_t slot, mask + cdef char *qname + cdef uint32_t l_qname + cdef int qpos, i, n_ref = 0, n_obs = 0, is_match + cdef int prev_state + cdef char base + + if n > (max_slots >> 1): + return None + if ref_base: + ref_char = ord(ref_base[0]) + else: + ref_char = 0 + self._reserve(n) + self.cur_epoch += 1 + if self.cur_epoch == 0: + memset(self.epoch, 0, self.cap * sizeof(uint32_t)) + self.cur_epoch = 1 + epoch = self.cur_epoch + mask = self.cap - 1 + for i in range(n): + p = start + i + if p.is_refskip: + continue + qpos = p.qpos + if p.is_del or qpos < 0: + # a deletion is an observation but never an allele: any base + # or insertion from an overlapping mate replaces it + is_match = NO_ALLELE + qual = 0 + else: + base = seq_nt16_str[bam_seqi(bam_get_seq(p.b), qpos)] + is_match = 1 if (p.indel == 0 and base == ref_char) else 0 + qual = (bam_get_qual(p.b))[qpos] + if qual == MISSING_QUAL: + qual = 0 + qname = bam_get_qname(p.b) + l_qname = p.b.core.l_qname + h = _hash_qname(qname, l_qname - 1) + slot = h & mask + while self.epoch[slot] == epoch: + if (self.hashes[slot] == h + and self.name_lens[slot] == l_qname + and memcmp(self.qnames[slot], qname, + l_qname) == 0): + prev_state = self.is_ref[slot] + prev_q = self.best_q[slot] + if prev_state == NO_ALLELE: + if is_match != NO_ALLELE: + self.is_ref[slot] = is_match + self.best_q[slot] = qual + n_ref += is_match + elif (is_match != NO_ALLELE + and qual > prev_q): + n_ref -= prev_state + self.best_q[slot] = qual + self.is_ref[slot] = is_match + n_ref += is_match + break + slot = (slot + 1) & mask + else: + self.epoch[slot] = epoch + self.hashes[slot] = h + self.qnames[slot] = qname + self.name_lens[slot] = l_qname + self.best_q[slot] = qual + self.is_ref[slot] = is_match + if is_match != NO_ALLELE: + n_ref += is_match + n_obs += 1 + return (n_obs, n_ref, n_obs) + + +cdef class BcfLineWriter: + """Write consensus records to a BCF file through htslib. + + Records are built with htslib's ``bcf_update_*`` functions instead of + pysam ``VariantRecord`` objects (``new_record`` + per-sample assignments + costs ~20us per record, which dominates runtime at one record per + position). + """ + + cdef htsFile *fp + cdef bcf_hdr_t *hdr + cdef bcf1_t *rec + cdef int32_t gt_buf[16] + cdef int32_t ad_buf[64] + cdef bint closed + + def __cinit__(self, VariantHeader header, str path): + cdef bytes bpath + self.fp = NULL + self.rec = NULL + self.closed = True + self.hdr = header.ptr + if self.hdr is NULL: + raise ValueError("header is not initialized") + header._hdr_sync() + bpath = path.encode("utf-8") + self.fp = hts_open(bpath, b"wb") + if self.fp is NULL: + raise OSError("could not open '%s' for writing" % path) + if bcf_hdr_write(self.fp, self.hdr) < 0: + hts_close(self.fp) + self.fp = NULL + raise OSError("could not write header to '%s'" % path) + self.rec = bcf_init1() + if self.rec is NULL: + raise MemoryError("could not allocate BCF record") + self.closed = False + + cdef int _write(self, int rid, int pos0, const char *alleles, + const int32_t *gt, int n_gt, const int32_t *ad, + int n_ad, int dp, int end_pos=-1) except -1: + cdef int32_t dp_value = dp + cdef int32_t end_value + cdef bcf1_t *rec = self.rec + bcf_clear1(rec) + rec.rid = rid + rec.pos = pos0 + if bcf_update_alleles_str(self.hdr, rec, alleles) < 0: + raise MemoryError("could not set alleles") + if end_pos >= 0: + # symbolic alleles carry INFO/END (1-based, inclusive) + end_value = end_pos + if bcf_update_info_int32(self.hdr, rec, b"END", &end_value, 1) < 0: + raise MemoryError("could not set END") + # rlen drives CSI overlap queries; without it an indexed multi-base + # block is invisible to fetches that start inside the block. + rec.rlen = end_pos - pos0 + if n_gt > 0 and bcf_update_genotypes(self.hdr, rec, gt, n_gt) < 0: + raise MemoryError("could not set GT") + if n_ad > 0 and bcf_update_format_int32(self.hdr, rec, b"AD", ad, + n_ad) < 0: + raise MemoryError("could not set AD") + if bcf_update_format_int32(self.hdr, rec, b"DP", &dp_value, 1) < 0: + raise MemoryError("could not set DP") + if bcf_write(self.fp, self.hdr, rec) < 0: + raise OSError("could not write record") + return 0 + + def write_monomorphic(self, int rid, int pos0, bytes alleles, + int ad_ref, int ad_other, int dp, int end_pos=-1): + """Write a gVCF-style reference record or block ``alleles`` + (precomputed, e.g. b'A,<*>'), GT 0/0, AD (ad_ref, ad_other) and DP. + + ``ad_other`` is the depth of reads that do not support the reference + base (the ``<*>`` slot). ``end_pos`` is the 1-based inclusive end; a + negative value means the single base at ``pos0``. + """ + cdef int32_t ad[2] + if end_pos < 0: + end_pos = pos0 + 1 + self.gt_buf[0] = 2 # bcf_gt_unphased(0) + self.gt_buf[1] = 2 + ad[0] = ad_ref + ad[1] = ad_other + return self._write(rid, pos0, alleles, self.gt_buf, 2, + ad, 2, dp, end_pos) + + def write_reference_block(self, int rid, int pos0, bytes alleles, int dp, + int end_pos): + """Write a multi-base gVCF reference block without FORMAT/AD. + + A single AD value cannot describe every position of the block, and + ``bcftools merge --gvcf`` reports missing depth when the block is used + to fill a hom-reference sample at another sample's variant. + """ + self.gt_buf[0] = 2 # bcf_gt_unphased(0) + self.gt_buf[1] = 2 + return self._write(rid, pos0, alleles, self.gt_buf, 2, + NULL, 0, dp, end_pos) + + def write_record(self, int rid, int pos0, object alleles, object gt, + object ad, int dp): + """Write an arbitrary consensus record. + + ``alleles`` is the REF allele followed by ALTs (sequence of str or + pre-joined bytes), ``gt`` a sequence of allele indices (None for a + missing allele) and ``ad`` a sequence of per-allele depths. + """ + cdef int32_t *gt_ptr = self.gt_buf + cdef int32_t *ad_ptr = self.ad_buf + cdef int32_t *heap = NULL + cdef bytes alleles_bytes + cdef object value + cdef int i, n_gt, n_ad + if isinstance(alleles, bytes): + alleles_bytes = alleles + else: + alleles_bytes = ",".join(alleles).encode("ascii") + gt_ = list(gt) + ad_ = list(ad) + n_gt = len(gt_) + n_ad = len(ad_) + if n_gt > 16 or n_ad > 64: + heap = malloc((n_gt + n_ad) * sizeof(int32_t)) + if heap is NULL: + raise MemoryError() + gt_ptr = heap + ad_ptr = heap + n_gt + try: + for i in range(n_gt): + value = gt_[i] + gt_ptr[i] = 0 if value is None else ((value + 1) << 1) + for i in range(n_ad): + ad_ptr[i] = ad_[i] + return self._write(rid, pos0, alleles_bytes, gt_ptr, + n_gt, ad_ptr, n_ad, dp) + finally: + free(heap) + + def close(self): + cdef htsFile *fp = self.fp + self.fp = NULL + if self.rec is not NULL: + bcf_destroy1(self.rec) + self.rec = NULL + self.closed = True + # hts_close flushes the final BGZF block; a failure (e.g. disk full) + # would otherwise leave a truncated BCF and a successful exit status. + if fp is not NULL and hts_close(fp) < 0: + raise OSError("error closing BCF output") diff --git a/bin/drop_refonly_duplicates.py b/bin/drop_refonly_duplicates.py new file mode 100755 index 0000000..fc39630 --- /dev/null +++ b/bin/drop_refonly_duplicates.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Drop reference-only records that duplicate a real variant at the same +position. + +Per-sample consensus files contain a gVCF-style reference record (ALT="<*>", +GT 0/0) at every covered position. When `bcftools merge --gvcf` finds a real +variant at the same position in another sample, it keeps the `<*>` allele in +the merged ALT list (e.g. ``C,T,<*>``); the following ``bcftools norm -m -any`` +splits that into two records at the same position: the variant, and a redundant +reference-only record carrying the hom-reference calls. + +This filter reads a position-sorted BCF stream (merged and split by ``norm +-m -any``) on stdin and writes it on stdout, keeping the variant records of an +affected position and dropping its reference-only duplicate. Sample values +that ``bcftools merge --gvcf`` assigned only to the dropped record are copied +to the surviving records first, so every split ALT keeps the genotype of a +hom-reference sample. Reference-only records at positions without any real +allele are monomorphic sites and are kept: the consensus output contains all +sites, monomorphic ones included. + +Used by ``MERGE_OUTPUTS`` between ``bcftools norm -m -any`` and the invariant +filter. Records are buffered one chromosome position at a time, so the memory +use is constant. +""" + +import pysam + +# ALT alleles that carry no sequence: the plain "." placeholder and the gVCF +# "any other allele" symbol. +_REFERENCE_ONLY_ALTS = ('.', '<*>') + + +def is_ref_only(record): + """True for a record without a real alternate allele.""" + if record.alts is None: + return True + return all(alt in _REFERENCE_ONLY_ALTS for alt in record.alts) + + +def fill_missing_samples(variant, references): + """Copy sample values from a reference-only record to a variant record. + + ``bcftools merge --gvcf`` can assign a hom-reference sample's ``<*>`` + record to only one of several split ALT records at a position. The + sample then has a missing genotype on the other split records, although + its per-sample consensus called the position hom-reference. The original + ``<*>`` record is about to be dropped as a duplicate, so its values are + the last copy of that sample's reference call. + + Args: + variant: Variant record that keeps its real ALT. + references: Reference-only records at the same position. + """ + for sample_name in variant.header.samples: + sample = variant.samples[sample_name] + gt = sample.get('GT') + if gt is not None and all(allele is not None for allele in gt): + continue + for reference in references: + ref_sample = reference.samples[sample_name] + ref_gt = ref_sample.get('GT') + if ref_gt is None or any(allele is None for allele in ref_gt): + continue + sample['GT'] = ref_gt + ref_ad = ref_sample.get('AD') + if (ref_ad is not None + and len(ref_ad) == len(variant.alts) + 1): + sample['AD'] = ref_ad + ref_dp = ref_sample.get('DP') + if ref_dp is not None: + sample['DP'] = ref_dp + break + + +def keep_records(records): + """Records of one position that should be written downstream. + + Args: + records: List of VariantRecord copies belonging to one position. + + Returns: + The variant records when the position carries at least one real allele; + otherwise the records themselves (a monomorphic position). + """ + variants = [record for record in records if not is_ref_only(record)] + if not variants: + return records + references = [record for record in records if is_ref_only(record)] + for variant in variants: + fill_missing_samples(variant, references) + return variants + + +def main(): + try: + with pysam.VariantFile('/dev/stdin') as inp, \ + pysam.VariantFile( + '/dev/stdout', 'wb0', header=inp.header) as out: + locus = None + pending = [] + for record in inp: + record_locus = (record.chrom, record.pos) + if locus is not None and record_locus != locus: + for kept in keep_records(pending): + out.write(kept) + pending = [] + locus = record_locus + pending.append(record.copy()) + for kept in keep_records(pending): + out.write(kept) + except BrokenPipeError: + pass + + +if __name__ == '__main__': + main() diff --git a/bin/fill_missing_ad.py b/bin/fill_missing_ad.py deleted file mode 100755 index 2d7ee8a..0000000 --- a/bin/fill_missing_ad.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -"""Replace missing FORMAT/AD allele depths with 0 for samples with a called GT. - -After multi-sample merge, hom-ref per-sample records (ALT=.) keep REF AD but lose the -placeholder ALT slot, leaving values like 0/0:4,.:4. Missing AD entries for called -samples are filled with 0. Samples with missing GT are left unchanged. -""" - -import pysam - - -def fill_record_ad(record): - for sample in record.samples.values(): - gt = sample.get('GT') - if gt is None or all(allele is None for allele in gt): - continue - ad = sample.get('AD') - if ad is None: - continue - if any(depth is None for depth in ad): - sample['AD'] = tuple(0 if depth is None else depth for depth in ad) - - -def main(): - try: - with pysam.VariantFile('/dev/stdin') as inp, \ - pysam.VariantFile('/dev/stdout', 'wb0', header=inp.header) as out: - for record in inp: - fill_record_ad(record) - out.write(record) - except BrokenPipeError: - pass - - -if __name__ == '__main__': - main() diff --git a/bin/filter_bed.py b/bin/filter_bed.py new file mode 100755 index 0000000..cee83dc --- /dev/null +++ b/bin/filter_bed.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Strip BED header lines and validate contigs against a reference .fai. + +BED inputs may contain UCSC ``track`` and ``browser`` lines and ``#`` +comments. ``sort-bed`` treats them as malformed records, so they must be +removed before sorting. Every remaining record is checked against the +reference ``.fai``: a BED built for another assembly used to produce an +empty intersection and a generic "empty coverage" error, while this script +names the offending contigs and the BED file. +""" + +import argparse +import sys + +HEADER_WORDS = ('track', 'browser') + + +def is_header(line): + """Return True for blank lines, comments and UCSC header lines.""" + text = line.rstrip('\r\n') + if not text.strip(): + return True + if text.startswith('#'): + return True + return any( + text == word or text.startswith(word + ' ') + or text.startswith(word + '\t') + for word in HEADER_WORDS + ) + + +def read_contigs(path): + """Return the contig names of a reference ``.fai`` index.""" + contigs = set() + with open(path, encoding='utf-8') as handle: + for line in handle: + fields = line.rstrip('\r\n').split('\t') + if len(fields) < 2 or not fields[0]: + continue + contigs.add(fields[0]) + return contigs + + +def main(): + parser = argparse.ArgumentParser( + description='Strip BED headers and validate contigs against a .fai.') + parser.add_argument('fai', help='Reference FASTA .fai index') + parser.add_argument('bed', help='BED input to filter') + args = parser.parse_args() + + contigs = read_contigs(args.fai) + unknown = set() + with open(args.bed, encoding='utf-8') as handle: + for line in handle: + if is_header(line): + continue + text = line.rstrip('\r\n') + contig = text.split('\t', 1)[0] + if contig not in contigs: + unknown.add(contig) + continue + sys.stdout.write(text + '\n') + + if unknown: + names = ', '.join(repr(name) for name in sorted(unknown)) + sys.exit( + "ERROR: BED input '{}' uses contig(s) absent from the " + 'reference .fai: {}. Use the contig names of the reference ' + 'FASTA.'.format(args.bed, names)) + + +if __name__ == '__main__': + main() diff --git a/bin/generate_consensus.py b/bin/generate_consensus.py index 992bf64..bcbd8f3 100755 --- a/bin/generate_consensus.py +++ b/bin/generate_consensus.py @@ -1,117 +1,593 @@ #!/usr/bin/env python3 +"""Generate the consensus BCF of one sample from its caller BCFs. + +By default every position of the (bgzipped, tabixed) coverage BED that +resolves to a reference or variant consensus is represented in the output, so +the consensus contains all sites, monomorphic ones included. Positions whose +reference base is an IUPAC ambiguity code (N included) and positions that +resolve to a no-call are omitted: the former cannot be written without an +ambiguity code, and the latter has no consensus genotype to report. Caller +alleles carrying an ambiguity code are omitted as well, so output sequence +alleles contain only A, C, G and T. A called deletion's REF span represents +the positions it covers, and ``--variants_only`` omits reference records +instead. At each position the callers vote with their +genotype in two stages: an ALT allele must first be supported by at least +``--consensus_threshold`` callers, then the majority zygosity among the +supporters of the winning ALT decides between heterozygous and homozygous +ALT. Ties are resolved by ``--tie_policy``: ``nocall`` (default, independent +of caller order) or ``first`` (the earliest supporter in configured caller +order). When several ALT alleles qualify, each becomes a split consensus +record, but their combined ALT copies must fit the site ploidy; otherwise the +position is a no-call. A position without ALT support is hom-reference when +at least ``--consensus_threshold`` callers vote reference, otherwise it is a +no-call. + +A caller without a record votes reference by default (``--absent_vote ref``). +With ``--absent_vote abstain`` it only votes reference when the sample pileup +has at least ``--min_coverage`` reads at the position; a caller record with a +missing genotype (``./.``, ``0/.``) always abstains. A caller record whose +FILTER is not ``PASS``/``.`` abstains as well by default +(``--filtered_vote abstain``); with ``--filtered_vote ref`` it votes reference +instead. Hom-reference calls of any ploidy vote reference. Positions +resolved as hom-reference are written as gVCF-style records with the +symbolic ALT ``<*>``: the reference allele plus "any other allele", whose +FORMAT/AD slot carries the depth of reads that do not support the reference +base. ``bcftools merge --gvcf`` maps that slot onto the real ALT allele of +another sample's variant, so merged hom-reference samples report their actual +non-reference depth instead of a fabricated 0. + +With ``--compress_reference_blocks``, runs of consecutive hom-reference +positions without caller records are written as one gVCF block carrying +``INFO/END``. FORMAT/DP is the minimum depth of the block; FORMAT/AD is omitted +because one value cannot honestly describe every position of the block and +``bcftools merge --gvcf`` then reports a missing depth at variant positions of +other samples instead of a fabricated zero. Single-base reference records keep +their per-position AD. Blocks are split at caller records, at positions that +cross ``--min_coverage``, at IUPAC-ambiguous reference positions and wherever +the BED coverage is not contiguous. + +With ``--variants_only``, reference records are not written at all: only +positions with a consensus ALT allele are emitted. The pipeline sets this flag +when per-sample output would discard reference records during export anyway; +the merged output still needs them for ``bcftools merge --gvcf``. + +FORMAT/AD and FORMAT/DP are recovered from the sample BAM: overlapping mate +pairs count once (the observation with the highest base quality), and DP counts +the collapsed observations, matching the coverage BED produced by mosdepth, +which also corrects for overlapping mates. + +Performance notes +----------------- +The default output has one record per BED position (one per reference run with +``--compress_reference_blocks``), so the per-position cost decides the runtime: + +* Genotypes, pileups and records are only touched where they are needed. + Positions without a caller record (the vast majority) take a fast path that + writes the reference record directly, without building genotype tuples or + the per-allele dictionaries. +* FORMAT/AD support counts come from pysam's bulk pileup accessors + (``get_query_names``/``get_query_sequences``) and overlapping mate pairs are + only collapsed when a column actually repeats a query name. If the optional + Cython extension ``consensus_accel`` is built (``bin/build_consensus_accel.py``) + the pileup is driven and tallied in C and the records are written through + htslib, which together is another ~5x faster. +* BED intervals are grouped into spans (gaps up to ``SPAN_MAX_GAP`` bases) that + share one pileup iterator and one caller-record scan. A BAM region query + costs ~2 ms because htslib decompresses a whole index bin, which dominates the + runtime on fragmented coverage BEDs. + +Set ``CHOCALLATE_NO_ACCEL=1`` in the environment to force the pure-pysam path +even when the Cython extension is importable (used by the test suite to check +that both implementations agree). +""" import argparse -from collections import Counter, namedtuple +import os +import sys +from collections import namedtuple from contextlib import ExitStack -from typing import NamedTuple import pysam + +def _warn_accel_unavailable(error): + print('[WARN] Cython accelerator unavailable, using pure pysam: ' + '{}'.format(error), file=sys.stderr) + + +if os.environ.get('CHOCALLATE_NO_ACCEL'): + BcfLineWriter = None + PileupTally = None +else: + try: + # Optional Cython helpers built by bin/build_consensus_accel.py. + from consensus_accel import BcfLineWriter, PileupTally + except ModuleNotFoundError as error: + # A checkout without the optional extension is the documented + # normal state, so stay silent: this module is imported once per + # consensus chunk and a warning would repeat for every chunk. A + # missing module other than the extension itself means the + # extension exists but its dependency does not, which is reported. + if error.name != 'consensus_accel': + _warn_accel_unavailable(error) + BcfLineWriter = None + PileupTally = None + except Exception as error: + # A stale extension built against another pysam raises ValueError + # ("size changed, may indicate binary incompatibility"), not + # ImportError; an extension that exists but fails to load is + # reported. + _warn_accel_unavailable(error) + BcfLineWriter = None + PileupTally = None + PileupSite = namedtuple('PileupSite', ['snp', 'ins', 'dele', 'dp']) -class Genotypes(NamedTuple): - index: int = 0 - ref: str = "" - alt: tuple[str, ...] = ("",) - gt: tuple[int, ...] = (0, 0) +# BED intervals separated by at most this many bases share a pileup iterator. +# Bridging a gap costs roughly 0.2us per covered column (advanced in C when the +# accelerator is built), a new region query ~1.6ms, so gaps of several kb are +# much cheaper to walk through than to re-query. +SPAN_MAX_GAP = 10000 + +# Sentinel for "no caller record left in this span". +_NO_RECORD = 1 << 60 + +# ALT placeholder of a reference call (no ALT allele). +_REFERENCE_ALTS = ('.',) + +# VCF allows only A, C, G, T and N in REF; IUPAC ambiguity codes from the +# reference FASTA (R, Y, ...) or caller records become N. N is itself an +# ambiguity code, so records carrying one are not written at all. +_VCF_REF_BASES = frozenset('ACGTN') + +# Output sequence alleles may not contain IUPAC ambiguity codes, N included. +_UNAMBIGUOUS_BASES = frozenset('ACGT') + +# Pre-encoded alleles for the monomorphic fast path (REF + ALT='<*>'). +_MONOMORPHIC_ALLELES = { + base: (base + ',<*>').encode('ascii') for base in 'ACGT'} + + +def to_vcf_ref(ref): + """Map every character of a REF allele to the VCF REF alphabet.""" + return ''.join(base if base in _VCF_REF_BASES else 'N' + for base in ref.upper()) + + +def has_ambiguous_bases(allele): + """True when a sequence allele contains an IUPAC ambiguity code. + Symbolic alleles (``<...>``), the spanning-deletion placeholder ``*`` and + the no-ALT placeholder ``.`` carry no sequence and are not flagged. + """ + if allele in ('.', '*') or allele.startswith('<'): + return False + return any(base not in _UNAMBIGUOUS_BASES for base in allele.upper()) + + +class BedIntervalError(ValueError): + """A BED interval cannot be read from the reference.""" -def genotype_at_position(records, index, pos): + +def validate_bed_interval(chrom, start, end, contig_lengths): + """Reject a BED interval that cannot be read from the reference. + + Args: + chrom: Contig name of the interval. + start: 0-based interval start. + end: 0-based, exclusive interval end. + contig_lengths: Mapping of contig name to reference length. + + Raises: + BedIntervalError: when the contig is absent from the reference, + the coordinates are negative or reversed, or the interval ends + past the contig length. """ - Get the genotype at a given position in a list of VCF records. + length = contig_lengths.get(chrom) + if length is None: + raise BedIntervalError( + "BED contig '{}' is not present in the reference".format(chrom)) + if start < 0 or end < start: + raise BedIntervalError( + 'invalid BED interval {}:{}-{}: start must be >= 0 and ' + 'end >= start'.format(chrom, start, end)) + if end > length: + raise BedIntervalError( + "BED interval {}:{}-{} extends past the end of contig " + "'{}' ({} bp); clip the interval to the contig".format( + chrom, start, end, chrom, length)) + + +def batched_intervals(bed_file, max_gap, contig_lengths): + """Group BED intervals into spans that share one pileup iterator. Args: - records: List of pysam.VariantRecord objects. - index: Integer index of the record to get the genotype from. - pos: Integer position to get the genotype from. + bed_file: pysam.TabixFile of the (bgzipped) coverage BED. + max_gap: Maximum gap between intervals of the same span. + contig_lengths: Mapping of contig name to reference length. - Returns: - A Genotypes named tuple containing the index, reference allele, alternate alleles, and numeric representation of the genotype. On a miss the index is the first record at or after pos, so the caller can resume the scan without rescanning from the start. + Yields: + Tuples of (contig, [(start, end), ...]) where start/end are the 0-based + half-open BED coordinates. + + Raises: + BedIntervalError: when an interval cannot be read from the reference. """ - i = index - while i < len(records) and records[i].pos < pos: - i += 1 - if (i < len(records)) and (records[i].pos == pos): - return Genotypes(index=i, - ref=records[i].ref.upper(), - alt=tuple(sorted(records[i].alts)), - gt=tuple(sorted(records[i].samples[0]['GT']))) - return Genotypes(index=i) - - -def get_consensus_genotype(genotypes, consensus_threshold): + contig = None + span = [] + span_end = None + for line in bed_file.fetch(): + chrom, start, end = line.split('\t')[:3] + start = int(start) + end = int(end) + validate_bed_interval(chrom, start, end, contig_lengths) + if span and (chrom != contig or start - span_end > max_gap): + yield contig, span + span = [] + span.append((start, end)) + contig = chrom + span_end = end + if span: + yield contig, span + + +class CallerCursor: + """Stream the records of one caller BCF over a span. + + Keeping only the current head record instead of materialising + ``list(fetch(...))`` bounds the memory use by the number of caller records + at one position rather than by the size of the span. + """ + + __slots__ = ('_head', '_iterator') + + def __init__(self, iterator): + self._iterator = iterator + self._head = next(iterator, None) + + def position(self): + """1-based position of the next record, or ``_NO_RECORD``.""" + if self._head is None: + return _NO_RECORD + return self._head.pos + + def skip_before(self, pos1, on_skip=None): + """Discard records that start before 1-based ``pos1``. + + ``on_skip`` is called with every discarded record. A deletion + anchored before ``pos1`` can still cover it, so the caller inspects + the discarded records before they are lost. + """ + head = self._head + iterator = self._iterator + while head is not None and head.pos < pos1: + if on_skip is not None: + on_skip(head) + head = next(iterator, None) + self._head = head + + def take_at(self, pos1): + """Consume and return all records at 1-based position ``pos1``.""" + records = [] + head = self._head + iterator = self._iterator + while head is not None and head.pos == pos1: + records.append(head) + head = next(iterator, None) + self._head = head + return records + + +def genotype_key(records, ref_base, filtered_vote='abstain'): + """Consensus voting key of one caller at a position. + + ``records`` are all records of the caller at the position: normally one, + but several after multi-allelic splitting or allele atomisation. Each + record contributes the copies of its own ``(REF, ALT)`` alleles, so + atomised records with different REF alleles at the same POS stay distinct + instead of forcing the caller to abstain. Hom-reference calls (any ploidy) + become the reference vote; records with a missing genotype are ignored. + Records whose FILTER is not ``PASS``/``.`` are ignored; when every record + is filtered the caller abstains (``filtered_vote='abstain'``) or votes + reference (``filtered_vote='ref'``). + + Split records must be consistent with one ploidy: when the ALT copies of + all records exceed the ploidy (for example a split ``1/1`` plus ``0/1`` in + a diploid caller), the caller abstains instead of inventing a genotype. + + Args: + records: Non-empty list of pysam.VariantRecord objects at one position. + ref_base: Reference base at the position, upper case. + filtered_vote: Vote of a caller whose records are all filtered. + + Returns: + ``(reference allele, {(ref, alt): copies}, ploidy)``, or ``None`` when + the records do not form a usable vote. A reference vote has an empty + copies dictionary. """ - Get the consensus genotype from a list of genotypes. + usable = [record for record in records + if not record.filter.keys() + or set(record.filter.keys()) == {'PASS'}] + if not usable: + if filtered_vote == 'ref': + return (ref_base, {}, 0) + return None + alt_copies = {} + ploidy = 0 + for record in usable: + gt = record.samples[0].get('GT') + if gt is None or any(allele is None for allele in gt): + continue + ploidy = max(ploidy, len(gt)) + alts = tuple(allele.upper() for allele in (record.alts or ())) + if not alts or alts == _REFERENCE_ALTS: + continue + ref = to_vcf_ref(record.ref) + for allele_index, alt in enumerate(alts, 1): + copies = sum(1 for allele in gt if allele == allele_index) + if copies: + allele = (ref, alt) + alt_copies[allele] = max(alt_copies.get(allele, 0), copies) + total_copies = sum(alt_copies.values()) + if total_copies > ploidy: + # inconsistent split genotypes (for example 1/1 plus 0/1 in a diploid + # caller): abstain instead of inventing a genotype + return None + if not alt_copies: + if ploidy: + return (ref_base, {}, ploidy) + return None + return (ref_base, alt_copies, ploidy) + + +def supported_deletion_spans(record, minimum_pos, filtered_vote='abstain'): + """REF spans of the deletion alleles a caller record carries. + + A record whose POS lies before a BED chunk is still fetched when its REF + span overlaps the chunk. The previous chunk already voted on it, but the + cursor would discard it before the walk reaches the covered positions. + Returning the spans lets the caller carry the deletion suppression across + the chunk boundary. Args: - genotypes: List of Genotypes named tuples. Each tuple contains the index, reference allele, - alternate alleles, and numeric representation of the genotype. - consensus_threshold: Integer threshold for the consensus genotype. + record: pysam.VariantRecord. + minimum_pos: 1-based position that a REF span must reach. + filtered_vote: Vote policy for filtered records. A filtered record + supplies no ALT copies under either policy, because ``ref`` turns + it into a reference vote. Returns: - The consensus genotype (reference allele, alternate alleles, numeric representation of the genotype). - If the consensus genotype is not found, returns None. + List of ``(REF, ALT, span_end)`` tuples for the deletion alleles the + genotype carries. ``span_end`` is the 1-based inclusive end of the + record's REF span. Empty when the record does not reach + ``minimum_pos`` or carries no usable deletion allele. + """ + if record.stop < minimum_pos: + return [] + key = genotype_key([record], '', filtered_vote) + if key is None: + return [] + _, copies, _ = key + return [(ref, alt, record.stop) + for ref, alt in copies if len(ref) > len(alt)] + + +def note_deletion_support(record, minimum_pos, caller_index, support, + filtered_vote): + """Add one caller's boundary-crossing deletions to the support map. + + ``support`` maps ``(POS, REF, ALT, span_end)`` to the set of caller + indexes carrying the allele, so the site-level consensus threshold can be + applied without retaining the records. + """ + for ref, alt, span_end in supported_deletion_spans( + record, minimum_pos, filtered_vote): + support.setdefault((record.pos, ref, alt, span_end), set()).add( + caller_index) + + +def _zygosity(key, allele): + """Classify a vote supporting ``allele`` as ``het`` or ``hom`` ALT. + + ``allele`` is a ``(ref, alt)`` pair, and copies are keyed by that pair, so + deletions whose anchor ALT is the same but whose REF differs are distinct. + Copies equal to the caller ploidy are homozygous ALT; fewer copies are + heterozygous. A haploid ALT call is therefore homozygous (one ALT copy + out of one). """ - genotypes = [(i.ref, i.alt, i.gt) for i in genotypes] - counter = Counter(genotypes) - value, count = counter.most_common(1)[0] - if count < consensus_threshold: + _, copies, ploidy = key + count = copies.get(allele, 0) + if count == 0: return None - return value + if count >= ploidy: + return 'hom' + return 'het' -def build_pileup_site(column): +def _genotype_for(zygosity, ploidy): + """Output GT for a zygosity and ploidy (ALT index 1).""" + if zygosity == 'hom': + return (1,) * ploidy + return (0,) * (ploidy - 1) + (1,) + + +def consensus_calls(keys, consensus_threshold, tie_policy='nocall'): + """Consensus records for one position. + + Stage 1 (site): every ALT allele supported by at least + ``consensus_threshold`` callers becomes a consensus candidate. Stage 2 + (genotype): for each candidate, the majority zygosity (heterozygous vs + homozygous ALT) among its supporters wins. A genotype tie is resolved by + ``tie_policy``: + + ``nocall`` + Skip that ALT (default; independent of caller order). + ``first`` + Use the earliest supporter in the configured caller order. + + Every qualifying ALT produces its own split consensus record; the output + stage re-joins them unless split output was requested. The split records + share one downstream genotype, so their combined ALT copies must fit the + site ploidy. When the qualifying ALTs would use more copies than the + ploidy (for example two hom-ALT alleles at a diploid site), the position is + a no-call and the list is empty. When no ALT reaches the threshold, the + position is hom-reference if at least ``consensus_threshold`` reference + votes remain, otherwise the list is empty. A caller key of ``None`` (an + abstention) does not support any allele; a caller whose split records carry + more ALT copies than its ploidy abstains in ``genotype_key`` and therefore + supports nothing here. + + Args: + keys: List of ``(ref, {(allele_ref, allele_alt): copies}, ploidy)`` + keys and ``None`` for abstentions, one entry per caller. An ALT is + identified by its ``(ref, alt)`` pair, not by the ALT string alone, + so deletions of different lengths are distinct alleles and a caller + may support several alleles at one position. + consensus_threshold: Minimum number of callers supporting an ALT (or + the reference) at the site stage. + tie_policy: ``nocall`` or ``first``. + + Returns: + A list of (ref, alt tuple, gt tuple) records; normally zero, one or one + split record per qualifying ALT, ordered by ``(ref, alt)``. """ - Summarise a single pileup column into per-allele counters. + reference_key = None + reference_count = 0 + alt_support = {} + for index, key in enumerate(keys): + if key is None: + continue + ref, copies, ploidy = key + if not copies: + reference_count += 1 + if reference_key is None: + reference_key = (ref, _REFERENCE_ALTS, (0, 0)) + continue + for allele in copies: + alt_support.setdefault(allele, []).append((index, key)) + + candidates = sorted( + (allele, supporters) for allele, supporters in alt_support.items() + if len(supporters) >= consensus_threshold) + if not candidates: + if reference_count >= consensus_threshold: + return [reference_key] + return [] - Overlapping PE mates are deduplicated for allele counts (one observation per - query name, keeping the highest base quality). DP counts every covering read - so it matches samtools depth used for the coverage BED. + calls = [] + for allele, supporters in candidates: + classes = {} + for index, key in supporters: + zygosity = _zygosity(key, allele) + if zygosity is not None: + classes.setdefault(zygosity, []).append((index, key)) + if not classes: + continue + max_class = max(len(members) for members in classes.values()) + winners = [(zygosity, members) for zygosity, members in classes.items() + if len(members) == max_class] + if len(winners) > 1: + if tie_policy == 'first': + winners = [min(winners, + key=lambda item: min(i for i, _ in item[1]))] + if len(winners) > 1: + # unresolved genotype tie: this ALT gets no consensus record + continue + zygosity, members = winners[0] + ploidy = max(key[2] for _, key in members) + allele_ref, allele_alt = allele + calls.append((allele_ref, (allele_alt,), + _genotype_for(zygosity, ploidy))) + # The split calls are re-joined into a single multiallelic genotype by the + # output stage, so their ALT copies must fit the site ploidy. For example, + # two hom-ALT alleles at a diploid site would contribute four copies and + # the joined genotype could keep only one of them. + site_ploidy = max((len(gt) for _, _, gt in calls), default=0) + alt_copies = sum(sum(1 for allele in gt if allele) + for _, _, gt in calls) + if alt_copies > site_ploidy: + return [] + return calls + + +def site_from_entries(names, seqs, quals): + """Per-allele counters from materialised pileup column entries. + + Overlapping PE mates are deduplicated (one observation per query name, + keeping the highest base quality), and DP is the number of collapsed + observations, matching the overlap correction of the mosdepth coverage BED. + Reference skips are not observations; a deletion is an observation without + an allele to count. Args: - column: pysam.PileupColumn. + names: Query names at the column. + seqs: Bases with samtools-style indel annotations ('A+2TT', 'A-2', + '*', '>', '<'). + quals: Base qualities (255 means missing). Returns: - PileupSite(snp Counter, insertion Counter, deletion-length Counter, depth). + PileupSite(snp dict, insertion dict, deletion-length dict, depth). """ best = {} - dp = 0 - for read in column.pileups: - if read.is_refskip: + observations = set() + for i, name in enumerate(names): + seq = seqs[i] + if seq == '>' or seq == '<': + # reference skip: no observation at this position continue - dp += 1 - aln = read.alignment - if read.is_del or read.query_position is None: + observations.add(name) + if seq == '*': + # deleted base: an observation, but no allele to count continue - bq = aln.query_qualities[read.query_position] if aln.query_qualities else 0 - if read.indel > 0: - allele = ( - 'ins', - aln.query_sequence[ - read.query_position + 1 : read.query_position + 1 + read.indel - ].upper(), - ) - elif read.indel < 0: - allele = ('del', -read.indel) - else: - allele = ('snp', aln.query_sequence[read.query_position].upper()) - qname = aln.query_name - prev = best.get(qname) - if prev is None or bq > prev[0]: - best[qname] = (bq, allele) - snp = Counter() - ins = Counter() - dele = Counter() + qual = quals[i] + if qual == 255: + # htslib's missing-quality sentinel; the original code used 0 + qual = 0 + prev = best.get(name) + if prev is None or qual > prev[0]: + best[name] = (qual, seq) + snp = {} + ins = {} + dele = {} for _, allele in best.values(): - kind, value = allele - if kind == 'snp': - snp[value] += 1 - elif kind == 'ins': - ins[value] += 1 + if '+' in allele: + sequence_start = allele.index('+') + 1 + length_end = sequence_start + while length_end < len(allele) and allele[length_end].isdigit(): + length_end += 1 + inserted = allele[length_end:].upper() + ins[inserted] = ins.get(inserted, 0) + 1 + elif '-' in allele: + length_start = allele.index('-') + 1 + length_end = length_start + while length_end < len(allele) and allele[length_end].isdigit(): + length_end += 1 + length = int(allele[length_start:length_end]) + dele[length] = dele.get(length, 0) + 1 else: - dele[value] += 1 - return PileupSite(snp=snp, ins=ins, dele=dele, dp=dp) + base = allele.upper() + snp[base] = snp.get(base, 0) + 1 + return PileupSite(snp=snp, ins=ins, dele=dele, dp=len(observations)) + + +def build_pileup_site(column): + """Summarise a single pileup column into per-allele counters. + + DP is the number of collapsed observations, matching the mosdepth coverage + BED, which corrects for overlapping mates by default. The per-read data + comes from pysam's bulk accessors, which are much cheaper than iterating + PileupRead/AlignedSegment objects. + + Args: + column: pysam.PileupColumn. + + Returns: + PileupSite(snp dict, insertion dict, deletion-length dict, depth). + """ + names = column.get_query_names() + seqs = column.get_query_sequences(add_indels=True) + return site_from_entries(names, seqs, column.get_query_qualities()) def count_allele_depth(site, ref, allele): @@ -128,9 +604,8 @@ def count_allele_depth(site, ref, allele): """ if site is None: return 0 - # ALT=. is not a real allele; keep a 0 placeholder so Number=R matches pysam's - # alleles=[REF, '.'] representation. Merge drops this slot and fill_missing_ad.py - # writes 0 for the real ALT(s) afterward. + # ALT="." is not a real allele. Hom-reference consensus is written as a + # gVCF-style <*> record, so this is only a defensive fallback. if allele == '.': return 0 if allele == ref: @@ -163,32 +638,255 @@ def recover_ad_dp(site, alleles): return ad, dp -def generate_mininimal_header(sample_name, - reference_file, - consensus_threshold, - split_multiallelic, - remove_invariant, - version): +def reference_support(column, ref_base): + """Reference-base support, depth and observation count of a pileup column. + + Equivalent to ``build_pileup_site`` followed by + ``count_allele_depth(site, ref_base, ref_base)``, but avoids building the + insertion/deletion counters and the allele tuples that the monomorphic fast + path never needs. + + Args: + column: pysam.PileupColumn. + ref_base: Reference base at the position (upper case). + + Returns: + (depth, number of collapsed reads supporting the reference base, number + of collapsed observations). Depth is the number of collapsed + observations, matching the mosdepth coverage BED; the last value feeds + the ``<*>`` AD slot. Reference skips are not observations. + """ + names = column.get_query_names() + if not names: + return 0, 0, 0 + seqs = column.get_query_sequences(add_indels=True) + if len(set(names)) == len(names): + # no overlapping mate pair at this column: nothing to collapse; every + # read that is not a reference skip is one observation + observations = len(seqs) - seqs.count('>') - seqs.count('<') + support = seqs.count(ref_base) + seqs.count(ref_base.lower()) + return observations, support, observations + support, observations = _collapsed_support( + names, seqs, column.get_query_qualities(), ref_base) + return observations, support, observations + + +def _collapsed_support(names, seqs, quals, ref_base): + """Collapse overlapping mates and count reference/other observations. + + Args: + names: Query names at the column. + seqs: Bases with samtools-style indel annotations. + quals: Base qualities (255 means missing). + ref_base: Reference base at the position (upper case). + + Returns: + (number of reads supporting the reference base, number of collapsed + observations). Reads with a base, an insertion or a deletion at the + position are one observation each; reference skips are ignored. A + deletion is not an allele, so an overlapping mate's base or insertion + is always preferred over it. + """ + best = {} + observations = set() + for i, name in enumerate(names): + seq = seqs[i] + if seq == '>' or seq == '<': + continue + observations.add(name) + if seq == '*': + continue + qual = quals[i] + if qual == 255: + qual = 0 + prev = best.get(name) + if prev is None or qual > prev[0]: + best[name] = (qual, seq) + ref_lower = ref_base.lower() + support = 0 + for _, seq in best.values(): + if seq == ref_base or seq == ref_lower: + support += 1 + return support, len(observations) + + +class ColumnWalker: + """Walk a pysam pileup column by column (no Cython accelerator).""" + + __slots__ = ('pileup', 'column') + + def __init__(self, pileup): + self.pileup = pileup + self.column = next(pileup, None) + + def sync(self, vcf_start): + """Advance to vcf_start; True if a pileup column is there.""" + column = self.column + pileup = self.pileup + while column is not None and column.reference_pos + 1 < vcf_start: + column = next(pileup, None) + self.column = column + return column is not None and column.reference_pos + 1 == vcf_start + + def reference_support(self, ref_base): + # type: (str) -> tuple + return reference_support(self.column, ref_base) + + def site(self): + return build_pileup_site(self.column) + + +class AccelWalker: + """Walk a pileup through the Cython tally. + + No PileupColumn object is created per reference position; columns without + coverage are skipped inside C. + """ + + __slots__ = ('tally',) + + def __init__(self, tally): + self.tally = tally + + def sync(self, vcf_start): + return self.tally.sync(vcf_start) + + def reference_support(self, ref_base): + counts = self.tally.simple(ref_base) + if counts is None: + # pathologically deep column: the C scratch table is too small + names, seqs, quals = self.tally.entries() + support, observations = _collapsed_support( + names, seqs, quals, ref_base) + return observations, support, observations + return counts + + def site(self): + return site_from_entries(*self.tally.entries()) + + +def write_monomorphic_record(output_file, contig, start0, end0, ref_base, + ad, dp): + """Write a gVCF-style reference record or reference block. + + The record carries the symbolic ALT ``<*>``; ``end0`` is the 1-based + inclusive end position (equal to ``start0 + 1`` for a single base). ``ad`` + is a ``(reference depth, other-allele depth)`` tuple, or ``None`` for a + multi-base block, whose per-position depth cannot be represented by one AD + value. + """ + # pysam keeps INFO/END for symbolic alleles even when stop == start + 1, + # matching BcfLineWriter, which sets it for every reference record. + if has_ambiguous_bases(ref_base): + return + new_record = output_file.new_record(contig=contig, start=start0, stop=end0, + alleles=(ref_base, '<*>'), qual=None) + sample = new_record.samples[0] + sample['GT'] = (0, 0) + if ad is not None: + sample['AD'] = ad + sample['DP'] = dp + output_file.write(new_record) + + +def write_consensus_record(output_file, contig, start0, stop0, alleles, gt, ad, dp): + """Write a consensus record from winning caller genotypes.""" + if any(has_ambiguous_bases(allele) for allele in alleles): + return + new_record = output_file.new_record(contig=contig, start=start0, stop=stop0, + alleles=alleles, qual=None) + sample = new_record.samples[0] + sample['GT'] = gt + sample['AD'] = ad + sample['DP'] = dp + output_file.write(new_record) + + +def write_reference_record(output_file, writer, use_accel, rid, contig, + start0, end0, ref_base, ad, dp): + """Write a single-base reference record or a multi-base block. + + ``end0`` is the 1-based inclusive end. ``ad`` is ``None`` for multi-base + blocks, whose FORMAT/AD is deliberately omitted. + """ + if has_ambiguous_bases(ref_base): + return + if use_accel: + alleles = _MONOMORPHIC_ALLELES.get(ref_base) + if alleles is None: + alleles = (ref_base + ',<*>').encode('ascii') + if ad is None: + writer.write_reference_block(rid, start0, alleles, dp, end0) + else: + writer.write_monomorphic(rid, start0, alleles, ad[0], ad[1], dp, + end0) + else: + write_monomorphic_record(output_file, contig, start0, end0, ref_base, + ad, dp) + + +def flush_reference_block(block, output_file, writer, use_accel): + """Write and forget a pending reference block.""" + if block is None: + return + contig, rid, start0, end0, ref_base, ad, dp = block[:7] + if end0 != start0 + 1: + # One AD value cannot describe every position of a multi-base block. + ad = None + write_reference_record(output_file, writer, use_accel, rid, contig, + start0, end0, ref_base, ad, dp) + + +def generate_minimal_header(sample_name, + reference_file, + consensus_threshold, + version, + callers, + callers_failed, + tie_policy, + absent_vote, + filtered_vote, + min_coverage, + compress_reference_blocks=False, + use_accel=False): """ Generate a minimal VCF header for the consensus file. Args: sample_name: String sample name. reference_file: pysam.FastaFile object. - consensus_threshold: Integer threshold for the consensus genotype. - split_multiallelic: Whether multiallelic sites were split. - remove_invariant: Whether monomorphic sites were removed. + consensus_threshold: Integer threshold for the site-level vote. version: String version of the ChoCallate pipeline. + callers: Callers that voted for this sample, in voting order. + callers_failed: Configured callers without a usable record set. + tie_policy: Tie policy applied by the voting model. + absent_vote: Policy for callers without a record at a position. + filtered_vote: Policy for caller records with a non-PASS FILTER. + min_coverage: Coverage threshold used by ``absent_vote=abstain``. + compress_reference_blocks: Whether reference blocks were requested. + use_accel: Whether the Cython accelerator wrote the records. Returns: A pysam.VariantHeader object. """ header = pysam.VariantHeader() header.add_sample(sample_name) - header.add_line(f'##tool=ChoCallate {version}') - header.add_line(f'##consensusThreshold={consensus_threshold}') - header.add_line(f'##multialleleSplit={split_multiallelic}') - header.add_line(f'##noMonomorphic={remove_invariant}') + meta = [f'Version={version}', + f'Threshold={consensus_threshold}', + f'Callers="{",".join(callers)}"', + 'Rule=two-stage', + f'Tie={tie_policy}', + f'AbsentVote={absent_vote}'] + if absent_vote == 'abstain': + meta.append(f'MinCoverage={min_coverage}') + meta.append(f'FilteredVote={filtered_vote}') + meta.append(f'ReferenceBlocks={compress_reference_blocks}') + meta.append(f'Accel={str(bool(use_accel)).lower()}') + if callers_failed: + meta.append(f'CallersFailed="{",".join(callers_failed)}"') + header.add_line('##ChoCallate=<' + ','.join(meta) + '>') + header.add_line('##ALT=') + header.add_line('##INFO=') for ref in reference_file.references: header.add_line(f'##contig=') header.add_line('##INFO=') @@ -207,93 +905,418 @@ def generate_mininimal_header(sample_name, return header +def resolve_inputs(input_specs, configured_callers): + """Map the configured callers onto the caller BCF files that exist. + + Args: + input_specs: ``--input`` values, either ``caller=path`` or a plain + path (the caller name is then taken from the file stem). + configured_callers: Comma-separated configured caller list, or None to + derive it from ``input_specs``. + + Returns: + (received callers in configured order, failed callers, paths). A + configured caller whose file is missing counts as failed; this is how + a caller that was skipped in opt-in degraded mode is detected. + """ + specs = [] + for spec in input_specs: + caller, separator, path = spec.partition('=') + if not separator: + caller = os.path.basename(spec) + if caller.endswith('.bcf'): + caller = caller[:-4] + path = spec + specs.append((caller, path)) + if configured_callers: + configured = [name.strip() for name in configured_callers.split(',') + if name.strip()] + else: + configured = [caller for caller, _ in specs] + present = {caller: path for caller, path in specs if os.path.exists(path)} + received = [caller for caller in configured if caller in present] + failed = [caller for caller in configured if caller not in present] + return received, failed, [present[caller] for caller in received] + + +def _bounded_int(value, minimum): + """argparse type for an integer of at least ``minimum``.""" + try: + number = int(value) + except ValueError: + raise argparse.ArgumentTypeError( + 'invalid integer: {!r}'.format(value)) from None + if number < minimum: + raise argparse.ArgumentTypeError( + 'must be at least {}, got {}'.format(minimum, number)) + return number + + +def positive_int(value): + """argparse type for an integer of at least 1.""" + return _bounded_int(value, 1) + + +def non_negative_int(value): + """argparse type for an integer of at least 0.""" + return _bounded_int(value, 0) + def main(): parser = argparse.ArgumentParser() - parser.add_argument('--input', required=True, nargs='+', help='Input BCF/VCF files') + parser.add_argument('--input', required=True, action='append', + metavar='[CALLER=]PATH', + help='Caller BCF; repeat once per caller') + parser.add_argument('--callers', default=None, + help='Configured callers in voting order (comma-separated)') parser.add_argument('--bed', required=True, help='Input BED file (gzipped and tabixed)') parser.add_argument('--bam', required=True, help='Input BAM file (indexed)') parser.add_argument('--output', required=True, help='Output BCF file') parser.add_argument('--sample_name', required=True, help='Sample name') parser.add_argument('--reference', required=True, help='Reference genome assembly (fasta file and fai index)') - parser.add_argument('--consensus_threshold', required=True, type=int, help='Number of matching calls for consensus') + parser.add_argument('--consensus_threshold', required=True, + type=positive_int, + help='Minimum caller support at the site level ' + '(at least 1)') parser.add_argument('--version', required=True, type=str, help='Version of ChoCallate pipeline') - parser.add_argument('--split_multiallelic', action='store_true') - parser.add_argument('--remove_invariant', action='store_true') + parser.add_argument('--absent_vote', choices=('ref', 'abstain'), default='ref', + help='Vote of a caller without a record at a position') + parser.add_argument('--filtered_vote', choices=('abstain', 'ref'), + default='abstain', + help='Vote of a caller record whose FILTER is not ' + 'PASS or .') + parser.add_argument('--tie_policy', choices=('nocall', 'first'), + default='nocall', help='How to break voting ties') + parser.add_argument('--min_coverage', type=non_negative_int, default=0, + help='Coverage required for an absent caller to vote ' + 'reference when --absent_vote is abstain ' + '(non-negative)') + parser.add_argument('--compress_reference_blocks', '--reference_blocks', + action='store_true', + dest='compress_reference_blocks', + help='Compress runs of hom-reference positions into ' + 'multi-base gVCF blocks') + parser.add_argument('--variants_only', action='store_true', + help='Write only consensus variant records, omitting ' + 'reference records') args = parser.parse_args() - with pysam.TabixFile(args.bed, index=args.bed + '.csi') as bed_file, \ - pysam.FastaFile(args.reference, filepath_index=args.reference + '.fai') as reference_file, \ - pysam.AlignmentFile(args.bam, 'rb') as bam_file, \ - pysam.VariantFile(args.output, mode='wb', - header=generate_mininimal_header(args.sample_name, - reference_file, - args.consensus_threshold, - args.split_multiallelic, - args.remove_invariant, - args.version)) as output_file, \ - ExitStack[bool | None]() as stack: - variant_files = [stack.enter_context(pysam.VariantFile(bcf, index_filename=bcf + '.csi')) for bcf in args.input] - for line in bed_file.fetch(): - contig, start, end = line.split('\t') - start0 = int(start) - end0 = int(end) + threshold = args.consensus_threshold + absent_vote = args.absent_vote + filtered_vote = args.filtered_vote + tie_policy = args.tie_policy + min_coverage = args.min_coverage + use_accel = BcfLineWriter is not None + variants_only = args.variants_only + compress_blocks = args.compress_reference_blocks and not variants_only + received, failed, input_paths = resolve_inputs(args.input, args.callers) + if not received: + sys.exit('ERROR: no caller BCF available for the configured callers') + if failed: + print('[WARN] sample {}: caller(s) without output: {}'.format( + args.sample_name, ','.join(failed)), file=sys.stderr) + if threshold > len(received): + sys.exit( + 'ERROR: consensus threshold {} exceeds the {} surviving ' + 'caller(s) ({}) after failure(s) of {}'.format( + threshold, len(received), ','.join(received), + ','.join(failed))) + + with ExitStack() as stack: + bed_file = stack.enter_context(pysam.TabixFile(args.bed, index=args.bed + '.csi')) + reference_file = stack.enter_context(pysam.FastaFile(args.reference, filepath_index=args.reference + '.fai')) + contig_lengths = dict(zip(reference_file.references, + reference_file.lengths)) + bam_file = stack.enter_context(pysam.AlignmentFile(args.bam, 'rb')) + header = generate_minimal_header(args.sample_name, + reference_file, + threshold, + args.version, + received, + failed, + tie_policy, + absent_vote, + filtered_vote, + min_coverage, + compress_blocks, + use_accel) + contig_ids = {name: header.contigs[name].id for name in header.contigs} + variant_files = [stack.enter_context(pysam.VariantFile(bcf, index_filename=bcf + '.csi')) + for bcf in input_paths] + n_files = len(variant_files) + output_file = None + writer = None + if use_accel: + writer = BcfLineWriter(header, args.output) + else: + output_file = stack.enter_context(pysam.VariantFile(args.output, mode='wb', header=header)) + + pending_block = None + # 1-based inclusive end of the furthest called deletion. Reference + # records must not be written over that span: the deletion record's + # REF already covers those positions. + deletion_end = 0 + current_contig = None + try: + for contig, intervals in batched_intervals( + bed_file, SPAN_MAX_GAP, contig_lengths): + if contig != current_contig: + current_contig = contig + deletion_end = 0 + span_start = intervals[0][0] + span_end = intervals[-1][1] + rid = contig_ids[contig] + # Caller records are streamed once per span (not per BED + # interval) through a per-file cursor: only the records at the + # current position are held in memory. + cursors = [CallerCursor(variant_file.fetch( + contig, span_start, span_end)) + for variant_file in variant_files] + min_next = min(cursor.position() for cursor in cursors) + pileup = bam_file.pileup(contig, span_start, span_end, + truncate=True, stepper='all', + min_base_quality=0, max_depth=1000000) + if use_accel: + walker = AccelWalker(PileupTally(pileup)) + else: + walker = ColumnWalker(pileup) + sync = walker.sync + reference_support_of = walker.reference_support + site_of = walker.site + + for start0, end0 in intervals: + ref_seq = to_vcf_ref( + reference_file.fetch(contig, start0, end0)) + rel = 0 + # BED 0-based half-open [start0, end0) covers 1-based positions + # start0+1 .. end0, so the walk stops after end0. vcf_start = start0 + 1 - vcf_end = end0 - ref_seq = reference_file.fetch(contig, start0, end0).upper() - records_per_file = [list(vf.fetch(contig, start0, end0)) for vf in variant_files] - indices = [0] * len(records_per_file) - ref_seq_rel_pos = 0 - # Stream the pileup in step with the genotype walk instead of - # materialising every covered position up front. AD/DP is built - # only at the positions that are actually written. - pileup = bam_file.pileup( - contig, - start0, - end0, - truncate=True, - stepper='all', - min_base_quality=0, - max_depth=1000000, - ) - column = next(pileup, None) - while vcf_start < vcf_end: - # advance the pileup to the current position - while column is not None and column.reference_pos + 1 < vcf_start: - column = next(pileup, None) - - # collect genotypes at current position - genotypes = [] - for fi in range(len(variant_files)): - res = genotype_at_position(records_per_file[fi], indices[fi], vcf_start) - indices[fi] = res.index - genotypes.append(res) - genotypes = [i if i.gt != (0, 0) else i._replace(ref=ref_seq[ref_seq_rel_pos].upper(), alt=('.',)) for i in genotypes] - - # get consensus genotype at current position - consensus_genotype = get_consensus_genotype(genotypes, args.consensus_threshold) - - shift = len(consensus_genotype[0]) if consensus_genotype is not None else 1 - - # write consensus genotype to output file - if consensus_genotype is not None: - alleles = [consensus_genotype[0]] + list(consensus_genotype[1]) - site = build_pileup_site(column) if (column is not None and column.reference_pos + 1 == vcf_start) else None + while vcf_start <= end0: + # keep the pileup in step with the walk + has_column = sync(vcf_start) + + if min_next < vcf_start: + # the start of the next interval, or a record that + # overlaps the span start, moved past some caller + # records + min_next = _NO_RECORD + # A deletion anchored in the previous BED chunk is + # fetched here because its REF span overlaps this + # chunk, but its POS is before the chunk start. + # Seed its span before the cursors discard the + # record, so reference records are not written + # over bases the previous chunk called deleted. + # Records skipped inside a BED gap are not seeded: + # no earlier interval wrote their deletion record. + seed_deletions = vcf_start == span_start + 1 + deletion_support = {} + for fi, cursor in enumerate(cursors): + if seed_deletions: + cursor.skip_before( + vcf_start, + lambda record, fi=fi: ( + note_deletion_support( + record, vcf_start, fi, + deletion_support, + filtered_vote))) + else: + cursor.skip_before(vcf_start) + position = cursor.position() + if position < min_next: + min_next = position + for key, supporters in deletion_support.items(): + if len(supporters) >= threshold: + deletion_end = max(deletion_end, key[3]) + + if min_next != vcf_start: + # fast path: no caller record here. With + # variants_only such positions are not written and + # need no reference support count. + if variants_only: + vcf_start += 1 + rel += 1 + continue + if vcf_start <= deletion_end: + # The position is inside a called deletion. + # Its REF span covers the site, so a + # hom-reference record would contradict the + # deletion genotype. + flush_reference_block( + pending_block, output_file, writer, + use_accel) + pending_block = None + vcf_start += 1 + rel += 1 + continue + ref_base = ref_seq[rel] + if has_ambiguous_bases(ref_base): + # An IUPAC code (N included) cannot be + # represented in the output; end the block and + # omit the position. + flush_reference_block( + pending_block, output_file, writer, + use_accel) + pending_block = None + vcf_start += 1 + rel += 1 + continue + if has_column: + dp, ad_ref, observations = reference_support_of(ref_base) + else: + dp = 0 + ad_ref = 0 + observations = 0 + # Absent callers vote reference by default. In + # abstain mode they need at least min_coverage reads + # at the position; a position without a pileup + # column has no evidence. + if absent_vote == 'ref' or ( + has_column and dp >= min_coverage): + reference_votes = n_files + else: + reference_votes = 0 + if reference_votes >= threshold: + ad_other = observations - ad_ref + ad = (ad_ref, ad_other) + if compress_blocks: + if (pending_block is not None + and pending_block[0] == contig + and vcf_start - 1 + == pending_block[3] + and (dp >= min_coverage) + == pending_block[7]): + pending_block[3] = vcf_start + if dp < pending_block[6]: + pending_block[6] = dp + else: + flush_reference_block( + pending_block, output_file, + writer, use_accel) + pending_block = [ + contig, rid, vcf_start - 1, + vcf_start, ref_base, ad, dp, + dp >= min_coverage] + else: + write_reference_record( + output_file, writer, use_accel, + rid, contig, vcf_start - 1, + vcf_start, ref_base, ad, dp) + else: + flush_reference_block( + pending_block, output_file, writer, + use_accel) + pending_block = None + vcf_start += 1 + rel += 1 + continue + + # A caller record bounds a reference block: the + # position is evaluated individually below. + flush_reference_block(pending_block, output_file, + writer, use_accel) + pending_block = None + # some caller has a record here: collect all of its + # records at this position (one per split ALT after + # normalisation) and build one vote per caller + ref_base = ref_seq[rel] + keys = [None] * n_files + present = [False] * n_files + for fi, cursor in enumerate(cursors): + caller_records = cursor.take_at(vcf_start) + if caller_records: + key = genotype_key(caller_records, ref_base, + filtered_vote) + keys[fi] = key + present[fi] = True + min_next = _NO_RECORD + for cursor in cursors: + position = cursor.position() + if position < min_next: + min_next = position + + reference_detail = None + if absent_vote == 'ref': + absent_key = (ref_base, {}, 0) + elif any(not seen for seen in present): + if has_column: + reference_detail = reference_support_of(ref_base) + absent_key = ( + (ref_base, {}, 0) + if reference_detail[0] >= min_coverage + else None) + else: + absent_key = None + else: + absent_key = None + for fi in range(n_files): + if not present[fi]: + keys[fi] = absent_key + + calls = consensus_calls(keys, threshold, tie_policy) + if not calls: + vcf_start += 1 + rel += 1 + continue + + if len(calls) == 1 and calls[0][1] == _REFERENCE_ALTS \ + and all(allele == 0 for allele in calls[0][2]): + # hom-reference consensus: write the same gVCF-style + # record as the fast path, with honest <*> depth + if (not variants_only + and vcf_start > deletion_end + and not has_ambiguous_bases(ref_base)): + if reference_detail is None: + if has_column: + reference_detail = \ + reference_support_of(ref_base) + else: + reference_detail = (0, 0, 0) + dp, ad_ref, observations = reference_detail + ad_other = observations - ad_ref + write_reference_record( + output_file, writer, use_accel, rid, + contig, vcf_start - 1, vcf_start, ref_base, + (ad_ref, ad_other), dp) + vcf_start += 1 + rel += 1 + continue + + # one split record per qualifying ALT; every BED + # position is evaluated, including positions covered + # by a deletion, so caller records inside the deletion + # span are not discarded. Reference records inside + # that span are suppressed above. + site = site_of() if has_column else None + for ref, alts, gt in calls: + alleles = [ref] + list(alts) + if any(has_ambiguous_bases(allele) + for allele in alleles): + # IUPAC codes cannot be written to the output. + continue ad, dp = recover_ad_dp(site, alleles) - new_record = output_file.new_record( - contig=contig, - start=vcf_start - 1, - stop=vcf_start, - alleles=alleles, - qual=None, - ) - new_record.ref = consensus_genotype[0] - new_record.samples[0]['GT'] = consensus_genotype[2] - new_record.samples[0]['AD'] = ad - new_record.samples[0]['DP'] = dp - output_file.write(new_record) - vcf_start += shift # shift on reference genotype length - ref_seq_rel_pos += shift # shift on reference genotype length + if use_accel: + writer.write_record(rid, vcf_start - 1, alleles, + gt, ad, dp) + else: + write_consensus_record(output_file, contig, + vcf_start - 1, + vcf_start, alleles, gt, + ad, dp) + if (len(ref) > len(alts[0]) + and any(allele > 0 for allele in gt)): + deletion_end = max( + deletion_end, vcf_start + len(ref) - 1) + vcf_start += 1 + rel += 1 + + flush_reference_block(pending_block, output_file, writer, + use_accel) + pending_block = None + except BedIntervalError as error: + sys.exit('ERROR: {}'.format(error)) + finally: + if writer is not None: + writer.close() if __name__ == '__main__': diff --git a/bin/mapping_stats.py b/bin/mapping_stats.py new file mode 100755 index 0000000..48cc455 --- /dev/null +++ b/bin/mapping_stats.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Collect per-sample mapping statistics into one TSV summary. + +The pipeline reports ``samtools flagstat`` of the aligner output before MAPQ +filtering and duplicate handling (``.raw.flagstat``), plus +``samtools flagstat``, ``stats`` and ``idxstats`` of the final BAM +(``.final.flagstat``, ``.stats``, ``.idxstats``). +This script reads those files in samplesheet order and writes one row per +sample. Counts are alignment records; a paired-end read pair contributes +two records. +""" + +import argparse +import csv +import re +import sys +from pathlib import Path + +COLUMNS = [ + 'sample', + 'raw_total', + 'raw_primary', + 'raw_secondary', + 'raw_supplementary', + 'raw_mapped', + 'raw_mapped_pct', + 'raw_paired', + 'raw_properly_paired', + 'raw_properly_paired_pct', + 'raw_duplicates', + 'raw_duplicate_pct', + 'raw_singletons', + 'raw_mate_different_chr', + 'final_total', + 'final_secondary', + 'final_supplementary', + 'final_duplicates', + 'final_duplicate_pct', + 'final_properly_paired', + 'final_properly_paired_pct', + 'final_mq0', + 'final_mq0_pct', + 'average_read_length', + 'average_mapq', + 'insert_size_mean', + 'insert_size_sd', + 'error_rate', + 'mapped_bases', + 'contigs_with_reads', + 'contigs_without_reads', +] + +# Flagstat labels that carry a read count. The parenthetical suffix (a +# percentage or the QC explanation) is removed before the lookup. +_FLAGSTAT_LABELS = { + 'in total': 'total', + 'primary': 'primary', + 'secondary': 'secondary', + 'supplementary': 'supplementary', + 'duplicates': 'duplicates', + 'mapped': 'mapped', + 'paired in sequencing': 'paired', + 'properly paired': 'properly_paired', + 'singletons': 'singletons', + 'with mate mapped to a different chr': 'mate_different_chr', +} +_FLAGSTAT_LINE = re.compile(r'^(\d+) \+ (\d+) (.+)$') +_PARENTHETICAL = re.compile(r'\s+\(.*\)$') + + +def parse_flagstat(path): + """Return flagstat read counts for one BAM. + + Args: + path: ``samtools flagstat`` output. + + Returns: + A dict with the labels from ``_FLAGSTAT_LABELS`` that are present. + """ + counts = {} + for line in path.read_text(encoding='utf-8').splitlines(): + match = _FLAGSTAT_LINE.match(line) + if match is None: + continue + passed, failed, rest = match.groups() + # The mapQ>=5 mate line has the same prefix as the line above it. + if 'mapQ>=5' in rest: + continue + label = _PARENTHETICAL.sub('', rest) + key = _FLAGSTAT_LABELS.get(label) + if key is not None: + counts[key] = int(passed) + int(failed) + missing = [key for key in ('total', 'mapped') if key not in counts] + if missing: + raise ValueError( + f"{path}: missing flagstat line(s): {', '.join(missing)}" + ) + return counts + + +def parse_stats(path): + """Return ``samtools stats`` SN values keyed by a normalised name.""" + values = {} + for line in path.read_text(encoding='utf-8').splitlines(): + if not line.startswith('SN\t'): + continue + fields = line.split('\t') + if len(fields) < 3: + continue + key = fields[1].rstrip(':').strip().lower() + key = re.sub(r'[^a-z0-9]+', '_', key).strip('_') + values[key] = fields[2].strip() + return values + + +def parse_idxstats(path): + """Return the number of contigs and the number with mapped reads. + + The ``*`` line reports unmapped reads and is not a reference sequence. + """ + contigs = 0 + with_reads = 0 + for line in path.read_text(encoding='utf-8').splitlines(): + fields = line.split('\t') + if len(fields) < 4 or fields[0] == '*': + continue + contigs += 1 + if int(fields[2]) > 0: + with_reads += 1 + return contigs, with_reads + + +def percent(numerator, denominator): + """Format a percentage with two decimals, or ``NA`` without reads.""" + if not denominator: + return 'NA' + return f'{100.0 * numerator / denominator:.2f}' + + +def integer(value): + """Convert a stats value to int, tolerating floats and missing values.""" + try: + return int(float(value)) + except (TypeError, ValueError): + return 0 + + +def build_row(sample, raw, final, stats, contigs): + """Build one summary row from the parsed per-sample reports.""" + raw_total = raw.get('total', 0) + raw_mapped = raw.get('mapped', 0) + raw_paired = raw.get('paired', 0) + raw_proper = raw.get('properly_paired', 0) + raw_dup = raw.get('duplicates', 0) + + final_total = final.get('total', 0) + final_dup = final.get('duplicates', 0) + final_proper = final.get('properly_paired', 0) + sequences = integer(stats.get('sequences')) + mq0 = integer(stats.get('reads_mq0')) + contigs_total, contigs_with = contigs + + return { + 'sample': sample, + 'raw_total': raw_total, + 'raw_primary': raw.get('primary', 0), + 'raw_secondary': raw.get('secondary', 0), + 'raw_supplementary': raw.get('supplementary', 0), + 'raw_mapped': raw_mapped, + 'raw_mapped_pct': percent(raw_mapped, raw_total), + 'raw_paired': raw_paired, + 'raw_properly_paired': raw_proper, + 'raw_properly_paired_pct': percent(raw_proper, raw_total), + 'raw_duplicates': raw_dup, + 'raw_duplicate_pct': percent(raw_dup, raw_total), + 'raw_singletons': raw.get('singletons', 0), + 'raw_mate_different_chr': raw.get('mate_different_chr', 0), + 'final_total': final_total, + 'final_secondary': final.get('secondary', 0), + 'final_supplementary': final.get('supplementary', 0), + 'final_duplicates': final_dup, + 'final_duplicate_pct': percent(final_dup, final_total), + 'final_properly_paired': final_proper, + 'final_properly_paired_pct': percent(final_proper, final_total), + 'final_mq0': mq0, + 'final_mq0_pct': percent(mq0, sequences), + 'average_read_length': stats.get('average_length', 'NA'), + 'average_mapq': stats.get('average_quality', 'NA'), + 'insert_size_mean': stats.get('insert_size_average', 'NA'), + 'insert_size_sd': stats.get( + 'insert_size_standard_deviation', 'NA' + ), + 'error_rate': stats.get('error_rate', 'NA'), + 'mapped_bases': stats.get('bases_mapped_cigar', 'NA'), + 'contigs_with_reads': contigs_with, + 'contigs_without_reads': contigs_total - contigs_with, + } + + +def main(argv=None): + """Parse the report directories and write ``mapping_summary.tsv``.""" + parser = argparse.ArgumentParser( + description='Summarise per-sample mapping statistics.' + ) + parser.add_argument('--sample-order', required=True, type=Path) + parser.add_argument('--raw-dir', required=True, type=Path) + parser.add_argument('--flagstat-dir', required=True, type=Path) + parser.add_argument('--stats-dir', required=True, type=Path) + parser.add_argument('--idxstats-dir', required=True, type=Path) + parser.add_argument('--output', required=True, type=Path) + args = parser.parse_args(argv) + + sample_ids = [ + line.strip() + for line in args.sample_order.read_text( + encoding='utf-8' + ).splitlines() + if line.strip() + ] + if not sample_ids: + parser.error('sample order is empty') + + rows = [] + for sample in sample_ids: + raw_path = args.raw_dir / f'{sample}.raw.flagstat' + final_path = args.flagstat_dir / f'{sample}.final.flagstat' + stats_path = args.stats_dir / f'{sample}.stats' + idxstats_path = args.idxstats_dir / f'{sample}.idxstats' + missing = [ + str(path) + for path in (raw_path, final_path, stats_path, idxstats_path) + if not path.is_file() + ] + if missing: + raise SystemExit( + 'missing mapping statistics: ' + ', '.join(missing) + ) + rows.append(build_row( + sample, + parse_flagstat(raw_path), + parse_flagstat(final_path), + parse_stats(stats_path), + parse_idxstats(idxstats_path), + )) + + with args.output.open('w', encoding='utf-8', newline='') as handle: + writer = csv.writer(handle, delimiter='\t', lineterminator='\n') + writer.writerow(COLUMNS) + for row in rows: + writer.writerow([row[column] for column in COLUMNS]) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/bin/merge_bcf_tree.py b/bin/merge_bcf_tree.py index 9a267b6..0bcfbae 100755 --- a/bin/merge_bcf_tree.py +++ b/bin/merge_bcf_tree.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +"""Merge per-sample BCF files through named pipes. + +The merge runs in two levels: each group of inputs is merged into a FIFO by +one ``bcftools merge --gvcf`` process, and a final merge combines the FIFOs. +This bounds the memory footprint and writes no intermediate file to disk. +""" import argparse import atexit @@ -8,23 +14,29 @@ import subprocess import sys import tempfile +import time class BcfChunkMerge: - def __init__(self, cpus, pipe_dir=None): + def __init__(self, cpus, gvcf, pipe_dir=None): self.cpus = cpus + self.gvcf = gvcf self.pipe_dir = pipe_dir or os.environ.get("TMPDIR", "/tmp") self.pipes = [] self.processes = [] + self.unblock_fds = [] def cleanup(self): - for proc in self.processes: - if proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() + alive = [proc for proc in self.processes + if proc.poll() is None] + if alive: + self._abort(alive) + for fd in self.unblock_fds: + try: + os.close(fd) + except OSError: + pass + self.unblock_fds = [] for pipe in self.pipes: try: os.unlink(pipe) @@ -62,25 +74,60 @@ def _chunk_cmd(self, files, out_pipe): return f"bcftools view -Ou {shlex.quote(files[0])} > {out}" inputs = " ".join(shlex.quote(path) for path in files) return ( - "bcftools merge -Ou -m all --force-samples --force-single --threads 1 " + "bcftools merge -Ou -m all --force-samples --force-single " + f"--gvcf {shlex.quote(self.gvcf)} --threads 1 " f"{inputs} > {out}" ) def _final_cmd(self, pipes): inputs = " ".join(shlex.quote(pipe) for pipe in pipes) + # --force-single: with --cpus 1 there is a single chunk pipe, and + # `bcftools merge` refuses a single input without the flag. return ( - f"bcftools merge --no-index -Ou -m all --force-samples --threads {self.cpus} " - f"{inputs}" + f"bcftools merge --no-index -Ou -m all --force-samples --force-single " + f"--gvcf {shlex.quote(self.gvcf)} --threads {self.cpus} {inputs}" ) + def _unblock_writers(self): + """Open the pipes so writers blocked in ``open`` can proceed.""" + for pipe in self.pipes: + try: + fd = os.open(pipe, os.O_RDONLY | os.O_NONBLOCK) + except OSError: + continue + self.unblock_fds.append(fd) + + def _abort(self, pending): + for proc in pending: + if proc.poll() is None: + proc.terminate() + # Open readers before reaping: a shell may leave a child behind that + # is blocked opening its FIFO, and only a reader releases it. + self._unblock_writers() + for proc in pending: + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + def _wait_all(self): - errors = [] - for proc in self.processes: - rc = proc.wait() - if rc != 0: - errors.append(rc) - if errors: - raise SystemExit(f"Merge failed with exit codes: {errors}") + pending = list(self.processes) + while pending: + failed = [] + for proc in list(pending): + rc = proc.poll() + if rc is None: + continue + pending.remove(proc) + if rc != 0: + failed.append(rc) + if failed: + self._abort(pending) + raise SystemExit( + f"Merge failed with exit codes: {failed}") + if pending: + time.sleep(0.05) def run(self, files): for path in files: @@ -107,6 +154,11 @@ def parse_args(): required=True, help="Input BCF paths, one per line", ) + parser.add_argument( + "--gvcf", + required=True, + help="Reference FASTA (with .fai) for `bcftools merge --gvcf`", + ) parser.add_argument( "--cpus", type=int, @@ -131,7 +183,7 @@ def main(): print("--cpus must be >= 1", file=sys.stderr) sys.exit(1) - merger = BcfChunkMerge(cpus=args.cpus) + merger = BcfChunkMerge(cpus=args.cpus, gvcf=args.gvcf) def _cleanup_handler(signum=None, frame=None): merger.cleanup() diff --git a/bin/merge_cho_headers.py b/bin/merge_cho_headers.py new file mode 100755 index 0000000..ce62c26 --- /dev/null +++ b/bin/merge_cho_headers.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Aggregate per-sample ``##ChoCallate=`` metadata for the merged output. + +``bcftools merge`` drops generic meta lines, so ``MERGE_OUTPUTS`` rebuilds the +structured line from the per-sample headers. The run-level fields are +identical across samples, but ``Callers`` and ``CallersFailed`` can differ in +degraded mode (``calling.allow_caller_failure: true``). The merged line +therefore records the union of the surviving and failed callers, and one +``##ChoCallateSamples=<...>`` line per sample preserves the per-sample values +whenever they differ from the aggregate. +""" + +import argparse +import subprocess + + +def split_fields(line): + """Split a ``##ChoCallate=<...>`` line into ``(key, raw_value)`` pairs. + + Values may be quoted and contain commas, so splitting on commas alone is + not enough. + """ + body = line[line.index('<') + 1:line.rindex('>')] + parts = [] + current = [] + quoted = False + for char in body: + if char == '"': + quoted = not quoted + current.append(char) + elif char == ',' and not quoted: + parts.append(''.join(current)) + current = [] + else: + current.append(char) + parts.append(''.join(current)) + + fields = [] + for part in parts: + key, separator, raw = part.partition('=') + if separator: + fields.append((key.strip(), raw.strip())) + return fields + + +def unquote(raw): + """Strip the surrounding quotes of a structured-field value.""" + if len(raw) >= 2 and raw[0] == raw[-1] == '"': + return raw[1:-1] + return raw + + +def caller_list(fields, key): + """Names in a quoted comma-separated field, or an empty list.""" + for field_key, raw in fields: + if field_key == key: + return [name for name in unquote(raw).split(',') if name] + return [] + + +def ordered_union(lists): + """Union of lists, keeping the first occurrence order.""" + seen = set() + merged = [] + for values in lists: + for value in values: + if value not in seen: + seen.add(value) + merged.append(value) + return merged + + +def set_field(fields, key, value): + """Replace a field's value, appending it when the key is absent.""" + for index, (field_key, _raw) in enumerate(fields): + if field_key == key: + fields[index] = (key, f'"{value}"') + return + fields.append((key, f'"{value}"')) + + +def remove_field(fields, key): + """Drop every occurrence of a field.""" + fields[:] = [field for field in fields if field[0] != key] + + +def read_header(bcftools, path): + """Return the text header of a BCF/VCF file.""" + result = subprocess.run( + [bcftools, 'view', '-h', path], + check=True, + capture_output=True, + text=True, + ) + return result.stdout + + +def sample_name(header_text, path): + """Sample name from the ``#CHROM`` line, falling back to the file stem.""" + for line in header_text.splitlines(): + if line.startswith('#CHROM'): + columns = line.split('\t') + if len(columns) > 9: + return columns[9] + name = path.rsplit('/', 1)[-1] + for suffix in ('.bcf', '.vcf.gz', '.vcf'): + if name.endswith(suffix): + return name[:-len(suffix)] + return name + + +def chocallate_line(header_text): + """The structured ChoCallate line, or None when the header lacks one.""" + for line in header_text.splitlines(): + if line.startswith('##ChoCallate=<'): + return line + return None + + +def collect(paths, bcftools): + """Metadata of every input that carries a ``##ChoCallate=`` line.""" + entries = [] + for path in paths: + header_text = read_header(bcftools, path) + line = chocallate_line(header_text) + if line is None: + continue + fields = split_fields(line) + entries.append({ + 'sample': sample_name(header_text, path), + 'fields': fields, + 'callers': caller_list(fields, 'Callers'), + 'failed': caller_list(fields, 'CallersFailed'), + }) + return entries + + +def build_header(entries, samples_dropped): + """Render the merged ``##ChoCallate=`` header lines.""" + if not entries: + return [] + + union_callers = ordered_union( + entry['callers'] for entry in entries + ) + union_failed = ordered_union( + entry['failed'] for entry in entries + ) + + fields = list(entries[0]['fields']) + set_field(fields, 'Callers', ','.join(union_callers)) + remove_field(fields, 'CallersFailed') + if union_failed: + fields.append(('CallersFailed', f'"{",".join(union_failed)}"')) + remove_field(fields, 'SamplesDropped') + if samples_dropped: + fields.append(('SamplesDropped', f'"{samples_dropped}"')) + + lines = [ + '##ChoCallate=<' + ','.join( + f'{key}={raw}' for key, raw in fields + ) + '>' + ] + + differs = union_failed or any( + entry['callers'] != union_callers for entry in entries + ) + if differs: + for entry in entries: + parts = [ + f'Sample={entry["sample"]}', + 'Callers="' + ','.join(entry['callers']) + '"', + ] + if entry['failed']: + parts.append( + 'CallersFailed="' + ','.join(entry['failed']) + '"' + ) + lines.append('##ChoCallateSamples=<' + ','.join(parts) + '>') + return lines + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Aggregate per-sample ##ChoCallate headers for a merge.' + ) + parser.add_argument( + '--file-list', + required=True, + help='Input BCF paths, one per line', + ) + parser.add_argument( + '--bcftools', + default='bcftools', + help='bcftools executable (default: bcftools)', + ) + parser.add_argument( + '--samples-dropped', + default='', + help='Comma-separated sample ids removed before the merge', + ) + return parser.parse_args() + + +def main(): + args = parse_args() + with open(args.file_list, encoding='utf-8') as handle: + paths = [line.strip() for line in handle if line.strip()] + entries = collect(paths, args.bcftools) + for line in build_header(entries, args.samples_dropped): + print(line) + + +if __name__ == '__main__': + main() diff --git a/bin/merge_outputs.sh b/bin/merge_outputs.sh new file mode 100755 index 0000000..8465b58 --- /dev/null +++ b/bin/merge_outputs.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: merge_outputs.sh --file-list FILE --reference FASTA --output PATH + + --format bcf|vcf Output format (default: bcf) + --cpus INT Threads for bcftools and the chunked merge (default: 1) + --remove-invariant Drop monomorphic records + --split-multiallelic Keep one record per ALT + --samples-dropped LIST Comma-separated ids added to ##ChoCallate + --bcftools PATH bcftools executable (default: bcftools) + --python PATH Python interpreter for helper scripts (default: python3) +EOF +} + +file_list="" +reference="" +output="" +format="bcf" +cpus=1 +remove_invariant=false +split_multiallelic=false +samples_dropped="" +bcftools="${BCFTOOLS:-bcftools}" +python="${PYTHON:-python3}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --file-list) + file_list="$2" + shift 2 + ;; + --reference) + reference="$2" + shift 2 + ;; + --output) + output="$2" + shift 2 + ;; + --format) + format="$2" + shift 2 + ;; + --cpus) + cpus="$2" + shift 2 + ;; + --remove-invariant) + remove_invariant=true + shift + ;; + --split-multiallelic) + split_multiallelic=true + shift + ;; + --samples-dropped) + samples_dropped="$2" + shift 2 + ;; + --bcftools) + bcftools="$2" + shift 2 + ;; + --python) + python="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$file_list" || -z "$reference" || -z "$output" ]]; then + echo "--file-list, --reference and --output are required" >&2 + exit 2 +fi +if [[ "$format" != "bcf" && "$format" != "vcf" ]]; then + echo "--format must be bcf or vcf" >&2 + exit 2 +fi +if [[ "$cpus" -lt 1 ]]; then + echo "--cpus must be >= 1" >&2 + exit 2 +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +tmpdir="$(mktemp -d -p "${TMPDIR:-/tmp}" merge_outputs.XXXXXX)" +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +mapfile -t files < "$file_list" +if [[ ${#files[@]} -eq 0 ]]; then + echo "No input BCF files provided" >&2 + exit 1 +fi + +first="${files[0]}" + +fill_tags="AN,AC,AF,NS,AC_Hom,AC_Het,MAF,TYPE,F_MISSING,DP:1=int(sum(FORMAT/DP))" +output_flags=(-Ob -o "$output") +if [[ "$format" == "vcf" ]]; then + output_flags=(-Oz -o "$output") +fi + +invariant_stream() { + if [[ "$remove_invariant" == true ]]; then + "$bcftools" filter --threads "$cpus" \ + -e 'ALT="." || ALT="<*>" || COUNT(GT="alt")=0' -Ou + else + "$bcftools" view --threads "$cpus" -Ou + fi +} + +join_stream() { + if [[ "$split_multiallelic" == true ]]; then + cat + else + "$bcftools" norm --threads "$cpus" -m +any -Ou + fi +} + +tag_stream() { + "$bcftools" +fill-tags -Ou -- -t "$fill_tags" +} + +annotate_stream() { + if [[ -s "$tmpdir/cho_meta.hdr" ]]; then + "$bcftools" annotate -h "$tmpdir/cho_meta.hdr" -Ou + else + cat + fi +} + +output_stream() { + "$bcftools" view "${output_flags[@]}" +} + +if [[ ${#files[@]} -eq 1 ]]; then + if [[ -n "$samples_dropped" ]]; then + "$bcftools" view -h "$first" > "$tmpdir/first.hdr" + awk -v dropped="$samples_dropped" \ + '/^##ChoCallate=$/, ",SamplesDropped=\"" dropped "\">") + } 1' \ + "$tmpdir/first.hdr" > "$tmpdir/first.mod.hdr" + "$bcftools" reheader -h "$tmpdir/first.mod.hdr" \ + -o "$tmpdir/first.bcf" "$first" + first="$tmpdir/first.bcf" + fi + "$bcftools" view --threads "$cpus" -Ou "$first" \ + | invariant_stream \ + | join_stream \ + | tag_stream \ + | output_stream +else + for bcf in "${files[@]}"; do + "$bcftools" index -f --csi "$bcf" + done + + # Aggregate the structured metadata of every sample so that per-sample + # caller failures are not reduced to the first sample's provenance. + "$python" "$script_dir/merge_cho_headers.py" \ + --file-list "$file_list" \ + --bcftools "$bcftools" \ + --samples-dropped "$samples_dropped" \ + > "$tmpdir/cho_meta.hdr" + + "$python" "$script_dir/merge_bcf_tree.py" \ + --gvcf "$reference" \ + --cpus "$cpus" \ + --file-list "$file_list" \ + | "$bcftools" norm -m -any --threads "$cpus" -Ou \ + | "$python" "$script_dir/drop_refonly_duplicates.py" \ + | invariant_stream \ + | join_stream \ + | tag_stream \ + | annotate_stream \ + | output_stream +fi diff --git a/bin/postprocess_bam.sh b/bin/postprocess_bam.sh new file mode 100755 index 0000000..c4cd539 --- /dev/null +++ b/bin/postprocess_bam.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: postprocess_bam.sh --input BAM|- + + --output PATH Final BAM (default: mapping.bam) + --reference FASTA Reference FASTA (required with --left-align) + --dict DICT Sequence dictionary (required with --left-align) + --fai PATH Reference .fai index. Verify the output BAM + @SQ entries against it; with --dict, compare + M5 checksums too. + --mapq INT Minimum mapping quality (default: 0) + --threads INT Worker threads, split across pipeline stages + (default: 1) + --memory-mb INT Task memory in MiB (sort uses half of the + remainder after --reserve-mb) + --reserve-mb INT Memory in MiB reserved for the aligner index + and other co-resident processes; excluded + before splitting sort memory (default: 0) + --java-options OPTS JVM options for GATK LeftAlignIndels + --rmdup-mode MODE Duplicate handling: none, mark or remove + (default: none) + --rmdup Alias for --rmdup-mode remove + --left-align Left-align indels with GATK LeftAlignIndels +EOF +} + +input="" +output="mapping.bam" +reference="" +dict="" +fai="" +mapq=0 +threads=1 +memory_mb=0 +reserve_mb=0 +java_options="" +rmdup_mode="none" +left_align=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --input) + input="$2" + shift 2 + ;; + --output) + output="$2" + shift 2 + ;; + --reference) + reference="$2" + shift 2 + ;; + --dict) + dict="$2" + shift 2 + ;; + --fai) + fai="$2" + shift 2 + ;; + --mapq) + mapq="$2" + shift 2 + ;; + --threads) + threads="$2" + shift 2 + ;; + --memory-mb) + memory_mb="$2" + shift 2 + ;; + --reserve-mb) + reserve_mb="$2" + shift 2 + ;; + --java-options) + java_options="$2" + shift 2 + ;; + --rmdup-mode) + rmdup_mode="$2" + shift 2 + ;; + --rmdup) + rmdup_mode="remove" + shift + ;; + --left-align) + left_align=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$input" ]]; then + echo "--input is required" >&2 + exit 2 +fi +if [[ "$input" != "-" && ! -f "$input" ]]; then + echo "Input BAM not found: $input" >&2 + exit 2 +fi +if [[ "$left_align" == true && ( -z "$reference" || -z "$dict" ) ]]; then + echo "--reference and --dict are required with --left-align" >&2 + exit 2 +fi +if [[ -n "$fai" && ! -f "$fai" ]]; then + echo "Reference index not found: $fai" >&2 + exit 2 +fi +case "$rmdup_mode" in + none|mark|remove) ;; + *) + echo "--rmdup-mode must be one of none, mark or remove" >&2 + exit 2 + ;; +esac +if [[ "$threads" -lt 1 ]]; then + echo "--threads must be >= 1" >&2 + exit 2 +fi +if [[ "$memory_mb" -lt 0 ]]; then + echo "--memory-mb must be >= 0" >&2 + exit 2 +fi +if [[ "$reserve_mb" -lt 0 ]]; then + echo "--reserve-mb must be >= 0" >&2 + exit 2 +fi +if [[ "$memory_mb" -gt 0 && "$reserve_mb" -ge "$memory_mb" ]]; then + echo "--reserve-mb ($reserve_mb) must be smaller than" \ + "--memory-mb ($memory_mb)" >&2 + exit 2 +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +tmpdir="$(mktemp -d -p "$PWD" .postprocess.XXXXXX)" +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +# The stages of each pipeline below run concurrently, so divide the requested +# threads between them instead of giving every stage the full count. The +# streaming stages get one thread (or none when the budget is smaller than the +# stage count) and sort, the CPU-heavy stage, receives the remainder. +if [[ "$rmdup_mode" != none ]]; then + streaming_stages=4 # collate, fixmate, markdup and view +else + streaming_stages=1 # view +fi +if [[ "$threads" -gt "$streaming_stages" ]]; then + stage_threads=1 + sort_threads=$(( threads - streaming_stages )) +else + stage_threads=0 + sort_threads="$threads" +fi + +# The aligner index is resident while sort runs in the same mapping task. +# Subtract the reserved memory first, then share half of the remainder +# between the sorting threads and leave the rest for the aligner working set +# and the other BAM-processing stages. +sort_memory_mb=768 +if [[ "$memory_mb" -gt 0 ]]; then + available_mb=$(( memory_mb - reserve_mb )) + sort_memory_mb=$(( available_mb / 2 / sort_threads )) + if [[ "$sort_memory_mb" -lt 1 ]]; then + sort_memory_mb=1 + fi +fi + +filtered_bam="$tmpdir/filtered.bam" +if [[ "$rmdup_mode" != none ]]; then + markdup_flags=() + if [[ "$rmdup_mode" == remove ]]; then + markdup_flags=(-r) + fi + # Stream markdup into the MAPQ filter instead of writing dedup.bam. + samtools collate --threads "$stage_threads" -O -u \ + -T "$tmpdir/collate" "$input" \ + | samtools fixmate --threads "$stage_threads" -m -u - - \ + | samtools sort --threads "$sort_threads" \ + -m "${sort_memory_mb}M" -u -T "$tmpdir/sort" - \ + | samtools markdup --threads "$stage_threads" -u \ + "${markdup_flags[@]}" -T "$tmpdir/markdup" - - \ + | samtools view --threads "$stage_threads" -F 4 -q "$mapq" \ + -b -o "$filtered_bam" - +else + # Filtering before sorting reduces the number of records sort has to + # order; no duplicate information is needed in this path. + samtools view --threads "$stage_threads" -F 4 -q "$mapq" -u \ + "$input" \ + | samtools sort --threads "$sort_threads" \ + -m "${sort_memory_mb}M" -T "$tmpdir/sort" \ + -o "$filtered_bam" - +fi + +# Fail before left-alignment and indexing when the aligner used an index +# built for another assembly. +if [[ -n "$fai" ]]; then + check_args=("$filtered_bam" "$fai") + if [[ -n "$dict" ]]; then + check_args+=(--dict "$dict") + fi + "$script_dir/check_bam_reference.py" "${check_args[@]}" +fi + +if [[ "$left_align" == true ]]; then + rm -f "$output" + gatk_opts=() + if [[ -n "$java_options" ]]; then + gatk_opts=(--java-options "$java_options") + fi + gatk LeftAlignIndels "${gatk_opts[@]}" \ + -I "$filtered_bam" \ + -O "$output" \ + -R "$reference" \ + --sequence-dictionary "$dict" \ + -OBI false +else + mv "$filtered_bam" "$output" +fi + +samtools index --threads "$threads" --csi "$output" diff --git a/bin/stage_cleanup.sh b/bin/stage_cleanup.sh index 1677e84..03d53a4 100755 --- a/bin/stage_cleanup.sh +++ b/bin/stage_cleanup.sh @@ -1,4 +1,21 @@ -#!/bin/bash +#!/usr/bin/env bash +set -u -rm -rf tmp/ -rm -rf ${TMPDIR} \ No newline at end of file +# The exit status of the task script is passed by the afterScript directive. +# Nextflow only sets it on failure, so an absent argument means success. +exit_status="${1:-0}" +if [ "${exit_status}" -ne 0 ]; then + exit 0 +fi + +# Staged inputs that are no longer needed after a successful task. +rm -rf -- tmp/ index/ + +# Only remove a temporary directory created inside this work directory. +if [ -n "${TMPDIR:-}" ] && [ "${TMPDIR}" != "/" ]; then + case "${TMPDIR}" in + "${PWD}"/*) + rm -rf -- "${TMPDIR}" + ;; + esac +fi diff --git a/conf/base.config b/conf/base.config new file mode 100644 index 0000000..1efc3c1 --- /dev/null +++ b/conf/base.config @@ -0,0 +1,76 @@ +/* + * Resource labels shared by all pipeline processes. Values live in the + * params map so they can be overridden with -params-file like every other + * pipeline option. + */ +process { + // Values are closures so overrides from a later config file or a params + // file are resolved at task scheduling time. + maxRetries = { params.resources.max_retries } + + // Retries escalate memory and time, so stop at the resources the + // executor can actually provide. + resourceLimits = [ + cpus: params.resources.limits.cpus, + memory: params.resources.limits.memory_mb.MB, + time: Duration.of(params.resources.limits.time), + ] + + // Transient resource failures are retried with the escalated memory of + // the next attempt. Any other error terminates the task. + errorStrategy = { + task.exitStatus in [137, 140] && + task.attempt <= params.resources.max_retries + ? 'retry' + : 'terminate' + } + + withLabel: process_low { + cpus = { params.resources.low.cpus } + memory = { params.resources.low.memory_mb.MB * task.attempt } + time = { Duration.of(params.resources.low.time) * task.attempt } + } + + withLabel: process_medium { + cpus = { params.resources.medium.cpus } + memory = { params.resources.medium.memory_mb.MB * task.attempt } + time = { Duration.of(params.resources.medium.time) * task.attempt } + } + + withLabel: process_high { + cpus = { params.resources.high.cpus } + memory = { params.resources.high.memory_mb.MB * task.attempt } + time = { Duration.of(params.resources.high.time) * task.attempt } + } + + // A label sets the CPU count for processes without a stage-specific + // option. These selectors take precedence and keep mapping.cpu, + // calling.cpu and consensus.cpu authoritative for their stages. + withName: 'MAPPING_BOWTIE2|MAPPING_BWA|MAPPING_MINIMAP2|PREPARE_BAM' { + cpus = { params.mapping?.cpu ?: params.resources.medium.cpus } + } + + withName: 'CALLING_BCFTOOLS|CALLING_FREEBAYES|CALLING_GATK' { + cpus = { params.calling?.cpu ?: params.resources.medium.cpus } + // Only transient resource failures are retried. A deterministic + // caller error (bad arguments, invalid input) fails immediately + // instead of repeating until max_retries. + errorStrategy = { + if (task.exitStatus in [137, 140] && + task.attempt <= params.resources.max_retries) { + return 'retry' + } + return params.calling?.allow_caller_failure + ? 'ignore' + : 'terminate' + } + } + + withName: 'SCATTER_INTERVALS' { + cpus = { params.calling?.cpu ?: params.resources.medium.cpus } + } + + withName: 'GENERATE_CONSENSUS|MERGE_OUTPUTS|EXPORT_OUTPUT' { + cpus = { params.consensus?.cpu ?: params.resources.medium.cpus } + } +} diff --git a/environment.yaml b/environment.yaml index ba8f51e..def63e2 100644 --- a/environment.yaml +++ b/environment.yaml @@ -4,17 +4,28 @@ channels: - bioconda dependencies: - bwa=0.7.19 - - minimap2>=2.30 - - bowtie2>=2.5.5 - - freebayes>=1.3.10 - - bcftools>=1.23.1 - - gatk4>=4.6.2.0 - - samtools>=1.23 - - python - - tabix>=1.11 - - bedops>=2.4.42 - - nextflow>=26.04.6 - - bedtools>=2.31.1 - - parallel - - pysam>=0.23.3 - - picard>=3.4.0 + - minimap2=2.31 + - bowtie2=2.5.5 + - freebayes=1.3.10 + - bcftools=1.24 + - gatk4=4.6.2.0 + - samtools=1.24 + - mosdepth=0.3.14 + - htslib=1.24 + # pysam has no free-threaded build, so keep the GIL-enabled CPython build. + - python=3.13.16=*_cp313 + - bedops=2.4.42 + - nextflow=26.04.7 + - parallel=20260922 + - pysam=0.24.1 + # test suite (tests/python, run with `pytest -q tests/python`) + - pytest=9.1.1 + - nf-test=0.9.5 + - ruff=0.17.1 + - actionlint=1.7.12 + - picard=3.5.0 + # build dependencies of the optional consensus accelerator + # (bin/build_consensus_accel.py) + - cython=3.3.0 + - setuptools=84.1.0 + - c-compiler=2.0.0 diff --git a/lib/CLIParamsValidation.groovy b/lib/CLIParamsValidation.groovy index 07136f2..43ccfe0 100644 --- a/lib/CLIParamsValidation.groovy +++ b/lib/CLIParamsValidation.groovy @@ -1,72 +1,567 @@ +/* + * Configuration and samplesheet validation used by the pipeline entry point. + */ +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +import nextflow.exception.AbortOperationException + class CLIParamsValidation { - static void reference_genome_validation(String reference_genome) { - if (reference_genome == null) { - println "ERROR: Reference genome is required" - System.exit(1) - } + + static final List AVAILABLE_CALLERS = + ['bcftools', 'freebayes', 'gatk'] + static final String SAMPLE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/ + static final String READ_GROUP_PATTERN = /^[A-Za-z0-9_.:-]+$/ + + // Caller options that select ploidy or copy number. A true value means + // that an explicit diploid setting (2) is accepted; a false value means + // the option is rejected because its contents are not inspected. + static final Map> PLOIDY_OPTIONS = [ + bcftools: [ + '--ploidy': true, + '--ploidy-file': false + ], + freebayes: [ + '--ploidy': true, + '-p': true, + '--cnv-map': false, + '-A': false + ], + gatk: [ + '--sample-ploidy': true, + '--ploidy': true, + '-ploidy': true, + '--ploidy-regions': false + ] + ] + + private static void fail(String message) { + throw new AbortOperationException("ERROR: ${message}") } - static void samples_tsv_validation(String samples_tsv) { - if (samples_tsv == null) { - println "ERROR: Samples TSV is required" - System.exit(1) + static List parse_callers(String callers) { + if (callers == null) { + return [] } + // trimmed, non-empty and deduplicated, preserving the configured order + return callers.split(',') + .collect { it.trim() } + .findAll { it } + .unique() } - static void reference_index_dir_validation(String reference_index_dir) { - if (reference_index_dir == null) { - println "ERROR: Reference index directory is required" - System.exit(1) + /** + * Resolves the effective duplicate-handling mode. + */ + static String rmdup_mode(Map rmdup) { + def mode = rmdup?.mode ?: 'mark' + if (!(mode in ['none', 'mark', 'remove'])) { + fail( + "rmdup.mode: unknown mode '${mode}'. Accepted values are: " + + 'none, mark, remove' + ) } + return mode } static void effective_callers_validation(String callers) { - if (callers == null) { - println "ERROR: Effective callers are not specified" - System.exit(1) + def callersList = parse_callers(callers) + if (!callersList) { + fail( + 'calling.callers is required and must name at least one ' + + "caller (available: ${AVAILABLE_CALLERS.join(', ')})" + ) + } + def unknown = callersList.findAll { !(it in AVAILABLE_CALLERS) } + if (unknown) { + fail( + "calling.callers: unknown caller(s) ${unknown.join(', ')}. " + + "Available callers are: ${AVAILABLE_CALLERS.join(', ')}" + ) + } + } + + static void mixed_reads_validation( + String mapper, + String reads_type, + String format + ) { + if (format == 'bam') { + return + } + if (reads_type == 'mx' && mapper != 'bowtie2') { + fail( + "input.reads_type: 'mx' is only supported with " + + "mapping.mapper 'bowtie2' (got '${mapper}')" + ) + } + } + + /** + * Rejects read-group values that are unsafe in the mapper command line + * or in the SAM header. + * + * The schema applies the same pattern, but JSON Schema pattern matching + * can accept a trailing line terminator. A full-string match keeps + * values such as `platform: "ILLUMINA\n"` out of the @RG tag. + */ + static void read_group_validation(Map mapping) { + def values = [ + 'mapping.platform': mapping?.platform ?: 'ILLUMINA', + 'mapping.library': mapping?.library + ] + values.each { key, value -> + if (value != null && !(value ==~ READ_GROUP_PATTERN)) { + fail( + "${key}: '${value}' is not safe for a SAM read group; " + + 'accepted characters are letters, digits, underscore, ' + + 'dot, colon and hyphen' + ) + } } - def available_callers = ['bcftools', 'gatk', 'freebayes'] - def callersList = callers.split(',').collect { it.trim() } - def unknown_callers = callersList.findAll { !(it in available_callers) } - if (unknown_callers) { - println "ERROR: Unknown caller(s): ${unknown_callers}. Available callers are: ${available_callers}" - System.exit(1) + } + + /** + * Rejects an index prefix that selects a subdirectory. + * + * Index files are resolved from reference_index_dir before staging but + * staged under a flat `index/` directory. A prefix containing a path + * separator would therefore point at the files during validation and at + * a non-existent nested path in the mapping command. + */ + static void reference_index_prefix_validation(String prefix) { + if (prefix == null) { + return + } + if (prefix.contains('/') || prefix.contains('\\')) { + fail( + "input.reference_index_prefix: '${prefix}' must not " + + 'contain a path separator' + ) + } + } + + /** + * Rejects caller arguments that request a non-diploid genotype. + * + * Consensus reference records are always diploid (0/0), while a caller + * that runs with another ploidy emits variants with a different GT arity. + * Only an explicit diploid setting is accepted. Custom ploidy, + * copy-number and region files are rejected because their contents are + * not inspected. + */ + static void ploidy_validation(Map calling) { + if (calling == null) { + return } + def bcftools = calling['bcftools'] ?: [:] + validate_ploidy_args( + 'calling.bcftools.call.extra_args', + bcftools['call']?.extra_args, + PLOIDY_OPTIONS['bcftools'] + ) + def freebayes = calling['freebayes'] ?: [:] + validate_ploidy_args( + 'calling.freebayes.extra_args', + freebayes['extra_args'], + PLOIDY_OPTIONS['freebayes'] + ) + def gatk = calling['gatk'] ?: [:] + validate_ploidy_args( + 'calling.gatk.extra_args', + gatk['extra_args'], + PLOIDY_OPTIONS['gatk'] + ) } - static void mapper_validation(String mapper, String reads_type) { - def available_mappers = ['bowtie2', 'bwa', 'minimap2'] - if (mapper == null) { - println "ERROR: Mapper is required" - System.exit(1) + private static void validate_ploidy_args( + String key, + String args, + Map options + ) { + if (args == null || args.trim().isEmpty()) { + return } - if (!(mapper in available_mappers)) { - println "ERROR: Invalid mapper: ${mapper}. Available mappers are: ${available_mappers}" - System.exit(1) + options.each { option, diploid_allowed -> + def value = option_value(args, option) + if (value == null) { + return + } + if (diploid_allowed && unquote(value) == '2') { + return + } + def found = value ? "${option} ${value}" : option + fail( + "${key}: ploidy is not supported yet except diploid (2); " + + "remove '${found}'" + ) + } + } + + /** + * Returns the value of an option in a shell-like argument string. + * Returns an empty string when the option has no value and null when the + * option is absent. + */ + private static String option_value(String args, String option) { + def with_value = args =~ ( + /(? callersList.size()) { + fail( + "consensus.threshold (${threshold}) must be less than or " + + 'equal to the number of configured callers ' + + "(${callersList.size()})" + ) + } + } + + /** + * Reads and validates the samplesheet before any task runs. + * + * Blank lines and lines starting with '#' are ignored. Only the first + * data line is treated as a possible header: if its first column is + * sample, sample_id, sampleid or id it is skipped. The row width depends + * on input.format and input.reads_type. Relative read paths are resolved + * from the samplesheet directory and must exist. + * + * @return list of maps with sample id and resolved read paths + */ + static List> parse_samplesheet( + Path samplesheet, + String format, + String reads_type + ) { + if (!Files.isRegularFile(samplesheet)) { + fail("input.samples_tsv: file not found: ${samplesheet}") + } + int expected = expected_columns(format, reads_type) + Path base_dir = samplesheet.toAbsolutePath().normalize().getParent() + + def samples = [] + def seen = new LinkedHashMap() + int line_number = 0 + boolean first_data_line = true + + Files.newBufferedReader(samplesheet, StandardCharsets.UTF_8) + .withCloseable { reader -> + String line + while ((line = reader.readLine()) != null) { + line_number++ + if (line_number == 1 && line.startsWith('\uFEFF')) { + line = line.substring(1) + } + if (line.trim().isEmpty() || + line.trim().startsWith('#')) { + continue + } + def columns = line.split('\t', -1) + .collect { it.replaceAll(/\r$/, '') } + if (first_data_line) { + first_data_line = false + if (is_header_row(columns)) { + continue + } + } + if (columns.size() != expected) { + fail( + "input.samples_tsv line ${line_number}: " + + "expected ${expected} tab-separated columns " + + "for format='${format}', " + + "reads_type='${reads_type}' but found " + + "${columns.size()}" + ) + } + + String sample_id = columns[0] + if (!sample_id) { + fail( + "input.samples_tsv line ${line_number}: " + + 'sample id is empty' + ) + } + if (!(sample_id ==~ SAMPLE_ID_PATTERN)) { + fail( + "input.samples_tsv line ${line_number}: " + + "sample id '${sample_id}' is not shell-safe. " + + "Use letters, digits, '.', '_' or '-' and " + + 'start with a letter or digit.' + ) + } + if (seen.containsKey(sample_id)) { + fail( + "input.samples_tsv line ${line_number}: " + + "duplicate sample id '${sample_id}' " + + "(first seen on line ${seen[sample_id]})" + ) + } + seen[sample_id] = line_number + + def reads = (1.. + resolve_read_path( + base_dir, + columns[index], + line_number, + index + 1 + ) + } + samples << [id: sample_id, reads: reads] + } + } + + if (samples.isEmpty()) { + fail("input.samples_tsv: no samples found in ${samplesheet}") + } + return samples + } + + private static int expected_columns(String format, String reads_type) { + if (format == 'bam') { + return 2 + } + switch (reads_type) { + case 'se': + return 2 + case 'pe': + return 3 + case 'mx': + return 4 + default: + fail( + "input.reads_type: unsupported value '${reads_type}' " + + '(accepted: se, pe, mx)' + ) + } + } + + private static boolean is_header_row(List columns) { + def first = columns[0].trim().toLowerCase() + return first in ['sample', 'sample_id', 'sampleid', 'id'] + } + + private static String resolve_read_path( + Path base_dir, + String value, + int line_number, + int column_number + ) { + if (value == null || value.trim().isEmpty()) { + fail( + "input.samples_tsv line ${line_number}: " + + "column ${column_number} is empty" + ) + } + def raw = value.trim() + // Leave remote URIs to Nextflow staging; local paths are checked here. + if (raw ==~ /^[A-Za-z][A-Za-z0-9+.-]*:\/\/.*/) { + return raw + } + Path candidate = new File(raw).isAbsolute() + ? Paths.get(raw) + : base_dir.resolve(raw) + Path resolved = candidate.toAbsolutePath().normalize() + if (resolved.toUri().scheme != 'file') { + return resolved.toUri().toString() + } + if (!Files.isRegularFile(resolved)) { + fail( + "input.samples_tsv line ${line_number}: " + + "column ${column_number} file not found: ${resolved}" + ) + } + return resolved.toRealPath().toString() + } + + /** + * Detects BGZF from the gzip header. A plain gzip stream has the FEXTRA + * flag unset; BGZF carries an extra field with the 'BC' subfield. + */ + static boolean is_bgzip(Path fasta) { + try { + return Files.newInputStream(fasta).withCloseable { stream -> + byte[] header = new byte[12] + int read = stream.readNBytes(header, 0, header.length) + if (read < header.length) { + return false + } + if ((header[0] & 0xff) != 0x1f || + (header[1] & 0xff) != 0x8b) { + return false + } + int flags = header[3] & 0xff + if ((flags & 0x04) == 0) { + return false + } + int xlen = (header[10] & 0xff) | + ((header[11] & 0xff) << 8) + if (xlen == 0) { + return false + } + byte[] extra = new byte[xlen] + int extra_read = stream.readNBytes(extra, 0, xlen) + int i = 0 + while (i + 3 < extra_read) { + if ((extra[i] & 0xff) == 0x42 && + (extra[i + 1] & 0xff) == 0x43) { + int slen = (extra[i + 2] & 0xff) | + ((extra[i + 3] & 0xff) << 8) + return slen == 2 + } + int slen = (extra[i + 2] & 0xff) | + ((extra[i + 3] & 0xff) << 8) + i += 4 + slen + } + return false + } + } catch (IOException exception) { + fail( + "input.reference_genome: cannot read '${fasta}': " + + exception.message + ) + } + return false + } + + /** + * Detects plain gzip from the magic bytes, ignoring the extra fields. + * A gzip-compressed reference that is not BGZF cannot be indexed by + * samtools faidx, so the workflow rejects it before any task starts. + */ + static boolean is_gzip(Path fasta) { + try { + return Files.newInputStream(fasta).withCloseable { stream -> + byte[] magic = new byte[2] + int read = stream.readNBytes(magic, 0, magic.length) + return read == magic.length && + (magic[0] & 0xff) == 0x1f && + (magic[1] & 0xff) == 0x8b + } + } catch (IOException exception) { + fail( + "input.reference_genome: cannot read '${fasta}': " + + exception.message + ) + } + return false + } + + /** + * Fails early when the merge step cannot open the descriptors it needs. + * Each chunk opens its BCFs plus their indexes; the final merge opens one + * named pipe per chunk. A small fixed margin covers stdio and htslib + * temporary handles. + */ + static void merge_ulimit_validation(int sample_count, int cpus) { + if (sample_count < 1) { + return + } + int chunk_count = Math.min(Math.max(cpus, 1), sample_count) + long largest_chunk = (long) Math.ceil( + sample_count / (double) chunk_count + ) + long required = Math.max( + largest_chunk * 2L, + (long) chunk_count + ) + 64L + long limit = open_file_limit() + if (limit > 0 && limit < required) { + fail( + 'The merge step needs at least ' + + "${required} open file descriptors for " + + "${sample_count} sample(s) with " + + "consensus.cpu=${cpus}, but the current soft limit is " + + "${limit}. Raise it with 'ulimit -n ${required}' or " + + 'lower consensus.cpu.' + ) } - def callersList = callers.split(',').collect { it.trim() } - if (cons_threshold > callersList.size()) { - println "ERROR: Consensus threshold must be less or equal to the number of effective callers" - System.exit(1) + } + + /** + * Returns the soft open-file limit of the current process, or -1 when it + * cannot be determined. + */ + static long open_file_limit() { + def limits = new File('/proc/self/limits') + if (limits.isFile()) { + try { + def line = limits.readLines('UTF-8').find { + it.trim().startsWith('Max open files') + } + if (line) { + def parts = line.trim().split(/\s+/) + int files_index = -1 + for (int i = 0; i < parts.length; i++) { + if (parts[i] == 'files') { + files_index = i + } + } + if (files_index >= 3) { + return parts[files_index - 2] as long + } + } + } catch (Exception ignored) { + // fall back to ulimit below + } } - if (cons_threshold < 1) { - println "ERROR: Consensus threshold must be greater than 0" - System.exit(1) + try { + def process = ['sh', '-c', 'ulimit -n'].execute() + process.waitFor() + def text = process.text.trim() + if (process.exitValue() == 0 && text ==~ /\d+/) { + return text as long + } + } catch (Exception ignored) { + // unknown limit: do not block the run } + return -1L } -} \ No newline at end of file +} diff --git a/lib/ResourceUtils.groovy b/lib/ResourceUtils.groovy new file mode 100644 index 0000000..ab6ee57 --- /dev/null +++ b/lib/ResourceUtils.groovy @@ -0,0 +1,32 @@ +/* + * Helpers shared by process scripts to size JVM heaps from task resources. + */ +class ResourceUtils { + + /** + * Heap size in MiB for one JVM. + * + * A task may start several JVMs in parallel (GNU parallel), in which + * case each receives an equal share. Twenty percent of the task + * memory is left for native allocations, stacks and the file cache. + * + * @param memory task memory, as exposed by Nextflow + * @param slots number of JVMs that may run concurrently + * @return heap size in MiB, or 0 when no memory is configured + */ + static int javaHeapMb(memory, int slots = 1) { + if (memory == null) { + return 0 + } + int heapSlots = slots > 0 ? slots : 1 + return (memory.toMega() * 0.8 / heapSlots) as int + } + + /** + * GATK/Picard {@code -Xmx} option for one JVM. + */ + static String javaHeapOption(memory, int slots = 1) { + int heapMb = javaHeapMb(memory, slots) + return heapMb > 0 ? "-Xmx${heapMb}m" : '' + } +} diff --git a/main.nf b/main.nf index 26ff0fb..b658570 100755 --- a/main.nf +++ b/main.nf @@ -6,114 +6,363 @@ include { CREATE_SEQ_DICT } from './modules/local/create_seq_dict' include { MAPPING_BOWTIE2 } from './modules/local/mapping_bowtie2' include { MAPPING_BWA } from './modules/local/mapping_bwa' include { MAPPING_MINIMAP2 } from './modules/local/mapping_minimap2' +include { MAPPING_STATS } from './modules/local/mapping_stats' +include { MAPPING_SUMMARY } from './modules/local/mapping_summary' include { PREPARE_BAM } from './modules/local/prepare_bam' include { GENERATE_COVERAGE } from './modules/local/generate_coverage' +include { SCATTER_INTERVALS } from './modules/local/scatter_intervals' include { CALLING_BCFTOOLS } from './modules/local/calling_bcftools' include { CALLING_FREEBAYES } from './modules/local/calling_freebayes' include { CALLING_GATK } from './modules/local/calling_gatk' include { GENERATE_CONSENSUS } from './modules/local/generate_consensus' include { MERGE_OUTPUTS } from './modules/local/merge_outputs' +include { REPORT_DROPPED_SAMPLES } from './modules/local/report_dropped_samples' +include { EXPORT_OUTPUT } from './modules/local/export_output' +include { validateParameters } from 'plugin/nf-schema' + workflow { - - // Validate required parameters - CLIParamsValidation.reference_genome_validation(params.input.reference_genome) - CLIParamsValidation.samples_tsv_validation(params.input.samples_tsv) + + // Unknown keys are rejected before any process starts. + validateParameters() + + // Caller selection: trimmed, deduplicated, in voting order. The same list + // drives process selection, threshold validation and the consensus header. + def selected_callers = + CLIParamsValidation.parse_callers(params.calling.callers) CLIParamsValidation.effective_callers_validation(params.calling.callers) - CLIParamsValidation.cons_threshold_validation(params.consensus.threshold, params.calling.callers) - CLIParamsValidation.mapper_validation(params.mapping.mapper, params.input.reads_type) - CLIParamsValidation.reference_index_dir_validation(params.input.reference_index_dir) - - // Create sample channel - sample_run_ch = channel - .fromPath(params.input.samples_tsv) - .splitCsv(header: false, sep: '\t') - .filter { row -> - if (row.size() == 1) { - println "[WARN] Dropping row with only one column: ${row[0]}" - return false - } - return true - } - .map { - row -> if (params.input.format == 'bam' || params.input.reads_type == 'se') { - tuple(row[0..1].toArray() + ['/dev/null'] * 2) - } else if (params.input.reads_type == 'pe' && params.input.format != 'bam') { - tuple(row[0..2].toArray() + ['/dev/null']) - } else if (params.input.reads_type == 'mx' && params.input.format != 'bam') { - tuple(row[0..3].toArray()) - } - } - - // Prepare genome assembly, fai index and sequence dictionary - ref_genome = params.ref_genome.bgzip ? DECOMPRESS_ASSEMBLY(file(params.input.reference_genome)) : file(params.input.reference_genome) + CLIParamsValidation.cons_threshold_validation( + params.consensus.threshold, + params.calling.callers + ) + CLIParamsValidation.mixed_reads_validation( + params.mapping.mapper, + params.input.reads_type, + params.input.format + ) + CLIParamsValidation.ploidy_validation(params.calling) + CLIParamsValidation.read_group_validation(params.mapping) + + // Parse the samplesheet up front: unique shell-safe ids, exact row widths, + // optional header/comment lines, and read paths relative to the sheet. + def samples = CLIParamsValidation.parse_samplesheet( + file(params.input.samples_tsv, checkIfExists: true), + params.input.format, + params.input.reads_type + ) + def sample_ids = samples.collect { sample -> sample.id } + + // A merge opens every sample BCF plus the named pipes; fail before the + // first task when the process descriptor limit cannot support that. + if (params.output.type == 'single') { + CLIParamsValidation.merge_ulimit_validation( + samples.size(), + params.consensus.cpu as int + ) + } + + // Compression is detected from the header. BGZF is decompressed below; + // plain gzip cannot be indexed by samtools faidx or read by GATK, so it + // is rejected before any task starts. + def reference_file = + file(params.input.reference_genome, checkIfExists: true) + def reference_is_bgzip = CLIParamsValidation.is_bgzip(reference_file) + if (!reference_is_bgzip && CLIParamsValidation.is_gzip(reference_file)) { + error( + "reference is gzip-compressed but not BGZF; recompress with bgzip" + ) + } + + // Mapper indexes are staged as task inputs. The prefix is resolved here, + // so the command never depends on a host path, and a missing index aborts + // before the first task starts. Exact suffixes keep the FASTA, its + // .fai/.dict and indexes of other genomes that share the prefix out of + // the staged set. An index is usable only when every file of one set is + // present. + def index_suffix_sets = [ + bowtie2: [ + ['.1.bt2', '.2.bt2', '.3.bt2', '.4.bt2', + '.rev.1.bt2', '.rev.2.bt2'], + ['.1.bt2l', '.2.bt2l', '.3.bt2l', '.4.bt2l', + '.rev.1.bt2l', '.rev.2.bt2l'] + ], + bwa: [ + ['.amb', '.ann', '.bwt', '.pac', '.sa'] + ], + minimap2: [ + ['.mmi'] + ] + ] + def index_files = channel.empty() + if (params.input.format != 'bam') { + def mapper = params.mapping.mapper + def index_dir = params.input.reference_index_dir + CLIParamsValidation.reference_index_prefix_validation( + params.input.reference_index_prefix + ) + def index_prefix = params.input.reference_index_prefix ?: + file(params.input.reference_genome).name + def suffix_sets = index_suffix_sets[mapper] + if (suffix_sets == null) { + error("Unknown mapper '${mapper}'") + } + def candidate_sets = suffix_sets.collect { suffixes -> + suffixes.collect { suffix -> + file("${index_dir}/${index_prefix}${suffix}") + } + } + def index_matches = candidate_sets.find { candidates -> + candidates.every { candidate -> candidate.exists() } + } + if (index_matches == null) { + def missing = candidate_sets.collect { candidates -> + candidates + .findAll { candidate -> !candidate.exists() } + .collect { candidate -> candidate.name } + .join(', ') + } + error( + "No ${mapper} index files found in '${index_dir}' for " + + "prefix '${index_prefix}'; missing: ${missing.join('; ')}" + ) + } + index_files = channel.value(index_matches) + } + + // Create the sample channel. Absent read slots stage the empty NO_FILE + // instead of /dev/null so every input is a real, portable file. + def no_file = file("${projectDir}/assets/NO_FILE", checkIfExists: true) + sample_run_ch = channel.fromList( + samples.collect { sample -> + def reads = sample.reads.collect { read_path -> file(read_path) } + reads = reads + ([no_file] * (3 - reads.size())) + tuple(sample.id, reads[0], reads[1], reads[2]) + } + ) + + // Prepare genome assembly, fai index and sequence dictionary. + ref_genome = reference_is_bgzip + ? DECOMPRESS_ASSEMBLY(reference_file) + : reference_file fai_index = CREATE_FAI_INDEX(ref_genome) gen_dict = CREATE_SEQ_DICT(ref_genome) // Prepare BAM files if (params.mapping.mapper == 'bowtie2' && params.input.format != 'bam') { bam_file = MAPPING_BOWTIE2(sample_run_ch, - file(params.input.reference_genome), + index_files, ref_genome, gen_dict, fai_index) - } else if (params.mapping.mapper == 'bwa' && params.input.format != 'bam') { - bam_file = MAPPING_BWA(sample_run_ch, - file(params.input.reference_genome), - ref_genome, - gen_dict, - fai_index) - } else if (params.mapping.mapper == 'minimap2' && params.input.format != 'bam') { - bam_file = MAPPING_MINIMAP2(sample_run_ch, - file(params.input.reference_genome), - ref_genome, - gen_dict, + } else if (params.mapping.mapper == 'bwa' && + params.input.format != 'bam') { + bam_file = MAPPING_BWA(sample_run_ch, + index_files, + ref_genome, + gen_dict, + fai_index) + } else if (params.mapping.mapper == 'minimap2' && + params.input.format != 'bam') { + bam_file = MAPPING_MINIMAP2(sample_run_ch, + index_files, + ref_genome, + gen_dict, fai_index) } else if (params.input.format == 'bam') { bam_file = PREPARE_BAM(sample_run_ch, - ref_genome, - gen_dict, + ref_genome, + gen_dict, fai_index) } else { error("Unknown input format or mapper") } + // The raw flagstat is captured by the mapping stage before MAPQ filtering + // and duplicate handling; the other reports describe the final BAM. + mapping_stats = MAPPING_STATS(bam_file) + raw_flagstat_files = mapping_stats + .map { _sample_id, raw, _flagstat, _stats, _idxstats -> raw } + .collect() + final_flagstat_files = mapping_stats + .map { _sample_id, _raw, flagstat, _stats, _idxstats -> flagstat } + .collect() + mapping_stats_files = mapping_stats + .map { _sample_id, _raw, _flagstat, stats, _idxstats -> stats } + .collect() + mapping_idxstats_files = mapping_stats + .map { _sample_id, _raw, _flagstat, _stats, idxstats -> idxstats } + .collect() + MAPPING_SUMMARY( + channel.value(sample_ids), + raw_flagstat_files, + final_flagstat_files, + mapping_stats_files, + mapping_idxstats_files + ) + + // Coverage, calling and consensus only need the BAM and its index. + bam_for_analysis = bam_file.map { sample_id, bam, csi, _raw -> + tuple(sample_id, bam, csi) + } + // Generate BED coverage - include_bed = params.input.include_bed == null ? file("${projectDir}/assets/N1_FILE", checkIfExists: true) : file(params.input.include_bed, checkIfExists: true) - exclude_bed = params.input.exclude_bed == null ? file("${projectDir}/assets/NO_FILE", checkIfExists: true) : file(params.input.exclude_bed, checkIfExists: true) - bed_coverage = GENERATE_COVERAGE(bam_file, include_bed, exclude_bed) + include_bed = params.input.include_bed == null + ? file("${projectDir}/assets/N1_FILE", checkIfExists: true) + : file(params.input.include_bed, checkIfExists: true) + exclude_bed = params.input.exclude_bed == null + ? file("${projectDir}/assets/NO_FILE", checkIfExists: true) + : file(params.input.exclude_bed, checkIfExists: true) + bed_coverage = GENERATE_COVERAGE( + bam_for_analysis, include_bed, exclude_bed, fai_index + ) - // Join each sample's BAM with its own coverage BED by sample_id. This keeps the - // pair coupled per sample: passing the two channels independently would combine - // them positionally and could mix a BAM with another sample's coverage BED. - bam_coverage = bam_file.join(bed_coverage) + // With coverage.on_empty="skip", an empty coverage BED removes the + // sample before scattering. The default "fail" mode is handled inside + // GENERATE_COVERAGE, which aborts with the sample id and threshold. + if (params.coverage.on_empty == 'skip') { + coverage_branch = bed_coverage.branch { _sample_id, bed -> + empty: bed.size() == 0 + covered: true + } + empty_coverage_samples = coverage_branch.empty + .map { sample_id, _bed -> + log.warn( + "Sample ${sample_id}: no region at or above " + + "coverage.min_coverage=" + + "${params.coverage.min_coverage}; sample dropped" + ) + sample_id + } + .toList() + bed_coverage = coverage_branch.covered + } else { + empty_coverage_samples = channel.value([]) + } + + // Scatter the covered intervals once per sample. Every caller and the + // consensus step consume the same chunk list instead of rebuilding it. + intervals = SCATTER_INTERVALS( + bed_coverage, ref_genome, fai_index, gen_dict + ) + + // Join each sample's BAM, BAM index and interval chunks by sample_id. This + // keeps all per-sample inputs coupled and avoids positional mixing. + bam_intervals = bam_for_analysis.join(intervals) // Perform variant calling - bcftools = params.calling.callers.contains('bcftools') ? CALLING_BCFTOOLS(bam_coverage, ref_genome, fai_index, gen_dict) : channel.empty() - freebayes = params.calling.callers.contains('freebayes') ? CALLING_FREEBAYES(bam_coverage, ref_genome, fai_index, gen_dict) : channel.empty() - gatk = params.calling.callers.contains('gatk') ? CALLING_GATK(bam_coverage, ref_genome, fai_index, gen_dict) : channel.empty() + bcftools = selected_callers.contains('bcftools') + ? CALLING_BCFTOOLS(bam_intervals, ref_genome, fai_index, gen_dict) + : channel.empty() + freebayes = selected_callers.contains('freebayes') + ? CALLING_FREEBAYES(bam_intervals, ref_genome, fai_index, gen_dict) + : channel.empty() + gatk = selected_callers.contains('gatk') + ? CALLING_GATK(bam_intervals, ref_genome, fai_index, gen_dict) + : channel.empty() all_calls = bcftools .join(freebayes, remainder: true) .join(gatk, remainder: true) .map { list -> list.findAll { item -> item != null } } - .map {tuple -> [tuple[0], tuple[1..-1]]} + .map { tuple -> + def paths = tuple[1..-1] + def received = paths.collect { path -> + path.name.replaceAll(/\.bcf$/, '') + } + def failed = selected_callers.findAll { caller -> + !(caller in received) + } + if (failed) { + log.warn( + "Sample ${tuple[0]}: caller(s) failed and were " + + "skipped: ${failed.join(',')}" + ) + } + [tuple[0], received.join(','), paths] + } + + // A sample whose every caller failed produces no joined tuple and would + // otherwise disappear from the output without a message. Compare the ids + // that reached the join with the ids that passed the coverage filter. + coverage_ids = bed_coverage.map { item -> item[0] }.toList() + call_ids = all_calls.map { item -> item[0] }.toList() + dropped_samples = coverage_ids + .concat(empty_coverage_samples) + .concat(call_ids) + .toList() + .map { lists -> + def covered = lists[0] + def empty = lists[1] + def called = lists[2] + def failed = covered.findAll { id -> !(id in called) } + failed.each { id -> + def message = "Sample ${id}: no caller produced calls " + + "(${selected_callers.join(',')} configured); " + + "sample dropped from consensus" + if (params.calling.allow_caller_failure) { + log.warn(message) + } else { + error(message) + } + } + (empty + failed).unique().sort() + } + + // Always publish the dropped list, even when it is empty. The output is + // used below to order the all-dropped error after the file is published. + dropped_report = REPORT_DROPPED_SAMPLES(dropped_samples) - // Generate consensus - GENERATE_CONSENSUS(all_calls.join(bam_coverage), ref_genome, fai_index, gen_dict) + // Generate consensus. The per-sample consensus is always an internal BCF. + // Reference records are written whenever a later merge may need them; + // per-sample output that discards them sets --variants_only instead. + GENERATE_CONSENSUS( + all_calls.join(bam_intervals), + selected_callers.join(','), + ref_genome, + fai_index + ) - // Merge consensuses from different samples into single VCF of BCF + // A single output is one multi-sample file: merge the internal BCFs first. + // Both paths end in EXPORT_OUTPUT, which applies output.format, output.type + // and the final record transformations. + def all_dropped_message = + "All samples were dropped; see dropped_samples.txt" if (params.output.type == 'single') { - consensus_ch = params.output.format == 'vcf' ? - GENERATE_CONSENSUS.out.consensus_vcf : - GENERATE_CONSENSUS.out.consensus_bcf sample_rank = sample_run_ch .map { item -> item[0] } .collect() - .flatMap { ids -> ids.withIndex().collect { id, i -> tuple(id, i) } } - consensus = consensus_ch + .flatMap { ids -> + ids.withIndex().collect { id, i -> + tuple(id, i) + } + } + consensus = GENERATE_CONSENSUS.out.consensus_bcf + .combine(dropped_report) + .map { sample_id, bcf, _report -> + tuple(sample_id, bcf) + } .join(sample_rank) .toSortedList { item -> item[2] } - .map { items -> items.collect { item -> item[1] } } - MERGE_OUTPUTS(consensus) + .map { items -> + def ids = items.collect { item -> item[0] } + [ids.join(','), items.collect { item -> item[1] }] + } + .filter { item -> !item[1].isEmpty() } + .ifEmpty { error(all_dropped_message) } + merged = MERGE_OUTPUTS( + consensus, ref_genome, fai_index, dropped_samples + ) + EXPORT_OUTPUT( + merged.map { bcf -> tuple('consensus', bcf) }, + ref_genome + ) + } else { + consensus_bcf = GENERATE_CONSENSUS.out.consensus_bcf + .combine(dropped_report) + .map { sample_id, bcf, _report -> tuple(sample_id, bcf) } + .ifEmpty { error(all_dropped_message) } + EXPORT_OUTPUT( + consensus_bcf, + ref_genome + ) } } diff --git a/modules/local/calling_bcftools/main.nf b/modules/local/calling_bcftools/main.nf index 466c3a1..32747c2 100644 --- a/modules/local/calling_bcftools/main.nf +++ b/modules/local/calling_bcftools/main.nf @@ -1,14 +1,12 @@ process CALLING_BCFTOOLS { - maxForks 1 - cpus params.calling.cpu - errorStrategy 'ignore' - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' tag "${sample_id}" input: - tuple val(sample_id), path("tmp/input.bam"), path("tmp/coverage.bed") + tuple val(sample_id), path("tmp/input.bam"), path("tmp/input.bam.csi"), path("tmp/bed_chunks/*", arity: '1..*') path("tmp/ref_genome.fasta") path("tmp/ref_genome.fasta.fai") path("tmp/ref_genome.dict") @@ -17,23 +15,55 @@ process CALLING_BCFTOOLS { tuple val(sample_id), path("bcftools.bcf"), emit: calling_result script: + def caller = params.calling.bcftools ?: [:] + def call_args = caller.call?.extra_args ?: '' + def mpileup_args = caller.mpileup?.extra_args ?: '' + def max_depth = caller.mpileup?.max_depth + def max_idepth = caller.mpileup?.max_idepth + def min_snp = caller.min_snp_qual + def min_indel = caller.min_indel_qual + assert max_depth != null : 'calling.bcftools.mpileup.max_depth is required' + assert max_idepth != null : 'calling.bcftools.mpileup.max_idepth is required' + assert min_snp != null : 'calling.bcftools.min_snp_qual is required' + assert min_indel != null : 'calling.bcftools.min_indel_qual is required' + def qual_filter = "(TYPE=\\\"snp\\\" && QUAL<${min_snp}) || (TYPE!=\\\"snp\\\" && QUAL<${min_indel})" """ - mkdir -p tmp/bed_chunks/ mkdir -p tmp/calling_chunks/ - gatk SplitIntervals --java-options "-Djava.io.tmpdir=\$TMPDIR" --tmp-dir \$TMPDIR -R tmp/ref_genome.fasta -L tmp/coverage.bed --scatter-count ${task.cpus} -O tmp/bed_chunks/ - parallel -j ${task.cpus} 'gatk IntervalListToBed -I {} -O {//}/{/.}.bed.tmp; cut -f 1-3 {//}/{/.}.bed.tmp | tee {//}/{/.}.bed; rm {} {//}/{/.}.bed.tmp' ::: tmp/bed_chunks/* - - samtools index --threads ${task.cpus} --csi tmp/input.bam - cat < tmp/calling_chunks/sample_id.txt ${sample_id} EOF - parallel -j ${task.cpus} \ - 'bcftools mpileup ${params.calling.bcftools.mpileup.extra_args} -a FORMAT/AD,FORMAT/DP --count-orphans -Ou --fasta-ref tmp/ref_genome.fasta --regions-file {} tmp/input.bam | bcftools call ${params.calling.bcftools.call.extra_args} --multiallelic-caller --variants-only -Ou | bcftools reheader -s tmp/calling_chunks/sample_id.txt -f tmp/ref_genome.fasta.fai | bcftools filter -e"QUAL<${params.calling.min_snp_qual}" -Ou | bcftools norm --check-ref x -Ob --fasta-ref tmp/ref_genome.fasta -o tmp/calling_chunks/{#}.bcf; - bcftools index --csi tmp/calling_chunks/{#}.bcf' ::: tmp/bed_chunks/*.bed + cat > tmp/run_chunk.sh <<'EOF' + set -euo pipefail + bcftools mpileup --max-depth ${max_depth} --max-idepth ${max_idepth} \ + ${mpileup_args} \ + -a FORMAT/AD,FORMAT/DP --count-orphans -Ou \ + --fasta-ref tmp/ref_genome.fasta --regions-file "\$1" \ + tmp/input.bam \ + | bcftools call ${call_args} --multiallelic-caller --variants-only -Ou \ + | bcftools reheader -s tmp/calling_chunks/sample_id.txt \ + -f tmp/ref_genome.fasta.fai \ + | bcftools view -Ob -o "tmp/calling_chunks/\$2.bcf" + EOF + + # SIGKILLed children appear as signal 9, exit 137, or GATK's 247. + parallel --halt now,fail=1 --joblog tmp/parallel.log \ + -j ${task.cpus} bash tmp/run_chunk.sh {} {#} \ + ::: tmp/bed_chunks/*.bed || { + status=\$? + if awk 'NR > 1 && (\$7 == 137 || \$7 == 247 || \$8 == 9) { + found = 1 } END { exit !found }' tmp/parallel.log; then + exit 137 + fi + exit "\${status}" + } - bcftools concat --naive --threads ${task.cpus} -Ob tmp/calling_chunks/*.bcf | bcftools view --threads ${task.cpus} -Ou --min-alleles 2 --max-alleles 2 | bcftools sort -Ou -o bcftools.bcf + # The chunks are emitted in genomic order; ls -v keeps that order for + # concat, so no full sort is needed afterwards. + bcftools concat --naive --threads ${task.cpus} -Ob \$(ls -v tmp/calling_chunks/*.bcf) \ + | bcftools norm --threads ${task.cpus} -f tmp/ref_genome.fasta -m -any --atomize --check-ref x -Ou \ + | bcftools norm --threads ${task.cpus} -d exact -Ou \ + | bcftools filter -s LowQual -m + -e"${qual_filter}" -Ob -o bcftools.bcf """ } diff --git a/modules/local/calling_freebayes/main.nf b/modules/local/calling_freebayes/main.nf index 7fb311f..39cb18c 100644 --- a/modules/local/calling_freebayes/main.nf +++ b/modules/local/calling_freebayes/main.nf @@ -1,14 +1,12 @@ process CALLING_FREEBAYES { - maxForks 1 - cpus params.calling.cpu - errorStrategy 'ignore' - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' tag "${sample_id}" input: - tuple val(sample_id), path("tmp/input.bam"), path("tmp/coverage.bed") + tuple val(sample_id), path("tmp/input.bam"), path("tmp/input.bam.csi"), path("tmp/bed_chunks/*", arity: '1..*') path("tmp/ref_genome.fasta") path("tmp/ref_genome.fasta.fai") path("tmp/ref_genome.dict") @@ -17,23 +15,55 @@ process CALLING_FREEBAYES { tuple val(sample_id), path("freebayes.bcf"), emit: calling_result script: + def caller = params.calling.freebayes ?: [:] + def extra_args = caller.extra_args ?: '' + def use_best_n_alleles = caller.use_best_n_alleles + def limit_coverage = caller.limit_coverage + def min_snp = caller.min_snp_qual + def min_indel = caller.min_indel_qual + assert use_best_n_alleles != null : + 'calling.freebayes.use_best_n_alleles is required' + assert limit_coverage != null : + 'calling.freebayes.limit_coverage is required' + assert min_snp != null : 'calling.freebayes.min_snp_qual is required' + assert min_indel != null : 'calling.freebayes.min_indel_qual is required' + def qual_filter = "(TYPE=\\\"snp\\\" && QUAL<${min_snp}) || (TYPE!=\\\"snp\\\" && QUAL<${min_indel})" """ - mkdir -p tmp/bed_chunks/ mkdir -p tmp/calling_chunks/ - gatk SplitIntervals --java-options "-Djava.io.tmpdir=\$TMPDIR" --tmp-dir \$TMPDIR -R tmp/ref_genome.fasta -L tmp/coverage.bed --scatter-count ${task.cpus} -O tmp/bed_chunks/ - parallel -j ${task.cpus} 'gatk IntervalListToBed -I {} -O {//}/{/.}.bed.tmp; cut -f 1-3 {//}/{/.}.bed.tmp | tee {//}/{/.}.bed; rm {} {//}/{/.}.bed.tmp' ::: tmp/bed_chunks/* - - samtools index --threads ${task.cpus} --csi tmp/input.bam - cat < tmp/calling_chunks/sample_id.txt ${sample_id} EOF - parallel -j ${task.cpus} \ - 'freebayes --fasta-reference tmp/ref_genome.fasta --targets {} --hwe-priors-off --no-population-priors --dont-left-align-indels ${params.calling.freebayes.extra_args} --bam tmp/input.bam | bcftools reheader -s tmp/calling_chunks/sample_id.txt -f tmp/ref_genome.fasta.fai | bcftools filter -e"QUAL<${params.calling.min_snp_qual}" -Ou | bcftools norm --check-ref x -Ob --fasta-ref tmp/ref_genome.fasta -o tmp/calling_chunks/{#}.bcf - bcftools index --csi tmp/calling_chunks/{#}.bcf' ::: tmp/bed_chunks/*.bed + cat > tmp/run_chunk.sh <<'EOF' + set -euo pipefail + freebayes --fasta-reference tmp/ref_genome.fasta --targets "\$1" \ + --use-best-n-alleles ${use_best_n_alleles} \ + --limit-coverage ${limit_coverage} \ + --hwe-priors-off --no-population-priors \ + --dont-left-align-indels ${extra_args} --bam tmp/input.bam \ + | bcftools reheader -s tmp/calling_chunks/sample_id.txt \ + -f tmp/ref_genome.fasta.fai \ + | bcftools view -Ob -o "tmp/calling_chunks/\$2.bcf" + EOF + + # SIGKILLed children appear as signal 9, exit 137, or GATK's 247. + parallel --halt now,fail=1 --joblog tmp/parallel.log \ + -j ${task.cpus} bash tmp/run_chunk.sh {} {#} \ + ::: tmp/bed_chunks/*.bed || { + status=\$? + if awk 'NR > 1 && (\$7 == 137 || \$7 == 247 || \$8 == 9) { + found = 1 } END { exit !found }' tmp/parallel.log; then + exit 137 + fi + exit "\${status}" + } - bcftools concat --naive --threads ${task.cpus} -Ob tmp/calling_chunks/*.bcf | bcftools view --threads ${task.cpus} -Ou --min-alleles 2 --max-alleles 2 | bcftools sort -Ou -o freebayes.bcf + # The chunks are emitted in genomic order; ls -v keeps that order for + # concat, so no full sort is needed afterwards. + bcftools concat --naive --threads ${task.cpus} -Ob \$(ls -v tmp/calling_chunks/*.bcf) \ + | bcftools norm --threads ${task.cpus} -f tmp/ref_genome.fasta -m -any --atomize --check-ref x -Ou \ + | bcftools norm --threads ${task.cpus} -d exact -Ou \ + | bcftools filter -s LowQual -m + -e"${qual_filter}" -Ob -o freebayes.bcf """ } diff --git a/modules/local/calling_gatk/main.nf b/modules/local/calling_gatk/main.nf index 71d7029..795411c 100644 --- a/modules/local/calling_gatk/main.nf +++ b/modules/local/calling_gatk/main.nf @@ -1,14 +1,12 @@ process CALLING_GATK { - maxForks 1 - cpus params.calling.cpu - errorStrategy 'ignore' - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_high' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' tag "${sample_id}" input: - tuple val(sample_id), path("tmp/input.bam"), path("tmp/coverage.bed") + tuple val(sample_id), path("tmp/input.bam"), path("tmp/input.bam.csi"), path("tmp/bed_chunks/*", arity: '1..*') path("tmp/ref_genome.fasta") path("tmp/ref_genome.fasta.fai") path("tmp/ref_genome.dict") @@ -17,23 +15,68 @@ process CALLING_GATK { tuple val(sample_id), path("gatk.bcf"), emit: calling_result script: + def caller = params.calling.gatk ?: [:] + def extra_args = caller.extra_args ?: '' + def interval_padding = caller.interval_padding != null ? caller.interval_padding : 150 + def min_snp = caller.min_snp_qual + def min_indel = caller.min_indel_qual + def heap_option = ResourceUtils.javaHeapOption(task.memory, task.cpus) + assert min_snp != null : 'calling.gatk.min_snp_qual is required' + assert min_indel != null : 'calling.gatk.min_indel_qual is required' + def call_confidence = [min_snp, min_indel].min() + def qual_filter = "(TYPE=\\\"snp\\\" && QUAL<${min_snp}) || (TYPE!=\\\"snp\\\" && QUAL<${min_indel})" """ - mkdir -p tmp/bed_chunks/ mkdir -p tmp/calling_chunks/ - gatk SplitIntervals --java-options "-Djava.io.tmpdir=\$TMPDIR" --tmp-dir \$TMPDIR -R tmp/ref_genome.fasta -L tmp/coverage.bed --scatter-count ${task.cpus} -O tmp/bed_chunks/ - parallel -j ${task.cpus} 'gatk IntervalListToBed -I {} -O {//}/{/.}.bed.tmp; cut -f 1-3 {//}/{/.}.bed.tmp | tee {//}/{/.}.bed; rm {} {//}/{/.}.bed.tmp' ::: tmp/bed_chunks/* - - samtools index --threads ${task.cpus} --csi tmp/input.bam - cat < tmp/calling_chunks/sample_id.txt ${sample_id} EOF - parallel -j ${task.cpus} \ - 'gatk HaplotypeCaller ${params.calling.gatk.extra_args} --java-options "-Djava.io.tmpdir=\$TMPDIR" --do-not-run-physical-phasing true --smith-waterman FASTEST_AVAILABLE --create-output-variant-index false --native-pair-hmm-threads 1 --input tmp/input.bam --reference tmp/ref_genome.fasta --output /dev/stdout --intervals {} --sequence-dictionary tmp/ref_genome.dict --tmp-dir \$TMPDIR | bcftools reheader -s tmp/calling_chunks/sample_id.txt -f tmp/ref_genome.fasta.fai | bcftools filter -e"QUAL<${params.calling.min_snp_qual}" -Ou | bcftools norm --check-ref x -Ob --fasta-ref tmp/ref_genome.fasta -o tmp/calling_chunks/{#}.bcf - bcftools index --csi tmp/calling_chunks/{#}.bcf' ::: tmp/bed_chunks/*.bed + # The padding lets GATK assemble across chunk edges, but it also makes + # neighbouring chunks call the same boundary sites (possibly with + # different allele representations, which -d exact cannot deduplicate). + # Restrict each chunk to its own unpadded interval before concatenating; + # -T streams the BED and needs no index. + cat > tmp/run_chunk.sh <<'EOF' + set -euo pipefail + gatk HaplotypeCaller \ + --standard-min-confidence-threshold-for-calling ${call_confidence} \ + ${extra_args} \ + --java-options "-Djava.io.tmpdir=\$TMPDIR ${heap_option}" \ + --do-not-run-physical-phasing true \ + --smith-waterman FASTEST_AVAILABLE \ + --create-output-variant-index false \ + --native-pair-hmm-threads 1 \ + --input tmp/input.bam --reference tmp/ref_genome.fasta \ + --output /dev/stdout --intervals "\$1" \ + --interval-padding ${interval_padding} \ + --sequence-dictionary tmp/ref_genome.dict \ + --tmp-dir "\$TMPDIR" \ + | bcftools reheader -s tmp/calling_chunks/sample_id.txt \ + -f tmp/ref_genome.fasta.fai \ + | bcftools view -T "\$1" -Ob -o "tmp/calling_chunks/\$2.bcf" + EOF + + # SIGKILLed children appear as signal 9, exit 137, or GATK's 247. + parallel --halt now,fail=1 --joblog tmp/parallel.log \ + -j ${task.cpus} bash tmp/run_chunk.sh {} {#} \ + ::: tmp/bed_chunks/*.bed || { + status=\$? + if awk 'NR > 1 && (\$7 == 137 || \$7 == 247 || \$8 == 9) { + found = 1 } END { exit !found }' tmp/parallel.log; then + exit 137 + fi + exit "\${status}" + } - bcftools concat --naive --threads ${task.cpus} -Ob tmp/calling_chunks/*.bcf | bcftools view --threads ${task.cpus} -Ou --min-alleles 2 --max-alleles 2 | bcftools sort -Ou -o gatk.bcf + # Each chunk now contributes only its own interval, so the concatenated + # records are disjoint and ordered. Sort only this caller as a guard for + # any remaining boundary order; the exact chunk targets of + # bcftools/freebayes are already ordered. + bcftools concat --naive --threads ${task.cpus} -Ob \$(ls -v tmp/calling_chunks/*.bcf) \ + | bcftools norm --threads ${task.cpus} -f tmp/ref_genome.fasta -m -any --atomize --check-ref x -Ou \ + | bcftools sort -Ou \ + | bcftools norm --threads ${task.cpus} -d exact -Ou \ + | bcftools filter -s LowQual -m + -e"${qual_filter}" -Ob -o gatk.bcf """ } diff --git a/modules/local/create_fai_index/main.nf b/modules/local/create_fai_index/main.nf index ead6dcb..96c8280 100644 --- a/modules/local/create_fai_index/main.nf +++ b/modules/local/create_fai_index/main.nf @@ -1,8 +1,7 @@ process CREATE_FAI_INDEX { - maxForks 1 - cpus Runtime.runtime.availableProcessors() - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_low' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' input: path(ref_genome), name: "tmp/ref_genome.fasta" @@ -12,7 +11,9 @@ process CREATE_FAI_INDEX { script: """ - samtools faidx --threads ${task.cpus} --output ref_genome.fasta.fai tmp/ref_genome.fasta + samtools faidx --threads ${task.cpus} \ + --output ref_genome.fasta.fai tmp/ref_genome.fasta + check_contig_lengths.py ref_genome.fasta.fai """ } diff --git a/modules/local/create_seq_dict/main.nf b/modules/local/create_seq_dict/main.nf index b157b56..108b7bd 100644 --- a/modules/local/create_seq_dict/main.nf +++ b/modules/local/create_seq_dict/main.nf @@ -1,8 +1,8 @@ process CREATE_SEQ_DICT { - maxForks 1 + label 'process_low' cpus 1 - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' input: path(ref_genome), name: "tmp/ref_genome.fasta" @@ -11,8 +11,12 @@ process CREATE_SEQ_DICT { path("ref_genome.dict"), emit: gen_dict script: + def heap_option = ResourceUtils.javaHeapOption(task.memory) """ - gatk CreateSequenceDictionary -R tmp/ref_genome.fasta -O ref_genome.dict + gatk CreateSequenceDictionary \ + --java-options "-Djava.io.tmpdir=\$TMPDIR ${heap_option}" \ + --TMP_DIR "\$TMPDIR" \ + -R tmp/ref_genome.fasta -O ref_genome.dict """ } diff --git a/modules/local/decompress_assembly/main.nf b/modules/local/decompress_assembly/main.nf index c8f25da..48d9c36 100644 --- a/modules/local/decompress_assembly/main.nf +++ b/modules/local/decompress_assembly/main.nf @@ -1,8 +1,7 @@ process DECOMPRESS_ASSEMBLY { - maxForks 1 - cpus Runtime.runtime.availableProcessors() - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_low' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' input: path(ref_genome), name: "tmp/ref_genome.fasta.gz" @@ -12,6 +11,7 @@ process DECOMPRESS_ASSEMBLY { script: """ - bgzip --threads ${task.cpus} --decompress -o ref_genome.fasta tmp/ref_genome.fasta.gz + bgzip --threads ${task.cpus} --decompress \ + -o ref_genome.fasta tmp/ref_genome.fasta.gz """ } \ No newline at end of file diff --git a/modules/local/export_output/main.nf b/modules/local/export_output/main.nf new file mode 100644 index 0000000..11f23fa --- /dev/null +++ b/modules/local/export_output/main.nf @@ -0,0 +1,57 @@ +process EXPORT_OUTPUT { + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' + + tag "${sample_id}" + + // Only the final export uses output.format and output.type. Every file + // consumed here is an internal BCF produced by GENERATE_CONSENSUS or + // MERGE_OUTPUTS. + publishDir( + params.output.type == 'single' + ? params.output.directory + : "${params.output.directory}/per_sample", + mode: params.output.publish_mode, + pattern: '*.bcf', + enabled: params.output.format == 'bcf' + ) + publishDir( + params.output.type == 'single' + ? params.output.directory + : "${params.output.directory}/per_sample", + mode: params.output.publish_mode, + pattern: '*.vcf.gz', + enabled: params.output.format == 'vcf' + ) + + input: + tuple val(sample_id), path("tmp/input.bcf") + path("tmp/ref_genome.fasta") + + output: + tuple val(sample_id), path("${sample_id}.bcf"), optional: true, emit: bcf + tuple val(sample_id), path("${sample_id}.vcf.gz"), optional: true, emit: vcf + + script: + def extension = params.output.format == 'vcf' ? 'vcf.gz' : 'bcf' + def output_name = "${sample_id}.${extension}" + def remove_invariant = params.output.remove_invariant + ? '--remove-invariant' + : '' + def split_multiallelic = params.output.split_multiallelic + ? '--split-multiallelic' + : '' + """ + printf '%s\n' tmp/input.bcf > tmp/export_list.txt + + merge_outputs.sh \ + --file-list tmp/export_list.txt \ + --reference tmp/ref_genome.fasta \ + --output ${output_name} \ + --format ${params.output.format} \ + --cpus ${task.cpus} \ + ${remove_invariant} \ + ${split_multiallelic} + """ +} diff --git a/modules/local/generate_consensus/main.nf b/modules/local/generate_consensus/main.nf index 398e39a..854d3f1 100644 --- a/modules/local/generate_consensus/main.nf +++ b/modules/local/generate_consensus/main.nf @@ -1,49 +1,104 @@ process GENERATE_CONSENSUS { - cpus params.consensus.cpu - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' tag "${sample_id}" - publishDir "${params.output.directory}/per_sample", mode: 'move', pattern: '*.bcf', enabled: params.output.type == 'sample' && params.output.format == 'bcf' - publishDir "${params.output.directory}/per_sample", mode: 'move', pattern: '*.vcf.gz', enabled: params.output.type == 'sample' && params.output.format == 'vcf' - input: - tuple val(sample_id), path('tmp/?.bcf', arity: '1..*'), path("tmp/input.bam"), path("tmp/coverage.bed") + // The `dir/*` staging pattern preserves the caller file names (bcftools.bcf, + // freebayes.bcf, gatk.bcf), so the consensus step knows which caller each + // file came from even when a caller is missing. + tuple val(sample_id), val(received_callers), path('tmp/callers/*', arity: '1..*'), path("tmp/input.bam"), path("tmp/input.bam.csi"), path("tmp/bed_chunks/*", arity: '1..*') + val configured_callers path("tmp/ref_genome.fasta") path("tmp/ref_genome.fasta.fai") - path("tmp/ref_genome.dict") output: - tuple val(sample_id), path("${sample_id}.bcf"), emit: consensus_bcf, optional: true - tuple val(sample_id), path("${sample_id}.vcf.gz"), emit: consensus_vcf, optional: true + // Internal representation only. The requested output format is applied by + // EXPORT_OUTPUT after the optional multi-sample merge. + tuple val(sample_id), path("${sample_id}.bcf"), emit: consensus_bcf script: - def output_format = params.output.format == 'vcf' ? "-Oz -o ${sample_id}.vcf.gz" : "-Ob -o ${sample_id}.bcf" - def filter_invariant = params.output.remove_invariant && params.output.type == 'sample' ? "| bcftools filter --threads ${task.cpus} -e 'COUNT(GT=\"RR\")=N_SAMPLES' -Ou" : "" - def split_multiallelic = params.output.split_multiallelic ? "--split_multiallelic" : "" - def remove_invariant = params.output.remove_invariant ? "--remove_invariant" : "" - def fill_tags = "AN,AC,AF,NS,AC_Hom,AC_Het,MAF,TYPE,F_MISSING,'DP:1=int(sum(FORMAT/DP))'" + def configured = configured_callers.split(',').collect { caller -> caller.trim() }.findAll { caller -> caller } + def received = received_callers.split(',').collect { caller -> caller.trim() }.findAll { caller -> caller } + def failed = configured.findAll { caller -> !(caller in received) } + if (failed && params.consensus.threshold > received.size()) { + error "Sample ${sample_id}: consensus.threshold (${params.consensus.threshold}) exceeds the ${received.size()} surviving caller(s) [${received.join(',')}] after failure(s) of ${failed.join(',')}" + } + def input_args = received.collect { caller -> "--input ${caller}=tmp/callers/${caller}.bcf" }.join(' ') + def compress_blocks = params.output.compress_reference_blocks + ? "--compress_reference_blocks" + : "" + // Per-sample export discards reference records when remove_invariant is + // true, so skip writing them at all. A merged output still needs them for + // bcftools merge --gvcf to assign ALT depths across samples. + def variants_only = params.output.type == 'sample' + && params.output.remove_invariant + def variants_only_arg = variants_only ? "--variants_only" : "" + def absent_vote = params.consensus.absent_vote != null ? params.consensus.absent_vote : 'ref' + def filtered_vote = params.consensus.filtered_vote != null + ? params.consensus.filtered_vote + : 'abstain' + def tie_policy = params.consensus.tie_policy != null ? params.consensus.tie_policy : 'nocall' + def min_coverage = params.coverage.min_coverage != null + ? params.coverage.min_coverage + : 5 """ - mkdir -p tmp/bed_chunks/ - mkdir -p tmp/vcf_chunks/ - - gatk SplitIntervals --java-options "-Djava.io.tmpdir=\$TMPDIR" --tmp-dir \$TMPDIR -R tmp/ref_genome.fasta -L tmp/coverage.bed --scatter-count ${task.cpus} -O tmp/bed_chunks/ - parallel -j ${task.cpus} 'gatk IntervalListToBed -I {} -O {//}/{/.}.bed.tmp; cut -f 1-3 {//}/{/.}.bed.tmp | tee {//}/{/.}.bed; rm {} {//}/{/.}.bed.tmp' ::: tmp/bed_chunks/* + mkdir -p tmp/bcf_chunks/ - samtools index --threads ${task.cpus} --csi tmp/input.bam + # SIGKILLed children appear as signal 9, exit 137, or GATK's 247. + parallel --halt now,fail=1 --joblog tmp/parallel.log \ + -j ${task.cpus} 'bcftools index --threads 1 --csi {}' \ + ::: tmp/callers/*.bcf || { + status=\$? + if awk 'NR > 1 && (\$7 == 137 || \$7 == 247 || \$8 == 9) { + found = 1 } END { exit !found }' tmp/parallel.log; then + exit 137 + fi + exit "\${status}" + } - parallel -j ${task.cpus} 'bcftools index --threads 1 --csi {}' ::: tmp/*.bcf + cat > tmp/run_chunk.sh <<'EOF' + set -euo pipefail + bgzip --threads 1 "\$1" + tabix --threads 1 --csi -p bed "\$1.gz" + generate_consensus.py ${input_args} \ + --callers "${configured_callers}" \ + --bed "\$1.gz" \ + --bam tmp/input.bam \ + --output "tmp/bcf_chunks/\$2.bcf" \ + --sample_name "${sample_id}" \ + --reference tmp/ref_genome.fasta \ + --consensus_threshold "${params.consensus.threshold}" \ + --absent_vote "${absent_vote}" \ + --filtered_vote "${filtered_vote}" \ + --tie_policy "${tie_policy}" \ + --min_coverage "${min_coverage}" \ + --version "${workflow.manifest.version}" \ + ${compress_blocks} \ + ${variants_only_arg} + EOF - parallel -j ${task.cpus} \ - 'bgzip --threads 1 {} - tabix --threads 1 --csi -p bed {}.gz - generate_consensus.py --input tmp/*.bcf --bed {}.gz --bam tmp/input.bam --output tmp/vcf_chunks/{#}.bcf --sample_name "${sample_id}" --reference tmp/ref_genome.fasta --consensus_threshold "${params.consensus.threshold}" --version "${workflow.manifest.version}" ${split_multiallelic} ${remove_invariant} - bcftools index --threads 1 --csi tmp/vcf_chunks/{#}.bcf' ::: tmp/bed_chunks/*.bed + # SIGKILLed children appear as signal 9, exit 137, or GATK's 247. + parallel --halt now,fail=1 --joblog tmp/parallel.log \ + -j ${task.cpus} bash tmp/run_chunk.sh {} {#} \ + ::: tmp/bed_chunks/*.bed || { + status=\$? + if awk 'NR > 1 && (\$7 == 137 || \$7 == 247 || \$8 == 9) { + found = 1 } END { exit !found }' tmp/parallel.log; then + exit 137 + fi + exit "\${status}" + } - bcftools concat --naive --threads ${task.cpus} -Ob tmp/vcf_chunks/*.bcf ${filter_invariant} \ - | bcftools +fill-tags -Ou -- -t ${fill_tags} \ - | bcftools sort ${output_format} + # The interval chunks are consumed in BED order; ls -v preserves that order + # through concat, so no full sort is needed afterwards. The per-sample + # consensus stays split by ALT; reference records are written unless the + # per-sample export would drop them. Output filtering and re-joining happen + # once in EXPORT_OUTPUT. + bcftools concat --naive --threads ${task.cpus} -Ob \ + -o ${sample_id}.bcf \$(ls -v tmp/bcf_chunks/*.bcf) """ } diff --git a/modules/local/generate_coverage/main.nf b/modules/local/generate_coverage/main.nf index 26dc760..be07502 100644 --- a/modules/local/generate_coverage/main.nf +++ b/modules/local/generate_coverage/main.nf @@ -1,22 +1,91 @@ process GENERATE_COVERAGE { + label 'process_low' cpus 1 - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' tag "${sample_id}" input: - tuple val(sample_id), path("tmp/input.bam") - path(include_bed) - path(exclude_bed) + tuple val(sample_id), path("tmp/input.bam"), path("tmp/input.bam.csi") + path(include_bed, stageAs: 'include/*') + path(exclude_bed, stageAs: 'exclude/*') + path("tmp/ref_genome.fasta.fai") output: tuple val(sample_id), path("coverage.bed"), emit: coverage script: - def ibed = include_bed.name != "N1_FILE" ? "-b ${include_bed}" : "" - def ebed = exclude_bed.name != "NO_FILE" ? "| bedops --difference - ${exclude_bed}" : "" + def min_coverage = params.coverage.min_coverage != null + ? params.coverage.min_coverage as int + : 5 + def quantize = min_coverage > 0 ? "0:${min_coverage}:" : "0:1:" + def keep_bin = min_coverage > 0 ? "${min_coverage}:inf" : "1:inf" + def on_empty = params.coverage.on_empty ?: 'fail' + def empty_message = "Sample ${sample_id}: coverage.bed is empty: " + + "no region reaches coverage.min_coverage=${min_coverage}" + // sort-bed rejects track/browser lines and # comments. filter_bed.py + // drops them and rejects contigs that are not present in the reference + // .fai with an actionable error. + def filter_bed = 'filter_bed.py tmp/ref_genome.fasta.fai' + // stageAs prefixes the path with the staging directory, so sentinel + // inputs are detected from the last path component. + def include_name = include_bed.toString().tokenize('/')[-1] + def exclude_name = exclude_bed.toString().tokenize('/')[-1] + def sort_include = include_name != "N1_FILE" + ? "${filter_bed} \"${include_bed}\" | " + + "sort-bed - > tmp/include.sorted.bed" + : "" + def sort_exclude = exclude_name != "NO_FILE" + ? "${filter_bed} \"${exclude_bed}\" | " + + "sort-bed - > tmp/exclude.sorted.bed" + : "" """ - samtools depth -J ${ibed} --threads ${task.cpus} tmp/input.bam | awk '\$3 >= ${params.coverage.min_coverage} {print \$1,\$2-1,\$2}' ${ebed} | bedops --merge - > coverage.bed + set -o pipefail + mkdir -p tmp + + ${sort_include} + ${sort_exclude} + + # mosdepth emits interval runs with a quantized depth label. Keep the run + # whose lower bound is coverage.min_coverage, then restrict to the requested + # regions. --no-per-base avoids the per-base intermediate file. + mosdepth \ + --no-per-base \ + --quantize ${quantize} \ + --threads ${task.cpus} \ + tmp/mosdepth \ + tmp/input.bam + + zcat tmp/mosdepth.quantized.bed.gz \\ + | awk -v OFS='\\t' -v keep="${keep_bin}" '\$4 == keep {print \$1, \$2, \$3}' \\ + > tmp/quantized.bed + sort-bed tmp/quantized.bed > tmp/quantized.sorted.bed + + if [ -f tmp/include.sorted.bed ]; then + bedops --intersect tmp/quantized.sorted.bed tmp/include.sorted.bed > tmp/selected.bed + else + cp tmp/quantized.sorted.bed tmp/selected.bed + fi + + if [ -f tmp/exclude.sorted.bed ]; then + bedops --difference tmp/selected.bed tmp/exclude.sorted.bed > tmp/filtered.bed + else + cp tmp/selected.bed tmp/filtered.bed + fi + + bedops --merge tmp/filtered.bed > coverage.bed + + # bedops writes an empty file when no run reaches the threshold. Fail with + # the sample id and threshold, or let main.nf drop the sample in skip mode. + if [ ! -s coverage.bed ]; then + if [ "${on_empty}" = "skip" ]; then + >&2 echo "WARNING: ${empty_message}" + else + >&2 echo "ERROR: ${empty_message}" + >&2 echo "Set coverage.on_empty: skip to drop this sample." + exit 1 + fi + fi """ } diff --git a/modules/local/mapping_bowtie2/main.nf b/modules/local/mapping_bowtie2/main.nf index 08bdf91..14901a1 100644 --- a/modules/local/mapping_bowtie2/main.nf +++ b/modules/local/mapping_bowtie2/main.nf @@ -1,43 +1,99 @@ process MAPPING_BOWTIE2 { - maxForks 1 - cpus params.mapping.cpu - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' tag "${sample_id}" input: tuple val(sample_id), path("tmp/read1"), path("tmp/read2"), path("tmp/read3") - path("tmp/ref_genome_real.fasta") + path(index_files, stageAs: 'index/*') path("tmp/ref_genome.fasta") path("tmp/ref_genome.dict") path("tmp/ref_genome.fasta.fai") output: - tuple val(sample_id), path("mapping.bam"), emit: bam + tuple val(sample_id), path("mapping.bam"), + path("mapping.bam.csi"), + path("mapping.raw.flagstat"), emit: bam script: - def rmdup = params.rmdup.enabled ? 'picard MarkDuplicates -I tmp/stage1.bam -O tmp/stage2.bam -M /dev/null --VALIDATION_STRINGENCY SILENT --REMOVE_DUPLICATES true' : 'mv tmp/stage1.bam tmp/stage2.bam' - def lai = params.left_align_indels.enabled ? 'gatk LeftAlignIndels -I tmp/stage2.bam -O mapping.bam -R tmp/ref_genome.fasta --sequence-dictionary tmp/ref_genome.dict -OBI false' : 'mv tmp/stage2.bam mapping.bam' - def fixmate = params.input.reads_type != 'se' ? "| samtools sort --threads ${task.cpus} -n | samtools fixmate --threads ${task.cpus} -m - -" : '' - def reads = params.input.reads_type == 'se' ? "-U tmp/read1" : - params.input.reads_type == 'pe' ? "-1 tmp/read1 -2 tmp/read2" : + def rmdup_mode = CLIParamsValidation.rmdup_mode(params.rmdup) + def rmdup = rmdup_mode == 'none' ? '' : "--rmdup-mode ${rmdup_mode}" + def left_align = params.left_align_indels.enabled ? '--left-align' : '' + def platform = params.mapping.platform ?: 'ILLUMINA' + def library = params.mapping.library ?: sample_id + def index_prefix = params.input.reference_index_prefix ?: + file(params.input.reference_genome).name + def java_options = "-Djava.io.tmpdir=\$TMPDIR " + + ResourceUtils.javaHeapOption(task.memory) + // The aligner and BAM post-processing run concurrently in one pipe, so + // split the requested CPUs. samtools view between them adds one thread. + def cpus = task.cpus as int + def aligner_cpus = Math.max(1, (cpus + 1).intdiv(2)) + def postprocess_cpus = Math.max(1, cpus - aligner_cpus) + // The aligner index is resident while BAM post-processing runs in the + // same pipe. Reserve the staged index size plus 1 GiB for aligner and + // pipe buffers before sort receives half of the remaining task memory; + // mapping.aligner_reserve_mb overrides the estimate. + def reserve_mb = params.mapping.aligner_reserve_mb ?: 0 + if (reserve_mb <= 0) { + def index_paths = index_files instanceof List + ? index_files : [index_files] + def index_bytes = index_paths + .collect { index_path -> index_path.size() } + .sum() ?: 0 + reserve_mb = index_bytes.intdiv(1048576) + 1024 + } + def memory_mb = task.memory ? task.memory.toMega() : 0 + def memory_option = memory_mb > 0 + ? "--memory-mb ${memory_mb} --reserve-mb ${reserve_mb}" : '' + def reads = params.input.reads_type == 'se' ? "-U tmp/read1" : + params.input.reads_type == 'pe' ? "-1 tmp/read1 -2 tmp/read2" : params.input.reads_type == 'mx' ? "-1 tmp/read1 -2 tmp/read2 -U tmp/read3" : '' """ - GENOME_BASENAME=\$(basename \$(realpath tmp/ref_genome_real.fasta)) - + set -o pipefail + + # flagstat reads a copy of the aligner output before the post-processing + # filters, so the report keeps the true mapping rate. The trap stops the + # reader if the pipeline fails before tee opens the FIFO; otherwise the + # blocked reader could keep the task from finishing. + mkfifo tmp/raw.flagstat.fifo + samtools flagstat - > mapping.raw.flagstat < tmp/raw.flagstat.fifo & + raw_flagstat_pid=\$! + cleanup_flagstat() { + if [ -n "\$raw_flagstat_pid" ]; then + kill "\$raw_flagstat_pid" 2>/dev/null || true + wait "\$raw_flagstat_pid" 2>/dev/null || true + fi + } + trap cleanup_flagstat EXIT + bowtie2 \ ${params.mapping.extra_args} \ - --threads ${task.cpus} \ - --rg-id ${sample_id} \ - --rg SM:${sample_id} \ - -x ${params.input.reference_index_dir}/\${GENOME_BASENAME} \ + --threads ${aligner_cpus} \ + --rg-id "${sample_id}" \ + --rg "SM:${sample_id}" \ + --rg "PL:${platform}" \ + --rg "LB:${library}" \ + -x "index/${index_prefix}" \ ${reads} | \ - samtools view -F 4 --threads ${task.cpus} -b -q ${params.bam_filter.min_map_qual} ${fixmate} | \ - samtools sort --threads ${task.cpus} -o tmp/stage1.bam - - ${rmdup} - - ${lai} + samtools view -u | \ + tee tmp/raw.flagstat.fifo | \ + postprocess_bam.sh \ + --input - \ + --output mapping.bam \ + --reference tmp/ref_genome.fasta \ + --dict tmp/ref_genome.dict \ + --fai tmp/ref_genome.fasta.fai \ + --mapq ${params.bam_filter.min_map_qual} \ + --threads ${postprocess_cpus} \ + ${memory_option} \ + --java-options "${java_options}" \ + ${rmdup} \ + ${left_align} + + wait "\${raw_flagstat_pid}" + raw_flagstat_pid='' """ } diff --git a/modules/local/mapping_bwa/main.nf b/modules/local/mapping_bwa/main.nf index 8b91db1..63dea79 100644 --- a/modules/local/mapping_bwa/main.nf +++ b/modules/local/mapping_bwa/main.nf @@ -1,42 +1,96 @@ process MAPPING_BWA { - maxForks 1 - cpus params.mapping.cpu - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' - + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' + tag "${sample_id}" input: tuple val(sample_id), path("tmp/read1"), path("tmp/read2"), path("tmp/read3") - path("tmp/ref_genome_real.fasta") + path(index_files, stageAs: 'index/*') path("tmp/ref_genome.fasta") path("tmp/ref_genome.dict") path("tmp/ref_genome.fasta.fai") output: - tuple val(sample_id), path("mapping.bam"), emit: bam + tuple val(sample_id), path("mapping.bam"), + path("mapping.bam.csi"), + path("mapping.raw.flagstat"), emit: bam script: - def rg_id = "@RG\\tID:${sample_id}\\tSM:${sample_id}" - def rmdup = params.rmdup.enabled ? 'picard MarkDuplicates -I tmp/stage1.bam -O tmp/stage2.bam -M /dev/null --VALIDATION_STRINGENCY SILENT --REMOVE_DUPLICATES true' : 'mv tmp/stage1.bam tmp/stage2.bam' - def lai = params.left_align_indels.enabled ? 'gatk LeftAlignIndels -I tmp/stage2.bam -O mapping.bam -R tmp/ref_genome.fasta --sequence-dictionary tmp/ref_genome.dict -OBI false' : 'mv tmp/stage2.bam mapping.bam' - def fixmate = params.input.reads_type != 'se' ? "| samtools sort --threads ${task.cpus} -n | samtools fixmate --threads ${task.cpus} -m - -" : '' - def reads = params.input.reads_type == 'se' ? "tmp/read1" : + def rmdup_mode = CLIParamsValidation.rmdup_mode(params.rmdup) + def rmdup = rmdup_mode == 'none' ? '' : "--rmdup-mode ${rmdup_mode}" + def left_align = params.left_align_indels.enabled ? '--left-align' : '' + def platform = params.mapping.platform ?: 'ILLUMINA' + def library = params.mapping.library ?: sample_id + def rg_id = "@RG\\tID:${sample_id}\\tSM:${sample_id}\\tPL:${platform}\\tLB:${library}" + def index_prefix = params.input.reference_index_prefix ?: + file(params.input.reference_genome).name + def java_options = "-Djava.io.tmpdir=\$TMPDIR " + + ResourceUtils.javaHeapOption(task.memory) + // The aligner and BAM post-processing run concurrently in one pipe, so + // split the requested CPUs. samtools view between them adds one thread. + def cpus = task.cpus as int + def aligner_cpus = Math.max(1, (cpus + 1).intdiv(2)) + def postprocess_cpus = Math.max(1, cpus - aligner_cpus) + // The aligner index is resident while BAM post-processing runs in the + // same pipe. Reserve the staged index size plus 1 GiB for aligner and + // pipe buffers before sort receives half of the remaining task memory; + // mapping.aligner_reserve_mb overrides the estimate. + def reserve_mb = params.mapping.aligner_reserve_mb ?: 0 + if (reserve_mb <= 0) { + def index_paths = index_files instanceof List + ? index_files : [index_files] + def index_bytes = index_paths + .collect { index_path -> index_path.size() } + .sum() ?: 0 + reserve_mb = index_bytes.intdiv(1048576) + 1024 + } + def memory_mb = task.memory ? task.memory.toMega() : 0 + def memory_option = memory_mb > 0 + ? "--memory-mb ${memory_mb} --reserve-mb ${reserve_mb}" : '' + def reads = params.input.reads_type == 'se' ? "tmp/read1" : params.input.reads_type == 'pe' ? "tmp/read1 tmp/read2" : '' """ - GENOME_BASENAME=\$(basename \$(realpath tmp/ref_genome_real.fasta)) - + set -o pipefail + + # flagstat reads a copy of the aligner output before the post-processing + # filters, so the report keeps the true mapping rate. The trap stops the + # reader if the pipeline fails before tee opens the FIFO; otherwise the + # blocked reader could keep the task from finishing. + mkfifo tmp/raw.flagstat.fifo + samtools flagstat - > mapping.raw.flagstat < tmp/raw.flagstat.fifo & + raw_flagstat_pid=\$! + cleanup_flagstat() { + if [ -n "\$raw_flagstat_pid" ]; then + kill "\$raw_flagstat_pid" 2>/dev/null || true + wait "\$raw_flagstat_pid" 2>/dev/null || true + fi + } + trap cleanup_flagstat EXIT + bwa mem \ ${params.mapping.extra_args} \ - -t ${task.cpus} \ + -t ${aligner_cpus} \ -R "${rg_id}" \ - ${params.input.reference_index_dir}/\${GENOME_BASENAME} \ + "index/${index_prefix}" \ ${reads} | \ - samtools view -F 4 --threads ${task.cpus} -b -q ${params.bam_filter.min_map_qual} ${fixmate} | \ - samtools sort --threads ${task.cpus} -o tmp/stage1.bam - - ${rmdup} - - ${lai} + samtools view -u | \ + tee tmp/raw.flagstat.fifo | \ + postprocess_bam.sh \ + --input - \ + --output mapping.bam \ + --reference tmp/ref_genome.fasta \ + --dict tmp/ref_genome.dict \ + --fai tmp/ref_genome.fasta.fai \ + --mapq ${params.bam_filter.min_map_qual} \ + --threads ${postprocess_cpus} \ + ${memory_option} \ + --java-options "${java_options}" \ + ${rmdup} \ + ${left_align} + + wait "\${raw_flagstat_pid}" + raw_flagstat_pid='' """ } diff --git a/modules/local/mapping_minimap2/main.nf b/modules/local/mapping_minimap2/main.nf index 93895e3..88e7c68 100644 --- a/modules/local/mapping_minimap2/main.nf +++ b/modules/local/mapping_minimap2/main.nf @@ -1,43 +1,99 @@ process MAPPING_MINIMAP2 { - maxForks 1 - cpus params.mapping.cpu - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' - + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' + tag "${sample_id}" input: tuple val(sample_id), path("tmp/read1"), path("tmp/read2"), path("tmp/read3") - path("tmp/ref_genome_real.fasta") + path(index_files, stageAs: 'index/*') path("tmp/ref_genome.fasta") path("tmp/ref_genome.dict") path("tmp/ref_genome.fasta.fai") output: - tuple val(sample_id), path("mapping.bam"), emit: bam + tuple val(sample_id), path("mapping.bam"), + path("mapping.bam.csi"), + path("mapping.raw.flagstat"), emit: bam script: - def rg_id = "@RG\\tID:${sample_id}\\tSM:${sample_id}" - def rmdup = params.rmdup.enabled ? 'picard MarkDuplicates -I tmp/stage1.bam -O tmp/stage2.bam -M /dev/null --VALIDATION_STRINGENCY SILENT --REMOVE_DUPLICATES true' : 'mv tmp/stage1.bam tmp/stage2.bam' - def lai = params.left_align_indels.enabled ? 'gatk LeftAlignIndels -I tmp/stage2.bam -O mapping.bam -R tmp/ref_genome.fasta --sequence-dictionary tmp/ref_genome.dict -OBI false' : 'mv tmp/stage2.bam mapping.bam' - def fixmate = params.input.reads_type != 'se' ? "| samtools sort --threads ${task.cpus} -n | samtools fixmate --threads ${task.cpus} -m - -" : '' - def reads = params.input.reads_type == 'se' ? "tmp/read1" : + def rmdup_mode = CLIParamsValidation.rmdup_mode(params.rmdup) + def rmdup = rmdup_mode == 'none' ? '' : "--rmdup-mode ${rmdup_mode}" + def left_align = params.left_align_indels.enabled ? '--left-align' : '' + def platform = params.mapping.platform ?: 'ILLUMINA' + def library = params.mapping.library ?: sample_id + def rg_id = "@RG\\tID:${sample_id}\\tSM:${sample_id}\\tPL:${platform}\\tLB:${library}" + def index_prefix = params.input.reference_index_prefix ?: + file(params.input.reference_genome).name + def java_options = "-Djava.io.tmpdir=\$TMPDIR " + + ResourceUtils.javaHeapOption(task.memory) + // The aligner and BAM post-processing run concurrently in one pipe, so + // split the requested CPUs. samtools view between them adds one thread. + def cpus = task.cpus as int + def aligner_cpus = Math.max(1, (cpus + 1).intdiv(2)) + def postprocess_cpus = Math.max(1, cpus - aligner_cpus) + // The aligner index is resident while BAM post-processing runs in the + // same pipe. Reserve the staged index size plus 1 GiB for aligner and + // pipe buffers before sort receives half of the remaining task memory; + // mapping.aligner_reserve_mb overrides the estimate. + def reserve_mb = params.mapping.aligner_reserve_mb ?: 0 + if (reserve_mb <= 0) { + def index_paths = index_files instanceof List + ? index_files : [index_files] + def index_bytes = index_paths + .collect { index_path -> index_path.size() } + .sum() ?: 0 + reserve_mb = index_bytes.intdiv(1048576) + 1024 + } + def memory_mb = task.memory ? task.memory.toMega() : 0 + def memory_option = memory_mb > 0 + ? "--memory-mb ${memory_mb} --reserve-mb ${reserve_mb}" : '' + def reads = params.input.reads_type == 'se' ? "tmp/read1" : params.input.reads_type == 'pe' ? "tmp/read1 tmp/read2" : '' + // A .mmi stores its own k-mer size and window, which override -x sr at + // mapping time. The index must be built with `minimap2 -x sr -d`. """ - GENOME_BASENAME=\$(basename \$(realpath tmp/ref_genome_real.fasta)) - + set -o pipefail + + # flagstat reads a copy of the aligner output before the post-processing + # filters, so the report keeps the true mapping rate. The trap stops the + # reader if the pipeline fails before tee opens the FIFO; otherwise the + # blocked reader could keep the task from finishing. + mkfifo tmp/raw.flagstat.fifo + samtools flagstat - > mapping.raw.flagstat < tmp/raw.flagstat.fifo & + raw_flagstat_pid=\$! + cleanup_flagstat() { + if [ -n "\$raw_flagstat_pid" ]; then + kill "\$raw_flagstat_pid" 2>/dev/null || true + wait "\$raw_flagstat_pid" 2>/dev/null || true + fi + } + trap cleanup_flagstat EXIT + minimap2 \ ${params.mapping.extra_args} \ -ax sr \ - -t ${task.cpus} \ + -t ${aligner_cpus} \ -R "${rg_id}" \ - -a ${params.input.reference_index_dir}/\${GENOME_BASENAME}.mmi \ + "index/${index_prefix}.mmi" \ ${reads} | \ - samtools view -F 4 --threads ${task.cpus} -b -q ${params.bam_filter.min_map_qual} ${fixmate} | \ - samtools sort --threads ${task.cpus} -o tmp/stage1.bam - - ${rmdup} - - ${lai} + samtools view -u | \ + tee tmp/raw.flagstat.fifo | \ + postprocess_bam.sh \ + --input - \ + --output mapping.bam \ + --reference tmp/ref_genome.fasta \ + --dict tmp/ref_genome.dict \ + --fai tmp/ref_genome.fasta.fai \ + --mapq ${params.bam_filter.min_map_qual} \ + --threads ${postprocess_cpus} \ + ${memory_option} \ + --java-options "${java_options}" \ + ${rmdup} \ + ${left_align} + + wait "\${raw_flagstat_pid}" + raw_flagstat_pid='' """ } diff --git a/modules/local/mapping_stats/main.nf b/modules/local/mapping_stats/main.nf new file mode 100644 index 0000000..4ac8b3e --- /dev/null +++ b/modules/local/mapping_stats/main.nf @@ -0,0 +1,53 @@ +process MAPPING_STATS { + label 'process_low' + cpus 1 + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' + + tag "${sample_id}" + + // Raw aligner statistics, before MAPQ filtering and duplicate handling. + publishDir( + "${params.output.directory}/mapping_stats", + mode: params.output.publish_mode, + pattern: '*.raw.flagstat' + ) + // Statistics of the BAM consumed by coverage and calling. + publishDir( + "${params.output.directory}/mapping_stats", + mode: params.output.publish_mode, + pattern: '*.final.flagstat' + ) + publishDir( + "${params.output.directory}/mapping_stats", + mode: params.output.publish_mode, + pattern: '*.stats' + ) + publishDir( + "${params.output.directory}/mapping_stats", + mode: params.output.publish_mode, + pattern: '*.idxstats' + ) + + input: + tuple val(sample_id), path("tmp/input.bam"), path("tmp/input.bam.csi"), + path("tmp/input.raw.flagstat") + + output: + tuple val(sample_id), + path("${sample_id}.raw.flagstat"), + path("${sample_id}.final.flagstat"), + path("${sample_id}.stats"), + path("${sample_id}.idxstats"), emit: stats + + script: + """ + cp tmp/input.raw.flagstat ${sample_id}.raw.flagstat + + samtools flagstat --threads ${task.cpus} \ + tmp/input.bam > ${sample_id}.final.flagstat + samtools stats --threads ${task.cpus} \ + tmp/input.bam > ${sample_id}.stats + samtools idxstats tmp/input.bam > ${sample_id}.idxstats + """ +} diff --git a/modules/local/mapping_summary/main.nf b/modules/local/mapping_summary/main.nf new file mode 100644 index 0000000..2bc78c7 --- /dev/null +++ b/modules/local/mapping_summary/main.nf @@ -0,0 +1,34 @@ +process MAPPING_SUMMARY { + label 'process_low' + cpus 1 + tag "mapping_summary" + + publishDir( + "${params.output.directory}/mapping_stats", + mode: params.output.publish_mode, + pattern: 'mapping_summary.tsv' + ) + + input: + val sample_ids + path(raw_flagstats, stageAs: 'raw/*') + path(final_flagstats, stageAs: 'flagstat/*') + path(stats_files, stageAs: 'stats/*') + path(idxstats_files, stageAs: 'idxstats/*') + + output: + path 'mapping_summary.tsv' + + script: + """ + printf '%s\\n' ${sample_ids.join(' ')} > sample_order.txt + + mapping_stats.py \ + --sample-order sample_order.txt \ + --raw-dir raw \ + --flagstat-dir flagstat \ + --stats-dir stats \ + --idxstats-dir idxstats \ + --output mapping_summary.tsv + """ +} diff --git a/modules/local/merge_outputs/main.nf b/modules/local/merge_outputs/main.nf index a294cf2..0147e74 100644 --- a/modules/local/merge_outputs/main.nf +++ b/modules/local/merge_outputs/main.nf @@ -1,43 +1,42 @@ process MERGE_OUTPUTS { - maxForks 1 - cpus params.consensus.cpu - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' - publishDir "${params.output.directory}", mode: 'move', pattern: 'consensus.bcf', enabled: params.output.type == 'single' && params.output.format == 'bcf' - publishDir "${params.output.directory}", mode: 'move', pattern: 'consensus.vcf.gz', enabled: params.output.type == 'single' && params.output.format == 'vcf' + tag "consensus" input: - path('tmp/?.bcf', arity: '1..*') + // The sample ids are carried explicitly so the merge order follows + // samples_tsv even though `dir/*` staging preserves the file names. + tuple val(sample_ids), path('tmp/bcfs/*', arity: '1..*') + path("tmp/ref_genome.fasta") + path("tmp/ref_genome.fasta.fai") + val dropped_samples output: - path("consensus.bcf"), optional: true - path("consensus.vcf.gz"), optional: true + // Internal merged representation. Filtering, re-joining and the final + // BCF/VCF conversion are done by EXPORT_OUTPUT. + path("consensus.bcf"), emit: consensus script: - def regex = params.output.remove_invariant ? 'ALT~\"\\.\" || COUNT(GT=\"RR\")=N_SAMPLES' : 'ALT~\"\\.\"' - def output_format = params.output.format == 'vcf' ? "-Oz -o consensus.vcf.gz" : "-Ob -o consensus.bcf" - def split_multiallelic = params.output.split_multiallelic ? "" : "| bcftools norm --threads ${task.cpus} -m +any -Ou" - def fill_tags = "AN,AC,AF,NS,AC_Hom,AC_Het,MAF,TYPE,F_MISSING,'DP:1=int(sum(FORMAT/DP))'" + def sample_id_list = sample_ids.split(',').collect { id -> id.trim() } + def merge_paths = sample_id_list.collect { id -> "tmp/bcfs/${id}.bcf" }.join(' ') + def dropped = dropped_samples ? dropped_samples.join(',') : '' + def dropped_arg = dropped + ? "--samples-dropped '${dropped}'" + : '' """ - n_bcf=\$(ls -1 tmp/*.bcf | wc -l) + printf '%s\n' ${merge_paths} > tmp/merge_list.txt - if [ "\$n_bcf" -gt 1 ]; then - parallel -j ${task.cpus} 'bcftools index --threads 1 --csi {}' ::: tmp/*.bcf - fi - - if [ "\$n_bcf" -eq 1 ]; then - bcftools filter --threads ${task.cpus} -e '${regex}' -Ou tmp/1.bcf \ - | bcftools +fill-tags -Ou -- -t ${fill_tags} \ - | bcftools view ${output_format} - else - ls -1v tmp/*.bcf > tmp/merge_list.txt - merge_bcf_tree.py --cpus ${task.cpus} --file-list tmp/merge_list.txt \ - | bcftools norm -m -any --threads ${task.cpus} -Ou \ - | bcftools filter --threads ${task.cpus} -e '${regex}' -Ou ${split_multiallelic} \ - | fill_missing_ad.py \ - | bcftools +fill-tags -Ou -- -t ${fill_tags} \ - | bcftools sort ${output_format} - fi + # Keep one record per ALT in the internal merge. EXPORT_OUTPUT re-joins + # multi-allelic records when output.split_multiallelic is false. + merge_outputs.sh \ + --file-list tmp/merge_list.txt \ + --reference tmp/ref_genome.fasta \ + --output consensus.bcf \ + --format bcf \ + --split-multiallelic \ + --cpus ${task.cpus} \ + ${dropped_arg} """ } diff --git a/modules/local/prepare_bam/main.nf b/modules/local/prepare_bam/main.nf index 2941db9..77f4ab9 100644 --- a/modules/local/prepare_bam/main.nf +++ b/modules/local/prepare_bam/main.nf @@ -1,8 +1,7 @@ process PREPARE_BAM { - maxForks 1 - cpus params.mapping.cpu - beforeScript 'export TMPDIR=$(mktemp -d -p $PWD/)' - afterScript 'stage_cleanup.sh' + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' tag "${sample_id}" @@ -13,19 +12,51 @@ process PREPARE_BAM { path("tmp/ref_genome.fasta.fai") output: - tuple val(sample_id), path("output.bam"), emit: bam - + tuple val(sample_id), path("mapping.bam"), + path("mapping.bam.csi"), + path("mapping.raw.flagstat"), emit: bam + script: - def rmdup = params.rmdup.enabled ? 'picard MarkDuplicates -I tmp/stage1.bam -O tmp/stage2.bam -M /dev/null --VALIDATION_STRINGENCY SILENT --REMOVE_DUPLICATES true' : 'mv tmp/stage1.bam tmp/stage2.bam' - def lai = params.left_align_indels.enabled ? 'gatk LeftAlignIndels -I tmp/stage2.bam -O mapping.bam -R tmp/ref_genome.fasta --sequence-dictionary tmp/ref_genome.dict -OBI false' : 'mv tmp/stage2.bam mapping.bam' - def fixmate = params.input.reads_type != 'se' ? "| samtools sort -n - | samtools fixmate --threads ${task.cpus} -m - -" : '' - def reads = "tmp/read1" + def rmdup_mode = CLIParamsValidation.rmdup_mode(params.rmdup) + def rmdup = rmdup_mode == 'none' ? '' : "--rmdup-mode ${rmdup_mode}" + def left_align = params.left_align_indels.enabled ? '--left-align' : '' + def platform = params.mapping.platform ?: 'ILLUMINA' + def library = params.mapping.library ?: sample_id + def java_options = "-Djava.io.tmpdir=\$TMPDIR " + + ResourceUtils.javaHeapOption(task.memory) + def memory_mb = task.memory ? task.memory.toMega() : 0 + def memory_option = memory_mb > 0 ? "--memory-mb ${memory_mb}" : '' """ - samtools view -F 4 --threads ${task.cpus} -b -q ${params.bam_filter.min_map_qual} ${reads} ${fixmate} | \ - samtools sort --threads ${task.cpus} -o tmp/stage1.bam - - ${rmdup} - - ${lai} + set -o pipefail + + # Record the input BAM before MAPQ filtering and duplicate handling, so + # the raw report stays comparable with the mapping path. + samtools flagstat --threads ${task.cpus} \ + tmp/read1 > mapping.raw.flagstat + + check_bam_reference.py tmp/read1 tmp/ref_genome.fasta.fai \ + --dict tmp/ref_genome.dict + + samtools addreplacerg \ + -w \ + -r "ID:${sample_id}" \ + -r "SM:${sample_id}" \ + -r "PL:${platform}" \ + -r "LB:${library}" \ + -m overwrite_all \ + -u \ + -o - \ + tmp/read1 | \ + postprocess_bam.sh \ + --input - \ + --output mapping.bam \ + --reference tmp/ref_genome.fasta \ + --dict tmp/ref_genome.dict \ + --mapq ${params.bam_filter.min_map_qual} \ + --threads ${task.cpus} \ + ${memory_option} \ + --java-options "${java_options}" \ + ${rmdup} \ + ${left_align} """ -} \ No newline at end of file +} diff --git a/modules/local/report_dropped_samples/main.nf b/modules/local/report_dropped_samples/main.nf new file mode 100644 index 0000000..1665161 --- /dev/null +++ b/modules/local/report_dropped_samples/main.nf @@ -0,0 +1,22 @@ +process REPORT_DROPPED_SAMPLES { + label 'process_low' + tag "dropped_samples" + + publishDir( + params.output.directory, + mode: params.output.publish_mode, + pattern: 'dropped_samples.txt' + ) + + input: + val dropped_samples + + output: + path 'dropped_samples.txt' + + script: + def lines = dropped_samples ? dropped_samples.join('\n') + '\n' : '' + """ + printf '%s' '${lines}' > dropped_samples.txt + """ +} diff --git a/modules/local/scatter_intervals/main.nf b/modules/local/scatter_intervals/main.nf new file mode 100644 index 0000000..3345dcd --- /dev/null +++ b/modules/local/scatter_intervals/main.nf @@ -0,0 +1,65 @@ +process SCATTER_INTERVALS { + label 'process_medium' + beforeScript 'export TMPDIR="$(mktemp -d -p "${PWD}")"' + afterScript 'stage_cleanup.sh "${nxf_main_ret:-0}"' + + tag "${sample_id}" + + input: + tuple val(sample_id), path("tmp/coverage.bed") + path("tmp/ref_genome.fasta") + path("tmp/ref_genome.fasta.fai") + path("tmp/ref_genome.dict") + + output: + tuple val(sample_id), path("chunks/*.bed", arity: '1..*'), + emit: chunks + + script: + def scatter_count = Math.max(params.calling.cpu ?: 1, params.consensus.cpu ?: 1) + def split_heap_option = ResourceUtils.javaHeapOption(task.memory) + def parallel_heap_option = + ResourceUtils.javaHeapOption(task.memory, task.cpus) + """ + mkdir -p tmp/bed_chunks chunks + + gatk SplitIntervals \ + --java-options "-Djava.io.tmpdir=\$TMPDIR ${split_heap_option}" \ + --tmp-dir "\$TMPDIR" \ + -R tmp/ref_genome.fasta \ + -L tmp/coverage.bed \ + --scatter-count ${scatter_count} \ + -O tmp/bed_chunks/ + + cat > tmp/run_chunk.sh <<'EOF' + set -euo pipefail + dir=\${1%/*} + stem=\${1##*/} + stem=\${stem%.*} + gatk IntervalListToBed \ + --java-options "-Djava.io.tmpdir=\$TMPDIR ${parallel_heap_option}" \ + --TMP_DIR "\$TMPDIR" \ + -I "\$1" -O "\$dir/\$stem.bed.tmp" + cut -f 1-3 "\$dir/\$stem.bed.tmp" > "chunks/\$stem.bed" + rm -f "\$1" "\$dir/\$stem.bed.tmp" + EOF + + # SIGKILLed children appear as signal 9, exit 137, or GATK's 247. + parallel --halt now,fail=1 --joblog tmp/parallel.log \ + -j ${task.cpus} bash tmp/run_chunk.sh {} \ + ::: tmp/bed_chunks/* || { + status=\$? + if awk 'NR > 1 && (\$7 == 137 || \$7 == 247 || \$8 == 9) { + found = 1 } END { exit !found }' tmp/parallel.log; then + exit 137 + fi + exit "\${status}" + } + + # A failed IntervalListToBed can still leave an empty file behind; fail + # instead of emitting a region that no caller will process. + for f in chunks/*.bed; do + [ -s "\$f" ] || { echo "empty chunk \$f" >&2; exit 1; } + done + """ +} diff --git a/nextflow.config b/nextflow.config index 42e7b2c..990a8e7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,13 +1,150 @@ -nextflow.enable.dsl = 2 - plugins { - id 'nf-boost@0.7.0' + id 'nf-boost@0.8.0' + id 'nf-schema@3.0.0' } -boost { - cleanup = 'v2' +// Schema validation of the nested params map. Unknown keys (typos or options +// removed in 4.0.0) are reported as errors before any process starts. +validation { + logging { + unrecognisedParams = 'error' + } +} + +// Run every process script with errexit, nounset and pipefail. Without +// pipefail a failed producer (aligner, caller, decompressor) is masked by a +// successful last pipeline stage, which silently truncates the output. +process.shell = ['/bin/bash', '-euo', 'pipefail'] + +// Defaults for every pipeline option. Values supplied with -params-file are +// deep-merged into these maps, so a run only needs to override what differs +// from the template (assets/templates/config.yaml). +params { + input { + samples_tsv = null + reference_genome = null + reference_index_dir = null + reference_index_prefix = null + include_bed = null + exclude_bed = null + format = 'fastq' + reads_type = 'pe' + } + + output { + directory = 'ChoCallate_output' + type = 'single' + format = 'bcf' + remove_invariant = false + split_multiallelic = false + compress_reference_blocks = true + publish_mode = 'copy' + } + + resources { + max_retries = 3 + + // Hard ceilings for a single task after retry escalation. + limits { + cpus = 64 + memory_mb = 262144 + time = '7d' + } + + low { + cpus = 1 + memory_mb = 4096 + time = '4h' + } + medium { + cpus = 6 + memory_mb = 16384 + time = '24h' + } + high { + cpus = 8 + memory_mb = 32768 + time = '48h' + } + } + + mapping { + mapper = 'bowtie2' + extra_args = '' + cpu = 10 + aligner_reserve_mb = 0 + platform = 'ILLUMINA' + library = null + } + + bam_filter { + min_map_qual = 10 + } + + rmdup { + mode = 'mark' + } + + left_align_indels { + enabled = true + } + + coverage { + min_coverage = 5 + on_empty = 'fail' + } + + calling { + callers = 'bcftools,gatk,freebayes' + cpu = 4 + allow_caller_failure = false + bcftools { + min_snp_qual = 20 + min_indel_qual = 20 + call { + extra_args = '' + } + mpileup { + max_depth = 100000 + max_idepth = 100000 + extra_args = '' + } + } + freebayes { + min_snp_qual = 20 + min_indel_qual = 20 + use_best_n_alleles = 4 + limit_coverage = 100000 + extra_args = '' + } + gatk { + min_snp_qual = 20 + min_indel_qual = 20 + interval_padding = 150 + extra_args = '' + } + } + + consensus { + threshold = 2 + cpu = 4 + absent_vote = 'ref' + filtered_vote = 'abstain' + tie_policy = 'nocall' + } + + boost { + cleanup = 'v2' + } } +// nf-boost deletes the work files of a task once no downstream process needs +// them. Deleted files cannot be reused by -resume, so the cleanup observer is +// exposed as a parameter: set boost.cleanup to false in the params file to +// keep completed work directories. +boost { + cleanup = params.boost.cleanup +} // Reporting configuration report { @@ -31,5 +168,13 @@ trace { } manifest { - version = '3.2.3' + name = 'ChoCallate' + version = '4.0.0' + nextflowVersion = '>=26.04.6' + homePage = 'https://github.com/alermol/ChoCallate' + mainScript = 'main.nf' } + +// Process labels depend on the params defined above, so this file is loaded +// after them. +includeConfig 'conf/base.config' diff --git a/nextflow_schema.json b/nextflow_schema.json new file mode 100644 index 0000000..a6a2ea6 --- /dev/null +++ b/nextflow_schema.json @@ -0,0 +1,488 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/alermol/ChoCallate/develop/nextflow_schema.json", + "title": "ChoCallate pipeline parameters", + "description": "Consensus variant calling with bcftools, FreeBayes and GATK.", + "type": "object", + "properties": { + "input": { + "type": "object", + "title": "Input", + "description": "Samples, reference genome and optional target regions.", + "properties": { + "samples_tsv": { + "type": "string", + "description": "Tab-separated samplesheet. Relative read paths are resolved from the samplesheet directory.", + "format": "file-path", + "exists": true + }, + "reference_genome": { + "type": "string", + "description": "Reference FASTA, plain or bgzip-compressed (detected from the file header).", + "format": "file-path", + "exists": true + }, + "reference_index_dir": { + "type": "string", + "description": "Directory containing the mapper and caller reference indexes. Required when input.format is 'fastq'.", + "format": "directory-path", + "exists": true + }, + "reference_index_prefix": { + "type": ["string", "null"], + "description": "Index prefix inside reference_index_dir. Must be a plain file name without directory separators. Defaults to the file name of reference_genome." + }, + "include_bed": { + "type": ["string", "null"], + "description": "Optional BED file restricting calling to the listed regions.", + "format": "file-path", + "exists": true + }, + "exclude_bed": { + "type": ["string", "null"], + "description": "Optional BED file of regions to exclude from calling.", + "format": "file-path", + "exists": true + }, + "format": { + "type": "string", + "description": "Input molecule type.", + "enum": ["fastq", "bam"] + }, + "reads_type": { + "type": "string", + "description": "FASTQ layout: single-end, paired-end or mixed (paired plus unpaired reads).", + "enum": ["se", "pe", "mx"] + } + }, + "required": ["samples_tsv", "reference_genome"], + "if": { + "properties": { + "format": { + "const": "fastq" + } + } + }, + "then": { + "required": ["reference_index_dir"] + } + }, + "output": { + "type": "object", + "title": "Output", + "description": "Consensus output location, layout and filtering.", + "properties": { + "directory": { + "type": "string", + "description": "Directory for published results and Nextflow reports.", + "minLength": 1 + }, + "type": { + "type": "string", + "description": "Publish one merged multi-sample consensus ('single') or one file per sample ('sample').", + "enum": ["single", "sample"] + }, + "format": { + "type": "string", + "description": "Final published format. Internal consensus and merge files are always BCF.", + "enum": ["bcf", "vcf"] + }, + "remove_invariant": { + "type": "boolean", + "default": false, + "description": "Drop monomorphic records when exporting the final consensus. Default false keeps monomorphic/reference records." + }, + "split_multiallelic": { + "type": "boolean", + "description": "Keep one record per ALT allele in the final output instead of re-joining multi-allelic sites." + }, + "compress_reference_blocks": { + "type": "boolean", + "default": true, + "description": "Collapse runs of hom-reference positions into gVCF blocks with INFO/END in the final output. Reference records are always gVCF-style internally; FORMAT/DP is the block minimum and FORMAT/AD is omitted on multi-base blocks." + }, + "publish_mode": { + "type": "string", + "description": "Publish result files as independent copies or as symbolic links into the work directory.", + "enum": ["copy", "link"] + } + } + }, + "resources": { + "type": "object", + "title": "Resources", + "description": "CPU, memory and time limits for the process labels.", + "properties": { + "max_retries": { + "type": "integer", + "description": "Maximum number of retries after an exit status of 137 or 140.", + "minimum": 0 + }, + "limits": { + "type": "object", + "title": "Limits", + "description": "Hard ceilings for every task after retry escalation.", + "properties": { + "cpus": { + "type": "integer", + "description": "Maximum CPUs requested by one task.", + "minimum": 1 + }, + "memory_mb": { + "type": "integer", + "description": "Maximum memory requested by one task in MiB.", + "minimum": 1 + }, + "time": { + "type": "string", + "description": "Maximum task runtime, for example 7d or 168h.", + "minLength": 1 + } + }, + "required": ["cpus", "memory_mb", "time"] + }, + "low": { + "type": "object", + "title": "Low", + "description": "Index creation, decompression and coverage tasks.", + "properties": { + "cpus": { + "type": "integer", + "description": "CPUs requested per task.", + "minimum": 1 + }, + "memory_mb": { + "type": "integer", + "description": "Memory requested per task in MiB. Retries multiply it by the attempt number.", + "minimum": 1 + }, + "time": { + "type": "string", + "description": "Maximum task runtime, for example 4h or 90m.", + "minLength": 1 + } + }, + "required": ["cpus", "memory_mb", "time"] + }, + "medium": { + "type": "object", + "title": "Medium", + "description": "Mapping, variant calling, scattering and merging tasks.", + "properties": { + "cpus": { + "type": "integer", + "description": "CPUs requested per task.", + "minimum": 1 + }, + "memory_mb": { + "type": "integer", + "description": "Memory requested per task in MiB. Retries multiply it by the attempt number.", + "minimum": 1 + }, + "time": { + "type": "string", + "description": "Maximum task runtime, for example 24h or 90m.", + "minLength": 1 + } + }, + "required": ["cpus", "memory_mb", "time"] + }, + "high": { + "type": "object", + "title": "High", + "description": "GATK HaplotypeCaller tasks, which start one JVM per assigned CPU.", + "properties": { + "cpus": { + "type": "integer", + "description": "CPUs requested per task.", + "minimum": 1 + }, + "memory_mb": { + "type": "integer", + "description": "Memory requested per task in MiB. Retries multiply it by the attempt number.", + "minimum": 1 + }, + "time": { + "type": "string", + "description": "Maximum task runtime, for example 48h or 90m.", + "minLength": 1 + } + }, + "required": ["cpus", "memory_mb", "time"] + } + }, + "required": ["max_retries", "limits", "low", "medium", "high"] + }, + "mapping": { + "type": "object", + "title": "Mapping", + "description": "Read mapping options.", + "properties": { + "mapper": { + "type": "string", + "description": "Read mapper. Mixed reads are supported by bowtie2 only.", + "enum": ["bowtie2", "bwa", "minimap2"] + }, + "extra_args": { + "type": "string", + "description": "Extra arguments passed verbatim to the mapper." + }, + "cpu": { + "type": "integer", + "description": "CPUs per task, shared by mapper and sort.", + "minimum": 1 + }, + "aligner_reserve_mb": { + "type": "integer", + "description": "Memory in MiB reserved for the aligner index and working set before sort memory is derived. 0 estimates the reserve from the staged index files plus 1 GiB. The reserve must be smaller than the task memory.", + "minimum": 0 + }, + "platform": { + "type": "string", + "description": "Read-group PL value. Accepted characters: letters, digits, underscore, dot, colon and hyphen.", + "pattern": "^[A-Za-z0-9_.:-]+$" + }, + "library": { + "type": ["string", "null"], + "description": "Read-group LB value. Defaults to the sample id. Accepted characters: letters, digits, underscore, dot, colon and hyphen.", + "pattern": "^[A-Za-z0-9_.:-]+$" + } + } + }, + "bam_filter": { + "type": "object", + "title": "BAM filter", + "description": "Alignment filtering applied after mapping.", + "properties": { + "min_map_qual": { + "type": "integer", + "description": "Minimum mapping quality; alignments below it are discarded.", + "minimum": 0 + } + } + }, + "rmdup": { + "type": "object", + "title": "Duplicate handling", + "description": "Optical/PCR duplicate marking with samtools markdup.", + "properties": { + "mode": { + "type": "string", + "description": "Duplicate handling: none (no marking), mark (flag duplicates, do not remove) or remove (flag and remove). Callers and the consensus pileup skip duplicate-flagged reads.", + "enum": ["none", "mark", "remove"], + "default": "mark" + } + } + }, + "left_align_indels": { + "type": "object", + "title": "Left-align indels", + "description": "Normalise indel alignment with GATK LeftAlignIndels.", + "properties": { + "enabled": { + "type": "boolean", + "description": "Run GATK LeftAlignIndels on the final BAM." + } + } + }, + "coverage": { + "type": "object", + "title": "Coverage", + "description": "Coverage-based site selection.", + "properties": { + "min_coverage": { + "type": "integer", + "description": "Minimum depth (at least 1) for a base to be included in the coverage BED and consensus.", + "minimum": 1 + }, + "on_empty": { + "type": "string", + "description": "Action for a sample whose coverage BED is empty: fail or skip.", + "enum": [ + "fail", + "skip" + ] + } + } + }, + "calling": { + "type": "object", + "title": "Variant calling", + "description": "Caller selection and per-caller quality filters.", + "properties": { + "callers": { + "type": "string", + "description": "Comma-separated non-empty subset of bcftools, freebayes and gatk.", + "minLength": 1 + }, + "cpu": { + "type": "integer", + "description": "CPUs requested per calling task.", + "minimum": 1 + }, + "allow_caller_failure": { + "type": "boolean", + "description": "Continue with the surviving callers when a caller fails. When false, a failed caller aborts the run." + }, + "bcftools": { + "type": "object", + "title": "bcftools", + "description": "bcftools mpileup/call options.", + "properties": { + "min_snp_qual": { + "type": "number", + "description": "Minimum bcftools QUAL for SNPs.", + "minimum": 0 + }, + "min_indel_qual": { + "type": "number", + "description": "Minimum bcftools QUAL for indels and other non-SNP records.", + "minimum": 0 + }, + "call": { + "type": "object", + "title": "bcftools call", + "properties": { + "extra_args": { + "type": "string", + "description": "Extra arguments for bcftools call." + } + } + }, + "mpileup": { + "type": "object", + "title": "bcftools mpileup", + "properties": { + "max_depth": { + "type": "integer", + "default": 100000, + "description": "Maximum raw depth per input file. The bcftools default of 250 silently down-samples high-coverage libraries.", + "minimum": 1 + }, + "max_idepth": { + "type": "integer", + "default": 100000, + "description": "Maximum per-file depth for indel calling. The bcftools default of 250 silently down-samples high-coverage libraries.", + "minimum": 1 + }, + "extra_args": { + "type": "string", + "description": "Extra arguments for bcftools mpileup." + } + } + } + }, + "required": ["min_snp_qual", "min_indel_qual"] + }, + "freebayes": { + "type": "object", + "title": "FreeBayes", + "description": "FreeBayes options.", + "properties": { + "min_snp_qual": { + "type": "number", + "description": "Minimum freebayes QUAL for SNPs.", + "minimum": 0 + }, + "min_indel_qual": { + "type": "number", + "description": "Minimum freebayes QUAL for indels and other non-SNP records.", + "minimum": 0 + }, + "use_best_n_alleles": { + "type": "integer", + "default": 4, + "description": "Evaluate only the N best alleles per site, ranked by supporting quality. 0 evaluates all alleles, which can exhaust memory in repeat regions.", + "minimum": 0 + }, + "limit_coverage": { + "type": "integer", + "default": 100000, + "description": "Per-sample coverage cap; freebayes down-samples coverage above this depth.", + "minimum": 1 + }, + "extra_args": { + "type": "string", + "description": "Extra arguments for freebayes." + } + }, + "required": ["min_snp_qual", "min_indel_qual"] + }, + "gatk": { + "type": "object", + "title": "GATK", + "description": "GATK HaplotypeCaller options.", + "properties": { + "min_snp_qual": { + "type": "number", + "description": "Minimum GATK QUAL for SNPs.", + "minimum": 0 + }, + "min_indel_qual": { + "type": "number", + "description": "Minimum GATK QUAL for indels and other non-SNP records.", + "minimum": 0 + }, + "interval_padding": { + "type": "integer", + "description": "Padding added to each calling interval so indels at region edges are assembled.", + "minimum": 0 + }, + "extra_args": { + "type": "string", + "description": "Extra arguments for HaplotypeCaller." + } + }, + "required": ["min_snp_qual", "min_indel_qual"] + } + }, + "required": ["callers", "cpu"] + }, + "consensus": { + "type": "object", + "title": "Consensus", + "description": "Voting, output and resource options for the consensus stage.", + "properties": { + "threshold": { + "type": "integer", + "description": "Minimum number of callers that must support an ALT allele.", + "minimum": 1 + }, + "cpu": { + "type": "integer", + "description": "CPUs requested for consensus and merge tasks.", + "minimum": 1 + }, + "absent_vote": { + "type": "string", + "description": "How a caller without a record at a position votes.", + "enum": ["ref", "abstain"] + }, + "filtered_vote": { + "type": "string", + "description": "How a caller record whose FILTER is not PASS or . votes.", + "enum": ["abstain", "ref"] + }, + "tie_policy": { + "type": "string", + "description": "Zygosity tie-breaker among supporters of the winning ALT.", + "enum": ["nocall", "first"] + } + }, + "required": ["threshold", "cpu"] + }, + "boost": { + "type": "object", + "title": "nf-boost", + "description": "Work-directory cleanup performed by the nf-boost plugin.", + "properties": { + "cleanup": { + "type": ["string", "boolean"], + "description": "Cleanup observer for completed tasks: 'v1' or 'v2' delete work files as soon as no downstream process needs them, false keeps them so -resume can reuse the run.", + "enum": ["v1", "v2", false] + } + }, + "required": ["cleanup"] + } + } +} diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 0000000..c27eb98 --- /dev/null +++ b/nf-test.config @@ -0,0 +1,6 @@ +config { + testsDir "tests/nf" + workDir ".nf-test" + libDir "tests/lib" + autoSort false +} diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..6b95cde --- /dev/null +++ b/ruff.toml @@ -0,0 +1,5 @@ +target-version = "py313" +line-length = 80 + +[lint] +select = ["E4", "E7", "E9", "F", "I"] diff --git a/test_run/arth_chr1.fasta.gz b/test_run/arth_chr1.fasta.gz index 116cf1e..746b5ed 100644 Binary files a/test_run/arth_chr1.fasta.gz and b/test_run/arth_chr1.fasta.gz differ diff --git a/test_run/run_test.sh b/test_run/run_test.sh index 7f4709d..145c2f8 100755 --- a/test_run/run_test.sh +++ b/test_run/run_test.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -export NFX_OFFLINE='true' +export NXF_OFFLINE='true' echo -e "sample1\t$(readlink -f test_reads_R1.fq.gz)\t$(readlink -f test_reads_R2.fq.gz)" > samples.tsv @@ -10,6 +10,7 @@ input: samples_tsv: "$PWD/samples.tsv" reference_genome: "$PWD/arth_chr1.fasta.gz" reference_index_dir: "$PWD" + reference_index_prefix: null include_bed: null exclude_bed: null format: "fastq" @@ -19,46 +20,54 @@ output: directory: "$PWD/ChoCallate_output" type: "single" format: "bcf" - remove_invariant: true + remove_invariant: false split_multiallelic: false -ref_genome: - bgzip: true - mapping: mapper: "bowtie2" extra_args: "" cpu: 10 + platform: "ILLUMINA" + library: null bam_filter: - min_map_qual: 0 + min_map_qual: 10 rmdup: - enabled: false + mode: "none" left_align_indels: enabled: false coverage: - min_coverage: 2 + min_coverage: 5 calling: callers: "bcftools,gatk,freebayes" - min_snp_qual: 20 cpu: 4 + allow_caller_failure: false bcftools: + min_snp_qual: 20 + min_indel_qual: 20 call: extra_args: "" mpileup: extra_args: "" freebayes: + min_snp_qual: 20 + min_indel_qual: 20 extra_args: "" gatk: + min_snp_qual: 20 + min_indel_qual: 20 + interval_padding: 150 extra_args: "" consensus: threshold: 2 cpu: 4 + absent_vote: "ref" + tie_policy: "nocall" EOF echo "Preparing genome index..." diff --git a/test_run/test_reads_R1.fq.gz b/test_run/test_reads_R1.fq.gz index d96671a..a178f29 100644 Binary files a/test_run/test_reads_R1.fq.gz and b/test_run/test_reads_R1.fq.gz differ diff --git a/test_run/test_reads_R2.fq.gz b/test_run/test_reads_R2.fq.gz index 8bc6a7f..38f3f44 100644 Binary files a/test_run/test_reads_R2.fq.gz and b/test_run/test_reads_R2.fq.gz differ diff --git a/tests/lib/ChoCallateTestUtils.groovy b/tests/lib/ChoCallateTestUtils.groovy new file mode 100644 index 0000000..29c7921 --- /dev/null +++ b/tests/lib/ChoCallateTestUtils.groovy @@ -0,0 +1,254 @@ +import java.nio.file.Path + +/** + * Shared fixtures and assertions for the ChoCallate nf-test suite. + */ +class ChoCallateTestUtils { + + static final String REFERENCE_GZ = 'test_run/arth_chr1.fasta.gz' + static final String READ1 = 'test_run/test_reads_R1.fq.gz' + static final String READ2 = 'test_run/test_reads_R2.fq.gz' + static final String CONTIG = 'CP002684.1' + static final int CONTIG_LENGTH = 200001 + + static File projectFile(Object projectDir, String relative) { + return new File(projectDir.toString(), relative) + } + + static File outputFile(Object path) { + return new File(path.toString()) + } + + /** Runs a shell command and fails with its output on a non-zero exit. */ + static String run(File workDir, String command) { + def builder = new ProcessBuilder('bash', '-c', command) + builder.directory(workDir) + builder.redirectErrorStream(true) + def process = builder.start() + String output = process.inputStream.getText('UTF-8') + int status = process.waitFor() + if (status != 0) { + throw new RuntimeException( + "Command failed with exit ${status}: ${command}\n${output}" + ) + } + return output + } + + /** + * Decompresses the test reference and creates the fai index and sequence + * dictionary used by the mapping and calling modules. + */ + static Map referenceBundle(Object projectDir, File launchDir, + String name = 'reference') { + File dir = new File(launchDir, name) + dir.mkdirs() + File fasta = new File(dir, 'genome.fasta') + if (!fasta.isFile()) { + run( + dir, + "gzip -dc '${projectFile(projectDir, REFERENCE_GZ)}' " + + "> '${fasta}'" + ) + run(dir, "samtools faidx '${fasta}'") + run( + dir, + "gatk CreateSequenceDictionary -R '${fasta}' " + + "-O '${dir}/genome.dict'" + ) + } + return [ + dir: dir, + fasta: fasta, + fai: new File(dir, 'genome.fasta.fai'), + dict: new File(dir, 'genome.dict'), + prefix: 'genome.fasta' + ] + } + + static Map bowtie2Index(Object projectDir, File launchDir, + String name = 'reference', + boolean large = false) { + Map ref = referenceBundle(projectDir, launchDir, name) + String large_index = large ? '--large-index' : '' + run( + ref.dir, + "bowtie2-build --quiet ${large_index} --threads 2 " + + "'${ref.fasta}' '${ref.dir}/${ref.prefix}'" + ) + return ref + } + + static Map bwaIndex(Object projectDir, File launchDir, + String name = 'reference') { + Map ref = referenceBundle(projectDir, launchDir, name) + run(ref.dir, "bwa index '${ref.fasta}'") + return ref + } + + static Map minimap2Index(Object projectDir, File launchDir, + String name = 'reference') { + Map ref = referenceBundle(projectDir, launchDir, name) + run( + ref.dir, + "minimap2 -x sr -t 2 -d '${ref.dir}/genome.fasta.mmi' " + + "'${ref.fasta}'" + ) + return ref + } + + /** Maps the test reads with bowtie2 and returns a sorted, indexed BAM. */ + static File bowtie2Bam(Object projectDir, File launchDir, + String sampleId = 'sample1', + boolean singleEnd = false) { + Map ref = bowtie2Index(projectDir, launchDir) + File bam = new File(launchDir, "${sampleId}.bam") + File read1 = projectFile(projectDir, READ1) + File read2 = projectFile(projectDir, READ2) + String reads = singleEnd + ? "-U '${read1}'" + : "-1 '${read1}' -2 '${read2}'" + run( + launchDir, + "set -o pipefail; bowtie2 --quiet --threads 2 " + + "--rg-id '${sampleId}' --rg 'SM:${sampleId}' " + + "--rg 'PL:ILLUMINA' --rg 'LB:${sampleId}' " + + "-x '${ref.dir}/${ref.prefix}' ${reads} | " + + "samtools sort --threads 2 -o '${bam}' - && " + + "samtools index --csi '${bam}'" + ) + return bam + } + + /** Writes a coordinate-sorted, indexed BAM with no alignments. */ + static File emptyBam(File launchDir, String sampleId = 'empty') { + File bam = new File(launchDir, "${sampleId}.bam") + run( + launchDir, + "printf '@HD\\tVN:1.6\\tSO:coordinate\\n" + + "@SQ\\tSN:${CONTIG}\\tLN:${CONTIG_LENGTH}\\n' | " + + "samtools view -b -o '${bam}' - && " + + "samtools index --csi '${bam}'" + ) + return bam + } + + static File samplesheet(File dir, List> rows) { + File sheet = new File(dir, 'samples.tsv') + sheet.text = rows.collect { row -> row.join('\t') }.join('\n') + '\n' + return sheet + } + + private static File writeVcfText(File dir, String name, String sample, + List> records) { + dir.mkdirs() + def header = [ + '##fileformat=VCFv4.2', + '##contig=', + '##ALT=', + '##INFO=', + '##FORMAT=', + '##FORMAT=', + '##FORMAT=', + '##ChoCallate=', + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\t' + + "FORMAT\t${sample}" + ] + File vcf = new File(dir, "${name}.vcf") + vcf.text = (header + records.collect { it.join('\t') }) + .join('\n') + '\n' + return vcf + } + + /** Writes a minimal single-sample BCF against the test contig. */ + static File writeBcf(File dir, String name, String sample, + List> records) { + File vcf = writeVcfText(dir, name, sample, records) + File bcf = new File(dir, "${name}.bcf") + run(dir, "bcftools view -Ob -o '${bcf}' '${vcf}'") + return bcf + } + + /** Writes a minimal bgzipped single-sample VCF against the test contig. */ + static File writeVcfGz(File dir, String name, String sample, + List> records) { + File vcf = writeVcfText(dir, name, sample, records) + File gz = new File(dir, "${name}.vcf.gz") + run(dir, "bgzip -c '${vcf}' > '${gz}'") + return gz + } + + static String bamHeader(File bam) { + return run( + bam.absoluteFile.parentFile, + "samtools view -H '${bam.absolutePath}'" + ) + } + + static int bamRecords(File bam) { + String count = run( + bam.absoluteFile.parentFile, + "samtools view -c -F 4 '${bam.absolutePath}'" + ) + return count.trim() as int + } + + static String vcfHeader(File vcf) { + return run( + vcf.absoluteFile.parentFile, + "bcftools view -h '${vcf.absolutePath}'" + ) + } + + static List vcfSamples(File vcf) { + String chrom = vcfHeader(vcf).readLines().find { line -> + line.startsWith('#CHROM') + } + return chrom.split('\t', -1).drop(9).collect { it.trim() } + } + + /** Returns CHROM, POS, REF and ALT for every record in the VCF/BCF. */ + static List vcfRecords(File vcf) { + String output = run( + vcf.absoluteFile.parentFile, + "bcftools query -f '%CHROM\\t%POS\\t%REF\\t%ALT\\n' " + + "'${vcf.absolutePath}'" + ) + return output.readLines() + } + + static List statsLines(File vcf) { + String output = run( + vcf.absoluteFile.parentFile, + "bcftools stats '${vcf.absolutePath}' | " + + "grep -E '^(SN|TSTV)[[:space:]]'" + ) + return output.readLines() + } + + static int recordCount(File vcf) { + String line = statsLines(vcf).find { entry -> + entry.startsWith('SN\t0\tnumber of records:') + } + if (line == null) { + throw new IllegalStateException("No record count for ${vcf}") + } + return line.split('\t', -1)[3] as int + } + + static int statsValue(File vcf, String key) { + String line = statsLines(vcf).find { entry -> + entry.startsWith("SN\t0\t${key}:") + } + if (line == null) { + throw new IllegalStateException("No ${key} for ${vcf}") + } + return line.split('\t', -1)[3] as int + } +} diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 0000000..8c0b891 --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,39 @@ +// Additional Nextflow configuration for the nf-test runs. + +// Work directories must survive until assertions run, so disable the +// nf-boost cleanup observer in tests. This sets the boost scope directly: +// `boost.cleanup` is evaluated in nextflow.config before this file is loaded, +// so a params override here would not reach it. +boost { + cleanup = false +} + +params { + resources { + max_retries = 0 + low { + cpus = 1 + memory_mb = 2048 + } + medium { + cpus = 2 + memory_mb = 4096 + } + high { + cpus = 2 + memory_mb = 4096 + } + } + + mapping { + cpu = 2 + } + + calling { + cpu = 2 + } + + consensus { + cpu = 2 + } +} diff --git a/tests/nf/modules/local/calling_bcftools.nf.test b/tests/nf/modules/local/calling_bcftools.nf.test new file mode 100644 index 0000000..c5d468b --- /dev/null +++ b/tests/nf/modules/local/calling_bcftools.nf.test @@ -0,0 +1,103 @@ +nextflow_process { + + name "Test process CALLING_BCFTOOLS" + script "../../../../modules/local/calling_bcftools/main.nf" + process "CALLING_BCFTOOLS" + + test("calls variants from a BAM") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + def chunks = new File(launchDir, 'chunks') + chunks.mkdirs() + new File(chunks, 'full.bed').text = + "CP002684.1\t0\t200001\n" + } + + when { + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + file("${launchDir}/chunks/full.bed") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + input[2] = file("${launchDir}/reference/genome.fasta.fai") + input[3] = file("${launchDir}/reference/genome.dict") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile( + process.out.calling_result[0][1] + ) + assert bcf.isFile() + assert ChoCallateTestUtils.vcfSamples(bcf) == ['sample1'] + def header = ChoCallateTestUtils.vcfHeader(bcf) + assert header.contains( + '##contig=' + ) + // Low-QUAL records are retained with FILTER=LowQual, so the + // count includes both passing and soft-filtered records. + assert ChoCallateTestUtils.recordCount(bcf) == 313 + assert snapshot( + ChoCallateTestUtils.statsLines(bcf) + ).match() + } + } + + test("accepts quoted mpileup extra arguments") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + def chunks = new File(launchDir, 'chunks') + chunks.mkdirs() + new File(chunks, 'full.bed').text = + "CP002684.1\t0\t200001\n" + } + + when { + params { + calling { + bcftools { + mpileup { + extra_args = "--annotate 'FORMAT/SP'" + } + } + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + file("${launchDir}/chunks/full.bed") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + input[2] = file("${launchDir}/reference/genome.fasta.fai") + input[3] = file("${launchDir}/reference/genome.dict") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile( + process.out.calling_result[0][1] + ) + assert bcf.isFile() + assert ChoCallateTestUtils.vcfSamples(bcf) == ['sample1'] + assert ChoCallateTestUtils.vcfHeader(bcf).contains( + '##FORMAT=' + ) + // Low-QUAL records are retained with FILTER=LowQual, so the + // count includes both passing and soft-filtered records. + assert ChoCallateTestUtils.recordCount(bcf) == 1855 + assert snapshot( + ChoCallateTestUtils.statsLines(bcf) + ).match() + } + } + + test("filters atomised records by their resulting type") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + def chunks = new File(launchDir, 'chunks') + chunks.mkdirs() + new File(chunks, 'full.bed').text = + "CP002684.1\t0\t200001\n" + } + + when { + params { + calling { + freebayes { + min_snp_qual = 20 + min_indel_qual = 1000 + } + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + file("${launchDir}/chunks/full.bed") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + input[2] = file("${launchDir}/reference/genome.fasta.fai") + input[3] = file("${launchDir}/reference/genome.dict") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile( + process.out.calling_result[0][1] + ) + def rows = ChoCallateTestUtils.run( + bcf.parentFile, + "bcftools query -f '%TYPE\\t%QUAL\\t%FILTER\\n' " + + "'${bcf}'" + ).readLines().collect { line -> line.split('\t', -1) } + + // MNP and complex records are split before filtering, so an + // atomised SNP must use min_snp_qual even though the indel + // threshold is 1000. + def snps = rows.findAll { row -> row[0] == 'SNP' } + def indels = rows.findAll { row -> row[0] == 'INDEL' } + assert snps.every { row -> + (row[1] as double) < 20 || row[2] == 'PASS' + } + assert indels.every { row -> + (row[1] as double) >= 1000 || row[2] == 'LowQual' + } + assert snps.any { row -> (row[1] as double) < 1000 } + assert indels.any { row -> (row[1] as double) < 1000 } + } + } +} diff --git a/tests/nf/modules/local/calling_freebayes.nf.test.snap b/tests/nf/modules/local/calling_freebayes.nf.test.snap new file mode 100644 index 0000000..47eaa8c --- /dev/null +++ b/tests/nf/modules/local/calling_freebayes.nf.test.snap @@ -0,0 +1,24 @@ +{ + "calls variants from a BAM": { + "content": [ + [ + "[W::bcf_hdr_check_sanity] GQ should be declared as Type=Integer", + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t1855", + "SN\t0\tnumber of no-ALTs:\t0", + "SN\t0\tnumber of SNPs:\t1725", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t130", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t28", + "SN\t0\tnumber of multiallelic SNP sites:\t0", + "TSTV\t0\t867\t858\t1.01\t867\t858\t1.01" + ] + ], + "timestamp": "2026-10-18T23:15:10.460961383", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/calling_gatk.nf.test b/tests/nf/modules/local/calling_gatk.nf.test new file mode 100644 index 0000000..08c1b09 --- /dev/null +++ b/tests/nf/modules/local/calling_gatk.nf.test @@ -0,0 +1,102 @@ +nextflow_process { + + name "Test process CALLING_GATK" + script "../../../../modules/local/calling_gatk/main.nf" + process "CALLING_GATK" + + test("calls variants from a BAM") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + def chunks = new File(launchDir, 'chunks') + chunks.mkdirs() + new File(chunks, 'full.bed').text = + "CP002684.1\t0\t200001\n" + } + + when { + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + file("${launchDir}/chunks/full.bed") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + input[2] = file("${launchDir}/reference/genome.fasta.fai") + input[3] = file("${launchDir}/reference/genome.dict") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile( + process.out.calling_result[0][1] + ) + assert bcf.isFile() + assert ChoCallateTestUtils.vcfSamples(bcf) == ['sample1'] + def header = ChoCallateTestUtils.vcfHeader(bcf) + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.recordCount(bcf) == 716 + assert snapshot( + ChoCallateTestUtils.statsLines(bcf) + ).match() + } + } + + test("drops calls from the interval padding of adjacent chunks") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + def chunks = new File(launchDir, 'chunks') + chunks.mkdirs() + new File(chunks, 'left.bed').text = + "CP002684.1\t99500\t100000\n" + new File(chunks, 'right.bed').text = + "CP002684.1\t100000\t100500\n" + } + + when { + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + [ + file("${launchDir}/chunks/left.bed"), + file("${launchDir}/chunks/right.bed") + ] + ) + input[1] = file("${launchDir}/reference/genome.fasta") + input[2] = file("${launchDir}/reference/genome.fasta.fai") + input[3] = file("${launchDir}/reference/genome.dict") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile( + process.out.calling_result[0][1] + ) + def records = ChoCallateTestUtils.vcfRecords(bcf) + def positions = records.collect { line -> + line.split('\t')[1] as int + } + // POS 100030 is outside the left unpadded chunk (which ends at + // 100000) but inside its padding, so both chunks call it. + assert positions.count { pos -> pos == 100030 } == 1 + // Calls outside the unpadded chunks come only from padding: + // left starts at 99500 and right ends at 100500 (0-based BED). + assert positions.every { pos -> pos >= 99501 } + assert positions.every { pos -> pos <= 100500 } + } + } +} diff --git a/tests/nf/modules/local/calling_gatk.nf.test.snap b/tests/nf/modules/local/calling_gatk.nf.test.snap new file mode 100644 index 0000000..8388c6e --- /dev/null +++ b/tests/nf/modules/local/calling_gatk.nf.test.snap @@ -0,0 +1,23 @@ +{ + "calls variants from a BAM": { + "content": [ + [ + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t716", + "SN\t0\tnumber of no-ALTs:\t0", + "SN\t0\tnumber of SNPs:\t637", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t77", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t0", + "SN\t0\tnumber of multiallelic SNP sites:\t0", + "TSTV\t0\t379\t258\t1.47\t379\t258\t1.47" + ] + ], + "timestamp": "2026-10-18T23:18:21.249924049", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/create_fai_index.nf.test b/tests/nf/modules/local/create_fai_index.nf.test new file mode 100644 index 0000000..9a15c6c --- /dev/null +++ b/tests/nf/modules/local/create_fai_index.nf.test @@ -0,0 +1,36 @@ +nextflow_process { + + name "Test process CREATE_FAI_INDEX" + script "../../../../modules/local/create_fai_index/main.nf" + process "CREATE_FAI_INDEX" + + test("creates a fai index for the reference") { + setup { + ChoCallateTestUtils.referenceBundle( + projectDir, new File(launchDir) + ) + } + + when { + process { + """ + input[0] = file("${launchDir}/reference/genome.fasta") + """ + } + } + + then { + assert process.success + def fai = ChoCallateTestUtils.outputFile( + process.out.fai_index[0] + ) + assert fai.isFile() + def lines = fai.readLines() + assert lines.size() == 1 + def fields = lines[0].split('\t', -1) + assert fields[0] == 'CP002684.1' + assert fields[1] as int == 200001 + assert snapshot(lines).match() + } + } +} diff --git a/tests/nf/modules/local/create_fai_index.nf.test.snap b/tests/nf/modules/local/create_fai_index.nf.test.snap new file mode 100644 index 0000000..518d8d7 --- /dev/null +++ b/tests/nf/modules/local/create_fai_index.nf.test.snap @@ -0,0 +1,14 @@ +{ + "creates a fai index for the reference": { + "content": [ + [ + "CP002684.1\t200001\t73\t70\t71" + ] + ], + "timestamp": "2026-10-17T23:41:15.901525881", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/create_seq_dict.nf.test b/tests/nf/modules/local/create_seq_dict.nf.test new file mode 100644 index 0000000..a1c2d0e --- /dev/null +++ b/tests/nf/modules/local/create_seq_dict.nf.test @@ -0,0 +1,40 @@ +nextflow_process { + + name "Test process CREATE_SEQ_DICT" + script "../../../../modules/local/create_seq_dict/main.nf" + process "CREATE_SEQ_DICT" + + test("creates a sequence dictionary for the reference") { + setup { + ChoCallateTestUtils.referenceBundle( + projectDir, new File(launchDir) + ) + } + + when { + process { + """ + input[0] = file("${launchDir}/reference/genome.fasta") + """ + } + } + + then { + assert process.success + def dict = ChoCallateTestUtils.outputFile( + process.out.gen_dict[0] + ) + assert dict.isFile() + def lines = dict.readLines() + assert lines.any { it.startsWith('@HD') } + def sq = lines.find { it.startsWith('@SQ') } + assert sq.startsWith('@SQ\tSN:CP002684.1\tLN:200001') + def stable = lines.collect { line -> + line.startsWith('@SQ') + ? line.split('\t')[0..2].join('\t') + : line + } + assert snapshot(stable).match() + } + } +} diff --git a/tests/nf/modules/local/create_seq_dict.nf.test.snap b/tests/nf/modules/local/create_seq_dict.nf.test.snap new file mode 100644 index 0000000..b105cd4 --- /dev/null +++ b/tests/nf/modules/local/create_seq_dict.nf.test.snap @@ -0,0 +1,15 @@ +{ + "creates a sequence dictionary for the reference": { + "content": [ + [ + "@HD\tVN:1.6", + "@SQ\tSN:CP002684.1\tLN:200001" + ] + ], + "timestamp": "2026-10-17T23:53:21.266370039", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/decompress_assembly.nf.test b/tests/nf/modules/local/decompress_assembly.nf.test new file mode 100644 index 0000000..d73e472 --- /dev/null +++ b/tests/nf/modules/local/decompress_assembly.nf.test @@ -0,0 +1,33 @@ +nextflow_process { + + name "Test process DECOMPRESS_ASSEMBLY" + script "../../../../modules/local/decompress_assembly/main.nf" + process "DECOMPRESS_ASSEMBLY" + + test("decompresses the bgzip reference") { + when { + process { + """ + input[0] = file("${projectDir}/test_run/" + + "arth_chr1.fasta.gz") + """ + } + } + + then { + assert process.success + def fasta = ChoCallateTestUtils.outputFile( + process.out.ref_genome[0] + ) + assert fasta.isFile() + def lines = fasta.readLines() + assert lines[0].startsWith('>CP002684.1') + assert lines.size() == 2859 + assert lines[1].length() == 70 + assert lines[-1].length() == 11 + assert snapshot( + [lines[0], lines.size(), lines[1].length()] + ).match() + } + } +} diff --git a/tests/nf/modules/local/decompress_assembly.nf.test.snap b/tests/nf/modules/local/decompress_assembly.nf.test.snap new file mode 100644 index 0000000..4538f18 --- /dev/null +++ b/tests/nf/modules/local/decompress_assembly.nf.test.snap @@ -0,0 +1,16 @@ +{ + "decompresses the bgzip reference": { + "content": [ + [ + ">CP002684.1 Arabidopsis thaliana chromosome 1 CI slice 11800000-12000000", + 2859, + 70 + ] + ], + "timestamp": "2026-10-17T23:48:23.889114381", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/export_output.nf.test b/tests/nf/modules/local/export_output.nf.test new file mode 100644 index 0000000..e082984 --- /dev/null +++ b/tests/nf/modules/local/export_output.nf.test @@ -0,0 +1,212 @@ +nextflow_process { + + name "Test process EXPORT_OUTPUT" + script "../../../../modules/local/export_output/main.nf" + process "EXPORT_OUTPUT" + + test("exports a per-sample internal BCF as bgzipped VCF") { + setup { + ChoCallateTestUtils.referenceBundle( + projectDir, new File(launchDir) + ) + def bcfs = new File(launchDir, 'bcfs') + ChoCallateTestUtils.writeBcf( + bcfs, 'sample_a', 'sample_a', + [ + ['CP002684.1', '10', '.', 'G', 'A', '50', 'PASS', '.', + 'GT:AD:DP', '0/1:5,5:10'], + ['CP002684.1', '11', '.', 'T', '<*>', '.', 'PASS', + 'END=11', 'GT:AD:DP', '0/0:9,1:10'], + ['CP002684.1', '20', '.', 'T', 'C', '50', 'PASS', '.', + 'GT:AD:DP', '1/1:0,10:10'] + ] + ) + } + + when { + params { + output { + directory = "${launchDir}/output" + type = 'sample' + format = 'vcf' + remove_invariant = false + split_multiallelic = false + } + } + process { + """ + input[0] = tuple( + 'sample_a', + file("${launchDir}/bcfs/sample_a.bcf") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + """ + } + } + + then { + assert process.success + def vcf = ChoCallateTestUtils.outputFile(process.out.vcf[0][1]) + assert vcf.name == 'sample_a.vcf.gz' + assert vcf.isFile() + assert ChoCallateTestUtils.vcfSamples(vcf) == ['sample_a'] + assert ChoCallateTestUtils.recordCount(vcf) == 3 + assert ChoCallateTestUtils.vcfHeader(vcf).contains( + '##ChoCallate=', '.', 'PASS', + 'END=11', 'GT:AD:DP', '0/0:9,1:10'], + ['CP002684.1', '20', '.', 'T', 'C', '50', 'PASS', '.', + 'GT:AD:DP', '1/1:0,10:10'] + ] + ) + } + + when { + params { + output { + directory = "${launchDir}/output" + type = 'single' + format = 'bcf' + remove_invariant = true + split_multiallelic = false + } + } + process { + """ + input[0] = tuple( + 'consensus', + file("${launchDir}/bcfs/consensus.bcf") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile(process.out.bcf[0][1]) + assert bcf.name == 'consensus.bcf' + assert bcf.isFile() + assert ChoCallateTestUtils.vcfSamples(bcf) == ['sample_a'] + assert ChoCallateTestUtils.recordCount(bcf) == 2 + assert ChoCallateTestUtils.vcfHeader(bcf).contains( + '##ChoCallate=', '.', 'PASS', + 'END=15', 'GT:AD:DP', '0/0:10:10'], + ['CP002684.1', '20', '.', 'T', 'C', '50', 'PASS', '.', + 'GT:AD:DP', '1/1:0,10:10'] + ] + ) + } + + when { + params { + output { + directory = "${launchDir}/output" + type = 'single' + format = 'vcf' + remove_invariant = false + split_multiallelic = false + } + } + process { + """ + input[0] = tuple( + 'consensus', + file("${launchDir}/bcfs/consensus.bcf") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + """ + } + } + + then { + assert process.success + def vcf = ChoCallateTestUtils.outputFile(process.out.vcf[0][1]) + assert vcf.isFile() + def body = ChoCallateTestUtils.run( + vcf.parentFile, + "bcftools view -H '${vcf.absolutePath}'" + ) + assert body.contains('<*>') + assert body.contains('END=15') + assert ChoCallateTestUtils.recordCount(vcf) == 2 + } + } +} diff --git a/tests/nf/modules/local/generate_consensus.nf.test b/tests/nf/modules/local/generate_consensus.nf.test new file mode 100644 index 0000000..67ceaf0 --- /dev/null +++ b/tests/nf/modules/local/generate_consensus.nf.test @@ -0,0 +1,159 @@ +nextflow_process { + + name "Test process GENERATE_CONSENSUS" + script "../../../../modules/local/generate_consensus/main.nf" + process "GENERATE_CONSENSUS" + + test("builds a consensus BCF from caller records") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + def callers = new File(launchDir, 'callers') + callers.mkdirs() + def sampleId = new File(launchDir, 'sample_id.txt') + sampleId.text = 'sample1\n' + ChoCallateTestUtils.run( + new File(launchDir), + "set -o pipefail; " + + "bcftools mpileup -a FORMAT/AD,FORMAT/DP " + + "--count-orphans -Ou " + + "--fasta-ref '${launchDir}/reference/genome.fasta' " + + "'${launchDir}/sample1.bam' | " + + "bcftools call --multiallelic-caller --variants-only -Ou | " + + "bcftools reheader -s '${sampleId}' " + + "-f '${launchDir}/reference/genome.fasta.fai' " + + "-o '${callers}/bcftools.bcf'" + ) + def chunks = new File(launchDir, 'chunks') + chunks.mkdirs() + new File(chunks, 'full.bed').text = + "CP002684.1\t0\t200001\n" + } + + when { + params { + output { + type = 'single' + format = 'bcf' + remove_invariant = false + } + coverage { + min_coverage = 1 + } + consensus { + threshold = 1 + } + } + process { + """ + input[0] = tuple( + 'sample1', + 'bcftools', + [file("${launchDir}/callers/bcftools.bcf")], + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + [file("${launchDir}/chunks/full.bed")] + ) + input[1] = 'bcftools' + input[2] = file("${launchDir}/reference/genome.fasta") + input[3] = file("${launchDir}/reference/genome.fasta.fai") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile( + process.out.consensus_bcf[0][1] + ) + assert bcf.isFile() + assert ChoCallateTestUtils.vcfSamples(bcf) == ['sample1'] + def header = ChoCallateTestUtils.vcfHeader(bcf) + assert header.contains('##ChoCallate=' + ) + assert ChoCallateTestUtils.recordCount(bcf) == 614 + assert snapshot( + ChoCallateTestUtils.statsLines(bcf) + ).match() + } + } + + test("writes variants only for per-sample output") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + def callers = new File(launchDir, 'callers') + callers.mkdirs() + def sampleId = new File(launchDir, 'sample_id.txt') + sampleId.text = 'sample1\n' + ChoCallateTestUtils.run( + new File(launchDir), + "set -o pipefail; " + + "bcftools mpileup -a FORMAT/AD,FORMAT/DP " + + "--count-orphans -Ou " + + "--fasta-ref '${launchDir}/reference/genome.fasta' " + + "'${launchDir}/sample1.bam' | " + + "bcftools call --multiallelic-caller --variants-only -Ou | " + + "bcftools reheader -s '${sampleId}' " + + "-f '${launchDir}/reference/genome.fasta.fai' " + + "-o '${callers}/bcftools.bcf'" + ) + def chunks = new File(launchDir, 'chunks') + chunks.mkdirs() + new File(chunks, 'full.bed').text = + "CP002684.1\t0\t200001\n" + } + + when { + params { + output { + type = 'sample' + format = 'bcf' + remove_invariant = true + } + coverage { + min_coverage = 1 + } + consensus { + threshold = 1 + } + } + process { + """ + input[0] = tuple( + 'sample1', + 'bcftools', + [file("${launchDir}/callers/bcftools.bcf")], + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + [file("${launchDir}/chunks/full.bed")] + ) + input[1] = 'bcftools' + input[2] = file("${launchDir}/reference/genome.fasta") + input[3] = file("${launchDir}/reference/genome.fasta.fai") + """ + } + } + + then { + assert process.success + def bcf = ChoCallateTestUtils.outputFile( + process.out.consensus_bcf[0][1] + ) + assert ChoCallateTestUtils.recordCount(bcf) == 312 + assert ChoCallateTestUtils.statsValue( + bcf, 'number of no-ALTs' + ) == 0 + assert ChoCallateTestUtils.vcfHeader(bcf).contains( + 'ReferenceBlocks=False' + ) + } + } +} diff --git a/tests/nf/modules/local/generate_consensus.nf.test.snap b/tests/nf/modules/local/generate_consensus.nf.test.snap new file mode 100644 index 0000000..2e5bb1a --- /dev/null +++ b/tests/nf/modules/local/generate_consensus.nf.test.snap @@ -0,0 +1,23 @@ +{ + "builds a consensus BCF from caller records": { + "content": [ + [ + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t614", + "SN\t0\tnumber of no-ALTs:\t302", + "SN\t0\tnumber of SNPs:\t283", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t29", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t0", + "SN\t0\tnumber of multiallelic SNP sites:\t0", + "TSTV\t0\t151\t132\t1.14\t151\t132\t1.14" + ] + ], + "timestamp": "2026-10-18T19:52:00.028447631", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/generate_coverage.nf.test b/tests/nf/modules/local/generate_coverage.nf.test new file mode 100644 index 0000000..95fdb57 --- /dev/null +++ b/tests/nf/modules/local/generate_coverage.nf.test @@ -0,0 +1,222 @@ +nextflow_process { + + name "Test process GENERATE_COVERAGE" + script "../../../../modules/local/generate_coverage/main.nf" + process "GENERATE_COVERAGE" + + test("computes covered intervals from a BAM") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + } + + when { + params { + coverage { + min_coverage = 1 + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi") + ) + input[1] = file("${projectDir}/assets/N1_FILE") + input[2] = file("${projectDir}/assets/NO_FILE") + input[3] = file( + "${launchDir}/reference/genome.fasta.fai" + ) + """ + } + } + + then { + assert process.success + def bed = ChoCallateTestUtils.outputFile( + process.out.coverage[0][1] + ) + assert bed.isFile() + def lines = bed.readLines().findAll { it.trim() } + assert lines.size() > 0 + def intervals = lines.collect { line -> + def fields = line.split('\t', -1) + [ + fields[0], + fields[1] as int, + fields[2] as int + ] + } + assert intervals.every { it[0] == 'CP002684.1' } + assert intervals.every { it[1] >= 0 && it[2] <= 200001 } + assert intervals.every { it[1] < it[2] } + int covered = intervals.sum { it[2] - it[1] } + assert covered > 0 + assert snapshot( + [intervals.size(), covered] + ).match() + } + } + + test("stages include and exclude BEDs with the same file name") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + new File(launchDir, 'include').mkdirs() + new File(launchDir, 'exclude').mkdirs() + new File(launchDir, 'include/regions.bed').text = + 'CP002684.1\t0\t200000\n' + new File(launchDir, 'exclude/regions.bed').text = + 'CP002684.1\t0\t100\n' + } + + when { + params { + coverage { + min_coverage = 1 + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi") + ) + input[1] = file("${launchDir}/include/regions.bed") + input[2] = file("${launchDir}/exclude/regions.bed") + input[3] = file( + "${launchDir}/reference/genome.fasta.fai" + ) + """ + } + } + + then { + assert process.success + def bed = ChoCallateTestUtils.outputFile( + process.out.coverage[0][1] + ) + def intervals = bed.readLines() + .findAll { it.trim() } + .collect { line -> + def fields = line.split('\t', -1) + [fields[0], fields[1] as int, fields[2] as int] + } + assert intervals.size() > 0 + assert intervals.every { it[0] == 'CP002684.1' } + assert intervals.every { it[1] >= 100 } + assert intervals.every { it[2] <= 200000 } + } + } + + test("strips track, browser and comment lines from BED inputs") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + // Put the header lines after data to ensure they are removed + // rather than relying on sort-bed's own header handling. + new File(launchDir, 'include.bed').text = [ + '# include regions', + 'CP002684.1\t0\t200000', + 'track name=include', + 'browser position CP002684.1:1-200000' + ].join('\n') + '\n' + new File(launchDir, 'exclude.bed').text = [ + 'browser position CP002684.1:1-100', + '# exclude the first 100 bp', + 'CP002684.1\t0\t100', + 'track name=exclude' + ].join('\n') + '\n' + } + + when { + params { + coverage { + min_coverage = 1 + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi") + ) + input[1] = file("${launchDir}/include.bed") + input[2] = file("${launchDir}/exclude.bed") + input[3] = file( + "${launchDir}/reference/genome.fasta.fai" + ) + """ + } + } + + then { + assert process.success + def bed = ChoCallateTestUtils.outputFile( + process.out.coverage[0][1] + ) + def lines = bed.readLines().findAll { it.trim() } + assert lines.every { line -> + !line.startsWith('track') && + !line.startsWith('browser') + } + def intervals = lines.collect { line -> + def fields = line.split('\t', -1) + [fields[0], fields[1] as int, fields[2] as int] + } + assert intervals.every { it[0] == 'CP002684.1' } + assert intervals.every { it[1] < it[2] } + assert intervals.every { it[1] >= 100 } + assert intervals == intervals.sort { a, b -> + a[1] <=> b[1] + } + } + } + + test("fails when a BED contig is absent from the reference .fai") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + new File(launchDir, 'include.bed').text = + '# wrong assembly\nchr1\t0\t100\n' + new File(launchDir, 'exclude.bed').text = '' + } + + when { + params { + coverage { + min_coverage = 1 + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi") + ) + input[1] = file("${launchDir}/include.bed") + input[2] = file("${launchDir}/exclude.bed") + input[3] = file( + "${launchDir}/reference/genome.fasta.fai" + ) + """ + } + } + + then { + assert process.failed + def output = (process.stdout + process.stderr).join('\n') + assert output.contains('include.bed') + assert output.contains("'chr1'") + assert output.contains('reference .fai') + } + } +} diff --git a/tests/nf/modules/local/generate_coverage.nf.test.snap b/tests/nf/modules/local/generate_coverage.nf.test.snap new file mode 100644 index 0000000..24938f3 --- /dev/null +++ b/tests/nf/modules/local/generate_coverage.nf.test.snap @@ -0,0 +1,15 @@ +{ + "computes covered intervals from a BAM": { + "content": [ + [ + 3, + 199795 + ] + ], + "timestamp": "2026-10-17T23:49:13.332801396", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/mapping_bowtie2.nf.test b/tests/nf/modules/local/mapping_bowtie2.nf.test new file mode 100644 index 0000000..db3dfed --- /dev/null +++ b/tests/nf/modules/local/mapping_bowtie2.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test process MAPPING_BOWTIE2" + script "../../../../modules/local/mapping_bowtie2/main.nf" + process "MAPPING_BOWTIE2" + + test("maps paired-end reads and adds read groups") { + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir) + ) + } + + when { + params { + input { + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + reads_type = "pe" + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${projectDir}/test_run/test_reads_R1.fq.gz"), + file("${projectDir}/test_run/test_reads_R2.fq.gz"), + file("${projectDir}/assets/NO_FILE") + ) + input[1] = files("${launchDir}/reference/genome.fasta*") + input[2] = file("${launchDir}/reference/genome.fasta") + input[3] = file("${launchDir}/reference/genome.dict") + input[4] = file("${launchDir}/reference/genome.fasta.fai") + """ + } + } + + then { + assert process.success + def bam = ChoCallateTestUtils.outputFile( + process.out.bam[0][1] + ) + def csi = ChoCallateTestUtils.outputFile( + process.out.bam[0][2] + ) + def raw = ChoCallateTestUtils.outputFile( + process.out.bam[0][3] + ) + assert bam.isFile() + assert csi.isFile() + assert raw.isFile() + assert raw.readLines().any { it.contains('mapped') } + assert ChoCallateTestUtils.bamRecords(bam) > 0 + def header = ChoCallateTestUtils.bamHeader(bam) + assert header.contains('@RG') + assert header.contains('ID:sample1') + assert header.contains('SM:sample1') + assert header.contains('PL:ILLUMINA') + assert header.contains('LB:sample1') + def rg = header.readLines().find { it.startsWith('@RG') } + assert snapshot( + [ChoCallateTestUtils.bamRecords(bam), rg] + ).match() + } + } +} diff --git a/tests/nf/modules/local/mapping_bowtie2.nf.test.snap b/tests/nf/modules/local/mapping_bowtie2.nf.test.snap new file mode 100644 index 0000000..c6f6819 --- /dev/null +++ b/tests/nf/modules/local/mapping_bowtie2.nf.test.snap @@ -0,0 +1,15 @@ +{ + "maps paired-end reads and adds read groups": { + "content": [ + [ + 39943, + "@RG\tID:sample1\tSM:sample1\tPL:ILLUMINA\tLB:sample1" + ] + ], + "timestamp": "2026-10-17T23:49:44.513064875", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/mapping_bwa.nf.test b/tests/nf/modules/local/mapping_bwa.nf.test new file mode 100644 index 0000000..10bd448 --- /dev/null +++ b/tests/nf/modules/local/mapping_bwa.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test process MAPPING_BWA" + script "../../../../modules/local/mapping_bwa/main.nf" + process "MAPPING_BWA" + + test("maps paired-end reads and adds read groups") { + setup { + ChoCallateTestUtils.bwaIndex( + projectDir, new File(launchDir) + ) + } + + when { + params { + input { + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + reads_type = "pe" + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${projectDir}/test_run/test_reads_R1.fq.gz"), + file("${projectDir}/test_run/test_reads_R2.fq.gz"), + file("${projectDir}/assets/NO_FILE") + ) + input[1] = files("${launchDir}/reference/genome.fasta*") + input[2] = file("${launchDir}/reference/genome.fasta") + input[3] = file("${launchDir}/reference/genome.dict") + input[4] = file("${launchDir}/reference/genome.fasta.fai") + """ + } + } + + then { + assert process.success + def bam = ChoCallateTestUtils.outputFile( + process.out.bam[0][1] + ) + def csi = ChoCallateTestUtils.outputFile( + process.out.bam[0][2] + ) + def raw = ChoCallateTestUtils.outputFile( + process.out.bam[0][3] + ) + assert bam.isFile() + assert csi.isFile() + assert raw.isFile() + assert raw.readLines().any { it.contains('mapped') } + assert ChoCallateTestUtils.bamRecords(bam) > 0 + def header = ChoCallateTestUtils.bamHeader(bam) + assert header.contains('@RG') + assert header.contains('ID:sample1') + assert header.contains('SM:sample1') + assert header.contains('PL:ILLUMINA') + assert header.contains('LB:sample1') + def rg = header.readLines().find { it.startsWith('@RG') } + assert snapshot( + [ChoCallateTestUtils.bamRecords(bam), rg] + ).match() + } + } +} diff --git a/tests/nf/modules/local/mapping_bwa.nf.test.snap b/tests/nf/modules/local/mapping_bwa.nf.test.snap new file mode 100644 index 0000000..15831e8 --- /dev/null +++ b/tests/nf/modules/local/mapping_bwa.nf.test.snap @@ -0,0 +1,15 @@ +{ + "maps paired-end reads and adds read groups": { + "content": [ + [ + 39986, + "@RG\tID:sample1\tSM:sample1\tPL:ILLUMINA\tLB:sample1" + ] + ], + "timestamp": "2026-10-17T23:50:14.219398598", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/mapping_minimap2.nf.test b/tests/nf/modules/local/mapping_minimap2.nf.test new file mode 100644 index 0000000..4a33c0e --- /dev/null +++ b/tests/nf/modules/local/mapping_minimap2.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test process MAPPING_MINIMAP2" + script "../../../../modules/local/mapping_minimap2/main.nf" + process "MAPPING_MINIMAP2" + + test("maps paired-end reads and adds read groups") { + setup { + ChoCallateTestUtils.minimap2Index( + projectDir, new File(launchDir) + ) + } + + when { + params { + input { + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + reads_type = "pe" + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${projectDir}/test_run/test_reads_R1.fq.gz"), + file("${projectDir}/test_run/test_reads_R2.fq.gz"), + file("${projectDir}/assets/NO_FILE") + ) + input[1] = files("${launchDir}/reference/genome.fasta*") + input[2] = file("${launchDir}/reference/genome.fasta") + input[3] = file("${launchDir}/reference/genome.dict") + input[4] = file("${launchDir}/reference/genome.fasta.fai") + """ + } + } + + then { + assert process.success + def bam = ChoCallateTestUtils.outputFile( + process.out.bam[0][1] + ) + def csi = ChoCallateTestUtils.outputFile( + process.out.bam[0][2] + ) + def raw = ChoCallateTestUtils.outputFile( + process.out.bam[0][3] + ) + assert bam.isFile() + assert csi.isFile() + assert raw.isFile() + assert raw.readLines().any { it.contains('mapped') } + assert ChoCallateTestUtils.bamRecords(bam) > 0 + def header = ChoCallateTestUtils.bamHeader(bam) + assert header.contains('@RG') + assert header.contains('ID:sample1') + assert header.contains('SM:sample1') + assert header.contains('PL:ILLUMINA') + assert header.contains('LB:sample1') + def rg = header.readLines().find { it.startsWith('@RG') } + assert snapshot( + [ChoCallateTestUtils.bamRecords(bam), rg] + ).match() + } + } +} diff --git a/tests/nf/modules/local/mapping_minimap2.nf.test.snap b/tests/nf/modules/local/mapping_minimap2.nf.test.snap new file mode 100644 index 0000000..0520d68 --- /dev/null +++ b/tests/nf/modules/local/mapping_minimap2.nf.test.snap @@ -0,0 +1,15 @@ +{ + "maps paired-end reads and adds read groups": { + "content": [ + [ + 39922, + "@RG\tID:sample1\tSM:sample1\tPL:ILLUMINA\tLB:sample1" + ] + ], + "timestamp": "2026-10-17T23:50:47.820499809", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/mapping_stats.nf.test b/tests/nf/modules/local/mapping_stats.nf.test new file mode 100644 index 0000000..8cf510f --- /dev/null +++ b/tests/nf/modules/local/mapping_stats.nf.test @@ -0,0 +1,71 @@ +nextflow_process { + + name "Test process MAPPING_STATS" + script "../../../../modules/local/mapping_stats/main.nf" + process "MAPPING_STATS" + + test("reports raw and final BAM statistics") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + ChoCallateTestUtils.run( + new File(launchDir), + "samtools flagstat '${launchDir}/sample1.bam' " + + "> '${launchDir}/raw.flagstat'" + ) + } + + when { + params { + output { + directory = "${launchDir}/output" + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/sample1.bam"), + file("${launchDir}/sample1.bam.csi"), + file("${launchDir}/raw.flagstat") + ) + """ + } + } + + then { + assert process.success + def raw = ChoCallateTestUtils.outputFile( + process.out.stats[0][1] + ) + def flagstat = ChoCallateTestUtils.outputFile( + process.out.stats[0][2] + ) + def stats = ChoCallateTestUtils.outputFile( + process.out.stats[0][3] + ) + def idxstats = ChoCallateTestUtils.outputFile( + process.out.stats[0][4] + ) + assert raw.name == 'sample1.raw.flagstat' + assert flagstat.name == 'sample1.final.flagstat' + assert stats.name == 'sample1.stats' + assert idxstats.name == 'sample1.idxstats' + assert raw.readLines().any { it.contains('mapped') } + assert flagstat.readLines().any { it.contains('mapped') } + assert stats.readLines().any { + it.startsWith('SN\traw total sequences:') + } + assert idxstats.readLines().any { + it.startsWith('CP002684.1\t') + } + assert new File( + "${launchDir}/output/mapping_stats/sample1.raw.flagstat" + ).isFile() + assert new File( + "${launchDir}/output/mapping_stats/sample1.stats" + ).isFile() + } + } +} diff --git a/tests/nf/modules/local/mapping_summary.nf.test b/tests/nf/modules/local/mapping_summary.nf.test new file mode 100644 index 0000000..cf1a5a3 --- /dev/null +++ b/tests/nf/modules/local/mapping_summary.nf.test @@ -0,0 +1,105 @@ +nextflow_process { + + name "Test process MAPPING_SUMMARY" + script "../../../../modules/local/mapping_summary/main.nf" + process "MAPPING_SUMMARY" + + test("collects raw and final statistics in sample order") { + setup { + def write = { String name, String text -> + def target = new File(launchDir, name) + target.parentFile.mkdirs() + target.text = text + } + ['sample1', 'sample2'].each { sample -> + write("input/raw/${sample}.raw.flagstat", '''\ +10 + 0 in total (QC-passed reads + QC-failed reads) +10 + 0 primary +0 + 0 secondary +0 + 0 supplementary +0 + 0 duplicates +8 + 0 mapped (80.00% : N/A) +10 + 0 paired in sequencing +5 + 0 read1 +5 + 0 read2 +6 + 0 properly paired (60.00% : N/A) +6 + 0 with itself and mate mapped +1 + 0 singletons (10.00% : N/A) +1 + 0 with mate mapped to a different chr +''') + write("input/flagstat/${sample}.final.flagstat", '''\ +8 + 0 in total (QC-passed reads + QC-failed reads) +8 + 0 primary +0 + 0 secondary +0 + 0 supplementary +1 + 0 duplicates +8 + 0 mapped (100.00% : N/A) +8 + 0 paired in sequencing +4 + 0 read1 +4 + 0 read2 +6 + 0 properly paired (75.00% : N/A) +6 + 0 with itself and mate mapped +0 + 0 singletons (0.00% : N/A) +0 + 0 with mate mapped to a different chr +''') + write("input/stats/${sample}.stats", '''\ +SN\traw total sequences:\t8 +SN\tsequences:\t8 +SN\treads MQ0:\t2 +SN\taverage length:\t150 +SN\taverage quality:\t36.5 +SN\tinsert size average:\t250.5 +SN\tinsert size standard deviation:\t20.1 +SN\terror rate:\t1.25e-03 +SN\tbases mapped (cigar):\t1200 +''') + write("input/idxstats/${sample}.idxstats", '''\ +chr1\t1000\t8\t0 +chr2\t500\t0\t0 +*\t0\t0\t0 +''') + } + } + + when { + params { + output { + directory = "${launchDir}/output" + } + } + process { + """ + input[0] = ['sample2', 'sample1'] + input[1] = files("${launchDir}/input/raw/*") + input[2] = files("${launchDir}/input/flagstat/*") + input[3] = files("${launchDir}/input/stats/*") + input[4] = files("${launchDir}/input/idxstats/*") + """ + } + } + + then { + assert process.success + def summary = ChoCallateTestUtils.outputFile( + process.out[0][0] + ) + assert summary.name == 'mapping_summary.tsv' + def lines = summary.readLines() + def header = lines[0].split('\t', -1) + def rows = lines.drop(1).collect { line -> + [header, line.split('\t', -1)].transpose().collectEntries() + } + assert lines.size() == 3 + assert [rows[0]['sample'], rows[1]['sample']] == + ['sample2', 'sample1'] + def values = rows[0] + assert values['raw_mapped_pct'] == '80.00' + assert values['final_duplicate_pct'] == '12.50' + assert values['final_mq0_pct'] == '25.00' + assert values['contigs_with_reads'] == '1' + assert new File( + "${launchDir}/output/mapping_stats/mapping_summary.tsv" + ).isFile() + } + } +} diff --git a/tests/nf/modules/local/merge_outputs.nf.test b/tests/nf/modules/local/merge_outputs.nf.test new file mode 100644 index 0000000..622dc6e --- /dev/null +++ b/tests/nf/modules/local/merge_outputs.nf.test @@ -0,0 +1,151 @@ +nextflow_process { + + name "Test process MERGE_OUTPUTS" + script "../../../../modules/local/merge_outputs/main.nf" + process "MERGE_OUTPUTS" + + test("merges sample BCFs in samplesheet order") { + setup { + ChoCallateTestUtils.referenceBundle( + projectDir, new File(launchDir) + ) + def bcfs = new File(launchDir, 'bcfs') + ChoCallateTestUtils.writeBcf( + bcfs, 'sample_b', 'sample_b', + [ + ['CP002684.1', '10', '.', 'G', 'A', '50', 'PASS', '.', + 'GT:AD:DP', '0/1:5,5:10'], + ['CP002684.1', '20', '.', 'T', 'C', '50', 'PASS', '.', + 'GT:AD:DP', '1/1:0,10:10'] + ] + ) + ChoCallateTestUtils.writeBcf( + bcfs, 'sample_a', 'sample_a', + [ + ['CP002684.1', '10', '.', 'G', 'A', '50', 'PASS', '.', + 'GT:AD:DP', '0/1:4,6:10'], + ['CP002684.1', '30', '.', 'G', 'A', '50', 'PASS', '.', + 'GT:AD:DP', '0/1:5,5:10'] + ] + ) + } + + when { + params { + output { + type = 'single' + format = 'bcf' + remove_invariant = false + split_multiallelic = false + } + } + process { + """ + input[0] = tuple( + 'sample_b,sample_a', + [ + file("${launchDir}/bcfs/sample_b.bcf"), + file("${launchDir}/bcfs/sample_a.bcf") + ] + ) + input[1] = file("${launchDir}/reference/genome.fasta") + input[2] = file("${launchDir}/reference/genome.fasta.fai") + input[3] = [] + """ + } + } + + then { + assert process.success + def emitted = process.out[0] + def paths = emitted instanceof List ? emitted : [emitted] + def consensus = paths.collect { output -> + ChoCallateTestUtils.outputFile(output) + }.find { file -> file.name == 'consensus.bcf' } + assert consensus != null + assert consensus.isFile() + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample_b', 'sample_a'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains( + '##contig=' + ) + assert header.contains('##ChoCallate= + ChoCallateTestUtils.outputFile(output) + }.find { file -> file.name == 'consensus.bcf' } + assert consensus != null + assert consensus.isFile() + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample_b', 'sample_a'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains( + '##contig=' + ) + assert header.contains('##ChoCallate= 0 + def header = ChoCallateTestUtils.bamHeader(bam) + assert header.contains('ID:bam_sample') + assert header.contains('SM:bam_sample') + assert header.contains('PL:ILLUMINA') + assert header.contains('LB:bam_sample') + def rg = header.readLines().find { it.startsWith('@RG') } + assert snapshot( + [ChoCallateTestUtils.bamRecords(bam), rg] + ).match() + } + } +} diff --git a/tests/nf/modules/local/prepare_bam.nf.test.snap b/tests/nf/modules/local/prepare_bam.nf.test.snap new file mode 100644 index 0000000..f445c4c --- /dev/null +++ b/tests/nf/modules/local/prepare_bam.nf.test.snap @@ -0,0 +1,15 @@ +{ + "replaces read groups and indexes an input BAM": { + "content": [ + [ + 39943, + "@RG\tID:bam_sample\tSM:bam_sample\tPL:ILLUMINA\tLB:bam_sample" + ] + ], + "timestamp": "2026-10-17T23:51:45.57727183", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/modules/local/scatter_intervals.nf.test b/tests/nf/modules/local/scatter_intervals.nf.test new file mode 100644 index 0000000..db24abd --- /dev/null +++ b/tests/nf/modules/local/scatter_intervals.nf.test @@ -0,0 +1,58 @@ +nextflow_process { + + name "Test process SCATTER_INTERVALS" + script "../../../../modules/local/scatter_intervals/main.nf" + process "SCATTER_INTERVALS" + + test("splits covered intervals into caller chunks") { + setup { + ChoCallateTestUtils.referenceBundle( + projectDir, new File(launchDir) + ) + def bed = new File(launchDir, 'coverage.bed') + bed.text = "CP002684.1\t0\t200001\n" + } + + when { + params { + calling { + cpu = 2 + } + consensus { + cpu = 2 + } + } + process { + """ + input[0] = tuple( + 'sample1', + file("${launchDir}/coverage.bed") + ) + input[1] = file("${launchDir}/reference/genome.fasta") + input[2] = file("${launchDir}/reference/genome.fasta.fai") + input[3] = file("${launchDir}/reference/genome.dict") + """ + } + } + + then { + assert process.success + def emitted = process.out.chunks[0][1] + def chunks = emitted instanceof List ? emitted : [emitted] + assert chunks.size() == 2 + def summary = chunks.collect { chunk -> + def file = ChoCallateTestUtils.outputFile(chunk) + def lines = file.readLines().findAll { it.trim() } + assert lines.size() > 0 + lines.each { line -> + def fields = line.split('\t', -1) + assert fields[0] == 'CP002684.1' + assert fields[1] as int >= 0 + assert fields[2] as int <= 200001 + } + [file.name, lines] + }.sort { it[0] } + assert snapshot(summary).match() + } + } +} diff --git a/tests/nf/modules/local/scatter_intervals.nf.test.snap b/tests/nf/modules/local/scatter_intervals.nf.test.snap new file mode 100644 index 0000000..75938b7 --- /dev/null +++ b/tests/nf/modules/local/scatter_intervals.nf.test.snap @@ -0,0 +1,25 @@ +{ + "splits covered intervals into caller chunks": { + "content": [ + [ + [ + "0000-scattered.bed", + [ + "CP002684.1\t0\t100000" + ] + ], + [ + "0001-scattered.bed", + [ + "CP002684.1\t100000\t200001" + ] + ] + ] + ], + "timestamp": "2026-10-17T23:52:18.604607296", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} \ No newline at end of file diff --git a/tests/nf/pipeline.nf.test b/tests/nf/pipeline.nf.test new file mode 100644 index 0000000..11e067f --- /dev/null +++ b/tests/nf/pipeline.nf.test @@ -0,0 +1,1052 @@ +nextflow_pipeline { + + name "ChoCallate pipeline modes" + script "main.nf" + + test("test_pe_bowtie2_single") { + tag "mapping_stats" + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample1'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.recordCount(consensus) == 271 + assert snapshot( + ChoCallateTestUtils.statsLines(consensus) + ).match() + def mapping_dir = new File("${outputDir}/mapping_stats") + ['sample1.raw.flagstat', 'sample1.final.flagstat', + 'sample1.stats', 'sample1.idxstats', + 'mapping_summary.tsv'].each { name -> + assert new File(mapping_dir, name).isFile() + } + def summary = new File(mapping_dir, 'mapping_summary.tsv') + def lines = summary.readLines() + assert lines.size() == 2 + assert lines[0].contains('raw_mapped_pct') + assert lines[1].startsWith('sample1\t') + } + } + + test("test_pe_bowtie2_single_vcf") { + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "vcf" + remove_invariant = true + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.vcf.gz") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample1'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.recordCount(consensus) == 271 + } + } + + test("test_bam_input") { + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'bam_sample' + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + ['bam_sample', "${launchDir}/bam_sample.bam"] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "bam" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['bam_sample'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.recordCount(consensus) == 271 + assert snapshot( + ChoCallateTestUtils.statsLines(consensus) + ).match() + } + } + + test("test_se_bwa_sample_vcf") { + tag "index" + setup { + ChoCallateTestUtils.bwaIndex( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample_se', + "${projectDir}/test_run/test_reads_R1.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "se" + } + output { + directory = "${outputDir}" + type = "sample" + format = "vcf" + remove_invariant = true + } + mapping { + mapper = "bwa" + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def vcf = new File( + "${outputDir}/per_sample/sample_se.vcf.gz" + ) + assert vcf.isFile() + assert vcf.length() > 0 + assert ChoCallateTestUtils.vcfSamples(vcf) == ['sample_se'] + def header = ChoCallateTestUtils.vcfHeader(vcf) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.recordCount(vcf) == 303 + assert snapshot( + ChoCallateTestUtils.statsLines(vcf) + ).match() + } + } + + test("test_minimap2") { + tag "index" + setup { + ChoCallateTestUtils.minimap2Index( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + mapping { + mapper = "minimap2" + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample1'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.recordCount(consensus) == 305 + assert snapshot( + ChoCallateTestUtils.statsLines(consensus) + ).match() + } + } + + test("test_default_keeps_monomorphic") { + tag "all_sites" + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample1'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + // output.remove_invariant is intentionally not set: the default + // must keep monomorphic reference records in the published file. + def body = ChoCallateTestUtils.run( + consensus.parentFile, + "bcftools view -H '${consensus.absolutePath}'" + ) + assert body.contains('<*>') + // Soft-filtered caller records abstain, so positions where the + // only caller record is LowQual are not written as reference. + assert ChoCallateTestUtils.recordCount(consensus) == 578 + assert snapshot( + ChoCallateTestUtils.statsLines(consensus) + ).match() + } + } + + test("test_multi_sample_merge") { + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample_c', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ], + [ + 'sample_a', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ], + [ + 'sample_b', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample_c', 'sample_a', 'sample_b'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.statsValue( + consensus, 'number of samples' + ) == 3 + assert ChoCallateTestUtils.recordCount(consensus) == 271 + assert snapshot( + ChoCallateTestUtils.statsLines(consensus) + ).match() + } + } + + test("test_multi_sample_merge_vcf") { + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample_a', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ], + [ + 'sample_b', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "vcf" + remove_invariant = true + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.vcf.gz") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample_a', 'sample_b'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('##ChoCallate=<') + assert header.contains( + '##contig=' + ) + assert ChoCallateTestUtils.recordCount(consensus) == 271 + } + } + + test("test_stage_copy_default_prefix") { + tag "index" + tag "stage_copy" + config "./stage_copy.config" + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert consensus.length() > 0 + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample1'] + assert ChoCallateTestUtils.recordCount(consensus) == 271 + } + } + + test("test_empty_coverage_fails_with_sample_name") { + tag "dropped_samples" + tag "empty_coverage" + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample_ok' + ) + ChoCallateTestUtils.emptyBam( + new File(launchDir), 'sample_empty' + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + ['sample_ok', "${launchDir}/sample_ok.bam"], + ['sample_empty', "${launchDir}/sample_empty.bam"] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "bam" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + left_align_indels { + enabled = false + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.failed + def output = ([workflow.errorMessage ?: ''] + + workflow.stdout + workflow.stderr).join('\n') + assert output.contains('sample_empty') + assert output.contains('coverage.min_coverage=1') + } + } + + test("test_empty_coverage_can_be_skipped") { + tag "dropped_samples" + tag "empty_coverage" + tag "empty_coverage_skip" + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample_ok' + ) + ChoCallateTestUtils.emptyBam( + new File(launchDir), 'sample_empty' + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + ['sample_ok', "${launchDir}/sample_ok.bam"], + ['sample_empty', "${launchDir}/sample_empty.bam"] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "bam" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + left_align_indels { + enabled = false + } + coverage { + min_coverage = 1 + on_empty = "skip" + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert ChoCallateTestUtils.vcfSamples(consensus) == ['sample_ok'] + def header = ChoCallateTestUtils.vcfHeader(consensus) + assert header.contains('SamplesDropped="sample_empty"') + def dropped = new File("${outputDir}/dropped_samples.txt") + assert dropped.isFile() + assert dropped.readLines().findAll { it.trim() } == + ['sample_empty'] + def log = new File(launchDir, 'meta/nextflow.log') + assert log.text.contains( + 'Sample sample_empty: no region at or above' + ) + } + } + + test("test_all_callers_failed_warns_and_drops_sample") { + tag "dropped_samples" + tag "all_callers_failed" + setup { + ChoCallateTestUtils.bowtie2Bam( + projectDir, new File(launchDir), 'sample1' + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [['sample1', "${launchDir}/sample1.bam"]] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "bam" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + left_align_indels { + enabled = false + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + allow_caller_failure = true + bcftools { + mpileup { + extra_args = "--definitely-not-a-real-option" + } + } + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.failed + def log = new File(launchDir, 'meta/nextflow.log') + assert log.isFile() + assert log.text.contains( + 'Sample sample1: no caller produced calls' + ) + def output = ([workflow.errorMessage ?: ''] + + workflow.stdout + workflow.stderr).join('\n') + assert output.contains( + 'All samples were dropped; see dropped_samples.txt' + ) + def dropped = new File("${outputDir}/dropped_samples.txt") + assert dropped.isFile() + assert dropped.readLines().findAll { it.trim() } == ['sample1'] + } + } + + test("test_all_samples_dropped_per_sample_fails") { + tag "dropped_samples" + tag "all_samples_dropped" + setup { + ChoCallateTestUtils.referenceBundle( + projectDir, new File(launchDir) + ) + ChoCallateTestUtils.emptyBam( + new File(launchDir), 'sample_empty' + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [['sample_empty', "${launchDir}/sample_empty.bam"]] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference" + reference_index_prefix = "genome.fasta" + format = "bam" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "sample" + format = "vcf" + remove_invariant = true + } + left_align_indels { + enabled = false + } + coverage { + min_coverage = 1 + on_empty = "skip" + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.failed + def output = ([workflow.errorMessage ?: ''] + + workflow.stdout + workflow.stderr).join('\n') + assert output.contains( + 'All samples were dropped; see dropped_samples.txt' + ) + def dropped = new File("${outputDir}/dropped_samples.txt") + assert dropped.isFile() + assert dropped.readLines().findAll { it.trim() } == + ['sample_empty'] + } + } + + test("test_missing_index_prefix_fails_early") { + tag "index" + setup { + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${projectDir}/test_run/" + + "arth_chr1.fasta.gz" + reference_index_dir = "${launchDir}" + reference_index_prefix = "missing_index_prefix" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.failed + assert workflow.trace.tasks().isEmpty() + def output = ([workflow.errorMessage ?: ''] + + workflow.stdout + workflow.stderr).join('\n') + assert output.contains("No bowtie2 index files found") + assert output.contains("missing_index_prefix") + } + } + + test("test_decoy_index_prefix_fails_early") { + tag "index" + setup { + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + // A complete-looking index for a different prefix used to satisfy + // the '*' glob and defer the failure to the mapper. + new File( + launchDir, 'decoy_prefix.other.1.bt2' + ).text = '' + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${projectDir}/test_run/" + + "arth_chr1.fasta.gz" + reference_index_dir = "${launchDir}" + reference_index_prefix = "decoy_prefix" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.failed + assert workflow.trace.tasks().isEmpty() + def output = ([workflow.errorMessage ?: ''] + + workflow.stdout + workflow.stderr).join('\n') + assert output.contains("No bowtie2 index files found") + assert output.contains("decoy_prefix.2.bt2") + assert output.contains("decoy_prefix.rev.2.bt2") + assert !output.contains("decoy_prefix.other.1.bt2") + } + } + + test("test_bowtie2_large_index") { + tag "index" + setup { + ChoCallateTestUtils.bowtie2Index( + projectDir, new File(launchDir), 'reference_large', true + ) + ChoCallateTestUtils.samplesheet( + new File(launchDir), + [ + [ + 'sample1', + "${projectDir}/test_run/test_reads_R1.fq.gz", + "${projectDir}/test_run/test_reads_R2.fq.gz" + ] + ] + ) + } + + when { + params { + input { + samples_tsv = "${launchDir}/samples.tsv" + reference_genome = "${launchDir}/reference_large/" + + "genome.fasta" + reference_index_dir = "${launchDir}/reference_large" + reference_index_prefix = "genome.fasta" + format = "fastq" + reads_type = "pe" + } + output { + directory = "${outputDir}" + type = "single" + format = "bcf" + remove_invariant = true + } + coverage { + min_coverage = 1 + } + calling { + callers = "bcftools" + } + consensus { + threshold = 1 + } + } + } + + then { + assert workflow.success + assert new File( + launchDir, 'reference_large/genome.fasta.1.bt2l' + ).isFile() + def consensus = new File("${outputDir}/consensus.bcf") + assert consensus.isFile() + assert ChoCallateTestUtils.vcfSamples(consensus) == + ['sample1'] + } + } +} diff --git a/tests/nf/pipeline.nf.test.snap b/tests/nf/pipeline.nf.test.snap new file mode 100644 index 0000000..d140dc3 --- /dev/null +++ b/tests/nf/pipeline.nf.test.snap @@ -0,0 +1,128 @@ +{ + "test_minimap2": { + "content": [ + [ + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t305", + "SN\t0\tnumber of no-ALTs:\t0", + "SN\t0\tnumber of SNPs:\t287", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t18", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t4", + "SN\t0\tnumber of multiallelic SNP sites:\t2", + "TSTV\t0\t164\t125\t1.31\t164\t123\t1.33" + ] + ], + "timestamp": "2026-10-18T23:32:12.539506763", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + }, + "test_se_bwa_sample_vcf": { + "content": [ + [ + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t303", + "SN\t0\tnumber of no-ALTs:\t0", + "SN\t0\tnumber of SNPs:\t287", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t17", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t5", + "SN\t0\tnumber of multiallelic SNP sites:\t3", + "TSTV\t0\t166\t124\t1.34\t165\t122\t1.35" + ] + ], + "timestamp": "2026-10-18T23:30:36.724053717", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + }, + "test_pe_bowtie2_single": { + "content": [ + [ + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t271", + "SN\t0\tnumber of no-ALTs:\t0", + "SN\t0\tnumber of SNPs:\t256", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t15", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t3", + "SN\t0\tnumber of multiallelic SNP sites:\t2", + "TSTV\t0\t145\t113\t1.28\t145\t111\t1.31" + ] + ], + "timestamp": "2026-10-18T23:27:29.765635124", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + }, + "test_multi_sample_merge": { + "content": [ + [ + "SN\t0\tnumber of samples:\t3", + "SN\t0\tnumber of records:\t271", + "SN\t0\tnumber of no-ALTs:\t0", + "SN\t0\tnumber of SNPs:\t256", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t15", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t3", + "SN\t0\tnumber of multiallelic SNP sites:\t2", + "TSTV\t0\t145\t113\t1.28\t145\t111\t1.31" + ] + ], + "timestamp": "2026-10-18T23:35:39.517778135", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + }, + "test_default_keeps_monomorphic": { + "content": [ + [ + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t578", + "SN\t0\tnumber of no-ALTs:\t307", + "SN\t0\tnumber of SNPs:\t256", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t15", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t3", + "SN\t0\tnumber of multiallelic SNP sites:\t2", + "TSTV\t0\t145\t113\t1.28\t145\t111\t1.31" + ] + ], + "timestamp": "2026-10-18T23:33:57.991736434", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + }, + "test_bam_input": { + "content": [ + [ + "SN\t0\tnumber of samples:\t1", + "SN\t0\tnumber of records:\t271", + "SN\t0\tnumber of no-ALTs:\t0", + "SN\t0\tnumber of SNPs:\t256", + "SN\t0\tnumber of MNPs:\t0", + "SN\t0\tnumber of indels:\t15", + "SN\t0\tnumber of others:\t0", + "SN\t0\tnumber of multiallelic sites:\t3", + "SN\t0\tnumber of multiallelic SNP sites:\t2", + "TSTV\t0\t145\t113\t1.28\t145\t111\t1.31" + ] + ], + "timestamp": "2026-10-18T23:29:36.028215182", + "meta": { + "nf-test": "0.9.5", + "nextflow": "26.04.7" + } + } +} diff --git a/tests/nf/stage_copy.config b/tests/nf/stage_copy.config new file mode 100644 index 0000000..18bfa0b --- /dev/null +++ b/tests/nf/stage_copy.config @@ -0,0 +1,6 @@ +// Exercise index staging with copies instead of symlinks. In copy mode the +// staged file's realpath is the work-directory copy, so the index prefix must +// be resolved before staging. +process { + stageInMode = 'copy' +} diff --git a/tests/python/conftest.py b/tests/python/conftest.py new file mode 100644 index 0000000..3a1742f --- /dev/null +++ b/tests/python/conftest.py @@ -0,0 +1,328 @@ +"""Fixtures for the Python unit tests of the ChoCallate helper scripts. + +Tiny artificial datasets (reference, indexed BAM, bgzipped+tabixed coverage BED, +indexed caller BCFs) are built in code with pysam so that the tests run without +external test data. The ChoCallate conda environment provides pysam and +bcftools; run the suite with: + + pytest -q tests/python + +Both the pure-pysam and the Cython-accelerated consensus paths are exercised +(``use_accel`` parameter); the accelerated case is skipped when the optional +extension has not been built (``python bin/build_consensus_accel.py``). +""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pysam +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +BIN_DIR = REPO_ROOT / 'bin' +CONSENSUS_SCRIPT = BIN_DIR / 'generate_consensus.py' + +CHR = 'chr1' +# 100 bp of "ACGT" repeats: position i (0-based) holds 'ACGT'[i % 4] +REF_SEQ = 'ACGT' * 25 + +# Prefer the bcftools shipped next to the running interpreter (the ChoCallate +# environment): other installations on PATH may lack the plugins the pipeline +# uses (bcftools +fill-tags). +_ENV_BCFTOOLS = Path(sys.executable).with_name('bcftools') +BCFTOOLS = str(_ENV_BCFTOOLS) if _ENV_BCFTOOLS.exists() else (shutil.which('bcftools') or 'bcftools') + + +def accel_available(): + """True when bin/consensus_accel can be imported by this interpreter.""" + if str(BIN_DIR) not in sys.path: + sys.path.insert(0, str(BIN_DIR)) + try: + import consensus_accel # noqa: F401 + except ImportError: + return False + return True + + +ACCEL_AVAILABLE = accel_available() + + +def ref_base(pos1): + """Reference base at a 1-based position of the test contig.""" + return REF_SEQ[(pos1 - 1) % len(REF_SEQ)] + + +def alt_base(pos1): + """A different base than the reference at a 1-based position.""" + return {'A': 'G', 'C': 'T', 'G': 'C', 'T': 'A'}[ref_base(pos1)] + + +def write_reference(path, seq=REF_SEQ, contig=CHR): + """Write a FASTA file plus .fai index and return its path.""" + with open(path, 'w', encoding='utf-8') as handle: + handle.write(f'>{contig}\n') + for start in range(0, len(seq), 60): + handle.write(seq[start:start + 60] + '\n') + pysam.faidx(str(path)) + return path + + +def write_bam(path, reads, contig=CHR, length=len(REF_SEQ), csi=False): + """Write a coordinate-sorted indexed BAM. + + Args: + path: Output BAM path. + reads: Iterable of tuples, in ascending start order: + ``(query_name, 0-based start, sequence)`` optionally followed by a + CIGAR tuple, a SAM flag and a base-quality string. ``None`` + qualities are stored as the BAM missing-quality sentinel (255). + contig: Contig name of the single reference sequence. + length: Contig length. + csi: Write a CSI index instead of a BAI index (needed for positions + at or above 2**29). + + Returns: + The BAM path. + """ + header = {'HD': {'VN': '1.6', 'SO': 'coordinate'}, + 'SQ': [{'SN': contig, 'LN': length}]} + with pysam.AlignmentFile(str(path), 'wb', header=header) as bam: + for entry in reads: + name, start, seq = entry[:3] + cigar = entry[3] if len(entry) > 3 else ((0, len(seq)),) + flag = entry[4] if len(entry) > 4 else 0 + qualities = entry[5] if len(entry) > 5 else 'I' * len(seq) + read = pysam.AlignedSegment() + read.query_name = name + read.query_sequence = seq + read.flag = flag + read.reference_id = 0 + read.reference_start = start + read.mapping_quality = 60 + read.cigar = cigar + if qualities is not None: + read.query_qualities = pysam.qualitystring_to_array( + qualities) + bam.write(read) + if csi: + pysam.index('-c', str(path)) + else: + pysam.index(str(path)) + return path + + +def write_sparse_reference(path, contig, length, start0, seq, + line_bases=60): + """Write a mostly-sparse FASTA with known bases and a hand-made .fai. + + Only ``seq``, placed at 0-based ``start0``, is written to the file; the + gap before it is a filesystem hole, so a contig close to the 32-bit + coordinate limit costs almost no disk space. The ``.fai`` is written + directly because indexing would read the whole sparse file. + + Args: + path: Output FASTA path. + contig: Contig name. + length: Declared contig length. + start0: 0-based position of the first base of ``seq``. + seq: Known bases to place at ``start0``. + line_bases: FASTA line width used by the .fai arithmetic. + + Returns: + The FASTA path. + """ + line_width = line_bases + 1 + header = f'>{contig}\n' + offset = len(header) + with open(path, 'w', encoding='utf-8') as handle: + handle.write(header) + for index, base in enumerate(seq, start=start0): + line, column = divmod(index, line_bases) + handle.seek(offset + line * line_width + column) + handle.write(base) + with open(f'{path}.fai', 'w', encoding='utf-8') as handle: + handle.write(f'{contig}\t{length}\t{offset}\t{line_bases}\t' + f'{line_width}\n') + return path + + +def write_bed(path, intervals, preset='bed', csi=True): + """Write, bgzip and index (0-based half-open) intervals. + + Args: + path: Path of the plain text BED to create; the compressed file gets + the '.gz' suffix. + intervals: Iterable of (contig, start, end) tuples. + preset: tabix preset. + csi: Write a '.csi' index (instead of '.tbi') when true. + + Returns: + Path of the bgzipped BED (the file the tools actually read). + """ + plain = Path(path) + with open(plain, 'w', encoding='utf-8') as handle: + for contig, start, end in intervals: + handle.write(f'{contig}\t{start}\t{end}\n') + gz = f'{plain}.gz' + pysam.tabix_compress(str(plain), gz, force=True) + pysam.tabix_index(gz, preset=preset, csi=csi, force=True) + return Path(gz) + + +def write_caller_bcf(path, sample_name, records, contig=CHR, length=len(REF_SEQ)): + """Write an indexed single-sample caller BCF. + + Args: + path: Output BCF path. + sample_name: Name of the single sample. + records: Iterable of (start0, ref, alts, gt) tuples; ``gt`` is a tuple + such as (0, 1), (0, 0) or (None, None). An optional fifth item sets + the record QUAL. An optional sixth item sets the FILTER: a name + such as ``'PASS'``/``'LowQual'``, a sequence of names, or + ``None``/``'.'`` for no filter. An empty ``alts`` is written as the + VCF placeholder ALT ``.``, the same representation + generate_consensus.py itself emits for hom-reference records. + contig: Contig name. + length: Contig length. + + Returns: + The BCF path. + """ + header = pysam.VariantHeader() + header.add_line(f'##contig=') + header.add_line('##FORMAT=') + header.add_line('##FILTER=') + header.add_sample(sample_name) + with pysam.VariantFile(str(path), 'wb', header=header) as out: + for entry in records: + start0, ref, alts, gt = entry[:4] + qual = entry[4] if len(entry) > 4 else None + filters = entry[5] if len(entry) > 5 else None + alleles = (ref,) + (tuple(alts) if alts else ('.',)) + record = out.new_record(contig=contig, start=start0, + stop=start0 + len(ref), + alleles=alleles) + if qual is not None: + record.qual = qual + if isinstance(filters, str): + filters = (filters,) + for name in filters or (): + if name != '.': + record.filter.add(name) + record.samples[0]['GT'] = gt + out.write(record) + subprocess.run([BCFTOOLS, 'index', '--csi', '-f', str(path)], check=True) + return path + + +def run_consensus(tmp_path, bam, bed_gz, caller_bcfs, threshold, sample_name='sample1', + use_accel=False, output_name='out.bcf', reference=None, + callers=None, configured_callers=None, check=True, + absent_vote='ref', tie_policy='nocall', min_coverage=0, + filtered_vote='abstain', compress_reference_blocks=False, + variants_only=False): + """Run bin/generate_consensus.py as a subprocess and return its output path. + + Args: + caller_bcfs: Caller BCF paths, in the order they should be passed. + callers: Caller names matching ``caller_bcfs`` (defaults to + ``caller0``, ``caller1``, ...). + configured_callers: Caller list for ``--callers`` (defaults to + ``callers``); extra names count as failed callers. + check: Raise on a non-zero exit status when true. + absent_vote: ``ref`` or ``abstain`` voting for absent callers. + tie_policy: ``nocall`` or ``first`` tie-breaking. + min_coverage: Coverage required by ``absent_vote=abstain``. + filtered_vote: ``abstain`` or ``ref`` voting for filtered records. + compress_reference_blocks: Compress hom-reference runs into blocks. + variants_only: Write only consensus variant records, omitting reference + records. + """ + output = tmp_path / output_name + if reference is None: + reference = tmp_path / 'ref.fa' + if callers is None: + callers = [f'caller{index}' for index in range(len(caller_bcfs))] + input_args = [] + for caller, path in zip(callers, caller_bcfs): + input_args += ['--input', f'{caller}={path}'] + if configured_callers is None: + configured_callers = callers + env = os.environ.copy() + if not use_accel: + env['CHOCALLATE_NO_ACCEL'] = '1' + cmd = [ + sys.executable, str(CONSENSUS_SCRIPT), + *input_args, + '--callers', ','.join(configured_callers), + '--bed', str(bed_gz), + '--bam', str(bam), + '--output', str(output), + '--sample_name', sample_name, + '--reference', str(reference), + '--consensus_threshold', str(threshold), + '--absent_vote', absent_vote, + '--filtered_vote', filtered_vote, + '--tie_policy', tie_policy, + '--min_coverage', str(min_coverage), + '--version', 'test', + ] + if compress_reference_blocks: + cmd.append('--compress_reference_blocks') + if variants_only: + cmd.append('--variants_only') + subprocess.run(cmd, check=check, env=env, cwd=str(tmp_path), + capture_output=True, text=True) + return output + + +def read_positions(bcf_path): + """1-based positions (in file order) of a BCF written by the tests.""" + with pysam.VariantFile(str(bcf_path)) as variant_file: + return [record.pos for record in variant_file] + + +@pytest.fixture(params=[False, True], ids=['pure', 'accel']) +def use_accel(request): + """Consensus implementation under test (pure-pysam or Cython).""" + if request.param and not ACCEL_AVAILABLE: + pytest.skip('consensus_accel is not built for this interpreter') + return request.param + + +@pytest.fixture +def basic_dataset(tmp_path, use_accel): + """One reference/BAM and the paths needed to run generate_consensus.py.""" + reference = write_reference(tmp_path / 'ref.fa') + # A single read per position 20..24 (1-based), i.e. 0-based starts 19..23. + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + return { + 'reference': reference, + 'bam': bam, + 'tmp_path': tmp_path, + } + + +@pytest.fixture +def consensus_runner(tmp_path, use_accel, basic_dataset): + """Callable that writes caller BCFs and runs the consensus script.""" + def runner(bed_intervals, caller_records, threshold, sample_name='sample1', + **kwargs): + bed_gz = write_bed(tmp_path / 'coverage.bed', bed_intervals) + caller_bcfs = [] + for index, records in enumerate(caller_records): + caller_bcfs.append(write_caller_bcf( + tmp_path / f'caller{index}.bcf', 'sample1', records)) + output = run_consensus(tmp_path, basic_dataset['bam'], bed_gz, + caller_bcfs, threshold, + sample_name=sample_name, + use_accel=use_accel, + reference=basic_dataset['reference'], + **kwargs) + return output + + return runner diff --git a/tests/python/fixtures/caller_strategy/main.nf b/tests/python/fixtures/caller_strategy/main.nf new file mode 100644 index 0000000..9ec788f --- /dev/null +++ b/tests/python/fixtures/caller_strategy/main.nf @@ -0,0 +1,13 @@ +process CALLING_BCFTOOLS { + label 'process_medium' + + script: + """ + printf '%s\\n' "${task.attempt}" >> "${params.attempt_log}" + exit ${params.caller_exit_status ?: 1} + """ +} + +workflow { + CALLING_BCFTOOLS() +} diff --git a/tests/python/fixtures/resource_precedence/main.nf b/tests/python/fixtures/resource_precedence/main.nf new file mode 100644 index 0000000..1ba47ad --- /dev/null +++ b/tests/python/fixtures/resource_precedence/main.nf @@ -0,0 +1,18 @@ +process MAPPING_BOWTIE2 { + label 'process_medium' + cpus 99 + publishDir params.outdir, mode: 'copy' + + output: + path 'resources.txt' + + script: + """ + printf 'cpus=%s\\n' "${task.cpus}" > resources.txt + printf 'memory=%s\\n' "${task.memory.toMega()}" >> resources.txt + """ +} + +workflow { + MAPPING_BOWTIE2() +} diff --git a/tests/python/fixtures/retry_flaky/main.nf b/tests/python/fixtures/retry_flaky/main.nf new file mode 100644 index 0000000..ef7fbf8 --- /dev/null +++ b/tests/python/fixtures/retry_flaky/main.nf @@ -0,0 +1,23 @@ +process FLAKY { + label 'process_low' + publishDir params.outdir, mode: 'copy' + + output: + path 'ok.txt' + path 'memory.txt' + path 'time.txt' + + script: + """ + if [ ${task.attempt} -lt 2 ]; then + kill -9 \$\$ + fi + printf '%s\\n' "${task.memory.toMega()}" > memory.txt + printf '%s\\n' "${task.time.toMinutes()}" > time.txt + touch ok.txt + """ +} + +workflow { + FLAKY() +} diff --git a/tests/python/test_bam_reference.py b/tests/python/test_bam_reference.py new file mode 100644 index 0000000..3cc2d17 --- /dev/null +++ b/tests/python/test_bam_reference.py @@ -0,0 +1,150 @@ +"""Tests for the BAM header vs. reference .fai guard.""" + +import subprocess +import sys + +import pysam +import pytest + +# Tests run from this directory, so conftest is importable by name. +from conftest import BIN_DIR # noqa: I100,I202 + +CHECK = BIN_DIR / 'check_bam_reference.py' + + +def write_bam(path, contigs, m5=None): + """Write a header-only BAM with the given (name, length) @SQ entries.""" + entries = [] + for name, length in contigs: + entry = {'SN': name, 'LN': length} + if m5 and name in m5: + entry['M5'] = m5[name] + entries.append(entry) + header = {'HD': {'VN': '1.6'}, 'SQ': entries} + with pysam.AlignmentFile(str(path), 'wb', header=header): + pass + return path + + +def write_fai(path, contigs): + """Write a minimal .fai with the given (name, length) pairs.""" + lines = [ + '{}\t{}\t6\t60\t61'.format(name, length) + for name, length in contigs + ] + path.write_text('\n'.join(lines) + '\n') + return path + + +def write_dict(path, contigs, m5=None): + """Write a minimal SAM sequence dictionary.""" + lines = ['@HD\tVN:1.6'] + for name, length in contigs: + line = '@SQ\tSN:{}\tLN:{}'.format(name, length) + if m5 and name in m5: + line += '\tM5:{}'.format(m5[name]) + lines.append(line) + path.write_text('\n'.join(lines) + '\n') + return path + + +def run_check(bam, fai, dict_path=None): + command = [sys.executable, str(CHECK), str(bam), str(fai)] + if dict_path is not None: + command += ['--dict', str(dict_path)] + return subprocess.run( + command, + capture_output=True, + text=True, + ) + + +def test_matching_names_and_lengths_are_accepted(tmp_path): + contigs = [('chr1', 100), ('chr2', 250)] + bam = write_bam(tmp_path / 'in.bam', contigs) + fai = write_fai(tmp_path / 'ref.fa.fai', contigs) + + result = run_check(bam, fai) + + assert result.returncode == 0 + assert result.stdout == '' + assert result.stderr == '' + + +def test_length_mismatch_fails_with_a_diff(tmp_path): + bam = write_bam(tmp_path / 'in.bam', [('chr1', 100), ('chr2', 200)]) + fai = write_fai(tmp_path / 'ref.fa.fai', [('chr1', 100), ('chr2', 201)]) + + result = run_check(bam, fai) + + assert result.returncode != 0 + assert 'do not match' in result.stderr + assert '-chr2\t200' in result.stderr + assert '+chr2\t201' in result.stderr + + +def test_missing_and_extra_contigs_are_reported(tmp_path): + bam = write_bam(tmp_path / 'in.bam', [('chr1', 100), ('chr2', 200)]) + fai = write_fai(tmp_path / 'ref.fa.fai', [('chr1', 100), ('chr3', 300)]) + + result = run_check(bam, fai) + + assert result.returncode != 0 + assert '-chr2\t200' in result.stderr + assert '+chr3\t300' in result.stderr + + +def test_matching_m5_checksums_are_accepted(tmp_path): + contigs = [('chr1', 100), ('chr2', 250)] + m5 = {'chr1': 'abcd', 'chr2': 'ef01'} + bam = write_bam(tmp_path / 'in.bam', contigs, m5=m5) + fai = write_fai(tmp_path / 'ref.fa.fai', contigs) + dictionary = write_dict(tmp_path / 'ref.dict', contigs, m5=m5) + + result = run_check(bam, fai, dictionary) + + assert result.returncode == 0 + assert result.stderr == '' + + +def test_m5_mismatch_fails_for_same_names_and_lengths(tmp_path): + contigs = [('chr1', 100)] + bam = write_bam(tmp_path / 'in.bam', contigs, m5={'chr1': 'aaaa'}) + fai = write_fai(tmp_path / 'ref.fa.fai', contigs) + dictionary = write_dict( + tmp_path / 'ref.dict', contigs, m5={'chr1': 'bbbb'}) + + result = run_check(bam, fai, dictionary) + + assert result.returncode != 0 + assert 'M5 checksums do not match' in result.stderr + assert 'chr1\tBAM aaaa != reference bbbb' in result.stderr + + +def test_missing_bam_m5_is_not_an_error(tmp_path): + contigs = [('chr1', 100)] + bam = write_bam(tmp_path / 'in.bam', contigs) + fai = write_fai(tmp_path / 'ref.fa.fai', contigs) + dictionary = write_dict( + tmp_path / 'ref.dict', contigs, m5={'chr1': 'bbbb'}) + + result = run_check(bam, fai, dictionary) + + assert result.returncode == 0 + assert result.stderr == '' + + +def test_prepare_bam_invokes_the_guard(): + module = (BIN_DIR.parent / 'modules' / 'local' / 'prepare_bam' + / 'main.nf').read_text() + assert ('check_bam_reference.py tmp/read1 ' + 'tmp/ref_genome.fasta.fai' in module) + assert '--dict tmp/ref_genome.dict' in module + + +@pytest.mark.parametrize('mapper', [ + 'mapping_bwa', 'mapping_bowtie2', 'mapping_minimap2']) +def test_mapping_passes_the_reference_index_to_the_guard(mapper): + module = (BIN_DIR.parent / 'modules' / 'local' / mapper + / 'main.nf').read_text() + assert '--fai tmp/ref_genome.fasta.fai' in module diff --git a/tests/python/test_boost_cleanup.py b/tests/python/test_boost_cleanup.py new file mode 100644 index 0000000..d9431e7 --- /dev/null +++ b/tests/python/test_boost_cleanup.py @@ -0,0 +1,119 @@ +"""End-to-end test for the enabled nf-boost cleanup observer. + +The nf-test suite disables ``boost.cleanup`` so that work directories survive +its assertions. This test runs a tiny BAM-only pipeline with cleanup enabled +and checks that the published result survives while a consumed task output is +removed from the work directory. +""" + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest +from conftest import ( # noqa: I100,I202 (tests run from this directory) + REF_SEQ, + write_bam, + write_reference, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +NEXTFLOW = shutil.which('nextflow') + +pytestmark = pytest.mark.skipif( + NEXTFLOW is None, + reason='nextflow is not installed', +) + + +def _environment(): + env = os.environ.copy() + if 'NXF_OFFLINE' not in env: + plugins = Path.home() / '.nextflow' / 'plugins' + has_plugins = ( + plugins.is_dir() + and any(plugins.glob('nf-schema-*')) + and any(plugins.glob('nf-boost-*')) + ) + if has_plugins: + env['NXF_OFFLINE'] = 'true' + return env + + +def _params_file(tmp_path, reference, samplesheet, output_dir): + params = { + 'input': { + 'samples_tsv': str(samplesheet), + 'reference_genome': str(reference), + 'format': 'bam', + }, + 'output': { + 'directory': str(output_dir), + 'type': 'single', + 'format': 'bcf', + }, + 'mapping': {'cpu': 1}, + 'bam_filter': {'min_map_qual': 0}, + 'rmdup': {'mode': 'none'}, + 'left_align_indels': {'enabled': False}, + 'coverage': {'min_coverage': 1}, + 'calling': {'callers': 'bcftools', 'cpu': 1}, + 'consensus': {'threshold': 1, 'cpu': 1}, + 'boost': {'cleanup': 'v2'}, + 'resources': { + 'max_retries': 0, + 'low': {'cpus': 1, 'memory_mb': 1024}, + 'medium': {'cpus': 1, 'memory_mb': 1024}, + 'high': {'cpus': 1, 'memory_mb': 1024}, + }, + } + path = tmp_path / 'params.json' + path.write_text(json.dumps(params)) + return path + + +def _trace_rows(path): + lines = path.read_text().splitlines() + header = lines[0].split('\t') + return [dict(zip(header, line.split('\t'))) for line in lines[1:]] + + +def test_cleanup_enabled_removes_consumed_task_output(tmp_path): + """Cleanup removes the per-sample BCF but keeps the published output.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 12]) for i in range(10, 30)] + bam = write_bam(tmp_path / 'sample.bam', reads) + samplesheet = tmp_path / 'samples.tsv' + samplesheet.write_text(f'sample1\t{bam}\n') + output_dir = tmp_path / 'results' + params = _params_file(tmp_path, reference, samplesheet, output_dir) + work_dir = tmp_path / 'work' + + result = subprocess.run( + [NEXTFLOW, '-log', str(tmp_path / 'nextflow.log'), 'run', + str(PROJECT_ROOT / 'main.nf'), '-params-file', str(params), + '-work-dir', str(work_dir), '-ansi-log', 'false'], + cwd=str(tmp_path), env=_environment(), capture_output=True, + text=True, timeout=900, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert (output_dir / 'consensus.bcf').is_file() + + # The trace always lands under the default output directory: it is opened + # while nextflow.config is parsed, before the params file is applied. + trace = tmp_path / 'ChoCallate_output' / 'trace.txt' + consensus_rows = [ + row for row in _trace_rows(trace) + if row['name'].startswith('GENERATE_CONSENSUS')] + assert len(consensus_rows) == 1 + assert consensus_rows[0]['status'] == 'COMPLETED' + # The trace records a shortened hash; the work directory name is full. + shard, prefix = consensus_rows[0]['hash'].split('/') + task_dirs = list((work_dir / shard).glob(prefix + '*')) + assert len(task_dirs) <= 1 + # The downstream merge consumed sample1.bcf, so the observer deletes it + # (and may remove the now-empty task directory as well). + assert all(not list(task_dir.glob('*.bcf')) for task_dir in task_dirs) diff --git a/tests/python/test_contig_lengths.py b/tests/python/test_contig_lengths.py new file mode 100644 index 0000000..cb120e4 --- /dev/null +++ b/tests/python/test_contig_lengths.py @@ -0,0 +1,47 @@ +"""Tests for the reference contig length guard.""" + +import subprocess +import sys + +from conftest import BIN_DIR # noqa: I100,I202 (tests run from this directory) + +CHECK = BIN_DIR / 'check_contig_lengths.py' +MAX_LENGTH = (1 << 31) - 1 + + +def write_fai(path, contigs): + """Write a minimal .fai with the given (name, length) pairs.""" + lines = [ + '{}\t{}\t6\t60\t61'.format(name, length) + for name, length in contigs + ] + path.write_text('\n'.join(lines) + '\n') + return path + + +def test_contigs_within_the_limit_are_accepted(tmp_path): + fai = write_fai(tmp_path / 'ref.fa.fai', + [('chr1', 100), ('chr2', MAX_LENGTH)]) + result = subprocess.run([sys.executable, str(CHECK), str(fai)], + capture_output=True, text=True) + assert result.returncode == 0 + assert result.stdout == '' + assert result.stderr == '' + + +def test_contig_over_the_limit_fails_with_a_clear_message(tmp_path): + fai = write_fai(tmp_path / 'ref.fa.fai', + [('chr1', 100), ('chrBig', MAX_LENGTH + 1)]) + result = subprocess.run([sys.executable, str(CHECK), str(fai)], + capture_output=True, text=True) + assert result.returncode != 0 + assert "'chrBig' ({} bp)".format(MAX_LENGTH + 1) in result.stderr + assert str(MAX_LENGTH) in result.stderr + assert '2^31-1' in result.stderr + assert '32-bit' in result.stderr + + +def test_fai_creation_process_invokes_the_guard(): + module = (BIN_DIR.parent / 'modules' / 'local' / 'create_fai_index' + / 'main.nf').read_text() + assert 'check_contig_lengths.py ref_genome.fasta.fai' in module diff --git a/tests/python/test_filter_bed.py b/tests/python/test_filter_bed.py new file mode 100644 index 0000000..5d2ce0e --- /dev/null +++ b/tests/python/test_filter_bed.py @@ -0,0 +1,71 @@ +"""Tests for the BED header stripping and contig validation guard.""" + +import subprocess +import sys + +# Tests run from this directory, so conftest is importable by name. +from conftest import BIN_DIR # noqa: I100,I202 + +FILTER = BIN_DIR / 'filter_bed.py' + + +def run_filter(tmp_path, bed_text, contigs=('chr1',)): + """Write a .fai and BED, then run filter_bed.py on them.""" + fai = tmp_path / 'ref.fa.fai' + fai.write_text(''.join( + '{}\t100\t6\t60\t61\n'.format(name) for name in contigs)) + bed = tmp_path / 'in.bed' + bed.write_text(bed_text) + return subprocess.run( + [sys.executable, str(FILTER), str(fai), str(bed)], + capture_output=True, + text=True, + ) + + +def test_headers_and_comments_are_removed(tmp_path): + bed = ( + 'track name=test\n' + 'browser position chr1:1-100\n' + '# leading comment\n' + '\n' + 'chr1\t0\t50\n' + '# trailing comment\n' + 'chr1\t60\t80\n' + ) + + result = run_filter(tmp_path, bed) + + assert result.returncode == 0 + assert result.stderr == '' + assert result.stdout == 'chr1\t0\t50\nchr1\t60\t80\n' + + +def test_header_words_are_only_removed_as_standalone_lines(tmp_path): + bed = 'track1\t0\t10\nbrowser2\t20\t30\n' + + result = run_filter(tmp_path, bed, contigs=('track1', 'browser2')) + + assert result.returncode == 0 + assert result.stdout == bed + + +def test_unknown_contig_fails_with_a_clear_message(tmp_path): + bed = 'chr1\t0\t50\nchr2\t0\t10\nchr3\t0\t10\n' + + result = run_filter(tmp_path, bed) + + assert result.returncode != 0 + assert result.stdout == 'chr1\t0\t50\n' + assert "'chr2'" in result.stderr + assert "'chr3'" in result.stderr + assert 'reference .fai' in result.stderr + + +def test_crlf_line_endings_are_normalised(tmp_path): + bed = 'track name=test\r\nchr1\t0\t50\r\n' + + result = run_filter(tmp_path, bed) + + assert result.returncode == 0 + assert result.stdout == 'chr1\t0\t50\n' diff --git a/tests/python/test_generate_consensus.py b/tests/python/test_generate_consensus.py new file mode 100644 index 0000000..c6fcf4b --- /dev/null +++ b/tests/python/test_generate_consensus.py @@ -0,0 +1,1709 @@ +"""End-to-end regression tests for bin/generate_consensus.py. + +Covers interval-end handling, missing genotypes, haploid calls, the voting +model, provenance headers, multi-allelic output and the pure/Cython equivalence +paths. Every test runs twice, on the pure-pysam path and (when built) on the +Cython path, and asserts the documented all-sites behaviour: every evaluated +BED base with an unambiguous reference base is covered by an output record +(reference blocks and the REF span of a called deletion included), while +IUPAC-ambiguous positions are omitted. +""" + +import os +import subprocess +import sys + +import pytest +from conftest import ( # noqa: I100,I202 (tests run from this directory) + ACCEL_AVAILABLE, + BCFTOOLS, + CHR, + CONSENSUS_SCRIPT, + REF_SEQ, + alt_base, + ref_base, + run_consensus, + write_bam, + write_bed, + write_caller_bcf, + write_reference, + write_sparse_reference, +) + +# 0-based indexes of two reference bases ('ACGT' repeats, so index % 4 picks it) +POS_20_0B = 19 # 1-based position 20, reference 'A' +POS_24_0B = 23 # 1-based position 24, covered by all five test reads +POS_30_0B = 29 # 1-based position 30, reference 'C' + + +def read_records(path): + """(pos, alleles, GT) triples of a BCF written by the tests.""" + import pysam + with pysam.VariantFile(str(path), index_filename=None) as variant_file: + return [(record.pos, (record.ref,) + tuple(record.alts), + tuple(record.samples[0]['GT'])) for record in variant_file] + + +def read_records_full(path): + """(pos, alleles, GT, AD, DP) tuples of a BCF written by the tests.""" + import pysam + with pysam.VariantFile(str(path), index_filename=None) as variant_file: + return [(record.pos, (record.ref,) + tuple(record.alts), + tuple(record.samples[0]['GT']), tuple(record.samples[0]['AD']), + record.samples[0]['DP']) for record in variant_file] + + +def read_spans(path): + """(pos, inclusive end, alleles, GT, AD or None, DP) of every record.""" + import pysam + spans = [] + with pysam.VariantFile(str(path), index_filename=None) as variant_file: + for record in variant_file: + sample = record.samples[0] + ad = sample.get('AD') + spans.append((record.pos, record.stop, + (record.ref,) + tuple(record.alts), + tuple(sample['GT']), + tuple(ad) if ad is not None else None, + sample['DP'])) + return spans + + +def span_positions(spans): + """Expand POS..END of each span into the covered 1-based positions.""" + return [position for start, end, *_ in spans + for position in range(start, end + 1)] + + +def test_interval_ends_are_written(consensus_runner): + """Every base of every BED interval, last base included.""" + # chr1 9 10 is a single base (1-based 10); chr1 19 25 covers 20..25. + output = consensus_runner( + bed_intervals=[('chr1', 9, 10), ('chr1', 19, 25)], + caller_records=[[], []], + threshold=2, + ) + assert [pos for pos, _, _ in read_records(output)] == [10, 20, 21, 22, 23, 24, 25] + + +def test_adjacent_intervals_are_not_truncated(consensus_runner): + """Adjacent intervals keep their last bases (1-based 10 and 20).""" + output = consensus_runner( + bed_intervals=[('chr1', 0, 10), ('chr1', 10, 20)], + caller_records=[[], []], + threshold=2, + ) + assert [pos for pos, _, _ in read_records(output)] == list(range(1, 21)) + + +def test_all_sites_written_once(consensus_runner): + """One record per BED base, in BED order, monomorphic sites included.""" + output = consensus_runner( + bed_intervals=[('chr1', 0, 10), ('chr1', 12, 20)], + caller_records=[[], []], + threshold=2, + ) + records = read_records(output) + assert [pos for pos, _, _ in records] == list(range(1, 11)) + list(range(13, 21)) + for pos, alleles, gt in records: + assert alleles == (ref_base(pos), '<*>') + assert gt == (0, 0) + + +def test_out_of_range_numeric_options_are_rejected(tmp_path): + """Invalid numeric options fail before the input files are opened.""" + base = [ + sys.executable, str(CONSENSUS_SCRIPT), + '--input', 'caller.bcf', + '--bed', str(tmp_path / 'coverage.bed.gz'), + '--bam', str(tmp_path / 'sample.bam'), + '--output', str(tmp_path / 'out.bcf'), + '--sample_name', 'sample1', + '--reference', str(tmp_path / 'ref.fa'), + '--consensus_threshold', '1', + '--min_coverage', '0', + '--version', 'test', + ] + for option, value, message in ( + ('--consensus_threshold', '0', 'must be at least 1'), + ('--min_coverage', '-1', 'must be at least 0')): + result = subprocess.run(base + [option, value], + capture_output=True, text=True) + assert result.returncode == 2 + assert option in result.stderr + assert message in result.stderr + + +def test_bed_interval_past_contig_end_fails_clearly( + tmp_path, use_accel, basic_dataset): + """A BED interval beyond the contig end is rejected explicitly.""" + bed_gz = write_bed(tmp_path / 'coverage.bed', + [(CHR, len(REF_SEQ) - 5, len(REF_SEQ) + 5)]) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', []) + with pytest.raises(subprocess.CalledProcessError) as excinfo: + run_consensus( + tmp_path, basic_dataset['bam'], bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=basic_dataset['reference']) + stderr = excinfo.value.stderr + assert 'Traceback' not in stderr + assert 'IndexError' not in stderr + assert 'extends past the end of contig' in stderr + assert "'{}' ({} bp)".format(CHR, len(REF_SEQ)) in stderr + + +def test_iupac_reference_positions_are_omitted(tmp_path, use_accel): + """Positions with an IUPAC reference base are not written.""" + seq = REF_SEQ[:19] + 'R' + REF_SEQ[20:23] + 'Y' + REF_SEQ[24:] + reference = write_reference(tmp_path / 'ref.fa', seq=seq) + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 25)]) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', []) + + output = run_consensus( + tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference, + compress_reference_blocks=True) + + # Positions 20 and 24 are IUPAC codes; the blocks around them are split. + spans = read_spans(output) + assert [(start, end, alleles) for start, end, alleles, *_ in spans] == [ + (21, 23, ('A', '<*>')), + (25, 25, ('A', '<*>')), + ] + + +def test_iupac_caller_ref_record_is_omitted(tmp_path, use_accel): + """A caller REF with an IUPAC code is not written.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 21)]) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', [ + (19, 'R', ['A'], (0, 1))]) + + output = run_consensus( + tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference) + + assert read_records(output) == [(21, (ref_base(21), '<*>'), (0, 0))] + + +def test_iupac_alt_record_is_omitted(tmp_path, use_accel): + """A caller ALT with an IUPAC code is not written.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 21)]) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', [ + (19, ref_base(20), ['N'], (0, 1))]) + + output = run_consensus( + tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference) + + assert read_records(output) == [(21, (ref_base(21), '<*>'), (0, 0))] + + +def test_reference_blocks_are_optional(tmp_path, use_accel): + """Blocks compress hom-reference runs and omit multi-base AD.""" + import pysam + + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 25)]) + callers = [write_caller_bcf(tmp_path / 'c0.bcf', 'sample1', []), + write_caller_bcf(tmp_path / 'c1.bcf', 'sample1', [])] + + plain = run_consensus(tmp_path, bam, bed_gz, callers, threshold=2, + use_accel=use_accel, reference=reference, + output_name='plain.bcf') + assert [span[0] for span in read_spans(plain)] == list(range(20, 26)) + + blocked = run_consensus( + tmp_path, bam, bed_gz, callers, threshold=2, + use_accel=use_accel, reference=reference, output_name='blocked.bcf', + compress_reference_blocks=True) + # 20..25 are one block; the minimum depth is 1 at position 20 and one AD + # value cannot describe the whole block, so FORMAT/AD is omitted. + assert read_spans(blocked) == [ + (20, 25, (ref_base(20), '<*>'), (0, 0), None, 1)] + + with pysam.VariantFile(str(blocked), index_filename=None) as variant_file: + header = str(variant_file.header) + assert 'ReferenceBlocks=True' in header + + # The block rlen must let an indexed fetch find a position inside it. + subprocess.run([BCFTOOLS, 'index', '--csi', '-f', str(blocked)], + check=True) + with pysam.VariantFile(str(blocked)) as variant_file: + assert [record.pos for record in variant_file.fetch('chr1', 23, 24)] \ + == [20] + + +def test_variants_only_skips_reference_records(tmp_path, use_accel): + """Only consensus variant records are written, with no reference blocks.""" + import pysam + + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 25)]) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', [ + (21, ref_base(22), [alt_base(22)], (0, 1))]) + + output = run_consensus( + tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference, + compress_reference_blocks=True, variants_only=True) + spans = read_spans(output) + assert [(start, end, alleles[:2]) for start, end, alleles, *_ in spans] == [ + (22, 22, (ref_base(22), alt_base(22)))] + with pysam.VariantFile(str(output), index_filename=None) as variant_file: + header = str(variant_file.header) + # Reference blocks are not written, so the header does not claim them. + assert 'ReferenceBlocks=False' in header + + +def test_reference_blocks_split_at_variant(tmp_path, use_accel): + """A caller record ends the current block and is written separately.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 25)]) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', [ + (21, ref_base(22), [alt_base(22)], (0, 1))]) + + output = run_consensus( + tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference, + compress_reference_blocks=True) + spans = read_spans(output) + assert [(start, end) for start, end, *_ in spans] == [ + (20, 21), (22, 22), (23, 25)] + assert span_positions(spans) == list(range(20, 26)) + assert spans[1][2] == (ref_base(22), alt_base(22)) + + +def test_reference_blocks_split_at_reference_caller_record(tmp_path, + use_accel): + """Any caller record splits a block, even a hom-reference call.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 25)]) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', [ + (21, ref_base(22), [], (0, 0))]) + + output = run_consensus( + tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference, + compress_reference_blocks=True) + assert [(start, end) for start, end, *_ in read_spans(output)] == [ + (20, 21), (22, 22), (23, 25)] + + +def test_reference_blocks_do_not_span_bed_gaps(tmp_path, use_accel): + """Discontiguous BED intervals become separate blocks.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', 19, 21), ('chr1', 23, 25)]) + callers = [write_caller_bcf(tmp_path / 'c0.bcf', 'sample1', [])] + + output = run_consensus( + tmp_path, bam, bed_gz, callers, threshold=1, + use_accel=use_accel, reference=reference, + compress_reference_blocks=True) + spans = read_spans(output) + assert [(start, end) for start, end, *_ in spans] == [(20, 21), (24, 25)] + assert span_positions(spans) == [20, 21, 24, 25] + + +def test_reference_blocks_split_when_depth_crosses_min_coverage( + tmp_path, use_accel): + """A block never spans positions on both sides of min_coverage.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 25)]) + callers = [write_caller_bcf(tmp_path / 'c0.bcf', 'sample1', [])] + + output = run_consensus( + tmp_path, bam, bed_gz, callers, threshold=1, min_coverage=3, + use_accel=use_accel, reference=reference, + compress_reference_blocks=True) + spans = read_spans(output) + # depths at 1-based 20..25 are 1, 2, 3, 4, 5, 5 + assert [(start, end, dp) for start, end, *_, dp in spans] == [ + (20, 21, 1), (22, 25, 3)] + assert span_positions(spans) == list(range(20, 26)) + + +def test_accel_and_pure_paths_agree_on_blocks(tmp_path): + """Both implementations write byte-identical reference blocks.""" + if not ACCEL_AVAILABLE: + pytest.skip('consensus_accel is not built for this interpreter') + + reference = write_reference(tmp_path / 'ref.fa') + reads = [('r1', 19, REF_SEQ[19:25]), ('r2', 29, REF_SEQ[29:35])] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 36)]) + caller_bcfs = [ + write_caller_bcf(tmp_path / 'caller0.bcf', 'sample1', + [(19, REF_SEQ[19], [], (0, 0)), + (POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))]), + write_caller_bcf(tmp_path / 'caller1.bcf', 'sample1', []), + ] + pure = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=1, use_accel=False, + output_name='pure.bcf', reference=reference, + compress_reference_blocks=True) + accel = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=1, use_accel=True, + output_name='accel.bcf', reference=reference, + compress_reference_blocks=True) + + def text(path): + return subprocess.run([BCFTOOLS, 'view', '-H', str(path)], + check=True, capture_output=True, + text=True).stdout + + assert text(pure) == text(accel) + assert text(pure) + + +def test_missing_genotype_does_not_crash(consensus_runner): + """A ``./.`` caller record is handled instead of raising TypeError.""" + output = consensus_runner( + bed_intervals=[('chr1', POS_20_0B, 25)], + caller_records=[[(POS_20_0B, REF_SEQ[POS_20_0B], [], (None, None))], [], []], + threshold=2, + ) + # The no-call abstains; the two other callers' placeholder votes reach the + # threshold, so the position is written as hom-reference. + assert [pos for pos, _, _ in read_records(output)] == [20, 21, 22, 23, 24, 25] + + +def test_missing_genotype_abstains_from_the_vote(consensus_runner): + """A no-call record is not a hom-reference vote. + + Two callers, threshold 2: where one caller emitted ``./.`` only the other + caller's placeholder vote is left, so the position stays below the threshold + and is not reported as hom-reference (it shows up as missing after the merge). + """ + output = consensus_runner( + bed_intervals=[('chr1', POS_20_0B, POS_20_0B + 2)], + caller_records=[[(POS_20_0B, REF_SEQ[POS_20_0B], [], (None, None))], []], + threshold=2, + ) + assert [pos for pos, _, _ in read_records(output)] == [21] + + +def test_missing_genotype_does_not_mask_variant(consensus_runner): + """A no-call must not out-vote a real call from another caller.""" + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 1)], + caller_records=[ + [(POS_30_0B, ref_base(30), [], (None, None))], + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))], + ], + threshold=1, + ) + assert read_records(output) == [(30, (ref_base(30), alt_base(30)), (0, 1))] + + +def test_partial_missing_genotype_abstains(consensus_runner): + """A ``0/.`` record abstains, it is not a hom-reference vote.""" + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 1)], + caller_records=[ + [(POS_30_0B, ref_base(30), [], (0, None))], + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))], + ], + threshold=1, + ) + assert read_records(output) == [(30, (ref_base(30), alt_base(30)), (0, 1))] + + +def test_haploid_hom_ref_is_reference(consensus_runner): + """A haploid ``0`` is a hom-reference vote, not a variant.""" + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, 31)], + caller_records=[[(POS_30_0B, ref_base(30), [], (0,))]], + threshold=1, + ) + assert read_records(output) == [ + (30, (ref_base(30), '<*>'), (0, 0)), + (31, (ref_base(31), '<*>'), (0, 0)), + ] + + +def test_haploid_alt_is_written(consensus_runner): + """A haploid ``1`` does not crash and is reported as a variant.""" + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, 31)], + caller_records=[[(POS_30_0B, ref_base(30), [alt_base(30)], (1,))]], + threshold=1, + ) + assert read_records(output) == [ + (30, (ref_base(30), alt_base(30)), (1,)), + (31, (ref_base(31), '<*>'), (0, 0)), + ] + + +def test_consensus_below_threshold_leaves_placeholder(consensus_runner): + """A caller 0/0 record loses to the placeholders of the other callers.""" + output = consensus_runner( + bed_intervals=[('chr1', POS_20_0B, POS_20_0B + 1)], + caller_records=[[(POS_20_0B, REF_SEQ[POS_20_0B], [], (0, 0))], []], + threshold=2, + ) + assert read_records(output) == [(20, (ref_base(20), '<*>'), (0, 0))] + + +def test_monomorphic_record_carries_other_allele_depth(tmp_path, use_accel): + """The <*> AD slot holds the non-reference read depth.""" + reference = write_reference(tmp_path / 'ref.fa') + ref = ref_base(30) + alt = alt_base(30) + reads = [(f'ref{start}', start, REF_SEQ[start:start + 10]) + for start in range(19, 30)] + for i in range(2): + seq = list(REF_SEQ[POS_30_0B:POS_30_0B + 10]) + seq[0] = alt + reads.append((f'alt{i}', POS_30_0B, ''.join(seq))) + reads.sort(key=lambda item: item[1]) + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', POS_30_0B, POS_30_0B + 1)]) + caller_bcfs = [write_caller_bcf(tmp_path / 'c0.bcf', 'sample1', []), + write_caller_bcf(tmp_path / 'c1.bcf', 'sample1', [])] + + output = run_consensus(tmp_path, bam, bed_gz, caller_bcfs, threshold=2, + use_accel=use_accel, reference=reference) + # 10 reads support the reference base, 2 support the ALT base; the <*> AD + # slot must report the non-reference depth instead of a fabricated 0. + assert read_records_full(output) == [(30, (ref, '<*>'), (0, 0), (10, 2), 12)] + + +def test_overlapping_mates_count_once_in_dp(tmp_path, use_accel): + """DP counts collapsed observations, including deletions, in both paths.""" + reference = write_reference(tmp_path / 'ref.fa') + match = REF_SEQ[POS_30_0B:POS_30_0B + 6] + # The unique read deletes 1-based position 31. + deleted = (REF_SEQ[POS_30_0B] + + REF_SEQ[POS_30_0B + 2:POS_30_0B + 6]) + # Two same-name mate pairs and one unique read cover 1-based 30..35, so a + # raw pileup has depth 5 while only 3 observations remain after collapsing. + reads = [('pair1', POS_30_0B, match), ('pair1', POS_30_0B, match), + ('pair2', POS_30_0B, match), ('pair2', POS_30_0B, match), + ('unique', POS_30_0B, deleted, + ((0, 1), (2, 1), (0, 4)))] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 2)]) + caller_bcfs = [ + write_caller_bcf(tmp_path / 'c0.bcf', 'sample1', [ + (POS_30_0B + 1, ref_base(31), [alt_base(31)], (0, 1))]), + write_caller_bcf(tmp_path / 'c1.bcf', 'sample1', []), + ] + + output = run_consensus(tmp_path, bam, bed_gz, caller_bcfs, threshold=1, + use_accel=use_accel, reference=reference) + records = read_records_full(output) + # 30 is hom-reference (fast path); 31 is a variant (full tally) and the + # unique read's deletion is still one observation, not a reference base. + assert [(pos, dp) for pos, *_, dp in records] == [(30, 3), (31, 3)] + assert records[1][3] == (2, 0) + + +def test_concordant_callers_are_merged(consensus_runner): + """Two matching ALT calls win over one placeholder vote. + + The genotype classes (0/1 and 1/1) tie, so the explicit ``first`` policy + picks the class of the earliest caller instead of relying on the default + no-call policy. + """ + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, 31), ('chr1', 40, 42)], + caller_records=[ + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1), 50)], + [(POS_30_0B, ref_base(30), [alt_base(30)], (1, 1), 20)], + ], + threshold=1, + tie_policy='first', + ) + records = read_records(output) + # The two callers agree on the ALT allele, so the site passes the site + # stage; the first caller's 0/1 class breaks the zygosity tie. Placeholder + # votes must not win, and the interval ends are written. + assert records[0] == (30, (ref_base(30), alt_base(30)), (0, 1)) + assert [pos for pos, _, _ in records] == [30, 31, 41, 42] + + +@pytest.mark.parametrize('order', [(0, 1), (1, 0)]) +@pytest.mark.parametrize('threshold,expected_count', [(2, 0), (1, 2)]) +def test_deletions_of_different_lengths_are_distinct_alleles( + consensus_runner, order, threshold, expected_count): + """Normalised deletion ALTs with the same anchor base do not pool. + + ``REF=CG ALT=C`` (1-bp deletion) and ``REF=CGT ALT=C`` (2-bp deletion) + share the ALT string but are different alleles. With threshold 2 neither + has enough supporters and no variant is written; with threshold 1 both + are written and the order is independent of the caller order. + """ + anchor = REF_SEQ[POS_30_0B].upper() + short = (POS_30_0B, REF_SEQ[POS_30_0B:POS_30_0B + 2].upper(), + [anchor], (0, 1)) + long = (POS_30_0B, REF_SEQ[POS_30_0B:POS_30_0B + 3].upper(), + [anchor], (0, 1)) + records = [short, long] + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 1)], + caller_records=[[records[index]] for index in order], + threshold=threshold, + ) + output_records = read_records(output) + assert len(output_records) == expected_count + if expected_count: + assert output_records == [ + (30, (short[1], short[2][0]), (0, 1)), + (30, (long[1], long[2][0]), (0, 1)), + ] + + +def test_deletion_does_not_skip_caller_records_inside_span(tmp_path, + use_accel): + """A het deletion at POS 100 must not hide a SNP at POS 102. + + Two callers support the heterozygous deletion and two callers (one of them + also carries the deletion) support the SNP on the other copy. The old + jump by the deletion REF length skipped POS 102 and discarded its caller + records, so the SNP never reached the vote. + """ + seq = REF_SEQ * 2 + reference = write_reference(tmp_path / 'ref.fa', seq=seq) + reads = [(f'read{i}', i, seq[i:i + 6]) for i in range(99, 109)] + bam = write_bam(tmp_path / 'sample.bam', reads, length=len(seq)) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 99, 110)]) + deletion_ref = seq[99:103].upper() + deletion_alt = seq[99].upper() + snp_ref = seq[101].upper() + snp_alt = {'A': 'G', 'C': 'T', 'G': 'C', 'T': 'A'}[snp_ref] + caller_bcfs = [ + write_caller_bcf(tmp_path / 'c0.bcf', 'sample1', [ + (99, deletion_ref, [deletion_alt], (0, 1))], + length=len(seq)), + write_caller_bcf(tmp_path / 'c1.bcf', 'sample1', [ + (99, deletion_ref, [deletion_alt], (0, 1)), + (101, snp_ref, [snp_alt], (0, 1))], length=len(seq)), + write_caller_bcf(tmp_path / 'c2.bcf', 'sample1', [ + (101, snp_ref, [snp_alt], (0, 1))], length=len(seq)), + ] + output = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=2, + use_accel=use_accel, reference=reference) + records = read_records(output) + assert (100, (deletion_ref, deletion_alt), (0, 1)) in records + assert (102, (snp_ref, snp_alt), (0, 1)) in records + + +@pytest.mark.parametrize('compress', [False, True]) +def test_deletion_suppresses_reference_records_inside_span( + consensus_runner, compress): + """A called deletion replaces reference records over its REF span. + + The deleted bases are covered by the deletion record's POS..END, so no + hom-reference ``0/0`` record may contradict the deletion. A hom-reference + caller record inside the span must not bypass that suppression either. + """ + deletion_ref = REF_SEQ[POS_20_0B:POS_20_0B + 4].upper() + deletion_alt = REF_SEQ[POS_20_0B].upper() + output = consensus_runner( + bed_intervals=[('chr1', POS_20_0B, 30)], + caller_records=[[ + (POS_20_0B, deletion_ref, [deletion_alt], (0, 1)), + (POS_20_0B + 2, ref_base(22), [], (0, 0)), + ]], + threshold=1, + compress_reference_blocks=compress, + ) + spans = read_spans(output) + assert spans[0][:3] == (20, 23, (deletion_ref, deletion_alt)) + if compress: + assert [(start, end) for start, end, *_ in spans] == [ + (20, 23), (24, 30)] + else: + assert [start for start, *_ in spans] == [20] + list(range(24, 31)) + # The deletion's REF span still covers 21..23, so every BED position is + # represented exactly once. + assert span_positions(spans) == list(range(20, 31)) + + +def test_deletion_does_not_duplicate_or_skip_interval_boundaries( + consensus_runner): + """A deletion spanning two BED intervals writes each site once.""" + deletion_ref = REF_SEQ[POS_20_0B:POS_20_0B + 4].upper() + deletion_alt = REF_SEQ[POS_20_0B].upper() + output = consensus_runner( + bed_intervals=[('chr1', POS_20_0B, POS_20_0B + 2), + ('chr1', POS_20_0B + 2, 24)], + caller_records=[[ + (POS_20_0B, deletion_ref, [deletion_alt], (0, 1))]], + threshold=1, + ) + assert [pos for pos, _, _ in read_records(output)] == [20, 24] + assert span_positions(read_spans(output)) == [20, 21, 22, 23, 24] + + +def _run_chunked_deletion(tmp_path, use_accel, caller_records, + threshold=2, filtered_vote='abstain', + compress_reference_blocks=False): + """Run two independent BED chunks around a deletion at POS 20. + + Chunk A covers 16..20 and evaluates the deletion anchored at POS 20; + chunk B covers 21..25 in a separate process. Returns both outputs. + """ + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + caller_bcfs = [] + for index, records in enumerate(caller_records): + caller_bcfs.append(write_caller_bcf( + tmp_path / f'caller{index}.bcf', 'sample1', records)) + chunk_a = write_bed(tmp_path / 'chunk_a.bed', + [('chr1', POS_20_0B - 4, POS_20_0B + 1)]) + chunk_b = write_bed(tmp_path / 'chunk_b.bed', + [('chr1', POS_20_0B + 1, POS_20_0B + 6)]) + outputs = [] + for name, bed_gz in (('a', chunk_a), ('b', chunk_b)): + outputs.append(run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold, + use_accel=use_accel, reference=reference, + filtered_vote=filtered_vote, + compress_reference_blocks=compress_reference_blocks, + output_name=f'chunk_{name}.bcf')) + return outputs + + +@pytest.mark.parametrize('compress', [False, True]) +def test_deletion_span_survives_chunk_boundary(tmp_path, use_accel, compress): + """A deletion crossing a chunk boundary suppresses the next chunk's + reference records. + + The second chunk fetches the deletion because its REF span overlaps the + chunk, but skips it because its POS is before the chunk start. Without + seeding the span, the chunk writes ``<*>`` records over the deleted bases + in addition to the deletion record written by the first chunk. + """ + deletion_ref = REF_SEQ[POS_20_0B:POS_20_0B + 4].upper() + deletion_alt = REF_SEQ[POS_20_0B].upper() + record = (POS_20_0B, deletion_ref, [deletion_alt], (0, 1)) + first, second = _run_chunked_deletion( + tmp_path, use_accel, [[record], [record]], + compress_reference_blocks=compress) + first_spans = read_spans(first) + second_spans = read_spans(second) + assert any(span[:4] == (20, 23, (deletion_ref, deletion_alt), (0, 1)) + for span in first_spans) + assert span_positions(second_spans) == [24, 25] + # The deletion record represents 20..23, so every chunk base is covered + # exactly once with no contradictory hom-reference call. + assert span_positions(first_spans + second_spans) == list(range(16, 26)) + + +def test_boundary_deletion_below_threshold_keeps_references( + tmp_path, use_accel): + """A skipped deletion with too little support must not suppress sites. + + The first chunk cannot call the deletion, so the second chunk must keep + writing reference records: otherwise the deleted bases would disappear + from the all-sites output without a deletion record covering them. + """ + deletion_ref = REF_SEQ[POS_20_0B:POS_20_0B + 4].upper() + deletion_alt = REF_SEQ[POS_20_0B].upper() + record = (POS_20_0B, deletion_ref, [deletion_alt], (0, 1)) + _, second = _run_chunked_deletion( + tmp_path, use_accel, [[record], []], threshold=2) + assert [start for start, *_ in read_spans(second)] == list(range(21, 26)) + + +@pytest.mark.parametrize('record,filtered_vote', [ + ((POS_20_0B, REF_SEQ[POS_20_0B:POS_20_0B + 4].upper(), + [REF_SEQ[POS_20_0B].upper()], (0, 0)), 'abstain'), + ((POS_20_0B, REF_SEQ[POS_20_0B:POS_20_0B + 4].upper(), + [REF_SEQ[POS_20_0B].upper()], (0, 1), None, 'LowQual'), 'abstain'), + ((POS_20_0B, REF_SEQ[POS_20_0B:POS_20_0B + 4].upper(), + [REF_SEQ[POS_20_0B].upper()], (0, 1), None, 'LowQual'), 'ref'), +], ids=['hom-ref-gt', 'filtered-abstain', 'filtered-ref']) +def test_boundary_deletion_respects_gt_and_filter(tmp_path, use_accel, + record, filtered_vote): + """A skipped record without usable ALT support does not suppress sites.""" + _, second = _run_chunked_deletion( + tmp_path, use_accel, [[record], [record]], + filtered_vote=filtered_vote) + assert [start for start, *_ in read_spans(second)] == list(range(21, 26)) + + +def _caller_record(ref, alts, gt): + """Minimal pysam.VariantRecord stand-in for genotype_key tests.""" + from types import SimpleNamespace + + return SimpleNamespace(ref=ref, alts=alts, samples=[{'GT': gt}], + filter=SimpleNamespace(keys=lambda: ())) + + +def test_consensus_calls_keys_alleles_by_ref_and_alt(): + """The documented reproduction: same anchor ALT, different REF.""" + import generate_consensus + + short = generate_consensus.genotype_key( + [_caller_record('AT', ('A',), (0, 1))], 'A') + long = generate_consensus.genotype_key( + [_caller_record('ATT', ('A',), (0, 1))], 'A') + reference = generate_consensus.genotype_key( + [_caller_record('A', (), (0, 0))], 'A') + # Each deletion has one supporter, so threshold 2 leaves only one + # reference vote: below the threshold, no consensus call at all. + assert generate_consensus.consensus_calls( + [short, long, reference], 2) == [] + assert generate_consensus.consensus_calls( + [long, short, reference], 2) == [] + expected_variants = [ + ('AT', ('A',), (0, 1)), + ('ATT', ('A',), (0, 1)), + ] + assert generate_consensus.consensus_calls( + [short, long, reference], 1) == expected_variants + assert generate_consensus.consensus_calls( + [long, short, reference], 1) == expected_variants + + +def test_genotype_key_votes_per_ref_alt_allele(): + """Atomised SNP and deletion at one POS stay separate alleles. + + A caller reporting ``A>G 0/1`` and ``AT>A 0/1`` at the same POS supports + both alleles with one copy each; the ploidy is 2. Two such callers reach + threshold 2 and produce one consensus record per allele. + """ + import generate_consensus + + key = generate_consensus.genotype_key([ + _caller_record('A', ('G',), (0, 1)), + _caller_record('AT', ('A',), (0, 1)), + ], 'A') + assert key == ('A', {('A', 'G'): 1, ('AT', 'A'): 1}, 2) + assert generate_consensus.consensus_calls([key, key], 2) == [ + ('A', ('G',), (0, 1)), + ('AT', ('A',), (0, 1)), + ] + + +def test_genotype_key_abstains_on_inconsistent_split_copies(): + """Split records whose ALT copies exceed the ploidy abstain. + + A diploid caller cannot be ``1/1`` for the SNP and ``0/1`` for the + deletion at the same time, so the combination is a no-call instead of a + fabricated genotype. Two such callers produce no consensus record. + """ + import generate_consensus + + key = generate_consensus.genotype_key([ + _caller_record('A', ('G',), (1, 1)), + _caller_record('AT', ('A',), (0, 1)), + ], 'A') + assert key is None + assert generate_consensus.consensus_calls([key, key], 2) == [] + + +def test_consensus_calls_abstains_when_alt_copies_exceed_ploidy(): + """Two hom-ALT alleles at one diploid site are a no-call. + + Each qualifying allele becomes a split record that is re-joined + downstream. Two hom-ALT calls would contribute four ALT copies to a + diploid genotype, so no combined genotype can represent both. + """ + import generate_consensus + + t_key = generate_consensus.genotype_key( + [_caller_record('A', ('T',), (1, 1))], 'A') + g_key = generate_consensus.genotype_key( + [_caller_record('A', ('G',), (1, 1))], 'A') + assert generate_consensus.consensus_calls([t_key, g_key], 1) == [] + assert generate_consensus.consensus_calls([g_key, t_key], 1) == [] + + +def test_consensus_calls_allows_two_heterozygous_alts(): + """Two het-ALT alleles at one diploid site fit a ``1/2`` genotype.""" + import generate_consensus + + t_key = generate_consensus.genotype_key( + [_caller_record('A', ('T',), (0, 1))], 'A') + g_key = generate_consensus.genotype_key( + [_caller_record('A', ('G',), (0, 1))], 'A') + assert generate_consensus.consensus_calls([t_key, g_key], 1) == [ + ('A', ('G',), (0, 1)), + ('A', ('T',), (0, 1)), + ] + + +def test_conflicting_hom_alts_are_a_no_call(consensus_runner): + """Two hom-ALT alleles at one diploid site write no records. + + Both alleles reach the site threshold, but their split records would + contribute four ALT copies to one diploid genotype. The position is a + no-call; the next position is still written normally. + """ + ref = ref_base(30) + first = alt_base(30) + second = ({'A', 'C', 'G', 'T'} - {ref, first}).pop() + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 2)], + caller_records=[ + [(POS_30_0B, ref, [first], (1, 1))], + [(POS_30_0B, ref, [second], (1, 1))], + ], + threshold=1, + ) + assert [pos for pos, _, _ in read_records(output)] == [31] + + +def test_atomized_snp_and_deletion_with_different_refs(tmp_path, use_accel): + """Both callers support A>G and AT>A at one POS; both reach consensus. + + Atomisation emits the SNP and the deletion as separate records with + different REF alleles at the same POS. Keying votes by ``(REF, ALT)`` + keeps the caller from abstaining, so threshold 2 writes one record per + allele. + """ + seq = 'AT' * 30 + reference = write_reference(tmp_path / 'ref.fa', seq=seq) + reads = [(f'read{i}', 18, seq[18:24]) for i in range(2)] + bam = write_bam(tmp_path / 'sample.bam', reads, length=len(seq)) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 20, 21)]) + callers = [ + write_caller_bcf(tmp_path / f'c{index}.bcf', 'sample1', [ + (20, 'A', ['G'], (0, 1)), + (20, 'AT', ['A'], (0, 1)), + ], length=len(seq)) + for index in range(2) + ] + output = run_consensus( + tmp_path, bam, bed_gz, callers, threshold=2, + use_accel=use_accel, reference=reference) + assert read_records(output) == [ + (21, ('A', 'G'), (0, 1)), + (21, ('AT', 'A'), (0, 1)), + ] + + +# Reference for the pileup edge-case dataset: upper-case bases, then a +# lower-case run and an N run, so both implementations must agree on case and +# unknown bases. +EDGE_REF_SEQ = (REF_SEQ[:30] + REF_SEQ[30:40].lower() + + 'N' * 10 + REF_SEQ[50:]) +EDGE_START0 = 19 +EDGE_END0 = 60 + + +def _bases(start, end): + """Upper-case reference bases at 0-based [start, end).""" + return EDGE_REF_SEQ[start:end].upper() + + +def write_edge_dataset(tmp_path): + """Write a BAM covering the pileup edge cases of the equivalence tests. + + The reads use htslib's 255 missing base quality, overlapping mates whose + qualities decide the observation, reference skips on both strands, + insertions and deletions at and after their anchor, and a reference that + mixes lower-case and N bases. + """ + reference = write_reference(tmp_path / 'ref.fa', seq=EDGE_REF_SEQ) + reads = [ + # full-length unique reads; the second has 255 qualities and an ALT + # base at the first caller record + ('normal', 19, _bases(19, 60), ((0, 41),), 0, 'I' * 41), + ('missingq', 19, _bases(19, 29) + 'A' + _bases(30, 60), + ((0, 41),), 0, None), + # same-name mates: the higher quality is the observation. The + # reference base wins the first pair, the ALT base the second. + ('mate1', 20, _bases(20, 26), ((0, 6),), 0, 'I' * 6), + ('mate1', 20, _bases(20, 21) + 'A' + _bases(22, 26), + ((0, 6),), 0, None), + ('mate2', 20, _bases(20, 26), ((0, 6),), 0, None), + ('mate2', 20, _bases(20, 23) + 'A' + _bases(24, 26), + ((0, 6),), 0, 'I' * 6), + # reference skips on the forward and reverse strand + ('skipf', 19, _bases(19, 24) + _bases(34, 40), + ((0, 5), (3, 10), (0, 6)), 0, 'I' * 11), + ('skipr', 20, _bases(20, 25) + _bases(35, 41), + ((0, 5), (3, 10), (0, 6)), 16, 'I' * 11), + # insertions and deletions at a caller record, and again away from + # any caller record so the fast path sees the affected columns + ('ins', 28, _bases(28, 34) + 'AA' + _bases(34, 38), + ((0, 6), (1, 2), (0, 4)), 0, 'I' * 12), + ('del', 35, _bases(35, 39) + _bases(41, 46), + ((0, 4), (2, 2), (0, 5)), 0, 'I' * 9), + ('del2', 44, _bases(44, 47) + _bases(48, 52), + ((0, 3), (2, 1), (0, 4)), 0, 'I' * 7), + ('ins2', 48, _bases(48, 51) + 'TT' + _bases(51, 54), + ((0, 3), (1, 2), (0, 3)), 0, 'I' * 8), + ] + reads.sort(key=lambda entry: entry[1]) + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed( + tmp_path / 'coverage.bed', + [(CHR, 9, 10), (CHR, EDGE_START0, EDGE_END0), (CHR, 70, 72)]) + callers = [ + write_caller_bcf(tmp_path / 'caller0.bcf', 'sample1', [ + (29, ref_base(30), [alt_base(30)], (0, 1)), + (33, ref_base(34), [ref_base(34) + 'AA'], (0, 1)), + (38, EDGE_REF_SEQ[38:41].upper(), + [EDGE_REF_SEQ[38].upper()], (0, 1)), + (44, 'N', ['A'], (0, 1)), + # two deletions with the same anchor ALT but different REF lengths, + # plus a SNP inside the longer deletion span + (51, _bases(51, 54), [_bases(51, 52)], (0, 1)), + (53, ref_base(54), [alt_base(54)], (0, 1)), + ]), + write_caller_bcf(tmp_path / 'caller1.bcf', 'sample1', [ + (51, _bases(51, 53), [_bases(51, 52)], (0, 1)), + ]), + ] + return {'reference': reference, 'bam': bam, 'bed': bed_gz, + 'callers': callers} + + +def test_accel_and_pure_paths_agree(tmp_path): + """The Cython and pure-pysam implementations produce identical text. + + The dataset exercises 255 base qualities, reference skips, deletions and + insertions after their anchor, deletions of different lengths with the + same anchor ALT, a SNP inside a deletion span, and a reference that mixes + lower-case and N bases. + """ + if not ACCEL_AVAILABLE: + pytest.skip('consensus_accel is not built for this interpreter') + + dataset = write_edge_dataset(tmp_path) + pure = run_consensus( + tmp_path, dataset['bam'], dataset['bed'], dataset['callers'], + threshold=1, use_accel=False, output_name='pure.bcf', + reference=dataset['reference']) + accel = run_consensus( + tmp_path, dataset['bam'], dataset['bed'], dataset['callers'], + threshold=1, use_accel=True, output_name='accel.bcf', + reference=dataset['reference']) + + def text(path): + return subprocess.run([BCFTOOLS, 'view', '-H', str(path)], + check=True, capture_output=True, text=True).stdout + + assert text(pure) == text(accel) + assert text(pure) # not trivially empty + records = read_records(pure) + assert (30, (ref_base(30), alt_base(30)), (0, 1)) in records + assert (34, (ref_base(34), ref_base(34) + 'AA'), (0, 1)) in records + # The deletion at POS 39 has an N in its REF span, so it is omitted. + assert (39, (EDGE_REF_SEQ[38:41].upper(), + EDGE_REF_SEQ[38].upper()), (0, 1)) not in records + # The N run in the reference cannot be written; its caller record is + # omitted along with the surrounding reference positions. + assert (45, ('N', 'A'), (0, 1)) not in records + for _, alleles, _ in records: + for allele in alleles: + if allele in ('.', '*') or allele.startswith('<'): + continue + assert set(allele) <= set('ACGT') + # The 1-bp and 2-bp deletions are distinct alleles at POS 52, and the SNP + # inside the longer deletion span at POS 54 is still written. + assert (52, (_bases(51, 53), _bases(51, 52)), (0, 1)) in records + assert (52, (_bases(51, 54), _bases(51, 52)), (0, 1)) in records + assert (54, (ref_base(54), alt_base(54)), (0, 1)) in records + + +def test_accel_and_pure_paths_agree_on_deletion_overlap_ad(tmp_path): + """A deletion does not hide the other mate's zero-quality base. + + Each mate pair overlaps at a deleted base: one mate deletes it and the + other reads a reference base with quality 0. The deletion is still one + observation, but it is not an allele, so reference AD must count the base. + The BAM lists the deletion first in one pair and last in the other, which + previously made the C tally and the pure fallback disagree. + """ + if not ACCEL_AVAILABLE: + pytest.skip('consensus_accel is not built for this interpreter') + + reference = write_reference(tmp_path / 'ref.fa') + reads = [ + # deletion before the zero-quality base at 0-based position 21 + ('del_first', 19, REF_SEQ[19:21] + REF_SEQ[22:24], + ((0, 2), (2, 1), (0, 2)), 0, 'IIII'), + ('del_first', 19, REF_SEQ[19:24], ((0, 5),), 0, 'II!II'), + # zero-quality base before the deletion at 0-based position 26 + ('base_first', 24, REF_SEQ[24:29], ((0, 5),), 0, 'II!II'), + ('base_first', 24, REF_SEQ[24:26] + REF_SEQ[27:29], + ((0, 2), (2, 1), (0, 2)), 0, 'IIII'), + ] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [(CHR, 19, 29)]) + caller_bcfs = [write_caller_bcf(tmp_path / 'c0.bcf', 'sample1', [])] + + pure = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=1, use_accel=False, + output_name='pure.bcf', reference=reference) + accel = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=1, use_accel=True, + output_name='accel.bcf', reference=reference) + + def text(path): + return subprocess.run([BCFTOOLS, 'view', '-H', str(path)], + check=True, capture_output=True, + text=True).stdout + + assert text(pure) == text(accel) + records = {pos: (alleles, ad, dp) + for pos, alleles, _, ad, dp in read_records_full(pure)} + # 1-based 22 and 27 are the deleted bases; AD is (reference, <*>). + for pos in (22, 27): + assert records[pos] == ((ref_base(pos), '<*>'), (1, 0), 1) + + +def test_accel_import_failure_falls_back(monkeypatch, capsys): + """A binary-incompatible extension warns and uses pure pysam.""" + import builtins + import importlib + + real_import = builtins.__import__ + + def broken_import(name, *args, **kwargs): + if name == 'consensus_accel': + raise ValueError( + 'size changed, may indicate binary incompatibility') + return real_import(name, *args, **kwargs) + + monkeypatch.delenv('CHOCALLATE_NO_ACCEL', raising=False) + monkeypatch.setattr(builtins, '__import__', broken_import) + original = sys.modules.pop('generate_consensus', None) + try: + module = importlib.import_module('generate_consensus') + finally: + sys.modules.pop('generate_consensus', None) + if original is not None: + sys.modules['generate_consensus'] = original + assert module.BcfLineWriter is None + assert module.PileupTally is None + assert 'Cython accelerator unavailable' in capsys.readouterr().err + + +def test_missing_accel_falls_back_silently(monkeypatch, capsys): + """A checkout without the optional extension falls back silently.""" + import builtins + import importlib + + real_import = builtins.__import__ + + def missing_import(name, *args, **kwargs): + if name == 'consensus_accel': + raise ModuleNotFoundError( + "No module named 'consensus_accel'", name='consensus_accel') + return real_import(name, *args, **kwargs) + + monkeypatch.delenv('CHOCALLATE_NO_ACCEL', raising=False) + monkeypatch.setattr(builtins, '__import__', missing_import) + original = sys.modules.pop('generate_consensus', None) + try: + module = importlib.import_module('generate_consensus') + finally: + sys.modules.pop('generate_consensus', None) + if original is not None: + sys.modules['generate_consensus'] = original + assert module.BcfLineWriter is None + assert module.PileupTally is None + assert capsys.readouterr().err == '' + + +def test_accel_close_reports_flush_failure(): + """A failed final flush raises instead of leaving a truncated BCF.""" + if not ACCEL_AVAILABLE: + pytest.skip('consensus_accel is not built for this interpreter') + if not os.path.exists('/dev/full'): + pytest.skip('/dev/full is not available') + + import consensus_accel + import pysam + + header = pysam.VariantHeader() + header.add_sample('sample1') + header.add_line('##contig=') + header.add_line('##INFO=') + header.add_line('##FORMAT=') + header.add_line('##FORMAT=') + header.add_line('##FORMAT=') + writer = consensus_accel.BcfLineWriter(header, '/dev/full') + writer.write_monomorphic(0, 0, b'A,<*>', 1, 0, 1) + with pytest.raises(OSError): + writer.close() + + +class _LimitedTally: + """PileupTally facade with a deliberately small scratch table.""" + + def __init__(self, tally, max_slots): + self.tally = tally + self.max_slots = max_slots + self.fallbacks = 0 + + def sync(self, pos1): + return self.tally.sync(pos1) + + def simple(self, ref_base): + counts = self.tally.simple(ref_base, self.max_slots) + if counts is None: + self.fallbacks += 1 + return counts + + def entries(self): + return self.tally.entries() + + +def test_accel_and_pure_paths_agree_beyond_scratch_table(tmp_path): + """A column too deep for the C scratch table falls back to Python.""" + if not ACCEL_AVAILABLE: + pytest.skip('consensus_accel is not built for this interpreter') + + import consensus_accel + import generate_consensus + import pysam + + dataset = write_edge_dataset(tmp_path) + reference = EDGE_REF_SEQ.upper() + # Each walker owns an independent pileup iterator: pysam's columns point + # into the read buffer of their file handle. + with pysam.AlignmentFile(str(dataset['bam']), 'rb') as pure_file, \ + pysam.AlignmentFile(str(dataset['bam']), 'rb') as accel_file: + def pileup(handle): + return handle.pileup( + CHR, EDGE_START0, EDGE_END0, truncate=True, stepper='all', + min_base_quality=0, max_depth=1000000) + + pure = generate_consensus.ColumnWalker(pileup(pure_file)) + limited = _LimitedTally( + consensus_accel.PileupTally(pileup(accel_file)), max_slots=4) + accel = generate_consensus.AccelWalker(limited) + for pos1 in range(EDGE_START0 + 1, EDGE_END0 + 1): + has_column = accel.sync(pos1) + assert has_column == pure.sync(pos1) + if not has_column: + continue + ref = reference[pos1 - 1] + assert accel.reference_support(ref) == pure.reference_support(ref) + assert accel.site() == pure.site() + # At least one column was deeper than the reduced table and took the + # Python fallback instead of the C tally. + assert limited.fallbacks + + +MAX_POSITION = (1 << 31) - 1 + + +def _sparse_files_supported(directory): + """True when a seek-created hole is stored compactly.""" + probe = directory / '.sparse-probe' + with open(probe, 'wb') as handle: + handle.seek(1 << 20) + handle.write(b'x') + try: + return probe.stat().st_blocks * 512 < (1 << 19) + finally: + probe.unlink() + + +def test_accel_and_pure_paths_agree_near_position_limit(tmp_path): + """Positions and columns just below 2**31 agree byte for byte.""" + if not ACCEL_AVAILABLE: + pytest.skip('consensus_accel is not built for this interpreter') + if not _sparse_files_supported(tmp_path): + pytest.skip('filesystem does not support sparse files') + + contig = 'chrBig' + length = MAX_POSITION + seq = 'ACGTACGTACGTAC' + start0 = length - len(seq) + reference = write_sparse_reference( + tmp_path / 'ref.fa', contig, length, start0, seq) + + # Three staggered reads cover all but the last two BED bases, so the walk + # also asks for columns past the end of the pileup near the limit. + reads = [] + for index in range(3): + reads.append((f'read{index}', start0 + index, seq[index:index + 10], + ((0, 10),), 0, 'I' * 10)) + bam = write_bam(tmp_path / 'sample.bam', reads, contig=contig, + length=length, csi=True) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [(contig, start0, length)]) + alt = 'A' if seq[5] != 'A' else 'C' + callers = [ + write_caller_bcf(tmp_path / 'caller0.bcf', 'sample1', + [(start0 + 5, seq[5], [alt], (0, 1))], + contig=contig, length=length), + write_caller_bcf(tmp_path / 'caller1.bcf', 'sample1', [], + contig=contig, length=length), + ] + + pure = run_consensus( + tmp_path, bam, bed_gz, callers, threshold=1, use_accel=False, + output_name='pure.bcf', reference=reference) + accel = run_consensus( + tmp_path, bam, bed_gz, callers, threshold=1, use_accel=True, + output_name='accel.bcf', reference=reference) + + def text(path): + return subprocess.run([BCFTOOLS, 'view', '-H', str(path)], + check=True, capture_output=True, text=True).stdout + + assert text(pure) == text(accel) + assert f'{contig}\t{MAX_POSITION}\t' in text(pure) + + +def test_provenance_headers_list_used_and_missing_callers(tmp_path, use_accel): + """The structured header describes the vote and its inputs.""" + import pysam + + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + caller_bcfs = [ + write_caller_bcf(tmp_path / 'bcftools.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))]), + write_caller_bcf(tmp_path / 'gatk.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))]), + ] + output = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=2, + callers=['bcftools', 'gatk'], + configured_callers=['bcftools', 'freebayes', 'gatk'], + use_accel=use_accel, reference=reference) + with pysam.VariantFile(str(output), index_filename=None) as variant_file: + header = str(variant_file.header) + records = list(variant_file) + accel = 'true' if use_accel else 'false' + expected = ('##ChoCallate=') + assert expected in header + assert [(record.pos, (record.ref,) + tuple(record.alts), + tuple(record.samples[0]['GT'])) + for record in records] == [ + (30, (ref_base(30), alt_base(30)), (0, 1))] + + + +def test_threshold_above_surviving_callers_fails(tmp_path, use_accel): + """Degraded mode refuses a threshold no surviving caller can reach.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + caller = write_caller_bcf(tmp_path / 'bcftools.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))]) + with pytest.raises(subprocess.CalledProcessError) as excinfo: + run_consensus(tmp_path, bam, bed_gz, [caller], threshold=2, + callers=['bcftools'], + configured_callers=['bcftools', 'freebayes'], + use_accel=use_accel, reference=reference) + assert 'surviving caller' in excinfo.value.stderr + + +def test_multiple_alts_are_emitted_regardless_of_caller_order(tmp_path, use_accel): + """Distinct supported ALTs become separate order-independent records.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + alt1 = alt_base(30) + alt2 = next(base for base in 'ACGT' + if base not in (ref_base(30), alt1)) + caller_bcfs = [ + write_caller_bcf(tmp_path / 'bcftools.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt1], (0, 1))]), + write_caller_bcf(tmp_path / 'freebayes.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt2], (0, 1))]), + ] + + def consensus_records(order, output_name): + output = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=1, + callers=['bcftools', 'freebayes'], configured_callers=order, + use_accel=use_accel, reference=reference, + output_name=output_name) + return sorted(read_records(output)) + + expected = sorted([ + (30, (ref_base(30), alt1), (0, 1)), + (30, (ref_base(30), alt2), (0, 1)), + ]) + assert consensus_records(['bcftools', 'freebayes'], 'first.bcf') == expected + assert consensus_records(['freebayes', 'bcftools'], 'second.bcf') == expected + + +@pytest.mark.parametrize('callers', [ + ('bcftools',), + ('freebayes',), + ('gatk',), + ('bcftools', 'freebayes'), + ('bcftools', 'gatk'), + ('freebayes', 'gatk'), + ('bcftools', 'freebayes', 'gatk'), +]) +def test_any_caller_subset(tmp_path, use_accel, callers): + """Any non-empty caller subset produces a consensus and a header.""" + import pysam + + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + caller_bcfs = [ + write_caller_bcf(tmp_path / f'{caller}.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))]) + for caller in callers] + output = run_consensus(tmp_path, bam, bed_gz, caller_bcfs, + threshold=len(callers), callers=list(callers), + use_accel=use_accel, reference=reference) + with pysam.VariantFile(str(output), index_filename=None) as variant_file: + header = str(variant_file.header) + records = list(variant_file) + assert f'Callers="{",".join(callers)}"' in header + assert 'Rule=two-stage' in header + assert [(record.pos, (record.ref,) + tuple(record.alts), + tuple(record.samples[0]['GT'])) + for record in records] == [ + (30, (ref_base(30), alt_base(30)), (0, 1))] + + +# Two-stage decision table with three configured callers. ``h`` is a 0/1 call, +# ``m`` a 1/1 call and ``-`` a caller without a record. The expected values +# use the defaults: absent_vote=ref and tie_policy=nocall. +VOTING_TABLE = { + ('--h', 1): 'variant', + ('--h', 2): 'reference', + ('--h', 3): None, + ('h--', 1): 'variant', + ('h--', 2): 'reference', + ('h--', 3): None, + ('hm-', 1): None, + ('hm-', 2): None, + ('hm-', 3): None, + ('-hm', 1): None, + ('-hm', 2): None, + ('-hm', 3): None, +} + + +def _voting_caller_records(placement, alt): + """Caller records for a row of the decision table.""" + records = [] + for marker in placement: + if marker == 'h': + records.append([(POS_30_0B, ref_base(30), [alt], (0, 1))]) + elif marker == 'm': + records.append([(POS_30_0B, ref_base(30), [alt], (1, 1))]) + else: + records.append([]) + return records + + +@pytest.mark.parametrize('placement,threshold', VOTING_TABLE) +def test_voting_table(consensus_runner, placement, threshold): + """The two-stage vote reproduces the documented table.""" + alt = alt_base(30) + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 1)], + caller_records=_voting_caller_records(placement, alt), + threshold=threshold, + ) + expected = VOTING_TABLE[(placement, threshold)] + if expected == 'variant': + assert read_records(output) == [ + (30, (ref_base(30), alt), (0, 1))] + elif expected == 'reference': + assert read_records(output) == [ + (30, (ref_base(30), '<*>'), (0, 0))] + else: + assert read_records(output) == [] + + +@pytest.mark.parametrize('placement', ['hm-', '-hm']) +def test_genotype_tie_with_first_policy(consensus_runner, placement): + """``first`` resolves a zygosity tie by configured caller order.""" + alt = alt_base(30) + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 1)], + caller_records=_voting_caller_records(placement, alt), + threshold=1, + tie_policy='first', + ) + assert read_records(output) == [(30, (ref_base(30), alt), (0, 1))] + + +def test_absent_vote_abstain_requires_coverage(consensus_runner): + """An absent caller only votes ref when the position is covered.""" + alt = alt_base(24) + covered_bed = [('chr1', POS_24_0B, POS_24_0B + 1)] + covered_variant = [(POS_24_0B, ref_base(24), [alt], (0, 1))] + + reference_vote = consensus_runner( + bed_intervals=covered_bed, + caller_records=[covered_variant, [], []], + threshold=2, absent_vote='ref') + assert read_records(reference_vote) == [ + (24, (ref_base(24), '<*>'), (0, 0))] + + abstain_with_coverage = consensus_runner( + bed_intervals=covered_bed, + caller_records=[covered_variant, [], []], + threshold=2, absent_vote='abstain', min_coverage=1, + output_name='abstain_covered.bcf') + assert read_records(abstain_with_coverage) == [ + (24, (ref_base(24), '<*>'), (0, 0))] + + abstain_too_strict = consensus_runner( + bed_intervals=covered_bed, + caller_records=[covered_variant, [], []], + threshold=2, absent_vote='abstain', min_coverage=100, + output_name='abstain_strict.bcf') + assert read_records(abstain_too_strict) == [] + + # A BED position without a pileup column has no coverage evidence, so an + # absent caller abstains even with min_coverage=0. + uncovered_bed = [('chr1', POS_30_0B, POS_30_0B + 1)] + uncovered_variant = [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))] + abstain_uncovered = consensus_runner( + bed_intervals=uncovered_bed, + caller_records=[uncovered_variant, [], []], + threshold=2, absent_vote='abstain', min_coverage=0, + output_name='abstain_uncovered.bcf') + assert read_records(abstain_uncovered) == [] + + ref_uncovered = consensus_runner( + bed_intervals=uncovered_bed, + caller_records=[uncovered_variant, [], []], + threshold=2, absent_vote='ref', output_name='ref_uncovered.bcf') + assert read_records(ref_uncovered) == [ + (30, (ref_base(30), '<*>'), (0, 0))] + + +def test_filtered_record_abstains_from_the_vote(consensus_runner): + """A soft-filtered record is not a reference vote. + + Caller A calls 1/1 with FILTER=PASS, caller B has the same variant with + FILTER=LowQual and caller C has no record. With threshold 2 the filtered + record abstains (the default), so the position has neither a variant nor a + hom-reference record. Counting it as reference would produce a confident + hom-reference call. + """ + alt = alt_base(30) + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 1)], + caller_records=[ + [(POS_30_0B, ref_base(30), [alt], (1, 1), None, 'PASS')], + [(POS_30_0B, ref_base(30), [alt], (1, 1), None, 'LowQual')], + [], + ], + threshold=2, + ) + assert read_records(output) == [] + + +def test_filtered_vote_ref_counts_as_reference(consensus_runner): + """``filtered_vote=ref`` turns a filtered record into a reference vote.""" + alt = alt_base(30) + output = consensus_runner( + bed_intervals=[('chr1', POS_30_0B, POS_30_0B + 1)], + caller_records=[ + [(POS_30_0B, ref_base(30), [alt], (1, 1), None, 'PASS')], + [(POS_30_0B, ref_base(30), [alt], (1, 1), None, 'LowQual')], + [], + ], + threshold=2, + filtered_vote='ref', + ) + assert read_records(output) == [ + (30, (ref_base(30), '<*>'), (0, 0))] + + +def test_header_describes_abstain_and_tie_policies(tmp_path, use_accel): + """The single header line records the applied voting rule.""" + import pysam + + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + caller_bcfs = [ + write_caller_bcf(tmp_path / 'bcftools.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))]), + write_caller_bcf(tmp_path / 'gatk.bcf', 'sample1', + [(POS_30_0B, ref_base(30), [alt_base(30)], (0, 1))]), + ] + output = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=2, + callers=['bcftools', 'gatk'], + configured_callers=['bcftools', 'freebayes', 'gatk'], + absent_vote='abstain', tie_policy='first', min_coverage=3, + use_accel=use_accel, reference=reference) + with pysam.VariantFile(str(output), index_filename=None) as variant_file: + header = str(variant_file.header) + accel = 'true' if use_accel else 'false' + expected = ('##ChoCallate=') + assert expected in header + + +def test_split_multiallelic_records_are_aggregated(tmp_path, use_accel): + """All records of a caller at one position form one vote.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + alt1 = alt_base(30) + alt2 = next(base for base in 'ACGT' + if base not in (ref_base(30), alt1)) + # Caller 0 carries two split records. The second one (alt1) must not be + # skipped: together with caller 1 it reaches threshold 2. + caller0 = write_caller_bcf(tmp_path / 'caller0.bcf', 'sample1', [ + (POS_30_0B, ref_base(30), [alt2], (0, 1)), + (POS_30_0B, ref_base(30), [alt1], (0, 1)), + ]) + caller1 = write_caller_bcf(tmp_path / 'caller1.bcf', 'sample1', [ + (POS_30_0B, ref_base(30), [alt1], (0, 1)), + ]) + output = run_consensus(tmp_path, bam, bed_gz, [caller0, caller1], + threshold=2, use_accel=use_accel, + reference=reference) + assert read_records(output) == [ + (30, (ref_base(30), alt1), (0, 1))] + + +@pytest.mark.parametrize('split_records', [True, False]) +def test_triallelic_caller_produces_split_consensus(tmp_path, use_accel, + split_records): + """A 1/2 call produces one consensus record per ALT.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + alt1 = alt_base(30) + alt2 = next(base for base in 'ACGT' + if base not in (ref_base(30), alt1)) + if split_records: + records = [(POS_30_0B, ref_base(30), [alt1], (0, 1)), + (POS_30_0B, ref_base(30), [alt2], (0, 1))] + else: + records = [(POS_30_0B, ref_base(30), [alt1, alt2], (1, 2))] + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', records) + output = run_consensus(tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference) + assert sorted(read_records(output)) == sorted([ + (30, (ref_base(30), alt1), (0, 1)), + (30, (ref_base(30), alt2), (0, 1)), + ]) + + +def test_atomized_mnp_matches_snp(tmp_path, use_accel): + """An atomised freebayes-style MNP votes with the matching SNP.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 3)]) + mnp_ref = REF_SEQ[POS_30_0B:POS_30_0B + 3].upper() + mnp_alt = ''.join({'A': 'T', 'C': 'A', 'G': 'T', 'T': 'G'}[base] + for base in mnp_ref) + mnp = write_caller_bcf(tmp_path / 'mnp.bcf', 'sample1', [ + (POS_30_0B, mnp_ref, [mnp_alt], (0, 1)), + ]) + normalized = tmp_path / 'mnp.normalized.bcf' + subprocess.run([BCFTOOLS, 'norm', '-f', str(reference), '-m', '-any', + '--atomize', '-Ob', '-o', str(normalized), str(mnp)], + check=True) + subprocess.run([BCFTOOLS, 'index', '--csi', str(normalized)], check=True) + snp = write_caller_bcf(tmp_path / 'snp.bcf', 'sample1', [ + (POS_30_0B, ref_base(30), [alt_base(30)], (0, 1)), + ]) + output = run_consensus(tmp_path, bam, bed_gz, [normalized, snp], + threshold=1, use_accel=use_accel, + reference=reference) + assert (30, (ref_base(30), alt_base(30)), (0, 1)) in read_records(output) + + +def test_norm_dedup_removes_chunk_boundary_duplicates(tmp_path): + """Exact duplicates from chunk boundaries are removed.""" + import pysam + + reference = write_reference(tmp_path / 'ref.fa') + duplicated = write_caller_bcf(tmp_path / 'duplicated.bcf', 'sample1', [ + (POS_30_0B, ref_base(30), [alt_base(30)], (0, 1)), + (POS_30_0B, ref_base(30), [alt_base(30)], (0, 1)), + ]) + output = tmp_path / 'deduplicated.bcf' + chain = ( + f'{BCFTOOLS} norm -f {reference} -m -any --atomize -Ou {duplicated}' + f' | {BCFTOOLS} sort -Ou' + f' | {BCFTOOLS} norm -d exact -Ob -o {output}' + ) + subprocess.run(['bash', '-c', chain], check=True) + with pysam.VariantFile(str(output), index_filename=None) as variant_file: + records = list(variant_file) + assert len(records) == 1 + assert (records[0].pos, records[0].ref, + tuple(records[0].alts)) == (30, ref_base(30), (alt_base(30),)) + + +def test_split_consensus_records_join_for_output(tmp_path, use_accel): + """Per-ALT consensus records re-join when split output is off.""" + import pysam + + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'read{i}', i, REF_SEQ[i:i + 6]) for i in range(19, 24)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', + [('chr1', POS_30_0B, POS_30_0B + 1)]) + alt1 = alt_base(30) + alt2 = next(base for base in 'ACGT' + if base not in (ref_base(30), alt1)) + caller = write_caller_bcf(tmp_path / 'caller.bcf', 'sample1', [ + (POS_30_0B, ref_base(30), [alt1], (0, 1)), + (POS_30_0B, ref_base(30), [alt2], (0, 1)), + ]) + split = run_consensus(tmp_path, bam, bed_gz, [caller], threshold=1, + use_accel=use_accel, reference=reference, + output_name='split.bcf') + joined = tmp_path / 'joined.bcf' + subprocess.run([BCFTOOLS, 'norm', '-m', '+any', '-Ob', '-o', str(joined), + str(split)], check=True) + with pysam.VariantFile(str(joined), index_filename=None) as variant_file: + records = list(variant_file) + assert len(records) == 1 + record = records[0] + assert record.pos == 30 + assert record.ref == ref_base(30) + assert tuple(record.alts) == tuple(sorted((alt1, alt2))) + assert tuple(sorted(record.samples[0]['GT'])) == (1, 2) diff --git a/tests/python/test_mapping_stats.py b/tests/python/test_mapping_stats.py new file mode 100644 index 0000000..91e4af1 --- /dev/null +++ b/tests/python/test_mapping_stats.py @@ -0,0 +1,167 @@ +"""Tests for the mapping statistics summary helper.""" + +import subprocess +import sys + +from conftest import BIN_DIR # noqa: I100,I202 (tests run from this directory) + +SCRIPT = BIN_DIR / 'mapping_stats.py' + +RAW_FLAGSTAT = """\ +10 + 0 in total (QC-passed reads + QC-failed reads) +10 + 0 primary +0 + 0 secondary +0 + 0 supplementary +0 + 0 duplicates +8 + 0 mapped (80.00% : N/A) +10 + 0 paired in sequencing +5 + 0 read1 +5 + 0 read2 +6 + 0 properly paired (60.00% : N/A) +6 + 0 with itself and mate mapped +1 + 0 singletons (10.00% : N/A) +1 + 0 with mate mapped to a different chr +1 + 0 with mate mapped to a different chr (mapQ>=5) +""" + +FINAL_FLAGSTAT = """\ +8 + 0 in total (QC-passed reads + QC-failed reads) +8 + 0 primary +0 + 0 secondary +0 + 0 supplementary +1 + 0 duplicates +8 + 0 mapped (100.00% : N/A) +8 + 0 paired in sequencing +4 + 0 read1 +4 + 0 read2 +6 + 0 properly paired (75.00% : N/A) +6 + 0 with itself and mate mapped +0 + 0 singletons (0.00% : N/A) +0 + 0 with mate mapped to a different chr +0 + 0 with mate mapped to a different chr (mapQ>=5) +""" + +STATS = """\ +SN\traw total sequences:\t8\t# excluding supplementary and secondary reads +SN\tfiltered sequences:\t0 +SN\tsequences:\t8 +SN\treads mapped:\t8 +SN\treads MQ0:\t2 +SN\taverage length:\t150 +SN\taverage quality:\t36.5 +SN\tinsert size average:\t250.5 +SN\tinsert size standard deviation:\t20.1 +SN\terror rate:\t1.25e-03 +SN\tbases mapped (cigar):\t1200 +""" + +IDXSTATS = """\ +chr1\t1000\t8\t0 +chr2\t500\t0\t0 +*\t0\t0\t0 +""" + + +def write_reports(root, sample, raw=RAW_FLAGSTAT, + final=FINAL_FLAGSTAT, stats=STATS, idxstats=IDXSTATS): + """Write the four reports used by the script for one sample.""" + dirs = { + 'raw': root / 'raw', + 'flagstat': root / 'flagstat', + 'stats': root / 'stats', + 'idxstats': root / 'idxstats', + } + for path in dirs.values(): + path.mkdir(parents=True, exist_ok=True) + (dirs['raw'] / f'{sample}.raw.flagstat').write_text(raw) + (dirs['flagstat'] / f'{sample}.final.flagstat').write_text(final) + (dirs['stats'] / f'{sample}.stats').write_text(stats) + (dirs['idxstats'] / f'{sample}.idxstats').write_text(idxstats) + return dirs + + +def run_summary(root, samples, dirs): + """Run the summary script and return its output path.""" + order = root / 'sample_order.txt' + order.write_text('\n'.join(samples) + '\n') + output = root / 'mapping_summary.tsv' + result = subprocess.run( + [ + sys.executable, str(SCRIPT), + '--sample-order', str(order), + '--raw-dir', str(dirs['raw']), + '--flagstat-dir', str(dirs['flagstat']), + '--stats-dir', str(dirs['stats']), + '--idxstats-dir', str(dirs['idxstats']), + '--output', str(output), + ], + capture_output=True, + text=True, + ) + return result, output + + +def table(path): + """Return the summary header and rows split on tabs.""" + lines = path.read_text().splitlines() + return lines[0].split('\t'), [ + dict(zip(lines[0].split('\t'), line.split('\t'))) + for line in lines[1:] + ] + + +def test_summary_reports_counts_and_rates(tmp_path): + dirs = write_reports(tmp_path, 'sample1') + + result, output = run_summary(tmp_path, ['sample1'], dirs) + + assert result.returncode == 0, result.stderr + header, rows = table(output) + assert header[0] == 'sample' + assert 'raw_mapped_pct' in header + assert len(rows) == 1 + row = rows[0] + assert row['sample'] == 'sample1' + assert row['raw_total'] == '10' + assert row['raw_mapped'] == '8' + assert row['raw_mapped_pct'] == '80.00' + assert row['raw_properly_paired_pct'] == '60.00' + assert row['raw_singletons'] == '1' + assert row['raw_mate_different_chr'] == '1' + assert row['final_total'] == '8' + assert row['final_duplicates'] == '1' + assert row['final_duplicate_pct'] == '12.50' + assert row['final_properly_paired_pct'] == '75.00' + assert row['final_mq0'] == '2' + assert row['final_mq0_pct'] == '25.00' + assert row['average_read_length'] == '150' + assert row['average_mapq'] == '36.5' + assert row['insert_size_mean'] == '250.5' + assert row['error_rate'] == '1.25e-03' + assert row['mapped_bases'] == '1200' + assert row['contigs_with_reads'] == '1' + assert row['contigs_without_reads'] == '1' + + +def test_summary_follows_samplesheet_order(tmp_path): + dirs = write_reports(tmp_path, 'sample_b') + write_reports(tmp_path, 'sample_a') + + result, output = run_summary( + tmp_path, ['sample_b', 'sample_a'], dirs + ) + + assert result.returncode == 0, result.stderr + _, rows = table(output) + assert [row['sample'] for row in rows] == ['sample_b', 'sample_a'] + + +def test_missing_report_is_an_error(tmp_path): + dirs = write_reports(tmp_path, 'sample1') + (dirs['stats'] / 'sample1.stats').unlink() + + result, _ = run_summary(tmp_path, ['sample1'], dirs) + + assert result.returncode != 0 + assert 'missing mapping statistics' in result.stderr + assert 'sample1.stats' in result.stderr diff --git a/tests/python/test_merge_outputs.py b/tests/python/test_merge_outputs.py new file mode 100644 index 0000000..39767df --- /dev/null +++ b/tests/python/test_merge_outputs.py @@ -0,0 +1,556 @@ +"""End-to-end regression tests for the merged (multi-sample) output stage. + +Covers: + +* all sites are kept when ``remove_invariant`` is false, including monomorphic + records; +* a hom-reference sample's non-reference read depth reaches the merged ALT AD; +* the merge path for N >= 2 samples as a whole: sample order, genotypes at + the variant position, the invariant filter when enabled, and multi-allelic + sites joined into one record or split per ALT. + +The command chain is exercised through `bin/merge_outputs.sh`, the same script +used by `MERGE_OUTPUTS`; changes to the production merge path are therefore +tested directly. +""" + +import os +import subprocess +import sys +import time + +import pysam +import pytest +from conftest import ( # noqa: I100,I202 (tests run from this directory) + BCFTOOLS, + BIN_DIR, + REF_SEQ, + REPO_ROOT, + alt_base, + ref_base, + run_consensus, + write_bam, + write_bed, + write_caller_bcf, + write_reference, +) + +# 1-based position and 0-based start of the variant carried by sampleA +VARIANT_POS = 30 +VARIANT_0B = 29 + +# The variant position carries 10 reference-supporting and 2 ALT-supporting +# reads; a hom-reference sample must report (10, 2), not (10, 0). +REF_DEPTH = 10 +ALT_DEPTH = 2 + + +def build_two_samples(tmp_path, use_accel, compress_reference_blocks=False): + """Per-sample consensus BCFs: sampleA is 0/1 at position 30, sampleB is ref.""" + reference = write_reference(tmp_path / 'ref.fa') + alt = alt_base(VARIANT_POS) + # reference-matching reads covering the whole BED window, plus ALT reads at + # the variant position + reads = [(f'ref{start}', start, REF_SEQ[start:start + 10]) + for start in range(19, 30)] + for i in range(ALT_DEPTH): + seq = list(REF_SEQ[VARIANT_0B:VARIANT_0B + 10]) + seq[0] = alt + reads.append((f'alt{i}', VARIANT_0B, ''.join(seq))) + reads.sort(key=lambda item: item[1]) + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 32)]) + + variant_calls = [ + write_caller_bcf(tmp_path / f'sampleA_caller{i}.bcf', 'sampleA', + [(VARIANT_0B, ref_base(VARIANT_POS), + [alt], (0, 1))]) + for i in range(2)] + reference_calls = [ + write_caller_bcf(tmp_path / f'sampleB_caller{i}.bcf', 'sampleB', []) + for i in range(2)] + + sample_files = {} + for sample, calls in (('sampleA', variant_calls), ('sampleB', reference_calls)): + sample_files[sample] = run_consensus( + tmp_path, bam, bed_gz, calls, threshold=2, sample_name=sample, + use_accel=use_accel, output_name=f'{sample}.bcf', + reference=reference, + compress_reference_blocks=compress_reference_blocks) + subprocess.run([BCFTOOLS, 'index', '--csi', str(sample_files[sample])], + check=True) + return sample_files, reference + + +def build_multiallelic_samples(tmp_path, use_accel): + """Per-sample BCFs: sampleA carries two ALT alleles at one site. + + Both callers report the heterozygous genotype 1/2, so the consensus + stage writes one split record per ALT and the output stage either joins + them into a multiallelic record or keeps them split. + """ + reference = write_reference(tmp_path / 'ref.fa') + ref = ref_base(VARIANT_POS) + alts = [base for base in 'ACGT' if base != ref][:2] + reads = [(f'ref{start}', start, REF_SEQ[start:start + 10]) + for start in range(19, 30)] + for alt_index, alt in enumerate(alts): + for copy in range(ALT_DEPTH + alt_index): + seq = list(REF_SEQ[VARIANT_0B:VARIANT_0B + 10]) + seq[0] = alt + reads.append((f'alt{alt_index}_{copy}', VARIANT_0B, ''.join(seq))) + reads.sort(key=lambda item: item[1]) + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 32)]) + + variant_calls = [ + write_caller_bcf(tmp_path / f'sampleA_caller{i}.bcf', 'sampleA', + [(VARIANT_0B, ref, alts, (1, 2))]) + for i in range(2)] + reference_calls = [ + write_caller_bcf(tmp_path / f'sampleB_caller{i}.bcf', 'sampleB', []) + for i in range(2)] + + sample_files = {} + for sample, calls in (('sampleA', variant_calls), + ('sampleB', reference_calls)): + sample_files[sample] = run_consensus( + tmp_path, bam, bed_gz, calls, threshold=2, sample_name=sample, + use_accel=use_accel, output_name=f'{sample}.bcf', + reference=reference) + subprocess.run([BCFTOOLS, 'index', '--csi', str(sample_files[sample])], + check=True) + return sample_files, reference, alts + + +def build_degraded_samples(tmp_path, use_accel): + """Per-sample BCFs where each sample lost a different caller.""" + reference = write_reference(tmp_path / 'ref.fa') + reads = [(f'ref{start}', start, REF_SEQ[start:start + 10]) + for start in range(19, 30)] + bam = write_bam(tmp_path / 'sample.bam', reads) + bed_gz = write_bed(tmp_path / 'coverage.bed', [('chr1', 19, 32)]) + + configured = ['bcftools', 'freebayes', 'gatk'] + surviving = { + 'sampleA': ['bcftools', 'gatk'], + 'sampleB': ['bcftools', 'freebayes'], + } + sample_files = {} + for sample, callers in surviving.items(): + caller_bcfs = [ + write_caller_bcf( + tmp_path / f'{sample}_{caller}.bcf', sample, []) + for caller in callers + ] + sample_files[sample] = run_consensus( + tmp_path, bam, bed_gz, caller_bcfs, threshold=1, + sample_name=sample, callers=callers, + configured_callers=configured, use_accel=use_accel, + output_name=f'{sample}.bcf', reference=reference) + subprocess.run( + [BCFTOOLS, 'index', '--csi', str(sample_files[sample])], + check=True) + return sample_files, reference + + +def merge_chain(tmp_path, sample_files, reference, remove_invariant, + split_multiallelic=False, output_format='bcf', cpus=1, + samples_dropped=None): + """Run bin/merge_outputs.sh and return the merged output path.""" + list_file = tmp_path / 'merge_list.txt' + list_file.write_text('\n'.join(str(path) for path in sample_files.values()) + '\n') + output = tmp_path / ('consensus.vcf.gz' if output_format == 'vcf' + else 'consensus.bcf') + cmd = [ + str(BIN_DIR / 'merge_outputs.sh'), + '--file-list', str(list_file), + '--reference', str(reference), + '--output', str(output), + '--format', output_format, + '--cpus', str(cpus), + '--bcftools', BCFTOOLS, + '--python', sys.executable, + ] + if remove_invariant: + cmd.append('--remove-invariant') + if split_multiallelic: + cmd.append('--split-multiallelic') + if samples_dropped: + cmd += ['--samples-dropped', samples_dropped] + env = os.environ.copy() + env['TMPDIR'] = str(tmp_path) + subprocess.run(cmd, check=True, cwd=str(REPO_ROOT), env=env) + return output + + +def read_merged(path): + with pysam.VariantFile(str(path), index_filename=None) as variant_file: + return [(record.pos, (record.ref,) + tuple(record.alts), + tuple(record.samples['sampleA']['GT']), + tuple(record.samples['sampleB']['GT'])) for record in variant_file] + + +def read_merged_ad(path): + with pysam.VariantFile(str(path), index_filename=None) as variant_file: + entries = [] + for record in variant_file: + ad_a = record.samples['sampleA'].get('AD') + ad_b = record.samples['sampleB'].get('AD') + entries.append((record.pos, + tuple(ad_a) if ad_a is not None else None, + tuple(ad_b) if ad_b is not None else None)) + return entries + + +def read_merged_spans(path): + """(pos, inclusive end, alleles, GT A, GT B, DP A, DP B) tuples.""" + with pysam.VariantFile(str(path), index_filename=None) as variant_file: + return [(record.pos, record.stop, + (record.ref,) + tuple(record.alts), + tuple(record.samples['sampleA']['GT']), + tuple(record.samples['sampleB']['GT']), + record.samples['sampleA']['DP'], + record.samples['sampleB']['DP']) + for record in variant_file] + + +def expanded_positions(spans): + return [position for start, end, *_ in spans + for position in range(start, end + 1)] + + +def test_all_sites_kept_without_remove_invariant(tmp_path, use_accel): + """With remove_invariant=false every covered position is kept exactly once.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, remove_invariant=False) + records = read_merged(merged) + assert [pos for pos, _, _, _ in records] == list(range(20, 33)) + variant = [record for record in records if record[0] == VARIANT_POS] + assert variant == [(VARIANT_POS, (ref_base(VARIANT_POS), alt_base(VARIANT_POS)), + (0, 1), (0, 0))] + for pos, alleles, gt_a, gt_b in records: + if pos != VARIANT_POS: + assert alleles[1] == '<*>' + assert (gt_a, gt_b) == ((0, 0), (0, 0)) + + +def test_other_allele_depth_reaches_the_merged_alt_slot(tmp_path, use_accel): + """A hom-reference sample's <*> depth becomes its AD for the merged ALT.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, remove_invariant=False) + variant_ad = [entry for entry in read_merged_ad(merged) if entry[0] == VARIANT_POS] + assert variant_ad == [(VARIANT_POS, (REF_DEPTH, ALT_DEPTH), + (REF_DEPTH, ALT_DEPTH))] + + +def test_all_sites_kept_without_remove_invariant_split(tmp_path, use_accel): + """Same, with output.split_multiallelic: true (no `norm -m +any` re-join).""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False, split_multiallelic=True) + assert [pos for pos, _, _, _ in read_merged(merged)] == list(range(20, 33)) + + +@pytest.mark.parametrize('split_multiallelic', [False, True], + ids=['join', 'split']) +def test_multiallelic_site_split_and_join(tmp_path, use_accel, + split_multiallelic): + """Both ALTs reach the output, joined as 1/2 or split by ALT.""" + sample_files, reference, alts = build_multiallelic_samples( + tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False, + split_multiallelic=split_multiallelic) + records = read_merged(merged) + positions = [pos for pos, _, _, _ in records] + if split_multiallelic: + expected = (list(range(20, 30)) + + [VARIANT_POS] * len(alts) + [31, 32]) + else: + expected = list(range(20, 33)) + assert positions == expected + + site = [record for record in records if record[0] == VARIANT_POS] + if split_multiallelic: + assert len(site) == len(alts) + assert [alleles for _, alleles, _, _ in site] == [ + (ref_base(VARIANT_POS), alt) for alt in alts] + for _, _, gt_a, gt_b in site: + assert sorted(gt_a) == [0, 1] + assert gt_b == (0, 0) + else: + assert len(site) == 1 + _, alleles, gt_a, gt_b = site[0] + assert alleles == (ref_base(VARIANT_POS), *alts) + assert sorted(gt_a) == [1, 2] + assert gt_b == (0, 0) + + depths = [ALT_DEPTH + index for index in range(len(alts))] + site_ad = [entry for entry in read_merged_ad(merged) + if entry[0] == VARIANT_POS] + if split_multiallelic: + assert [ad_a for _, ad_a, _ in site_ad] == [ + (REF_DEPTH, depth) for depth in depths] + else: + assert site_ad[0][1] == (REF_DEPTH,) + tuple(depths) + + +def test_remove_invariant_drops_monomorphic_sites(tmp_path, use_accel): + """remove_invariant=true still removes the monomorphic records.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, remove_invariant=True) + records = read_merged(merged) + assert [pos for pos, _, _, _ in records] == [VARIANT_POS] + assert records[0][1] == (ref_base(VARIANT_POS), alt_base(VARIANT_POS)) + assert records[0][2:] == ((0, 1), (0, 0)) + + +def test_reference_blocks_survive_the_merge(tmp_path, use_accel): + """Compressed per-sample input stays compressed and covers all sites.""" + sample_files, reference = build_two_samples( + tmp_path, use_accel, compress_reference_blocks=True) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False) + spans = read_merged_spans(merged) + assert expanded_positions(spans) == list(range(20, 33)) + # 20..29 is one block, 30 the only variant, 31..32 a trailing block. + assert [(start, end) for start, end, *_ in spans] == [ + (20, 29), (30, 30), (31, 32)] + assert any(end > start for start, end, *_ in spans) + with pysam.VariantFile(str(merged), index_filename=None) as variant_file: + assert 'ReferenceBlocks=True' in str(variant_file.header) + variant = [span for span in spans if span[0] == VARIANT_POS][0] + assert variant[2] == (ref_base(VARIANT_POS), alt_base(VARIANT_POS)) + assert variant[3:5] == ((0, 1), (0, 0)) + # Block DP is the minimum over the block (1 at position 20). + assert variant[6] == 1 + + +def test_reference_block_alt_depth_is_missing_not_zero(tmp_path, use_accel): + """A compressed hom-reference sample reports missing, not fake, ALT AD.""" + sample_files, reference = build_two_samples( + tmp_path, use_accel, compress_reference_blocks=True) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False) + variant_ad = [entry for entry in read_merged_ad(merged) + if entry[0] == VARIANT_POS] + assert len(variant_ad) == 1 + _, ad_a, ad_b = variant_ad[0] + assert ad_a == (REF_DEPTH, ALT_DEPTH) + assert ad_b == (None, None) + + +def test_reference_blocks_survive_a_chunked_merge(tmp_path, use_accel): + """Blocks pass through the chunked named-pipe merge as well.""" + sample_files, reference = build_two_samples( + tmp_path, use_accel, compress_reference_blocks=True) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False, cpus=2) + spans = read_merged_spans(merged) + assert expanded_positions(spans) == list(range(20, 33)) + variant = [span for span in spans if span[0] == VARIANT_POS][0] + assert variant[3:5] == ((0, 1), (0, 0)) + + +def test_reference_blocks_survive_remove_invariant(tmp_path, use_accel): + """The invariant filter sees block records and keeps only the variant.""" + sample_files, reference = build_two_samples( + tmp_path, use_accel, compress_reference_blocks=True) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=True) + spans = read_merged_spans(merged) + assert [(start, end) for start, end, *_ in spans] == [(VARIANT_POS, + VARIANT_POS)] + + +def test_merged_output_has_metadata_line(tmp_path, use_accel): + """The merged consensus keeps the ##ChoCallate metadata line.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, remove_invariant=False) + with pysam.VariantFile(str(merged), index_filename=None) as variant_file: + header = str(variant_file.header) + assert '##ChoCallate=' in header + assert 'Rule=two-stage' in header + assert header.count('##ChoCallate=') == 1 + assert '##ChoCallateSamples=' not in header + + +def test_merged_header_aggregates_degraded_callers(tmp_path, use_accel): + """Merged provenance keeps every sample's failed callers.""" + sample_files, reference = build_degraded_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False) + with pysam.VariantFile(str(merged), index_filename=None) as variant_file: + lines = str(variant_file.header).splitlines() + cho = [line for line in lines if line.startswith('##ChoCallate=<')] + per_sample = [line for line in lines + if line.startswith('##ChoCallateSamples=<')] + assert len(cho) == 1 + assert 'Callers="bcftools,gatk,freebayes"' in cho[0] + assert cho[0].endswith('CallersFailed="freebayes,gatk">') + assert per_sample == [ + '##ChoCallateSamples=', + '##ChoCallateSamples=', + ] + + +def test_merged_header_records_dropped_samples(tmp_path, use_accel): + """Dropped sample ids are appended to the merged ##ChoCallate line.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False, + samples_dropped='sampleC,sampleD') + with pysam.VariantFile(str(merged), index_filename=None) as variant_file: + header = str(variant_file.header) + assert 'SamplesDropped="sampleC,sampleD"' in header + assert header.count('##ChoCallate=') == 1 + + +def test_single_file_merge_records_dropped_samples(tmp_path, use_accel): + """The single-input merge path patches the header as well.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + one_sample = {'sampleA': sample_files['sampleA']} + merged = merge_chain(tmp_path, one_sample, reference, + remove_invariant=False, + samples_dropped='sampleB') + with pysam.VariantFile(str(merged), index_filename=None) as variant_file: + header = str(variant_file.header) + assert 'SamplesDropped="sampleB"' in header + assert header.count('##ChoCallate=') == 1 + + +def test_dropped_samples_patches_only_chocallate_line(tmp_path, use_accel): + """Other structured header lines must not gain SamplesDropped.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + with pysam.VariantFile( + str(sample_files['sampleA']), index_filename=None) as source: + source_header = str(source.header).splitlines() + merged = merge_chain(tmp_path, sample_files, reference, + remove_invariant=False, + samples_dropped='sampleC,sampleD') + with pysam.VariantFile(str(merged), index_filename=None) as variant_file: + header = str(variant_file.header) + merged_header = header.splitlines() + cho_callate = [line for line in merged_header + if line.startswith('##ChoCallate=<')] + assert len(cho_callate) == 1 + assert cho_callate[0].endswith(',SamplesDropped="sampleC,sampleD">') + for prefix in ('##INFO=<', '##FORMAT=<', '##ALT=<', '##contig=<'): + expected = [line for line in source_header + if line.startswith(prefix)] + actual = [line for line in merged_header if line.startswith(prefix)] + assert expected, f'test fixture lacks {prefix} lines' + assert actual == expected + + +def test_merged_vcf_output_is_readable(tmp_path, use_accel): + """The production merge script writes a valid VCF.gz as well as BCF.""" + sample_files, reference = build_two_samples(tmp_path, use_accel) + merged = merge_chain(tmp_path, sample_files, reference, remove_invariant=False, + output_format='vcf') + with pysam.VariantFile(str(merged), index_filename=None) as variant_file: + records = list(variant_file) + header = str(variant_file.header) + assert '##ChoCallate=' in header + assert [record.pos for record in records] == list(range(20, 33)) + + +def test_drop_refonly_duplicates_does_not_cross_contigs(tmp_path): + """Equal positions on different contigs are separate loci. + + The last record of chr1 (``chr1:5 <*>``) and the first record of chr2 + (``chr2:5 A>T``) share POS 5 and must both reach the output. + """ + header = pysam.VariantHeader() + header.add_line('##contig=') + header.add_line('##contig=') + header.add_line('##ALT=') + header.add_line('##INFO=') + header.add_line('##FORMAT=') + header.add_sample('sample1') + source = tmp_path / 'in.bcf' + with pysam.VariantFile(str(source), 'wb', header=header) as out: + record = out.new_record(contig='chr1', start=4, stop=5, + alleles=('A', '<*>')) + record.samples[0]['GT'] = (0, 0) + out.write(record) + record = out.new_record(contig='chr2', start=4, stop=5, + alleles=('A', 'T')) + record.samples[0]['GT'] = (0, 1) + out.write(record) + + result = subprocess.run( + [sys.executable, str(BIN_DIR / 'drop_refonly_duplicates.py')], + check=True, input=source.read_bytes(), capture_output=True) + out_path = tmp_path / 'out.bcf' + out_path.write_bytes(result.stdout) + with pysam.VariantFile(str(out_path), index_filename=None) as variant_file: + assert [(record.chrom, record.pos, tuple(record.alts)) + for record in variant_file] == [ + ('chr1', 5, ('<*>',)), ('chr2', 5, ('T',))] + + +def test_failed_final_merge_does_not_hang_on_fifos(tmp_path): + """A failing final merge must not leave chunk writers blocked on FIFOs.""" + inputs = [ + write_caller_bcf(tmp_path / f'sample{index}.bcf', + f'sample{index}', []) + for index in range(2)] + file_list = tmp_path / 'merge_list.txt' + file_list.write_text('\n'.join(str(path) for path in inputs) + '\n') + env = os.environ.copy() + env['TMPDIR'] = str(tmp_path) + + started = time.monotonic() + result = subprocess.run( + [sys.executable, str(BIN_DIR / 'merge_bcf_tree.py'), + '--file-list', str(file_list), + '--gvcf', str(tmp_path / 'missing.fa'), + '--cpus', '2'], + capture_output=True, text=True, timeout=10, + cwd=str(REPO_ROOT), env=env) + elapsed = time.monotonic() - started + + assert result.returncode != 0 + assert elapsed < 5 + + +def test_drop_refonly_duplicates_keeps_monomorphic_positions(tmp_path): + """The dedup helper drops '.'/'<*>' duplicates but keeps monomorphic sites.""" + header = pysam.VariantHeader() + header.add_line('##contig=') + header.add_line('##ALT=') + header.add_line('##INFO=') + header.add_line('##FORMAT=') + header.add_sample('sample1') + source = tmp_path / 'in.bcf' + with pysam.VariantFile(str(source), 'wb', header=header) as out: + # monomorphic positions with both reference-only symbols: kept + for start, alt in ((9, '.'), (14, '<*>')): + record = out.new_record(contig='chr1', start=start, stop=start + 1, + alleles=('A', alt)) + record.samples[0]['GT'] = (0, 0) + out.write(record) + # variant position with the duplicated reference-only record: only the + # variant record survives + for start, alleles, gt in ((29, ('C', '<*>'), (0, 0)), + (29, ('C', 'T'), (0, 1))): + record = out.new_record(contig='chr1', start=start, stop=start + 1, + alleles=alleles) + record.samples[0]['GT'] = gt + out.write(record) + + result = subprocess.run([sys.executable, str(BIN_DIR / 'drop_refonly_duplicates.py')], + check=True, input=source.read_bytes(), + capture_output=True) + out_path = tmp_path / 'out.bcf' + out_path.write_bytes(result.stdout) + with pysam.VariantFile(str(out_path), index_filename=None) as variant_file: + assert [(record.pos, tuple(record.alts)) for record in variant_file] == \ + [(10, ('.',)), (15, ('<*>',)), (30, ('T',))] diff --git a/tests/python/test_postprocess_bam.py b/tests/python/test_postprocess_bam.py new file mode 100644 index 0000000..b8cf2f7 --- /dev/null +++ b/tests/python/test_postprocess_bam.py @@ -0,0 +1,353 @@ +"""Tests for the shared BAM post-processing helper. + +The helper normalises aligner output before calling: it sorts, optionally +marks or removes duplicates, filters MAPQ after duplicate marking, optionally +left-aligns indels, and writes the BAM index exactly once. +""" + +import os +import shutil +import subprocess + +import pysam +import pytest +from conftest import BIN_DIR # noqa: I100,I202 (tests run from this directory) + +SAMTOOLS = shutil.which('samtools') +pytestmark = pytest.mark.skipif(SAMTOOLS is None, reason='samtools is not available') + + +def write_paired_bam(path, m5=None): + """Two duplicate pairs, one low-MAPQ pair and one unique pair.""" + sq = {'SN': 'chr1', 'LN': 1000} + if m5: + sq['M5'] = m5 + header = { + 'HD': {'VN': '1.6', 'SO': 'coordinate'}, + 'SQ': [sq], + 'RG': [{'ID': 's1', 'SM': 's1'}], + } + reads = [] + + def add(name, start, flag, mapq, mate_start, template_length): + read = pysam.AlignedSegment() + read.query_name = name + read.query_sequence = 'ACGTACGT' + read.flag = flag + read.reference_id = 0 + read.reference_start = start + read.mapping_quality = mapq + read.cigar = ((0, 8),) + read.query_qualities = pysam.qualitystring_to_array('IIIIIIII') + read.next_reference_id = 0 + read.next_reference_start = mate_start + read.template_length = template_length + read.set_tag('RG', 's1') + reads.append(read) + + # duplicate pair 1 + add('dup1', 10, 99, 60, 30, 28) + add('dup1', 30, 147, 60, 10, -28) + # duplicate pair 2 + add('dup2', 10, 99, 60, 30, 28) + add('dup2', 30, 147, 60, 10, -28) + # low MAPQ pair, kept by markdup and removed by the later MAPQ filter + add('low', 100, 99, 5, 120, 28) + add('low', 120, 147, 5, 100, -28) + # unique pair + add('uniq', 200, 99, 60, 220, 28) + add('uniq', 220, 147, 60, 200, -28) + + reads.sort(key=lambda read: (read.reference_start, read.query_name, read.flag)) + with pysam.AlignmentFile(str(path), 'wb', header=header) as out: + for read in reads: + out.write(read) + pysam.index(str(path)) + return path + + +def run_helper(tmp_path, *args, env=None, threads=1, memory_mb=None, + reserve_mb=None): + output = tmp_path / 'mapping.bam' + command = [ + str(BIN_DIR / 'postprocess_bam.sh'), + '--input', str(tmp_path / 'in.bam'), + '--output', str(output), + '--mapq', '10', + '--threads', str(threads), + ] + if memory_mb is not None: + command += ['--memory-mb', str(memory_mb)] + if reserve_mb is not None: + command += ['--reserve-mb', str(reserve_mb)] + command += list(args) + subprocess.run(command, check=True, cwd=str(tmp_path), env=env) + return output + + +def write_stub_samtools(tmp_path): + """Replace samtools with a wrapper that logs and forwards arguments.""" + real_samtools = shutil.which('samtools') + stub_dir = tmp_path / 'stub_samtools_bin' + stub_dir.mkdir() + log = tmp_path / 'samtools_args.txt' + stub = stub_dir / 'samtools' + stub.write_text( + '#!/bin/bash\n' + 'printf "%s\\n" "$*" >> "$SAMTOOLS_ARGS_LOG"\n' + 'exec "$REAL_SAMTOOLS" "$@"\n' + ) + stub.chmod(0o755) + env = os.environ.copy() + env['PATH'] = str(stub_dir) + os.pathsep + env['PATH'] + env['SAMTOOLS_ARGS_LOG'] = str(log) + env['REAL_SAMTOOLS'] = real_samtools + return env, log + + +def write_stub_gatk(tmp_path): + """Replace GATK with a stub that logs its arguments and copies the BAM.""" + stub_dir = tmp_path / 'stub_bin' + stub_dir.mkdir() + log = tmp_path / 'gatk_args.txt' + stub = stub_dir / 'gatk' + stub.write_text( + '#!/bin/bash\n' + 'printf "%s\\n" "$@" > "$GATK_ARGS_LOG"\n' + 'input=""\n' + 'output=""\n' + 'while [ "$#" -gt 0 ]; do\n' + ' case "$1" in\n' + ' -I) input="$2"; shift 2 ;;\n' + ' -O) output="$2"; shift 2 ;;\n' + ' *) shift ;;\n' + ' esac\n' + 'done\n' + 'cp "$input" "$output"\n' + ) + stub.chmod(0o755) + return stub_dir, log + + +def read_names(path): + with pysam.AlignmentFile(str(path), 'rb') as bam: + return [read.query_name for read in bam] + + +def test_postprocess_removes_duplicates_filters_mapq_and_indexes(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + # --rmdup is the backwards-compatible alias for --rmdup-mode remove. + output = run_helper(tmp_path, '--rmdup') + assert sorted(read_names(output)) == ['dup1', 'dup1', 'uniq', 'uniq'] + assert (tmp_path / 'mapping.bam.csi').is_file() + subprocess.run([SAMTOOLS, 'quickcheck', str(output)], check=True) + with pysam.AlignmentFile(str(output), 'rb') as bam: + assert bam.header.to_dict()['HD']['SO'] == 'coordinate' + + +def test_postprocess_marks_duplicates_without_removing_them(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + output = run_helper(tmp_path, '--rmdup-mode', 'mark') + assert sorted(read_names(output)) == [ + 'dup1', 'dup1', 'dup2', 'dup2', 'uniq', 'uniq'] + assert (tmp_path / 'mapping.bam.csi').is_file() + subprocess.run([SAMTOOLS, 'quickcheck', str(output)], check=True) + with pysam.AlignmentFile(str(output), 'rb') as bam: + flags = {} + for read in bam: + flags.setdefault(read.query_name, set()).add(read.is_duplicate) + assert flags['uniq'] == {False} + # Exactly one of the two identical pairs is flagged, both mates included. + assert len([name for name in ('dup1', 'dup2') + if flags[name] == {True}]) == 1 + + +@pytest.mark.parametrize('extra_args', [[], ['--rmdup-mode', 'none']], + ids=['default', 'none']) +def test_postprocess_without_rmdup_keeps_duplicates(tmp_path, extra_args): + write_paired_bam(tmp_path / 'in.bam') + output = run_helper(tmp_path, *extra_args) + assert sorted(read_names(output)) == [ + 'dup1', 'dup1', 'dup2', 'dup2', 'uniq', 'uniq'] + assert (tmp_path / 'mapping.bam.csi').is_file() + subprocess.run([SAMTOOLS, 'quickcheck', str(output)], check=True) + with pysam.AlignmentFile(str(output), 'rb') as bam: + assert not any(read.is_duplicate for read in bam) + + +def write_fai(path, length=1000): + path.write_text('chr1\t{}\t6\t60\t61\n'.format(length)) + return path + + +def test_postprocess_accepts_a_matching_reference_index(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + fai = write_fai(tmp_path / 'ref.fa.fai') + + output = run_helper(tmp_path, '--fai', str(fai)) + + assert output.is_file() + + +def test_postprocess_rejects_a_bam_from_another_reference(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + fai = write_fai(tmp_path / 'ref.fa.fai', length=999) + + with pytest.raises(subprocess.CalledProcessError): + run_helper(tmp_path, '--fai', str(fai)) + + +def test_postprocess_rejects_mismatched_m5_checksums(tmp_path): + write_paired_bam(tmp_path / 'in.bam', m5='aaaa') + fai = write_fai(tmp_path / 'ref.fa.fai') + dictionary = tmp_path / 'ref.dict' + dictionary.write_text( + '@HD\tVN:1.6\n@SQ\tSN:chr1\tLN:1000\tM5:bbbb\n') + + with pytest.raises(subprocess.CalledProcessError): + run_helper(tmp_path, '--fai', str(fai), + '--dict', str(dictionary)) + + +def test_postprocess_rejects_unknown_rmdup_mode(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + with pytest.raises(subprocess.CalledProcessError): + run_helper(tmp_path, '--rmdup-mode', 'dedup') + + +@pytest.mark.parametrize( + 'extra_args,expected', + [([], '292M'), + (['--rmdup-mode', 'mark'], '512M'), + (['--rmdup-mode', 'remove'], '512M')], + ids=['sort', 'mark', 'remove']) +def test_sort_memory_comes_from_task_memory(tmp_path, extra_args, expected): + write_paired_bam(tmp_path / 'in.bam') + env, log = write_stub_samtools(tmp_path) + + run_helper( + tmp_path, *extra_args, threads=8, memory_mb=4096, env=env) + + sort_calls = [ + line for line in log.read_text().splitlines() + if line.startswith('sort ') + ] + assert sort_calls + assert all(f'-m {expected}' in line for line in sort_calls) + + +@pytest.mark.parametrize( + 'extra_args,expected', + [([], '146M'), + (['--rmdup-mode', 'mark'], '256M'), + (['--rmdup-mode', 'remove'], '256M')], + ids=['sort', 'mark', 'remove']) +def test_sort_memory_excludes_the_aligner_reserve(tmp_path, extra_args, + expected): + write_paired_bam(tmp_path / 'in.bam') + env, log = write_stub_samtools(tmp_path) + + run_helper( + tmp_path, *extra_args, threads=8, memory_mb=4096, + reserve_mb=2048, env=env) + + sort_calls = [ + line for line in log.read_text().splitlines() + if line.startswith('sort ') + ] + assert sort_calls + assert all(f'-m {expected}' in line for line in sort_calls) + + +def test_postprocess_rejects_reserve_not_below_task_memory(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + with pytest.raises(subprocess.CalledProcessError): + run_helper(tmp_path, threads=1, memory_mb=1024, reserve_mb=1024) + + +@pytest.mark.parametrize( + 'extra_args,stages', + [([], 2), (['--rmdup-mode', 'mark'], 5), + (['--rmdup-mode', 'remove'], 5)], + ids=['sort', 'mark', 'remove']) +def test_threads_are_split_across_concurrent_stages( + tmp_path, extra_args, stages): + write_paired_bam(tmp_path / 'in.bam') + env, log = write_stub_samtools(tmp_path) + + run_helper( + tmp_path, *extra_args, threads=8, memory_mb=8192, env=env) + + subcommands = {'collate', 'fixmate', 'sort', 'markdup', 'view'} + calls = [ + line.split() for line in log.read_text().splitlines() + if line.split() and line.split()[0] in subcommands + ] + assert len(calls) == stages + allocations = [int(call[call.index('--threads') + 1]) for call in calls] + assert sum(allocations) == 8 + # Sort is the only stage that receives more than one thread. + sort_call = next(call for call in calls if call[0] == 'sort') + assert int(sort_call[sort_call.index('--threads') + 1]) == 8 - stages + 1 + assert sorted(allocations) == [1] * (stages - 1) + [8 - stages + 1] + + +def test_markdup_is_streamed_into_mapq_filter(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + env, log = write_stub_samtools(tmp_path) + + run_helper(tmp_path, '--rmdup-mode', 'mark', env=env) + + text = log.read_text() + assert 'dedup.bam' not in text + calls = [line.split() for line in text.splitlines()] + markdup = next(call for call in calls if call[0] == 'markdup') + view = next(call for call in calls if call[0] == 'view') + # markdup reads and writes the stream; view filters that stream. + assert markdup[-2:] == ['-', '-'] + assert view[-1] == '-' + + +def test_filter_runs_before_sort_without_rmdup(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + env, log = write_stub_samtools(tmp_path) + + run_helper(tmp_path, env=env) + + text = log.read_text() + assert 'sorted.bam' not in text + calls = [line.split() for line in text.splitlines()] + view = next(call for call in calls if call[0] == 'view') + sort = next(call for call in calls if call[0] == 'sort') + # The filter emits uncompressed BAM and sort consumes it from stdin. + assert '-u' in view + assert sort[-1] == '-' + assert '-o' in sort + + +def test_left_align_receives_task_java_options(tmp_path): + write_paired_bam(tmp_path / 'in.bam') + stub_dir, log = write_stub_gatk(tmp_path) + reference = tmp_path / 'ref.fa' + reference.write_text('>chr1\nACGT\n') + dictionary = tmp_path / 'ref.dict' + dictionary.write_text('') + + env = os.environ.copy() + env['PATH'] = str(stub_dir) + os.pathsep + env['PATH'] + env['GATK_ARGS_LOG'] = str(log) + java_options = '-Djava.io.tmpdir=/tmp/scratch -Xmx1024m' + + run_helper( + tmp_path, + '--left-align', + '--reference', str(reference), + '--dict', str(dictionary), + '--java-options', java_options, + env=env, + ) + + arguments = log.read_text().splitlines() + assert arguments[0] == 'LeftAlignIndels' + option_index = arguments.index('--java-options') + assert arguments[option_index + 1] == java_options diff --git a/tests/python/test_reproducibility.py b/tests/python/test_reproducibility.py new file mode 100644 index 0000000..d7d3635 --- /dev/null +++ b/tests/python/test_reproducibility.py @@ -0,0 +1,22 @@ +"""Reproducibility guards for the environment specification.""" + +from conftest import ( # noqa: I100,I202 (tests run from this directory) + REPO_ROOT, +) + + +def test_environment_pins_exact_dependency_versions(): + environment = (REPO_ROOT / 'environment.yaml').read_text() + dependencies = environment.split('dependencies:', 1)[1] + pins = [ + line.strip()[2:] + for line in dependencies.splitlines() + if line.strip().startswith('- ') + ] + assert pins + for pin in pins: + assert '>=' not in pin + name, version, *build = pin.split('=') + assert name + assert version[0].isdigit() + assert len(build) <= 1 diff --git a/tests/python/test_resources.py b/tests/python/test_resources.py new file mode 100644 index 0000000..38498df --- /dev/null +++ b/tests/python/test_resources.py @@ -0,0 +1,199 @@ +"""Tests for task labels, memory escalation and transient-failure retries. + +A process that exits with status 137 (SIGKILL, typically the scheduler +out-of-memory killer) must be retried with more memory. The fixture records +the task memory of the successful attempt, so the test also proves that the +retry escalated the request. +""" + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +BASE_CONFIG = PROJECT_ROOT / 'conf' / 'base.config' +FIXTURE = Path(__file__).parent / 'fixtures' / 'retry_flaky' / 'main.nf' +PRECEDENCE_FIXTURE = ( + Path(__file__).parent / 'fixtures' / 'resource_precedence' / 'main.nf' +) +CALLER_FIXTURE = ( + Path(__file__).parent / 'fixtures' / 'caller_strategy' / 'main.nf' +) +NEXTFLOW = shutil.which('nextflow') + +pytestmark = pytest.mark.skipif( + NEXTFLOW is None, + reason='nextflow is not installed', +) + + +def _merge(dst, src): + for key, value in src.items(): + if isinstance(value, dict) and isinstance(dst.get(key), dict): + _merge(dst[key], value) + else: + dst[key] = value + return dst + + +def _params(tmp_path, overrides=None): + params = { + 'outdir': str(tmp_path / 'results'), + 'resources': { + 'max_retries': 3, + 'limits': { + 'cpus': 64, + 'memory_mb': 262144, + 'time': '7d', + }, + 'low': {'cpus': 1, 'memory_mb': 256, 'time': '5m'}, + 'medium': {'cpus': 1, 'memory_mb': 256, 'time': '5m'}, + 'high': {'cpus': 1, 'memory_mb': 256, 'time': '5m'}, + }, + } + _merge(params, overrides or {}) + path = tmp_path / 'params.json' + path.write_text(json.dumps(params)) + return path + + +def _run_fixture(fixture, params, tmp_path): + return subprocess.run( + [ + NEXTFLOW, 'run', str(fixture), + '-c', str(BASE_CONFIG), + '-params-file', str(params), + '-ansi-log', 'false', + ], + cwd=tmp_path, + env=os.environ.copy(), + capture_output=True, + text=True, + timeout=180, + ) + + +def test_exit_137_is_retried_with_escalated_memory(tmp_path): + result = _run_fixture(FIXTURE, _params(tmp_path), tmp_path) + + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert 'error exit status (137)' in output + assert 'retried' in output + # The label requests 256 MiB and 5 minutes; attempt 2 doubles both. + assert (tmp_path / 'results' / 'memory.txt').read_text().strip() == '512' + assert (tmp_path / 'results' / 'time.txt').read_text().strip() == '10' + + +def test_retry_escalation_is_capped_by_resource_limits(tmp_path): + params = _params( + tmp_path, + { + 'resources': { + 'limits': { + 'cpus': 64, + 'memory_mb': 300, + 'time': '7m', + }, + }, + }, + ) + result = _run_fixture(FIXTURE, params, tmp_path) + + output = result.stdout + result.stderr + assert result.returncode == 0, output + results = tmp_path / 'results' + assert (results / 'memory.txt').read_text().strip() == '300' + assert (results / 'time.txt').read_text().strip() == '7' + + +def test_cpu_is_capped_by_resource_limits(tmp_path): + params = _params( + tmp_path, + { + 'mapping': {'cpu': 99}, + 'resources': { + 'limits': { + 'cpus': 4, + 'memory_mb': 256, + 'time': '7d', + }, + }, + }, + ) + result = _run_fixture(PRECEDENCE_FIXTURE, params, tmp_path) + + output = result.stdout + result.stderr + assert result.returncode == 0, output + resources = (tmp_path / 'results' / 'resources.txt').read_text() + assert 'cpus=4' in resources + + +def test_stage_cpu_overrides_label_and_task_defaults(tmp_path): + params = _params( + tmp_path, + { + 'mapping': {'cpu': 3}, + 'resources': {'medium': {'cpus': 6}}, + }, + ) + result = _run_fixture(PRECEDENCE_FIXTURE, params, tmp_path) + + output = result.stdout + result.stderr + assert result.returncode == 0, output + resources = (tmp_path / 'results' / 'resources.txt').read_text() + assert 'cpus=3' in resources + assert 'memory=256' in resources + + +def test_caller_failure_is_ignored_when_enabled(tmp_path): + params = _params( + tmp_path, + { + 'attempt_log': str(tmp_path / 'attempts.txt'), + 'calling': {'allow_caller_failure': True}, + }, + ) + result = _run_fixture(CALLER_FIXTURE, params, tmp_path) + + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert (tmp_path / 'attempts.txt').read_text().split() == ['1'] + + +@pytest.mark.parametrize('max_retries', [3, 1]) +def test_deterministic_caller_failure_is_not_retried(tmp_path, max_retries): + params = _params( + tmp_path, + { + 'attempt_log': str(tmp_path / 'attempts.txt'), + 'calling': {'allow_caller_failure': False}, + 'resources': {'max_retries': max_retries}, + }, + ) + result = _run_fixture(CALLER_FIXTURE, params, tmp_path) + + assert result.returncode != 0 + attempts = (tmp_path / 'attempts.txt').read_text().split() + assert attempts == ['1'] + + +def test_transient_caller_failure_is_retried(tmp_path): + params = _params( + tmp_path, + { + 'attempt_log': str(tmp_path / 'attempts.txt'), + 'caller_exit_status': 137, + 'calling': {'allow_caller_failure': False}, + 'resources': {'max_retries': 2}, + }, + ) + result = _run_fixture(CALLER_FIXTURE, params, tmp_path) + + assert result.returncode != 0 + attempts = (tmp_path / 'attempts.txt').read_text().split() + assert attempts == ['1', '2', '3'] diff --git a/tests/python/test_resume_cache.py b/tests/python/test_resume_cache.py new file mode 100644 index 0000000..fe3811b --- /dev/null +++ b/tests/python/test_resume_cache.py @@ -0,0 +1,119 @@ +"""Acceptance test: adding -resume must not re-run completed tasks.""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +NEXTFLOW = shutil.which('nextflow') +BCFTOOLS = shutil.which('bcftools') +BOWTIE2_BUILD = shutil.which('bowtie2-build') +TEST_RUN = PROJECT_ROOT / 'test_run' +REFERENCE = TEST_RUN / 'arth_chr1.fasta.gz' +READ1 = TEST_RUN / 'test_reads_R1.fq.gz' +READ2 = TEST_RUN / 'test_reads_R2.fq.gz' + +pytestmark = pytest.mark.skipif( + not all([NEXTFLOW, BCFTOOLS, BOWTIE2_BUILD, REFERENCE.is_file(), + READ1.is_file(), READ2.is_file()]), + reason='nextflow, bcftools, bowtie2 or the test dataset is unavailable', +) + + +def _reference_index(tmp_path): + index_dir = tmp_path / 'reference' + index_dir.mkdir() + subprocess.run( + [BOWTIE2_BUILD, '--quiet', '--threads', '2', str(REFERENCE), + str(index_dir / REFERENCE.name)], + check=True, capture_output=True, text=True, + ) + return index_dir + + +def _params_file(tmp_path, output_type, index_dir): + params = { + 'input': { + 'samples_tsv': str(tmp_path / 'samples.tsv'), + 'reference_genome': str(REFERENCE), + 'reference_index_dir': str(index_dir), + 'reference_index_prefix': REFERENCE.name, + 'format': 'fastq', + 'reads_type': 'pe', + }, + 'output': { + 'directory': str(tmp_path / 'results'), + 'type': output_type, + 'format': 'bcf', + 'remove_invariant': True, + }, + 'mapping': {'mapper': 'bowtie2'}, + 'left_align_indels': {'enabled': False}, + 'coverage': {'min_coverage': 1}, + 'calling': {'callers': 'bcftools'}, + 'consensus': {'threshold': 1}, + } + path = tmp_path / 'params.json' + path.write_text(json.dumps(params)) + return path + + +def _run(tmp_path, params_file, resume): + # tests/nextflow.config disables nf-boost cleanup: the plugin deletes work + # files once no downstream process needs them, so cleanup and -resume + # cannot be combined. The pipeline exposes the observer as + # boost.cleanup, but this test isolates cache-key stability from cleanup. + command = [ + NEXTFLOW, '-log', str(tmp_path / 'nextflow.log'), + 'run', str(PROJECT_ROOT / 'main.nf'), + '-c', str(PROJECT_ROOT / 'tests' / 'nextflow.config'), + '-params-file', str(params_file), + '-work-dir', str(tmp_path / 'work'), + '-ansi-log', 'false', + ] + if resume: + command.append('-resume') + return subprocess.run(command, cwd=str(tmp_path), text=True, + capture_output=True, timeout=1200) + + +def _trace_rows(trace): + lines = trace.read_text().splitlines() + header = lines[0].split('\t') + return [dict(zip(header, line.split('\t'))) for line in lines[1:]] + + +@pytest.mark.parametrize('output_type', ['single', 'sample']) +def test_resume_keeps_all_tasks_cached(tmp_path, output_type): + samples = tmp_path / 'samples.tsv' + samples.write_text(f'sample1\t{READ1}\t{READ2}\n') + params_file = _params_file(tmp_path, output_type, + _reference_index(tmp_path)) + + first = _run(tmp_path, params_file, resume=False) + assert first.returncode == 0, first.stdout + first.stderr + + second = _run(tmp_path, params_file, resume=True) + assert second.returncode == 0, second.stdout + second.stderr + + rows = _trace_rows(tmp_path / 'ChoCallate_output' / 'trace.txt') + assert rows + statuses = {row['name']: row['status'] for row in rows} + assert all(row['status'] == 'CACHED' for row in rows), statuses + final_step = ( + 'MERGE_OUTPUTS' if output_type == 'single' else 'EXPORT_OUTPUT' + ) + assert any(row['name'].startswith(final_step) for row in rows) + + if output_type == 'single': + output = tmp_path / 'results' / 'consensus.bcf' + else: + output = tmp_path / 'results' / 'per_sample' / 'sample1.bcf' + header = subprocess.run( + [BCFTOOLS, 'view', '-h', str(output)], + check=True, capture_output=True, text=True, + ).stdout + assert '##ChoCallate=<' in header diff --git a/tests/python/test_stage_cleanup.py b/tests/python/test_stage_cleanup.py new file mode 100644 index 0000000..7f8f306 --- /dev/null +++ b/tests/python/test_stage_cleanup.py @@ -0,0 +1,88 @@ +"""Tests for the per-task cleanup helper. + +The helper must remove staged files only after a successful task and must +never delete a TMPDIR that was not created inside the task work directory. +""" + +import os +import subprocess + +from conftest import BIN_DIR # noqa: I100,I202 (tests run from this directory) + +CLEANUP = BIN_DIR / 'stage_cleanup.sh' + + +def _workdir(tmp_path): + work = tmp_path / 'work' + (work / 'tmp').mkdir(parents=True) + (work / 'tmp' / 'payload.txt').write_text('payload') + task_tmp = work / 'task_tmp' + task_tmp.mkdir() + (task_tmp / 'scratch.txt').write_text('scratch') + return work, task_tmp + + +def _run(work, *args, tmpdir=None): + env = os.environ.copy() + env.pop('TMPDIR', None) + if tmpdir is not None: + env['TMPDIR'] = str(tmpdir) + return subprocess.run( + [str(CLEANUP), *args], + cwd=work, + env=env, + capture_output=True, + text=True, + ) + + +def test_failure_preserves_staged_and_temporary_files(tmp_path): + work, task_tmp = _workdir(tmp_path) + + result = _run(work, '137', tmpdir=task_tmp) + + assert result.returncode == 0 + assert (work / 'tmp' / 'payload.txt').is_file() + assert (task_tmp / 'scratch.txt').is_file() + + +def test_success_removes_staged_and_temporary_files(tmp_path): + work, task_tmp = _workdir(tmp_path) + + result = _run(work, '0', tmpdir=task_tmp) + + assert result.returncode == 0 + assert not (work / 'tmp').exists() + assert not task_tmp.exists() + + +def test_empty_tmpdir_is_ignored(tmp_path): + work, _ = _workdir(tmp_path) + + result = _run(work, '0', tmpdir='') + + assert result.returncode == 0 + assert not (work / 'tmp').exists() + + +def test_tmpdir_outside_the_workdir_is_preserved(tmp_path): + work, _ = _workdir(tmp_path) + outside = tmp_path / 'outside' + outside.mkdir() + (outside / 'keep.txt').write_text('keep') + + result = _run(work, '0', tmpdir=outside) + + assert result.returncode == 0 + assert not (work / 'tmp').exists() + assert (outside / 'keep.txt').is_file() + + +def test_missing_status_is_treated_as_success(tmp_path): + work, task_tmp = _workdir(tmp_path) + + result = _run(work, tmpdir=task_tmp) + + assert result.returncode == 0 + assert not (work / 'tmp').exists() + assert not task_tmp.exists() diff --git a/tests/python/test_validation.py b/tests/python/test_validation.py new file mode 100644 index 0000000..7a5accb --- /dev/null +++ b/tests/python/test_validation.py @@ -0,0 +1,435 @@ +"""Early-validation tests for the Nextflow entry point. + +Every case here is expected to fail before the first process starts, so the +suite does not need mapping or calling tools. A Nextflow installation and the +pipeline plugins are required; the tests are skipped otherwise. +""" + +import gzip +import json +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +NEXTFLOW = shutil.which("nextflow") + +pytestmark = pytest.mark.skipif( + NEXTFLOW is None, + reason="nextflow is not installed", +) + + +def _merge(dst, src): + for key, value in src.items(): + if isinstance(value, dict) and isinstance(dst.get(key), dict): + _merge(dst[key], value) + else: + dst[key] = value + return dst + + +def _prepare_inputs(tmp_path, rows=("sample1\tr1.fq\tr2.fq",)): + (tmp_path / "idx").mkdir(exist_ok=True) + (tmp_path / "ref.fa").write_text(">chr1\nACGTACGTACGT\n") + (tmp_path / "r1.fq").write_text("@r1\nACGT\n+\nIIII\n") + (tmp_path / "r2.fq").write_text("@r2\nACGT\n+\nIIII\n") + (tmp_path / "sample.bam").write_text("not a real bam") + samplesheet = tmp_path / "samples.tsv" + samplesheet.write_text("\n".join(rows) + "\n") + return samplesheet + + +def _config(tmp_path, samplesheet=None, overrides=None): + config = { + "input": { + "samples_tsv": str(samplesheet or tmp_path / "samples.tsv"), + "reference_genome": str(tmp_path / "ref.fa"), + "reference_index_dir": str(tmp_path / "idx"), + }, + "output": {"directory": str(tmp_path / "out")}, + "mapping": {"cpu": 1}, + "calling": {"cpu": 1}, + "consensus": {"cpu": 1, "threshold": 1}, + } + _merge(config, overrides or {}) + params = tmp_path / "params.json" + params.write_text(json.dumps(config)) + return params + + +def _environment(): + env = os.environ.copy() + if "NXF_OFFLINE" not in env: + plugins = Path.home() / ".nextflow" / "plugins" + has_plugins = ( + plugins.is_dir() + and any(plugins.glob("nf-schema-*")) + and any(plugins.glob("nf-boost-*")) + ) + if has_plugins: + env["NXF_OFFLINE"] = "true" + return env + + +def _run(tmp_path, params=None, ulimit=None): + command = [NEXTFLOW, "run", str(PROJECT_ROOT / "main.nf")] + if params is not None: + command += ["-params-file", str(params)] + command += ["-ansi-log", "false"] + if ulimit is not None: + shell_command = "ulimit -n {}; exec {}".format( + ulimit, " ".join(shlex.quote(part) for part in command) + ) + command = ["bash", "-c", shell_command] + return subprocess.run( + command, + cwd=tmp_path, + env=_environment(), + capture_output=True, + text=True, + timeout=180, + ) + + +def _output(result): + return result.stdout + result.stderr + + +def test_pipeline_defaults_for_depth_and_mapq(tmp_path): + result = subprocess.run( + [NEXTFLOW, "config", str(PROJECT_ROOT / "main.nf")], + cwd=tmp_path, + env=_environment(), + capture_output=True, + text=True, + timeout=60, + ) + + assert result.returncode == 0 + output = _output(result) + assert "min_map_qual = 10" in output + assert "min_coverage = 5" in output + assert "compress_reference_blocks = true" in output + assert "remove_invariant = false" in output + assert "mode = 'mark'" in output + + +def test_missing_required_parameter_is_named(tmp_path): + _prepare_inputs(tmp_path) + params = _config(tmp_path) + config = json.loads(params.read_text()) + del config["input"]["samples_tsv"] + params.write_text(json.dumps(config)) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "Missing required parameter(s): samples_tsv" in output + assert "--input" in output + + +def test_enum_reports_key_and_accepted_values(tmp_path): + _prepare_inputs(tmp_path) + params = _config(tmp_path, overrides={"input": {"format": "cram"}}) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "--input.format" in output + assert "Expected any of [fastq, bam]" in output + + +def test_removed_rmdup_enabled_is_rejected(tmp_path): + _prepare_inputs(tmp_path) + params = _config( + tmp_path, overrides={"rmdup": {"enabled": True}} + ) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + assert "--rmdup.enabled" in _output(result) + + +def test_negative_integer_is_rejected(tmp_path): + _prepare_inputs(tmp_path) + params = _config(tmp_path, overrides={"coverage": {"min_coverage": -1}}) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "--coverage.min_coverage" in output + assert "-1 is less than 1" in output + + +def test_zero_min_coverage_is_rejected(tmp_path): + _prepare_inputs(tmp_path) + params = _config(tmp_path, overrides={"coverage": {"min_coverage": 0}}) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "--coverage.min_coverage" in output + assert "0 is less than 1" in output + + +def test_unknown_nested_parameter_is_rejected(tmp_path): + _prepare_inputs(tmp_path) + params = _config(tmp_path, overrides={"input": {"samples_tvs": "typo"}}) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + assert "--input.samples_tvs" in _output(result) + + +def test_duplicate_sample_ids_are_rejected(tmp_path): + samplesheet = _prepare_inputs( + tmp_path, + rows=("sample1\tr1.fq\tr2.fq", "sample1\tr1.fq\tr2.fq"), + ) + params = _config(tmp_path, samplesheet=samplesheet) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + assert "duplicate sample id 'sample1'" in _output(result) + + +def test_unsafe_sample_id_is_rejected(tmp_path): + samplesheet = _prepare_inputs(tmp_path, rows=("bad id!\tr1.fq\tr2.fq",)) + params = _config(tmp_path, samplesheet=samplesheet) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + assert "not shell-safe" in _output(result) + + +@pytest.mark.parametrize( + "key, value", + [ + ("platform", 'ILLUMINA"'), + ("platform", "ILLUMINA SM"), + ("library", 'lib"ary'), + ("library", "lib ary"), + ("library", "lib\\ary"), + ], +) +def test_unsafe_read_group_values_are_rejected(tmp_path, key, value): + _prepare_inputs(tmp_path) + params = _config(tmp_path, overrides={"mapping": {key: value}}) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "--mapping.{}".format(key) in output + assert "does not match regular expression" in output + assert "^[A-Za-z0-9_.:-]+$" in output + + +@pytest.mark.parametrize("key", ["platform", "library"]) +def test_read_group_value_with_line_terminator_is_rejected(tmp_path, key): + _prepare_inputs(tmp_path) + params = _config( + tmp_path, overrides={"mapping": {key: "value\n"}} + ) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "mapping.{}".format(key) in output + assert "not safe for a SAM read group" in output + + +def test_wrong_samplesheet_width_is_rejected(tmp_path): + samplesheet = _prepare_inputs( + tmp_path, + rows=("sample1\tr1.fq\tr2.fq\textra.fq",), + ) + params = _config(tmp_path, samplesheet=samplesheet) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "expected 3 tab-separated columns" in output + assert "found 4" in output + + +def test_missing_read_file_is_rejected(tmp_path): + samplesheet = _prepare_inputs( + tmp_path, + rows=("sample1\tmissing.fq\tr2.fq",), + ) + params = _config(tmp_path, samplesheet=samplesheet) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "missing.fq" in output + assert "file not found" in output + + +@pytest.mark.parametrize("prefix", ["subdir/genome", "subdir\\genome"]) +def test_index_prefix_with_path_separator_is_rejected(tmp_path, prefix): + _prepare_inputs(tmp_path) + params = _config( + tmp_path, + overrides={"input": {"reference_index_prefix": prefix}}, + ) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "input.reference_index_prefix" in output + assert "must not contain a path separator" in output + assert "No bowtie2 index files found" not in output + + +def test_plain_gzip_reference_is_rejected(tmp_path): + _prepare_inputs(tmp_path) + reference = tmp_path / "ref.fa.gz" + with gzip.open(reference, "wb") as handle: + handle.write((tmp_path / "ref.fa").read_bytes()) + params = _config( + tmp_path, + overrides={"input": {"reference_genome": str(reference)}}, + ) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + assert ( + "reference is gzip-compressed but not BGZF; recompress with bgzip" + in _output(result) + ) + + +def test_header_and_comment_lines_are_skipped(tmp_path): + samplesheet = _prepare_inputs( + tmp_path, + rows=( + "# samples", + "sample_id\tr1\tr2", + "", + "sample1\tr1.fq\tr2.fq", + "sample1\tr1.fq\tr2.fq", + ), + ) + params = _config(tmp_path, samplesheet=samplesheet) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + # If the header or comment were parsed as data, this would be a width + # error instead of the duplicate-id error from the two data rows. + assert "duplicate sample id 'sample1'" in _output(result) + + +def test_header_keyword_after_first_data_line_is_data(tmp_path): + samplesheet = _prepare_inputs( + tmp_path, + rows=( + "sample1\tr1.fq\tr2.fq", + "id\tr1.fq", + ), + ) + params = _config(tmp_path, samplesheet=samplesheet) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + # Only the first data line may be a header. The second row starts with + # 'id' but must be parsed, so its missing column is a width error. + assert "expected 3 tab-separated columns" in output + assert "found 2" in output + + +def test_bam_samplesheet_uses_two_columns(tmp_path): + samplesheet = _prepare_inputs( + tmp_path, + rows=("sample1\tsample.bam", "sample1\tsample.bam"), + ) + params = _config( + tmp_path, + samplesheet=samplesheet, + overrides={"input": {"format": "bam"}}, + ) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert "duplicate sample id 'sample1'" in output + assert "expected 2 tab-separated columns" not in output + + +@pytest.mark.parametrize( + "caller, override, option", + [ + ( + "bcftools", + {"call": {"extra_args": "--ploidy 3"}}, + "--ploidy", + ), + ("freebayes", {"extra_args": "-p 1"}, "-p"), + ("freebayes", {"extra_args": "--ploidy=4"}, "--ploidy"), + ("gatk", {"extra_args": "--sample-ploidy 1"}, "--sample-ploidy"), + ("gatk", {"extra_args": "--ploidy=1"}, "--ploidy"), + ], +) +def test_non_diploid_ploidy_is_rejected(tmp_path, caller, override, option): + _prepare_inputs(tmp_path) + params = _config( + tmp_path, + overrides={ + "calling": { + "callers": caller, + caller: override, + }, + }, + ) + + result = _run(tmp_path, params) + + assert result.returncode != 0 + output = _output(result) + assert option in output + assert "ploidy is not supported yet except diploid" in output + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="uses a POSIX descriptor limit", +) +def test_merge_file_descriptor_guard(tmp_path): + rows = ["sample{}\tr1.fq\tr2.fq".format(index) for index in range(300)] + samplesheet = _prepare_inputs(tmp_path, rows=tuple(rows)) + params = _config( + tmp_path, + samplesheet=samplesheet, + overrides={"consensus": {"cpu": 1}}, + ) + + result = _run(tmp_path, params, ulimit=256) + + assert result.returncode != 0 + output = _output(result) + assert "merge step needs at least" in output + assert "soft limit is 256" in output