Bump version to 4.0.0

.dockerignore

@@ -8,3 +8,7 @@ ROADMAP.md
 *-workspace
 work
 test_run
+# accelerator build artifacts generated by bin/build_consensus_accel.py
+bin/consensus_accel*.so
+bin/consensus_accel.c
+bin/__pycache__

.github/workflows/ci.yml

@@ -0,0 +1,129 @@
+name: CI
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -el {0}
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up ChoCallate environment
+        uses: mamba-org/setup-micromamba@v3
+        with:
+          environment-file: environment.yaml
+          environment-name: ChoCallate
+          cache-environment: true
+          cache-downloads: true
+          init-shell: bash
+
+      - name: Ruff
+        run: ruff check .
+
+      - name: Nextflow lint
+        run: nextflow lint .
+
+      - name: Actionlint
+        run: actionlint
+
+  pytest:
+    name: pytest (${{ matrix.accelerator }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        accelerator: [pure, accel]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up ChoCallate environment
+        uses: mamba-org/setup-micromamba@v3
+        with:
+          environment-file: environment.yaml
+          environment-name: ChoCallate
+          cache-environment: true
+          cache-downloads: true
+          init-shell: bash
+
+      - name: Build consensus accelerator
+        if: matrix.accelerator == 'accel'
+        run: python bin/build_consensus_accel.py
+
+      - name: Verify accelerator import
+        if: matrix.accelerator == 'accel'
+        run: >-
+          python -c "import sys; sys.path.insert(0, 'bin');
+          import consensus_accel"
+
+      - name: Run pytest
+        run: pytest -q tests/python
+
+  docker:
+    name: Docker build
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          push: false
+          load: true
+          tags: chocallate:ci
+          cache-from: type=gha
+          cache-to: type=gha
+
+      - name: Verify consensus accelerator
+        run: >-
+          docker run --rm --entrypoint micromamba chocallate:ci
+          run -n ChoCallate python -c
+          "import sys; sys.path.insert(0, '/opt/chocallate/bin');
+          import consensus_accel"
+
+      - name: Verify offline plugins with arbitrary UID
+        run: >-
+          docker run --rm --network none -u 12345:12345 -w /tmp
+          --entrypoint micromamba chocallate:ci
+          run -n ChoCallate nextflow inspect /opt/chocallate/main.nf
+
+  nf-test:
+    name: nf-test
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up ChoCallate environment
+        uses: mamba-org/setup-micromamba@v3
+        with:
+          environment-file: environment.yaml
+          environment-name: ChoCallate
+          cache-environment: true
+          cache-downloads: true
+          init-shell: bash
+
+      - name: Run nf-test
+        run: nf-test test --ci

.github/workflows/docker-image.yml

@@ -14,9 +14,6 @@ jobs:
           username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v4
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 

.gitignore

@@ -9,3 +9,26 @@ test_data/*.bt2
 test_data/chocallate_test_*
 .vscode/
 ChoCallate.code-workspace
+bin/consensus_accel.c
+bin/consensus_accel*.so
+bin/.build_accel/
+test_run/ChoCallate_output*
+test_run/merge_*
+test_run/consensus_*
+test_run/bcfs
+test_run/ad_perf
+test_run/accs_list.txt
+test_run/ERR*
+test_run/*.bt2
+AGENTS.md
+test_run/samples.tsv
+test_run/arth_chr1.fasta.gz.*
+test_run/benchmark_*
+test_run/test_merge*
+test_run/test_config*
+test_run/run_test_2.sh
+test_run/include.bed
+.pytest_cache
+.nf-test/
+.nf-test.log
+.ruff_cache/

CHANGELOG.md

@@ -2,7 +2,91 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [4.0.0] - 2026-10-20
+
+### Added
+
+- `nextflow_schema.json` and the `nf-schema` plugin validate every parameter before the first task: enum values, numeric minimums, input-file existence and unknown keys abort with the offending key and the accepted values. Every parameter now has a default in `nextflow.config` and documentation in `assets/templates/config.yaml`.
+- Samplesheet validation: blank lines and comments are skipped, only the first non-comment line can be a header, sample ids must be unique and shell-safe, every row must have the exact column count for `input.format`/`input.reads_type`, and relative read paths are resolved from the samplesheet directory and must exist.
+- `calling.<caller>.min_snp_qual` and `calling.<caller>.min_indel_qual` provide per-caller, per-variant-type quality filters and are required for every configured caller.
+- `calling.allow_caller_failure` (default `false`) enables degraded mode: a failed caller is skipped with a warning, `CallersFailed` is recorded, the run continues while `consensus.threshold` is satisfiable, and samples with no surviving caller are listed in `dropped_samples.txt` (and in `SamplesDropped` in a merged output).
+- Caller `extra_args` that request non-diploid ploidy, a copy-number map or custom ploidy regions are rejected, because consensus genotypes are diploid.
+- `consensus.absent_vote` (`ref`/`abstain`), `consensus.filtered_vote` (`abstain`/`ref`) and `consensus.tie_policy` (`nocall`/`first`) configure the voting model.
+- `output.compress_reference_blocks` (default `true`) collapses runs of hom-reference positions into multi-base gVCF blocks with `INFO/END` when reference records are written, in both per-sample and merged output.
+- `output.publish_mode` (`copy` or `link`, default `copy`) copies or links results instead of moving them out of the work directory; `copy` keeps results independent of work-directory cleanup, while `link` requires the work directory to be retained.
+- `input.reference_index_prefix` sets the mapper index prefix when it differs from the reference file name; it must be a plain file name without path separators.
+- `mapping.platform` (default `ILLUMINA`) and `mapping.library` (default the sample id) set the read-group `PL` and `LB` values for mapping and BAM input; values are restricted to letters, digits, `_`, `.`, `:` and `-`.
+- `mapping.aligner_reserve_mb` reserves memory for the aligner index while `samtools sort` runs in the same task; `0` estimates the reserve from the staged index files plus 1 GiB.
+- `resources.low`, `resources.medium` and `resources.high` define per-label CPU, memory and time limits; `resources.limits` caps those values after retry escalation, and JVM heaps are derived from task memory.
+- `resources.max_retries` controls retries for transient failures (exit status 137 or 140), with memory and time multiplied by the attempt number.
+- `boost.cleanup` is now a params-file option (`"v1"`, `"v2"` or `false`) instead of a hard-coded `nextflow.config` setting.
+- `SCATTER_INTERVALS` computes the coverage BED chunks once per sample and shares them between every caller and consensus.
+- `EXPORT_OUTPUT` applies `output.format`, `output.remove_invariant` and `output.split_multiallelic` once at the end and publishes per-sample or merged files according to `output.type`; internal consensus and merge files are always BCF, including for `output.format: "vcf"`.
+- Mapping statistics are published under `mapping_stats/`: raw `samtools flagstat` plus final `samtools flagstat`, `samtools stats` and `samtools idxstats`, and `mapping_summary.tsv` with per-sample alignment, duplicate, MAPQ, insert-size and error-rate metrics.
+- `bin/postprocess_bam.sh` provides shared mapping/BAM post-processing: mate fixing, `samtools sort`, optional `samtools markdup`, MAPQ filtering, optional GATK `LeftAlignIndels` and one CSI index.
+- BAM input read groups are normalised with `samtools addreplacerg` (`ID`/`SM` set to the sample id, `PL`/`LB` from `mapping.platform`/`mapping.library`), so GATK and `bcftools reheader` see consistent read groups.
+- BAM inputs and mapping output are checked against the reference: `@SQ` names and lengths must match the `.fai`, and when the BAM header carries `M5` checksums they are compared with the sequence dictionary; mismatches abort before calling.
+- The generated `.fai` is checked for contigs longer than 2,147,483,647 bp (2^31-1), because BAM/BCF positions and GATK use 32-bit coordinates.
+- `bin/filter_bed.py` strips `track`, `browser` and `#` lines before `sort-bed` and rejects contigs absent from the reference `.fai`; consensus rejects coverage BED intervals with negative or reversed coordinates or ends past the contig.
+- `coverage.on_empty` (`fail` by default, or `skip`) controls what happens when no base of a sample reaches `coverage.min_coverage`; skipped samples are listed in `dropped_samples.txt` (and in `SamplesDropped` in a merged output).
+- `bin/merge_outputs.sh` is the production merge chain used by `MERGE_OUTPUTS` and the pytest merge tests; `bin/drop_refonly_duplicates.py` removes the redundant `<*>` record created by `bcftools merge --gvcf`.
+- `bin/merge_cho_headers.py` builds the merged `##ChoCallate=` line from the union of per-sample `Callers` and `CallersFailed`, adds `SamplesDropped` when samples were dropped, and writes `##ChoCallateSamples=` when caller sets differ or any caller failed.
+- The merge step checks the process open-file limit before the first task for `output.type: "single"` and aborts with the required `ulimit -n` value when the sample count and `consensus.cpu` would exceed it.
+- `bin/build_consensus_accel.py` builds an optional Cython accelerator for `generate_consensus.py`; the Docker image builds it automatically, and an equivalent pure-pysam implementation remains as a fallback.
+- One structured `##ChoCallate=` header line per consensus file records the version, threshold, callers, rule, tie policy, absence policy, filtered-record policy and reference-block flag.
+- The `tests/python` pytest suite covers consensus (pure and accelerated), merge, BAM post-processing, validation, resources, retries, resume/cache and cleanup; the `tests/nf` nf-test suite contains module tests and end-to-end pipeline cases on the bundled `test_run` data.
+- GitHub Actions CI runs `ruff`, `nextflow lint`, `actionlint`, pytest with and without the Cython accelerator, nf-test, and Docker image smoke tests including offline plugin resolution.
+
+### Changed
+
+- `output.remove_invariant` now defaults to `false`, so a default run keeps monomorphic/reference sites in both per-sample and merged BCF/VCF output.
+- Hom-reference positions are written as gVCF-style `ALT="<*>"` records with `INFO/END` instead of `ALT="."`; single-base records carry `AD=(reference depth, other-allele depth)`, while multi-base blocks omit `AD` and carry the block minimum in `FORMAT/DP`. `MERGE_OUTPUTS` uses `bcftools merge --gvcf`, so merged `FORMAT/AD` reports honest ALT depth for hom-reference samples.
+- `coverage.min_coverage` now defaults to `5` (was `2`), must be at least `1`, and `bam_filter.min_map_qual` now defaults to `10` (was `0`).
+- `rmdup.mode` (`none`, `mark` or `remove`; default `mark`) replaces `rmdup.enabled` and `picard MarkDuplicates`; `samtools markdup` flags duplicates by default, and callers, coverage and consensus skip duplicate-flagged reads.
+- Reference compression is detected from the BGZF header, so `ref_genome.bgzip` is no longer needed; plain gzip references are rejected while plain and BGZF-compressed FASTA are accepted.
+- `input.reference_index_dir` is required only for `format: "fastq"`, so BAM-only runs no longer need mapper indexes.
+- Mapper indexes are resolved from `input.reference_index_dir` and staged as task inputs instead of being read from that host path at task runtime, so mapping works in containers, on cloud executors and with `scratch`; an incomplete or missing index aborts before the first task.
+- Consensus inputs are staged as `tmp/callers/<caller>.bcf` and passed as `--input caller=path`, so caller identity and voting order no longer depend on positional file order when a caller fails.
+- Callers are normalised with `bcftools norm -f ref -m -any --atomize`; multi-allelic and complex records are split, low-QUAL records are kept with `FILTER=LowQual`, and the old `--min-alleles 2 --max-alleles 2` filter is no longer applied.
+- Consensus is now two-stage and caller-order independent by default: an ALT becomes a candidate once `consensus.threshold` callers support it, then the majority zygosity among its supporters decides the genotype; ties are a no-call unless `consensus.tie_policy: "first"`, and multiple supported ALT alleles produce ploidy-consistent split records.
+- Filtered caller records abstain from the vote by default (`consensus.filtered_vote: "abstain"`), no-calls no longer count as hom-reference votes, and positions with an IUPAC-ambiguous reference base or caller allele are omitted from consensus output.
+- `FORMAT/DP` counts collapsed observations (one per read name) instead of every covering read, matching the overlap-corrected coverage BED; a deletion is not an allele, so an overlapping mate's base is preferred when tallying reference `AD`.
+- `calling.bcftools.mpileup.max_depth` and `max_idepth` (default `100000`) override the bcftools default of 250; FreeBayes receives `--use-best-n-alleles` (default `4`) and `--limit-coverage` (default `100000`); GATK receives `--standard-min-confidence-threshold-for-calling min(min_snp_qual, min_indel_qual)`.
+- GATK calling intervals are padded by `calling.gatk.interval_padding` (default `150`), and each padded chunk's calls are restricted to its own unpadded BED interval before concatenation.
+- Coverage is computed with `mosdepth --no-per-base --quantize` and sorted with `sort-bed`/`bedops` instead of a per-base `samtools depth | awk` stream.
+- Mapping and BAM preparation share `bin/postprocess_bam.sh`; in mapping tasks `mapping.cpu` is split between mapper and post-processing, sort memory is derived from task memory after the aligner reserve, and each BAM is indexed once.
+- Every process now uses `process_low`, `process_medium` or `process_high`; every JVM receives an `-Xmx` derived from task memory, `Runtime.runtime.availableProcessors()` is no longer used, `maxForks` directives are gone, and caller failures are no longer ignored unconditionally.
+- All process scripts run under `bash -euo pipefail`, process-local `$TMPDIR` is used for temporary files, and cleanup removes `tmp/`, `index/` and `TMPDIR` only after a successful task.
+- The bundled `test_run` dataset is reduced to a 200-kb slice and 20,000 read pairs, so the end-to-end test takes about 3 minutes.
+- `environment.yaml` adds `mosdepth`, `cython`, `setuptools` and `c-compiler` and updates the tool stack; `htslib` provides `bgzip`/`tabix`.
+- The Docker image installs the pipeline under `/opt/chocallate`, pins the `nf-schema` and `nf-boost` plugins at build time for offline runs, and provides a world-writable `HOME`, so it works with Docker, Apptainer and Singularity under an arbitrary UID.
+- `nextflow.config` now declares defaults for every parameter plus `manifest.name`, `nextflowVersion`, `homePage` and `mainScript`.
+- `generate_consensus.py` streams one cursor per caller BCF instead of materialising all records of a coverage span and is about 20x faster on a fragmented 2.1 Mbp coverage BED through span grouping, a fast path for positions without caller records and bulk pileup accessors; an optional Cython accelerator speeds up the pileup and record writing further, with an equivalent pure-pysam fallback.
+- `merge_bcf_tree.py` handles a single input file and aborts pending FIFO readers/writers when the final merge fails.
+
+### Removed
+
+- `ref_genome.bgzip`; compression is detected automatically.
+- `rmdup.enabled`; use `rmdup.mode` instead.
+- The shared `calling.min_snp_qual` key; configure `calling.<caller>.min_snp_qual` and `calling.<caller>.min_indel_qual` instead.
+- `bin/fill_missing_ad.py`; the gVCF-style `<*>` representation makes post-merge AD filling unnecessary.
+- The standalone `tabix` and unused `bedtools` conda dependencies.
+- The `--remove_invariant` and `--split_multiallelic` options of `generate_consensus.py`; `EXPORT_OUTPUT` now applies both transformations.
+
+### Fixed
+
+- `generate_consensus.py` no longer drops the last base of every coverage BED interval, so 1-bp intervals produce output and variants at interval ends are retained.
+- Deletion handling no longer skips the deletion REF span or loses overlapping caller records: the walk advances one base at a time, caller records inside a deletion are still voted on, and reference records are not written over deleted bases.
+- ALT support is keyed by `(REF, ALT)` instead of ALT alone, so deletions of different lengths and atomised records with different REF alleles at the same position remain distinct and caller-order independent.
+- `generate_consensus.py` no longer crashes with `TypeError` on a caller record with a missing genotype (`./.`, `0/.`).
+- `MERGE_OUTPUTS` keeps monomorphic sites when `output.remove_invariant: false`, matching the documented meaning; the merged output no longer drops every reference-only record.
+- `output.type: "single"` with `output.format: "vcf"` no longer fails by passing per-sample `.vcf.gz` files to a merge stage that expects BCF; internal files are always BCF and the VCF is created only by the export stage.
+- `PREPARE_BAM` declares the BAM it actually writes (`mapping.bam`), so `input.format: "bam"` no longer fails with "Missing output file(s)".
+- `merge_bcf_tree.py` no longer fails with "Expected two or more files to merge" when it runs with a single chunk (`--cpus 1`), and a failed final merge no longer hangs on FIFOs.
+- `bin/stage_cleanup.sh` preserves staged inputs and temporary files when a task fails and never deletes a `TMPDIR` outside the task work directory.
+- `GENERATE_COVERAGE` stages `include_bed` and `exclude_bed` in separate directories, so BED inputs with the same file name no longer overwrite each other.
+- `test_run/run_test.sh` sets `NXF_OFFLINE` (the variable was misspelled `NFX_OFFLINE`), so the offline setting takes effect.
+
 
 ## [3.2.3] - 2026-10-01
 

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing to the ChoCallate
 
-We welcome you contributions to the ChoCallate project
+We welcome your contributions to the ChoCallate project
 
 ## How to report a bug
 
@@ -14,10 +14,10 @@ In general, we follow the "fork-and-pull" Git workflow.
  2. **Clone** the project to your own machine
  3. **Commit** changes to your own branch
  4. **Push** your work back up to your fork
- 5. Submit a **Pull request** so that we can review your changes
+ 5. Submit a **Pull request** with your changes
 
-Note: Be sure to merge the latest changes before making the pull request. You can increase your chances of a successfull pull request by indicating clearly what branch you are contributing to and making a detailed commit message. It also helps if you open an issue first to let us know you are working on this project.
+Note: Be sure to merge the latest changes before making the pull request. You can increase your chances of a successful pull request by indicating clearly what branch you are contributing to and making a detailed commit message. It also helps if you open an issue first to let us know you are working on this project.
 
 ## License
 
-ChoCallate is licensed undert [MIT License](https://github.com/alermol/ChoCallate/blob/main/LICENSE)
+ChoCallate is licensed under [MIT License](https://github.com/alermol/ChoCallate/blob/main/LICENSE)