diff --git a/README.md b/README.md index f2c54c0..3c73477 100644 --- a/README.md +++ b/README.md @@ -74,3 +74,6 @@ The main QC fastq output is named by prefix_filtered.fastq.gz. ## Link to Doc Site Please refer [here](https://docs.microbiomedata.org/workflows/chapters/3_Metagenome_Reads_QC/) for more information. + +## Required Testing +Contributions to this repository must undergo direct testing via jaws, using the test jsons in the required_tests folder. \ No newline at end of file diff --git a/required_tests/README b/required_tests/README new file mode 100644 index 0000000..e3b2958 --- /dev/null +++ b/required_tests/README @@ -0,0 +1,15 @@ +# Required Testing + +Jsons in this folder cover the various data types handled by this workflow, as detailed in the below table. + +Each PR to ReadsQC MUST come with successful jaws tests using the below jsons, in addition to any other relevant testing. Provide the jaws ids in a comment on your PR. If additional data types become available, you are REQUIRED TO ADD A TEST JSON to this folder. Beyond having a status of 'succeeded', you MUST MANUALLY CHECK that the output of these tests: +1) have a correctly annotated fastq, indicating which reads are 1 or 2 if they are non-interleaved +2) have correctly populated stats that match those existing in mongodb + + +| Use Case | Last successful run on NERSC in prod | ReadsQC Json | DG Id | +|--------------------------------------------------------|--------------------------------------|-----------------------------------------------|-----------------------------------------------------| +| Short Read - Non Interleave - SRA - Non Manifest | n/a | single_accession.json | n/a | +| Short Read - Non Interleave - non SRA - Non Manifest | 1/12/26 | rqcfilter_nonint_direct.json | nmdc:dgns-11-kc8d3v87 | +| Short Read - Non Interleave - non SRA - Manifest | 5/23/26 | rqcfilter_paired_recent_manifest_direct.json | nmdc:dgns-15-6m4kha03, nmdc:omprc-11-2vny0554 | +| Short Read - Interleave - non SRA - Non Manifest | 1/26/24 | rqcfilter_int_recent.json | nmdc:omprc-11-x221tr39 | \ No newline at end of file diff --git a/required_tests/rqcfilter_int_recent.json b/required_tests/rqcfilter_int_recent.json new file mode 100644 index 0000000..e0d80cc --- /dev/null +++ b/required_tests/rqcfilter_int_recent.json @@ -0,0 +1,6 @@ +{ + "rqcfilter.input_files": ["https://portal.nersc.gov/project/m3408/submission_data/80f5752b-e8d3-453f-9a47-f48210ed6ffc/HS177_72.fastq.gz"], + "rqcfilter.proj": "nmdc:xxxxxxx", + "rqcfilter.interleaved": true, + "rqcfilter.shortRead": true +} \ No newline at end of file diff --git a/required_tests/rqcfilter_nonint_direct.json b/required_tests/rqcfilter_nonint_direct.json new file mode 100644 index 0000000..6c0b41d --- /dev/null +++ b/required_tests/rqcfilter_nonint_direct.json @@ -0,0 +1,7 @@ +{ + "rqcfilter.input_fq1": ["https://storage.googleapis.com/neon-microbial-raw-seq-files/2023/BMI_HLGJLBGX5_mms_R1/BMI_HLGJLBGX5_Tube34_R1.fastq.gz"], + "rqcfilter.input_fq2": ["https://storage.googleapis.com/neon-microbial-raw-seq-files/2023/BMI_HLGJLBGX5_mms_R2/BMI_HLGJLBGX5_Tube34_R2.fastq.gz"], + "rqcfilter.proj": "nmdc:xxxxxxx", + "rqcfilter.interleaved": false, + "rqcfilter.shortRead": true +} \ No newline at end of file diff --git a/required_tests/rqcfilter_paired_recent_manifest_direct.json b/required_tests/rqcfilter_paired_recent_manifest_direct.json new file mode 100644 index 0000000..4c0bfb3 --- /dev/null +++ b/required_tests/rqcfilter_paired_recent_manifest_direct.json @@ -0,0 +1,13 @@ +{ + "rqcfilter.input_fq1": [ + "https://storage.googleapis.com/neon-microbial-raw-seq-files/2023/BMI_H7FFKBGX5_mms_R1/BMI_H7FFKBGX5_Plate10WellA2_R1.fastq.gz", + "https://storage.googleapis.com/neon-microbial-raw-seq-files/2023/BMI_HLGJLBGX5_mms_R1/BMI_HLGJLBGX5_Plate10WellA2_R1.fastq.gz" + ], + "rqcfilter.input_fq2": [ + "https://storage.googleapis.com/neon-microbial-raw-seq-files/2023/BMI_H7FFKBGX5_mms_R2/BMI_H7FFKBGX5_Plate10WellA2_R2.fastq.gz", + "https://storage.googleapis.com/neon-microbial-raw-seq-files/2023/BMI_HLGJLBGX5_mms_R2/BMI_HLGJLBGX5_Plate10WellA2_R2.fastq.gz" + ], + "rqcfilter.shortRead": true, + "rqcfilter.interleaved": false, + "rqcfilter.proj": "nmdc:xxxxxxx" +} \ No newline at end of file diff --git a/required_tests/single_accession.json b/required_tests/single_accession.json new file mode 100644 index 0000000..e6b2504 --- /dev/null +++ b/required_tests/single_accession.json @@ -0,0 +1,8 @@ +{ + "rqcfilter.accessions": [ + "ERX4811336" + ], + "rqcfilter.proj": "nmdc:xxxxxxx", + "rqcfilter.interleaved": false, + "rqcfilter.shortRead": true +} \ No newline at end of file diff --git a/shortReadsqc.wdl b/shortReadsqc.wdl index e5cb497..7813231 100644 --- a/shortReadsqc.wdl +++ b/shortReadsqc.wdl @@ -155,7 +155,7 @@ task stage_interleave { cat $fq2_name >> ~{target_reads_2} done - reformat.sh -Xmx~{memory}G trimreaddescription=t in1=~{target_reads_1} in2=~{target_reads_2} out=~{output_interleaved} + reformat.sh -Xmx~{memory}G in1=~{target_reads_1} in2=~{target_reads_2} out=~{output_interleaved} # Validate that the read1 and read2 files are sorted correctly reformat.sh -Xmx~{memory}G verifypaired=t in=~{output_interleaved} @@ -237,6 +237,9 @@ task rqcfilter { > >(tee -a ~{filename_outlog}) \ 2> >(tee -a ~{filename_errlog} >&2) + # Validate the filtered output is paired if not interleaved + reformat.sh -Xmx~{memory}G verifypaired=t in=filtered/raw.anqdpht.fastq.gz + >>> output { diff --git a/test/Ecoli_10x.1.fastq.gz b/test/Ecoli_10x.1.fastq.gz deleted file mode 100755 index 6c62d0e..0000000 Binary files a/test/Ecoli_10x.1.fastq.gz and /dev/null differ diff --git a/test/Ecoli_10x.2.fastq.gz b/test/Ecoli_10x.2.fastq.gz deleted file mode 100755 index a29bc7e..0000000 Binary files a/test/Ecoli_10x.2.fastq.gz and /dev/null differ diff --git a/test/README.md b/test/README.md deleted file mode 100644 index a6e5479..0000000 --- a/test/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Validation Workflow - -## Summary - -The ReadsQC validation workflow is meant to compare test data in json format available on the NMDC github against data generated by user test input. The purpose is to ensure user test data matches to or falls within an acceptable range of the NMDC test data in an effort to push towards reproducible data standards. The **rqc_test.wdl** file contains most of the neccesary inputs needed for the ReadsQC worfklow and will grab user generated json file, but will require user input for the path of the database in their local environment and local user paths for test data to be updated in the input.json. - - -## Input files for rqc_test.wdl -1. ReadsQC workflow contianer(provided), -2. Comparejson container(provided), -3. database (path to database provided by user), -4. memory (optional, can be changed by user), -5. threads (optional, can be changed by user), -6. url for test fasta (provided), -7. url for NMDC test json (provided for small test data) -``` -workflow rqctest { - String container="microbiomedata/bbtools:38.96" - String validate_container="microbiomedata/comparejson" - String database="/vol_b/nmdc_workflows/data/test_refdata" - Boolean flag=false - String? memory="60G" - String? threads="8" - String url="https://portal.nersc.gov/cfs/m3408/test_data/Ecoli_10x-int.fastq.gz" - String ref_json="https://raw.githubusercontent.com/microbiomedata/ReadsQC/master/test/small_test_filterStats.json" -``` -## Docker contianers can be found here: -Bbtools: [microbiomedata/bbtools:38.96](https://hub.docker.com/r/microbiomedata/bbtools) -Comparjson: [microbiomedata/comparejson](https://hub.docker.com/r/microbiomedata/comparejson) - -## Running Testing Validation Workflow - -The command for running test validation is similar to that found in the submit.sh file, with the exception of switching out rqcfilter.wdl for rqc_test.wdl. - - - `rqc_test.wdl` file: the WDL file for workflow definition - - `input.json` file: the test input for the workflow - - `cromwell.conf` file: the conf file for running Cromwell. - - `cromwell.jar` file: the jar file for running Cromwell. - - `metadata_out.json` file: file collects run data, will be created after run of command - -Example: -``` -java -Dconfig.file=cromwell.conf -jar cromwell.jar run -m metadata_out.json -i input.json rqc_test.wdl -``` - -## Validation Metric -Validation metric is determined through a printed command line statement that will read: -``` -"test.validate.result": ["No differences detected: test validated"] -``` -or -``` -"test.validate.result": ["Test Failed"] -``` - -If test fails, please check inputs or contact local system administrators to ensure there are no system issues causing discrepency in results. diff --git a/test/incorrect_pairs_R1.fastq.gz b/test/incorrect_pairs_R1.fastq.gz deleted file mode 100644 index 48d2b62..0000000 Binary files a/test/incorrect_pairs_R1.fastq.gz and /dev/null differ diff --git a/test/incorrect_pairs_R2.fastq.gz b/test/incorrect_pairs_R2.fastq.gz deleted file mode 100644 index 9f99316..0000000 Binary files a/test/incorrect_pairs_R2.fastq.gz and /dev/null differ diff --git a/test/large_test_filterStats.json b/test/large_test_filterStats.json deleted file mode 100755 index e35c44a..0000000 --- a/test/large_test_filterStats.json +++ /dev/null @@ -1 +0,0 @@ -{"inputReads": 44943418, "kfilteredBases": 14684762, "qfilteredReads": 200626, "ktrimmedReads": 6186690, "outputBases": 4868925674, "ktrimmedBases": 354478706, "kfilteredReads": 100360, "qtrimmedBases": 8690, "outputReads": 33510668, "gcPolymerRatio": 0.42046, "inputBases": 6741512700, "qtrimmedReads": 8583, "qfilteredBases": 29786796} \ No newline at end of file diff --git a/test/longreads_input.json b/test/longreads_input.json deleted file mode 100644 index ef3e51e..0000000 --- a/test/longreads_input.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "rqcfilter.input_files": ["https://portal.nersc.gov/project/m3408//test_data/SRR13128014.pacbio.subsample/SRR13128014.pacbio.subsample.ccs.fastq.gz"], - "rqcfilter.proj": "nmdc:xxxxxxx", - "rqcfilter.interleaved": true, - "rqcfilter.shortRead": false -} \ No newline at end of file diff --git a/test/rqc_test.wdl b/test/rqc_test.wdl deleted file mode 100644 index d0da5ca..0000000 --- a/test/rqc_test.wdl +++ /dev/null @@ -1,72 +0,0 @@ -version 1.0 -import "rqcfilter.wdl" as rqc -workflow rqctest { - input{ - String container="microbiomedata/bbtools:38.96" - String validate_container="microbiomedata/comparejson" - String database="/vol_b/nmdc_workflows/data/test_refdata" - Boolean flag=false - String memory="60G" - String threads="8" - String url="https://portal.nersc.gov/cfs/m3408/test_data/Ecoli_10x-int.fastq.gz" - String ref_json="https://raw.githubusercontent.com/microbiomedata/ReadsQC/master/test/small_test_filterStats.json" - } - call prepare { - input: container=container, - url=url, - ref_json=ref_json - } - call rqc.rqcfilter as filter { - input: input_files=prepare.fastq, - database=database, - container=container, - chastityfilter_flag=flag, - memory=memory, - threads=threads - } - call validate { - input: container=validate_container, - refjson=prepare.refjson, - user_json=filter.json_out - } -} -task prepare { - input{ - String container - String ref_json - String url - } - command{ - wget -O "input.fastq.gz" ${url} - wget -O "ref_json.json" ${ref_json} - } - output{ - File fastq = "input.fastq.gz" - File refjson = "ref_json.json" - } - runtime { - memory: "1 GiB" - cpu: 2 - maxRetries: 1 - docker: container - } -} -task validate { - input{ - String container - File refjson - File user_json - } - command { - compare_json.py -i ${refjson} -f ${user_json} - } - output { - Array[String] result = read_lines(stdout()) - } - runtime { - memory: "1 GiB" - cpu: 1 - maxRetries: 1 - docker: container - } -} diff --git a/test/shortread_interleave_input.json b/test/shortread_interleave_input.json deleted file mode 100644 index 426a9d0..0000000 --- a/test/shortread_interleave_input.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "rqcfilter.input_files": ["https://portal.nersc.gov/project/m3408//test_data/smalltest.int.fastq.gz"], - "rqcfilter.proj": "nmdc:xxxxxxx", - "rqcfilter.interleaved": true, - "rqcfilter.shortRead": true -} \ No newline at end of file diff --git a/test/shortread_noninterleaved_input.json b/test/shortread_noninterleaved_input.json deleted file mode 100644 index bce0ef9..0000000 --- a/test/shortread_noninterleaved_input.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "rqcfilter.input_fq1": ["https://portal.nersc.gov/project/m3408//test_data/smalltest.R1.fastq.gz"], - "rqcfilter.input_fq2": ["https://portal.nersc.gov/project/m3408//test_data/smalltest.R2.fastq.gz"], - "rqcfilter.accessions":[], - "rqcfilter.proj": "nmdc:xxxxxxx", - "rqcfilter.interleaved": false, - "rqcfilter.shortRead": true -} \ No newline at end of file diff --git a/test/small_test_filterStats.json b/test/small_test_filterStats.json deleted file mode 100755 index 1ff9132..0000000 --- a/test/small_test_filterStats.json +++ /dev/null @@ -1 +0,0 @@ -{"inputReads": 331126, "kfilteredBases": 138732, "qfilteredReads": 0, "ktrimmedReads": 478, "outputBases": 1680724, "ktrimmedBases": 25248, "kfilteredReads": 926, "qtrimmedBases": 0, "outputReads": 11212, "gcPolymerRatio": 0.182857, "inputBases": 50000026, "qtrimmedReads": 0, "qfilteredBases": 0} \ No newline at end of file diff --git a/version.txt b/version.txt index ce77b51..db97376 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v1.0.27 +v1.0.28