From 32a0f86ed6cabcd98b4ea20d8af9b094f30c43ae Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Thu, 28 May 2026 16:06:13 -0700 Subject: [PATCH 1/9] first pass extracting remaining elements from open PR 63 --- docs/index.rst | 5 +- interleave_rqcfilter.wdl | 263 --------------------------------------- longReadsqc.wdl | 13 +- shortReadsqc.wdl | 31 ++++- 4 files changed, 35 insertions(+), 277 deletions(-) delete mode 100644 interleave_rqcfilter.wdl diff --git a/docs/index.rst b/docs/index.rst index a31173f..bb2c905 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,7 +6,7 @@ site the file is incorporated into. You can learn more about the `github_url` field at: https://sphinx-rtd-theme.readthedocs.io/en/stable/configuring.html#confval-github_url -Reads QC Workflow (v1.0.14-alpha.1) +Reads QC Workflow ============================= .. image:: lrrqc_workflow2024.svg @@ -39,6 +39,7 @@ The following parameters are used for :literal:`rqcfilter2` in this workflow:: - removemouse=true : Remove mouse reads via mapping. - barcodefilter=false : Disable improper barcodes filter - chastityfilter=false: Remove illumina reads failing chastity filter. + - filterbytile=false : Disable filtering by Illumina tile information; automatically set to false when SRA-format reads are detected (reads with headers matching @SRR/ERR/DRR accessions). - trimfragadapter=true: Trim all known Illumina adapter sequences, including TruSeq and Nextera. - removemicrobes=true : Remove common contaminant microbial reads via mapping, and place them in a separate file. @@ -90,7 +91,7 @@ Workflow Dependencies Third party software (This is included in the Docker image.) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- `BBTools v38.96 `_ (License: `BSD-3-Clause-LBNL `_) +- `BBTools v39.80 `_ (License: `BSD-3-Clause-LBNL `_) Requisite database ~~~~~~~~~~~~~~~~~~ diff --git a/interleave_rqcfilter.wdl b/interleave_rqcfilter.wdl deleted file mode 100644 index 683453f..0000000 --- a/interleave_rqcfilter.wdl +++ /dev/null @@ -1,263 +0,0 @@ -# Interleaved fastq QC workflow -version 1.0 -workflow nmdc_rqcfilter { - input { - String container="bfoster1/img-omics:0.1.9" - String bbtools_container="microbiomedata/bbtools:38.96" - String workflowmeta_container="microbiomedata/workflowmeta:1.1.1" - String proj - String prefix=sub(proj, ":", "_") - Array[String] input_fastq1 - Array[String] input_fastq2 - String database="/refdata/" - Int rqc_mem = 180 - } - - call stage { - input: - container=bbtools_container, - memory=10, - input_fastq1=input_fastq1, - input_fastq2=input_fastq2 - } - # Estimate RQC runtime at an hour per compress GB - call rqcfilter as qc { - input: - input_files=stage.interleaved_reads, - threads=16, - database=database, - memory=rqc_mem, - container = bbtools_container - } - call make_info_file { - input: - info_file = qc.info_file, - container=container, - prefix = prefix - } - - call finish_rqc { - input: - container=workflowmeta_container, - prefix = prefix, - filtered = qc.filtered, - filtered_stats = qc.stat, - filtered_stats2 = qc.stat2 - } - output { - File filtered_final = finish_rqc.filtered_final - File filtered_stats_final = finish_rqc.filtered_stats_final - File filtered_stats2_final = finish_rqc.filtered_stats2_final - File rqc_info = make_info_file.rqc_info - } -} - -task stage { - input { - String container - Int memory - String target_reads_1="raw_reads_1.fastq.gz" - String target_reads_2="raw_reads_2.fastq.gz" - String output_interleaved="raw_interleaved.fastq.gz" - Array[String] input_fastq1 - Array[String] input_fastq2 - Int file_num = length(input_fastq1) - } - - command <<< - set -euo pipefail - - # load wdl array to shell array - FQ1_ARRAY=(~{sep=" " input_fastq1}) - FQ2_ARRAY=(~{sep=" " input_fastq2}) - - for (( i = 0; i < ~{file_num}; i++ )) ;do - fq1_name=$(basename ${FQ1_ARRAY[$i]}) - fq2_name=$(basename ${FQ2_ARRAY[$i]}) - if [ $( echo ${FQ1_ARRAY[$i]} | egrep -c "https*:") -gt 0 ] ; then - wget --no-check-certificate ${FQ1_ARRAY[$i]} -O $fq1_name - wget --no-check-certificate ${FQ2_ARRAY[$i]} -O $fq2_name - else - ln -s ${FQ1_ARRAY[$i]} $fq1_name || cp ${FQ1_ARRAY[$i]} $fq1_name - ln -s ${FQ2_ARRAY[$i]} $fq2_name || cp ${FQ2_ARRAY[$i]} $fq2_name - fi - - cat $fq1_name >> ~{target_reads_1} - cat $fq2_name >> ~{target_reads_2} - done - - reformat.sh -Xmx~{memory}G in1=~{target_reads_1} in2=~{target_reads_2} out=~{output_interleaved} - # Capture the start time - date --iso-8601=seconds > start.txt - >>> - - output{ - File interleaved_reads = "~{output_interleaved}" - String start = read_string("start.txt") - } - - runtime { - memory: "~{memory} GiB" - cpu: 2 - maxRetries: 1 - docker: container - } -} - - -task rqcfilter { - input { - File input_files - String container - String database - String rqcfilterdata = database + "/RQCFilterData" - Boolean chastityfilter_flag=true - Int memory - Int xmxmem = floor(memory * 0.75) - Int? threads - String filename_outlog="stdout.log" - String filename_errlog="stderr.log" - String filename_stat="filtered/filterStats.txt" - String filename_stat2="filtered/filterStats2.txt" - String filename_stat_json="filtered/filterStats.json" - String filename_reproduce="filtered/reproduce.sh" - String system_cpu="$(grep \"model name\" /proc/cpuinfo | wc -l)" - String jvm_threads=select_first([threads,system_cpu]) - String chastityfilter= if (chastityfilter_flag) then "cf=t" else "cf=f" - } - - command <<< - export TIME="time result\ncmd:%C\nreal %es\nuser %Us \nsys %Ss \nmemory:%MKB \ncpu %P" - set -euo pipefail - - rqcfilter2.sh \ - ~{if (defined(memory)) then "-Xmx" + xmxmem + "G" else "-Xmx60G" }\ - -da \ - threads=~{jvm_threads} \ - ~{chastityfilter} \ - jni=t \ - in=~{input_files} \ - path=filtered \ - rna=f \ - trimfragadapter=t \ - qtrim=r \ - trimq=0 \ - maxns=3 \ - maq=3 \ - minlen=51 \ - mlf=0.33 \ - phix=t \ - removehuman=t \ - removedog=t \ - removecat=t \ - removemouse=t \ - khist=t \ - removemicrobes=t \ - sketch \ - kapa=t \ - clumpify=t \ - barcodefilter=f \ - trimpolyg=5 \ - usejni=f \ - rqcfilterdata=~{rqcfilterdata} \ - > >(tee -a ~{filename_outlog}) \ - 2> >(tee -a ~{filename_errlog} >&2) - - python <>> - - output { - File stdout = filename_outlog - File stderr = filename_errlog - File stat = filename_stat - File stat2 = filename_stat2 - File info_file = filename_reproduce - File filtered = glob("filtered/*fastq.gz")[0] - File json_out = filename_stat_json - } - - runtime { - docker: container - memory: "~{memory} GiB" - cpu: 16 - } -} - -task make_info_file { - input { - File info_file - String prefix - String container - } - - command<<< - set -euo pipefail - sed -n 2,5p ~{info_file} 2>&1 | \ - perl -ne 's:in=/.*/(.*) :in=$1:; s/#//; s/BBTools/BBTools(1)/; print;' > \ - ~{prefix}_readsQC.info - echo -e "\n(1) B. Bushnell: BBTools software package, http://bbtools.jgi.doe.gov/" >> \ - ~{prefix}_readsQC.info - >>> - - output { - File rqc_info = "~{prefix}_readsQC.info" - } - - runtime { - memory: "1 GiB" - cpu: 1 - maxRetries: 1 - docker: container - } -} - -task finish_rqc { - input { - # File read - File filtered_stats - File filtered_stats2 - File filtered - String container - String prefix - # String start - } - - command<<< - - set -euo pipefail - end=`date --iso-8601=seconds` - # Generate QA objects - ln -s ~{filtered} ~{prefix}_filtered.fastq.gz - ln -s ~{filtered_stats} ~{prefix}_filterStats.txt - ln -s ~{filtered_stats2} ~{prefix}_filterStats2.txt - - # Generate stats but rename some fields untilt the script is - # fixed. - /scripts/rqcstats.py ~{filtered_stats} > stats.json - cp stats.json ~{prefix}_qa_stats.json - - >>> - - output { - File filtered_final = "~{prefix}_filtered.fastq.gz" - File filtered_stats_final = "~{prefix}_filterStats.txt" - File filtered_stats2_final = "~{prefix}_filterStats2.txt" - } - - runtime { - docker: container - memory: "1 GiB" - cpu: 1 - } -} diff --git a/longReadsqc.wdl b/longReadsqc.wdl index 10f476c..4ccef84 100644 --- a/longReadsqc.wdl +++ b/longReadsqc.wdl @@ -12,10 +12,9 @@ workflow LongReadsQC { Boolean rmdup = true Boolean overwrite = true File? reference - String container="bfoster1/img-omics:0.1.9" String pbmarkdup_container="microbiomedata/pbmarkdup:1.1" - String bbtools_container="microbiomedata/bbtools:39.03" - String jq_container="microbiomedata/jq:1.6" + String bbtools_container="microbiomedata/bbtools:39.80" + String workflowmeta_container = "microbiomedata/workflowmeta:1.1.1" # String outdir # String prefix = basename(file) } @@ -23,7 +22,7 @@ workflow LongReadsQC { call stage_longread { input: file = file, - container = container + container = workflowmeta_container } call pbmarkdup { @@ -72,7 +71,7 @@ call stage_longread { call finish_rqc { input: - container = jq_container, + container = workflowmeta_container, prefix = prefix, filtered = bbdukReads.out_fastq, pbmarkdup_stats = pbmarkdup.stats, @@ -312,7 +311,7 @@ task bbdukReads { task make_info_file { input { String prefix - String bbtools_container + String container File pbmarkdup_log } @@ -341,7 +340,7 @@ task make_info_file { memory: "1 GiB" cpu: 1 maxRetries: 1 - docker: bbtools_container + docker: container } } diff --git a/shortReadsqc.wdl b/shortReadsqc.wdl index 0d24f9b..0eae3cc 100644 --- a/shortReadsqc.wdl +++ b/shortReadsqc.wdl @@ -3,8 +3,7 @@ version 1.0 workflow ShortReadsQC { input{ - String container="bfoster1/img-omics:0.1.9" - String bbtools_container="microbiomedata/bbtools:38.96" + String bbtools_container="microbiomedata/bbtools:39.80" String workflow_container = "microbiomedata/workflowmeta:1.1.1" String proj String prefix=sub(proj, ":", "_") @@ -20,7 +19,7 @@ workflow ShortReadsQC { if (interleaved) { call stage_single { input: - container = container, + container = bbtools_container, input_file = input_files } } @@ -43,7 +42,8 @@ workflow ShortReadsQC { database = database, memory = rqc_mem, container = bbtools_container, - chastityfilter_flag = chastityfilter_flag + chastityfilter_flag = chastityfilter_flag, + filterbytile_flag = !select_first([stage_single.is_external_sra, stage_interleave.is_external_sra, false]) } call stats_jsons { @@ -55,7 +55,7 @@ workflow ShortReadsQC { call make_info_file { input: info_file = qc.info_file, - container = container, + container = workflow_container, prefix = prefix } @@ -103,11 +103,20 @@ task stage_single { # Capture the start time date --iso-8601=seconds > start.txt + # Check if reads are SRA format + first_line=$(zcat ~{target} 2>/dev/null | head -1 || head -1 ~{target}) + if echo "$first_line" | grep -qE '^@(SRR|ERR|DRR)[0-9]+(\.[0-9]+)?'; then + echo "true" > is_external_sra.txt + else + echo "false" > is_external_sra.txt + fi + >>> output{ File reads_fastq = "~{target}" String start = read_string("start.txt") + Boolean is_external_sra = read_boolean("is_external_sra.txt") } runtime { @@ -161,11 +170,20 @@ task stage_interleave { # Capture the start time date --iso-8601=seconds > start.txt + # Check if reads are SRA format + first_line=$(zcat ~{target_reads_1} 2>/dev/null | head -1 || head -1 ~{target_reads_1}) + if echo "$first_line" | grep -qE '^@(SRR|ERR|DRR)[0-9]+(\.[0-9]+)?'; then + echo "true" > is_external_sra.txt + else + echo "false" > is_external_sra.txt + fi + >>> output{ File reads_fastq = "~{output_interleaved}" String start = read_string("start.txt") + Boolean is_external_sra = read_boolean("is_external_sra.txt") } runtime { @@ -183,6 +201,7 @@ task rqcfilter { String database String rqcfilterdata = database + "/RQCFilterData" Boolean chastityfilter_flag=true + Boolean filterbytile_flag=true Int memory Int xmxmem = floor(memory * 0.75) Int threads @@ -192,6 +211,7 @@ task rqcfilter { String filename_stat2="filtered/filterStats2.txt" String filename_reproduce="filtered/reproduce.sh" String chastityfilter= if (chastityfilter_flag) then "cf=t" else "cf=f" + String filterbytile = if (filterbytile_flag) then "filterbytile=t" else "filterbytile=f" } command <<< @@ -221,6 +241,7 @@ task rqcfilter { removecat=t \ removemouse=t \ khist=t \ + ~{filterbytile} \ removemicrobes=t \ sketch \ kapa=t \ From 1cf0481a85d617f04ddecd97b82fad6e9263d16e Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Fri, 29 May 2026 08:52:33 -0700 Subject: [PATCH 2/9] revert bbtools version update. havent confirmed memory issue resolved --- longReadsqc.wdl | 13 +++++++------ shortReadsqc.wdl | 7 ++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/longReadsqc.wdl b/longReadsqc.wdl index 4ccef84..10f476c 100644 --- a/longReadsqc.wdl +++ b/longReadsqc.wdl @@ -12,9 +12,10 @@ workflow LongReadsQC { Boolean rmdup = true Boolean overwrite = true File? reference + String container="bfoster1/img-omics:0.1.9" String pbmarkdup_container="microbiomedata/pbmarkdup:1.1" - String bbtools_container="microbiomedata/bbtools:39.80" - String workflowmeta_container = "microbiomedata/workflowmeta:1.1.1" + String bbtools_container="microbiomedata/bbtools:39.03" + String jq_container="microbiomedata/jq:1.6" # String outdir # String prefix = basename(file) } @@ -22,7 +23,7 @@ workflow LongReadsQC { call stage_longread { input: file = file, - container = workflowmeta_container + container = container } call pbmarkdup { @@ -71,7 +72,7 @@ call stage_longread { call finish_rqc { input: - container = workflowmeta_container, + container = jq_container, prefix = prefix, filtered = bbdukReads.out_fastq, pbmarkdup_stats = pbmarkdup.stats, @@ -311,7 +312,7 @@ task bbdukReads { task make_info_file { input { String prefix - String container + String bbtools_container File pbmarkdup_log } @@ -340,7 +341,7 @@ task make_info_file { memory: "1 GiB" cpu: 1 maxRetries: 1 - docker: container + docker: bbtools_container } } diff --git a/shortReadsqc.wdl b/shortReadsqc.wdl index 0eae3cc..21f2766 100644 --- a/shortReadsqc.wdl +++ b/shortReadsqc.wdl @@ -3,7 +3,8 @@ version 1.0 workflow ShortReadsQC { input{ - String bbtools_container="microbiomedata/bbtools:39.80" + String container="bfoster1/img-omics:0.1.9" + String bbtools_container="microbiomedata/bbtools:38.96" String workflow_container = "microbiomedata/workflowmeta:1.1.1" String proj String prefix=sub(proj, ":", "_") @@ -19,7 +20,7 @@ workflow ShortReadsQC { if (interleaved) { call stage_single { input: - container = bbtools_container, + container = container, input_file = input_files } } @@ -55,7 +56,7 @@ workflow ShortReadsQC { call make_info_file { input: info_file = qc.info_file, - container = workflow_container, + container = container, prefix = prefix } From e7609dfaadae349a4b7bc7b7a57c5cfefa876a37 Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Fri, 29 May 2026 08:53:36 -0700 Subject: [PATCH 3/9] revert bbtools version update. havent confirmed memory issue resolved --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index bb2c905..099e00e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -91,7 +91,7 @@ Workflow Dependencies Third party software (This is included in the Docker image.) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- `BBTools v39.80 `_ (License: `BSD-3-Clause-LBNL `_) +- `BBTools v38.96 `_ (License: `BSD-3-Clause-LBNL `_) Requisite database ~~~~~~~~~~~~~~~~~~ From ef03eb4747c7ddb4107f45b1b91d8897a269ec33 Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Fri, 29 May 2026 08:59:17 -0700 Subject: [PATCH 4/9] . --- interleave_rqcfilter.wdl | 263 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 interleave_rqcfilter.wdl diff --git a/interleave_rqcfilter.wdl b/interleave_rqcfilter.wdl new file mode 100644 index 0000000..4f429f7 --- /dev/null +++ b/interleave_rqcfilter.wdl @@ -0,0 +1,263 @@ +# Interleaved fastq QC workflow +version 1.0 +workflow nmdc_rqcfilter { + input { + String container="bfoster1/img-omics:0.1.9" + String bbtools_container="microbiomedata/bbtools:38.96" + String workflowmeta_container="microbiomedata/workflowmeta:1.1.1" + String proj + String prefix=sub(proj, ":", "_") + Array[String] input_fastq1 + Array[String] input_fastq2 + String database="/refdata/" + Int rqc_mem = 180 + } + + call stage { + input: + container=bbtools_container, + memory=10, + input_fastq1=input_fastq1, + input_fastq2=input_fastq2 + } + # Estimate RQC runtime at an hour per compress GB + call rqcfilter as qc { + input: + input_files=stage.interleaved_reads, + threads=16, + database=database, + memory=rqc_mem, + container = bbtools_container + } + call make_info_file { + input: + info_file = qc.info_file, + container=container, + prefix = prefix + } + + call finish_rqc { + input: + container=workflowmeta_container, + prefix = prefix, + filtered = qc.filtered, + filtered_stats = qc.stat, + filtered_stats2 = qc.stat2 + } + output { + File filtered_final = finish_rqc.filtered_final + File filtered_stats_final = finish_rqc.filtered_stats_final + File filtered_stats2_final = finish_rqc.filtered_stats2_final + File rqc_info = make_info_file.rqc_info + } +} + +task stage { + input { + String container + Int memory + String target_reads_1="raw_reads_1.fastq.gz" + String target_reads_2="raw_reads_2.fastq.gz" + String output_interleaved="raw_interleaved.fastq.gz" + Array[String] input_fastq1 + Array[String] input_fastq2 + Int file_num = length(input_fastq1) + } + + command <<< + set -euo pipefail + + # load wdl array to shell array + FQ1_ARRAY=(~{sep=" " input_fastq1}) + FQ2_ARRAY=(~{sep=" " input_fastq2}) + + for (( i = 0; i < ~{file_num}; i++ )) ;do + fq1_name=$(basename ${FQ1_ARRAY[$i]}) + fq2_name=$(basename ${FQ2_ARRAY[$i]}) + if [ $( echo ${FQ1_ARRAY[$i]} | egrep -c "https*:") -gt 0 ] ; then + wget --no-check-certificate ${FQ1_ARRAY[$i]} -O $fq1_name + wget --no-check-certificate ${FQ2_ARRAY[$i]} -O $fq2_name + else + ln -s ${FQ1_ARRAY[$i]} $fq1_name || cp ${FQ1_ARRAY[$i]} $fq1_name + ln -s ${FQ2_ARRAY[$i]} $fq2_name || cp ${FQ2_ARRAY[$i]} $fq2_name + fi + + cat $fq1_name >> ~{target_reads_1} + cat $fq2_name >> ~{target_reads_2} + done + + reformat.sh -Xmx~{memory}G in1=~{target_reads_1} in2=~{target_reads_2} out=~{output_interleaved} + # Capture the start time + date --iso-8601=seconds > start.txt + >>> + + output{ + File interleaved_reads = "~{output_interleaved}" + String start = read_string("start.txt") + } + + runtime { + memory: "~{memory} GiB" + cpu: 2 + maxRetries: 1 + docker: container + } +} + + +task rqcfilter { + input { + File input_files + String container + String database + String rqcfilterdata = database + "/RQCFilterData" + Boolean chastityfilter_flag=true + Int memory + Int xmxmem = floor(memory * 0.75) + Int? threads + String filename_outlog="stdout.log" + String filename_errlog="stderr.log" + String filename_stat="filtered/filterStats.txt" + String filename_stat2="filtered/filterStats2.txt" + String filename_stat_json="filtered/filterStats.json" + String filename_reproduce="filtered/reproduce.sh" + String system_cpu="$(grep \"model name\" /proc/cpuinfo | wc -l)" + String jvm_threads=select_first([threads,system_cpu]) + String chastityfilter= if (chastityfilter_flag) then "cf=t" else "cf=f" + } + + command <<< + export TIME="time result\ncmd:%C\nreal %es\nuser %Us \nsys %Ss \nmemory:%MKB \ncpu %P" + set -euo pipefail + + rqcfilter2.sh \ + ~{if (defined(memory)) then "-Xmx" + xmxmem + "G" else "-Xmx60G" }\ + -da \ + threads=~{jvm_threads} \ + ~{chastityfilter} \ + jni=t \ + in=~{input_files} \ + path=filtered \ + rna=f \ + trimfragadapter=t \ + qtrim=r \ + trimq=0 \ + maxns=3 \ + maq=3 \ + minlen=51 \ + mlf=0.33 \ + phix=t \ + removehuman=t \ + removedog=t \ + removecat=t \ + removemouse=t \ + khist=t \ + removemicrobes=t \ + sketch \ + kapa=t \ + clumpify=t \ + barcodefilter=f \ + trimpolyg=5 \ + usejni=f \ + rqcfilterdata=~{rqcfilterdata} \ + > >(tee -a ~{filename_outlog}) \ + 2> >(tee -a ~{filename_errlog} >&2) + + python <>> + + output { + File stdout = filename_outlog + File stderr = filename_errlog + File stat = filename_stat + File stat2 = filename_stat2 + File info_file = filename_reproduce + File filtered = glob("filtered/*fastq.gz")[0] + File json_out = filename_stat_json + } + + runtime { + docker: container + memory: "~{memory} GiB" + cpu: 16 + } +} + +task make_info_file { + input { + File info_file + String prefix + String container + } + + command<<< + set -euo pipefail + sed -n 2,5p ~{info_file} 2>&1 | \ + perl -ne 's:in=/.*/(.*) :in=$1:; s/#//; s/BBTools/BBTools(1)/; print;' > \ + ~{prefix}_readsQC.info + echo -e "\n(1) B. Bushnell: BBTools software package, http://bbtools.jgi.doe.gov/" >> \ + ~{prefix}_readsQC.info + >>> + + output { + File rqc_info = "~{prefix}_readsQC.info" + } + + runtime { + memory: "1 GiB" + cpu: 1 + maxRetries: 1 + docker: container + } +} + +task finish_rqc { + input { + # File read + File filtered_stats + File filtered_stats2 + File filtered + String container + String prefix + # String start + } + + command<<< + + set -euo pipefail + end=`date --iso-8601=seconds` + # Generate QA objects + ln -s ~{filtered} ~{prefix}_filtered.fastq.gz + ln -s ~{filtered_stats} ~{prefix}_filterStats.txt + ln -s ~{filtered_stats2} ~{prefix}_filterStats2.txt + + # Generate stats but rename some fields untilt the script is + # fixed. + /scripts/rqcstats.py ~{filtered_stats} > stats.json + cp stats.json ~{prefix}_qa_stats.json + + >>> + + output { + File filtered_final = "~{prefix}_filtered.fastq.gz" + File filtered_stats_final = "~{prefix}_filterStats.txt" + File filtered_stats2_final = "~{prefix}_filterStats2.txt" + } + + runtime { + docker: container + memory: "1 GiB" + cpu: 1 + } +} \ No newline at end of file From 9e6f988f198d7ea7477defdec0a54c9fe19c0a6b Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Fri, 29 May 2026 09:02:00 -0700 Subject: [PATCH 5/9] . --- interleave_rqcfilter.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interleave_rqcfilter.wdl b/interleave_rqcfilter.wdl index 4f429f7..683453f 100644 --- a/interleave_rqcfilter.wdl +++ b/interleave_rqcfilter.wdl @@ -260,4 +260,4 @@ task finish_rqc { memory: "1 GiB" cpu: 1 } -} \ No newline at end of file +} From 131ef5b1da83c78de7ff2c4fe92b161f0d9ea72d Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Fri, 29 May 2026 13:23:10 -0700 Subject: [PATCH 6/9] revert sra header detection. use whether there are accessions instead --- rqcfilter.wdl | 1 + shortReadsqc.wdl | 24 ++---------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/rqcfilter.wdl b/rqcfilter.wdl index bb4b1c9..9655083 100644 --- a/rqcfilter.wdl +++ b/rqcfilter.wdl @@ -43,6 +43,7 @@ workflow rqcfilter{ interleaved = is_interleaved, proj = proj, chastityfilter_flag = if (has_accessions) then false else chastityfilter_flag + filterbytile_flag = if (has_accessions) then false else filterbytile_flag } } diff --git a/shortReadsqc.wdl b/shortReadsqc.wdl index 21f2766..61ffa73 100644 --- a/shortReadsqc.wdl +++ b/shortReadsqc.wdl @@ -15,6 +15,7 @@ workflow ShortReadsQC { String database="/refdata/" Int rqc_mem = 180 Boolean? chastityfilter_flag + Boolean? filterbytile_flag } if (interleaved) { @@ -44,7 +45,7 @@ workflow ShortReadsQC { memory = rqc_mem, container = bbtools_container, chastityfilter_flag = chastityfilter_flag, - filterbytile_flag = !select_first([stage_single.is_external_sra, stage_interleave.is_external_sra, false]) + filterbytile_flag = filterbytile_flag } call stats_jsons { @@ -104,20 +105,11 @@ task stage_single { # Capture the start time date --iso-8601=seconds > start.txt - # Check if reads are SRA format - first_line=$(zcat ~{target} 2>/dev/null | head -1 || head -1 ~{target}) - if echo "$first_line" | grep -qE '^@(SRR|ERR|DRR)[0-9]+(\.[0-9]+)?'; then - echo "true" > is_external_sra.txt - else - echo "false" > is_external_sra.txt - fi - >>> output{ File reads_fastq = "~{target}" String start = read_string("start.txt") - Boolean is_external_sra = read_boolean("is_external_sra.txt") } runtime { @@ -168,23 +160,11 @@ task stage_interleave { # Validate that the read1 and read2 files are sorted correctly reformat.sh -Xmx~{memory}G verifypaired=t in=~{output_interleaved} - # Capture the start time - date --iso-8601=seconds > start.txt - - # Check if reads are SRA format - first_line=$(zcat ~{target_reads_1} 2>/dev/null | head -1 || head -1 ~{target_reads_1}) - if echo "$first_line" | grep -qE '^@(SRR|ERR|DRR)[0-9]+(\.[0-9]+)?'; then - echo "true" > is_external_sra.txt - else - echo "false" > is_external_sra.txt - fi - >>> output{ File reads_fastq = "~{output_interleaved}" String start = read_string("start.txt") - Boolean is_external_sra = read_boolean("is_external_sra.txt") } runtime { From 6214cedd267db7fbdf5259c037aac4f9500db6e2 Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Fri, 29 May 2026 13:53:47 -0700 Subject: [PATCH 7/9] . --- docs/index.rst | 2 +- rqcfilter.wdl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 099e00e..b1b5559 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,7 +39,7 @@ The following parameters are used for :literal:`rqcfilter2` in this workflow:: - removemouse=true : Remove mouse reads via mapping. - barcodefilter=false : Disable improper barcodes filter - chastityfilter=false: Remove illumina reads failing chastity filter. - - filterbytile=false : Disable filtering by Illumina tile information; automatically set to false when SRA-format reads are detected (reads with headers matching @SRR/ERR/DRR accessions). + - filterbytile=false : Remove reads from low-quality parts of the flowcell. - trimfragadapter=true: Trim all known Illumina adapter sequences, including TruSeq and Nextera. - removemicrobes=true : Remove common contaminant microbial reads via mapping, and place them in a separate file. diff --git a/rqcfilter.wdl b/rqcfilter.wdl index 9655083..b5b2b37 100644 --- a/rqcfilter.wdl +++ b/rqcfilter.wdl @@ -14,6 +14,7 @@ workflow rqcfilter{ Boolean interleaved Boolean shortRead Boolean? chastityfilter_flag + Boolean? filterbytile_flag } Boolean has_accessions = defined(accessions) && length(select_first([accessions, []])) > 0 @@ -42,7 +43,7 @@ workflow rqcfilter{ input_fq2 = if is_interleaved then [] else select_first([sra2fastq.output_fq2, input_fq2]), interleaved = is_interleaved, proj = proj, - chastityfilter_flag = if (has_accessions) then false else chastityfilter_flag + chastityfilter_flag = if (has_accessions) then false else chastityfilter_flag, filterbytile_flag = if (has_accessions) then false else filterbytile_flag } } From 5e75edff979966d8f721c9a9dc3e99cdb423705c Mon Sep 17 00:00:00 2001 From: Samantha Obermiller Date: Fri, 29 May 2026 15:46:51 -0700 Subject: [PATCH 8/9] accidental deletion --- shortReadsqc.wdl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shortReadsqc.wdl b/shortReadsqc.wdl index 61ffa73..4072cc4 100644 --- a/shortReadsqc.wdl +++ b/shortReadsqc.wdl @@ -160,6 +160,9 @@ task stage_interleave { # Validate that the read1 and read2 files are sorted correctly reformat.sh -Xmx~{memory}G verifypaired=t in=~{output_interleaved} + # Capture the start time + date --iso-8601=seconds > start.txt + >>> output{ From 7d82d13e81abd6408ac170d1ee91980f2bc3a4e2 Mon Sep 17 00:00:00 2001 From: samobermiller <105182636+samobermiller@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:48:34 -0700 Subject: [PATCH 9/9] Bump version from v1.0.24 to v1.0.25 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 1727484..46904da 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v1.0.24 +v1.0.25