use detailed profiles in tests/nf-test.config (#7207)

* use detailed profiles in tests/nf-test.config * fix fastqc memory_in_mb calculation * fix docker fixOwnership * docker.userEmulation false * docker.userEmulation false * update profiles in tests/config/nextflow.config * try security-opt=no-new-privileges * try security-opt=no-new-privileges * fix fastq_fastqc_umitools_fastp subworkflow * revert to old config for pytests
nf-core · Dec 13, 2024 · 0810805 · 0810805
1 parent b5828f4
commit 0810805
Show file tree

Hide file tree

Showing 15 changed files with 968 additions and 921 deletions.
diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml
@@ -74,14 +74,14 @@ jobs:
 
   nf-test-gpu:
     runs-on: "gpu"
-    name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}"
+    name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }}"
     needs: nf-test-changes
     if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:
       fail-fast: false
       matrix:
         shard: [1, 2]
-        profile: [docker_self_hosted, singularity] # conda?
+        profile: [docker_self_hosted, singularity]
     env:
       NXF_ANSI_LOG: false
       TOTAL_SHARDS: 2

diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml
@@ -76,7 +76,7 @@ jobs:
           echo ${{ steps.outputs.outputs.subworkflows }}
   nf-test:
     runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
-    name: "${{ matrix.profile }} | ${{ matrix.shard }}"
+    name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}"
     needs: nf-test-changes
     if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:

diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf
@@ -1,5 +1,5 @@
 process FASTQC {
-    tag "$meta.id"
+    tag "${meta.id}"
     label 'process_medium'
 
     conda "${moduleDir}/environment.yml"
@@ -19,30 +19,30 @@ process FASTQC {
     task.ext.when == null || task.ext.when
 
     script:
-    def args = task.ext.args ?: ''
-    def prefix = task.ext.prefix ?: "${meta.id}"
+    def args          = task.ext.args ?: ''
+    def prefix        = task.ext.prefix ?: "${meta.id}"
     // Make list of old name and new name pairs to use for renaming in the bash while loop
     def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] }
-    def rename_to = old_new_pairs*.join(' ').join(' ')
+    def rename_to     = old_new_pairs*.join(' ').join(' ')
     def renamed_files = old_new_pairs.collect{ _old_name, new_name -> new_name }.join(' ')
 
     // The total amount of allocated RAM by FastQC is equal to the number of threads defined (--threads) time the amount of RAM defined (--memory)
     // https://github.com/s-andrews/FastQC/blob/1faeea0412093224d7f6a07f777fad60a5650795/fastqc#L211-L222
     // Dividing the task.memory by task.cpu allows to stick to requested amount of RAM in the label
-    def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') / task.cpus
+    def memory_in_mb = task.memory ? task.memory.toUnit('MB').toFloat() / task.cpus : null
     // FastQC memory value allowed range (100 - 10000)
     def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb)
 
     """
-    printf "%s %s\\n" $rename_to | while read old_name new_name; do
+    printf "%s %s\\n" ${rename_to} | while read old_name new_name; do
         [ -f "\${new_name}" ] || ln -s \$old_name \$new_name
     done
 
     fastqc \\
-        $args \\
-        --threads $task.cpus \\
-        --memory $fastqc_memory \\
-        $renamed_files
+        ${args} \\
+        --threads ${task.cpus} \\
+        --memory ${fastqc_memory} \\
+        ${renamed_files}
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

diff --git a/modules/nf-core/fastqc/tests/tags.yml b/modules/nf-core/fastqc/tests/tags.yml
diff --git a/modules/nf-core/parabricks/applybqsr/main.nf b/modules/nf-core/parabricks/applybqsr/main.nf
@@ -1,5 +1,5 @@
 process PARABRICKS_APPLYBQSR {
-    tag "$meta.id"
+    tag "${meta.id}"
     label 'process_high'
     label 'process_gpu'
     stageInMode 'copy'
@@ -26,21 +26,21 @@ process PARABRICKS_APPLYBQSR {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    def args = task.ext.args ?: ''
-    def prefix = task.ext.prefix ?: "${meta.id}"
+    def args             = task.ext.args ?: ''
+    def prefix           = task.ext.prefix ?: "${meta.id}"
     def interval_command = intervals ? intervals.collect{"--interval-file $it"}.join(' ') : ""
-    def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : ''
+    def num_gpus         = task.accelerator ? "--num-gpus $task.accelerator.request" : ''
     """
     pbrun \\
         applybqsr \\
-        --ref $fasta \\
-        --in-bam $bam \\
-        --in-recal-file $bqsr_table \\
-        $interval_command \\
+        --ref ${fasta} \\
+        --in-bam ${bam} \\
+        --in-recal-file ${bqsr_table} \\
+        ${interval_command} \\
         --out-bam ${prefix}.bam \\
-        --num-threads $task.cpus \\
-        $num_gpus \\
-        $args
+        --num-threads ${task.cpus} \\
+        ${num_gpus} \\
+        ${args}
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

diff --git a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test
@@ -10,7 +10,7 @@ nextflow_process {
     tag "parabricks/applybqsr"
     tag "gpu"
 
-    test("sarscov2 - bam - pe") {
+    test("sarscov2 | paired-end | [bam]") {
 
         when {
             process {
@@ -50,7 +50,7 @@ nextflow_process {
 
     }
 
-    test("sarscov2 - bam - pe - intervals") {
+    test("sarscov2 | paired-end | [bam] | intervals") {
 
         when {
             process {
@@ -91,7 +91,7 @@ nextflow_process {
         }
     }
 
-    test("human - cram - pe") {
+    test("homo_sapiens | paired-end | [cram]") {
 
         when {
             process {
@@ -131,7 +131,7 @@ nextflow_process {
 
     }
 
-    test("human - cram - pe - intervals - stub") {
+    test("homo_sapiens | paired-end | [cram] | intervals - stub") {
 
         options "-stub"
 
@@ -171,7 +171,7 @@ nextflow_process {
 
     }
 
-    test("sarscov2 - bam - pe - intervals - stub") {
+    test("sarscov2 | paired-end | [bam] | intervals - stub") {
 
         options "-stub"
 

diff --git a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test.snap b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test.snap
@@ -1,5 +1,5 @@
 {
-    "human - cram - pe": {
+    "homo_sapiens | paired-end | [cram]": {
         "content": [
             "2f11e4fe3390b8ad0a1852616fd1da04",
             "test.bam.bai",
@@ -13,7 +13,7 @@
         },
         "timestamp": "2024-12-10T08:44:34.058228768"
     },
-    "sarscov2 - bam - pe": {
+    "sarscov2 | paired-end | [bam]": {
         "content": [
             "894549ee3ced6b5ca2eed2563a985217",
             "test.bam.bai",
@@ -27,7 +27,7 @@
         },
         "timestamp": "2024-12-10T08:44:17.204736211"
     },
-    "sarscov2 - bam - pe - intervals": {
+    "sarscov2 | paired-end | [bam] | intervals": {
         "content": [
             "b194021b699cc5cf4b2b1f715e0b7b4c",
             "test.bam.bai",
@@ -41,7 +41,7 @@
         },
         "timestamp": "2024-12-10T08:44:25.661590103"
     },
-    "sarscov2 - bam - pe - intervals - stub": {
+    "sarscov2 | paired-end | [bam] | intervals - stub": {
         "content": [
             {
                 "0": [
@@ -90,7 +90,7 @@
         },
         "timestamp": "2024-12-10T08:44:47.700907318"
     },
-    "human - cram - pe - intervals - stub": {
+    "homo_sapiens | paired-end | [cram] | intervals - stub": {
         "content": [
             {
                 "0": [
@@ -139,4 +139,4 @@
         },
         "timestamp": "2024-12-10T08:44:41.08456539"
     }
-}
+}
diff --git a/subworkflows/nf-core/fastq_fastqc_umitools_fastp/main.nf b/subworkflows/nf-core/fastq_fastqc_umitools_fastp/main.nf
@@ -1,7 +1,6 @@
 //
 // Read QC, UMI extraction and trimming
 //
-
 include { FASTQC as FASTQC_RAW  } from '../../../modules/nf-core/fastqc/main'
 include { FASTQC as FASTQC_TRIM } from '../../../modules/nf-core/fastqc/main'
 include { UMITOOLS_EXTRACT      } from '../../../modules/nf-core/umitools/extract/main'
@@ -10,26 +9,31 @@ include { FASTP                 } from '../../../modules/nf-core/fastp/main'
 //
 // Function that parses fastp json output file to get total number of reads after trimming
 //
-import groovy.json.JsonSlurper
 
 def getFastpReadsAfterFiltering(json_file, min_num_reads) {
 
     if ( workflow.stubRun ) { return min_num_reads }
 
-    def Map json = (Map) new JsonSlurper().parseText(json_file.text).get('summary')
+    def json = new groovy.json.JsonSlurper().parseText(json_file.text).get('summary')
     return json['after_filtering']['total_reads'].toLong()
 }
 
-def getFastpAdapterSequence(json_file){
-
-    if ( workflow.stubRun ) { return "" }
+def getFastpAdapterSequence(json_file) {
+    // Handle stub runs
+    if (workflow.stubRun) {
+        return ""
+    }
 
-    def Map json = (Map) new JsonSlurper().parseText(json_file.text)
-    try{
-        adapter = json['adapter_cutting']['read1_adapter_sequence']
-    } catch(Exception ex){
-        adapter = ""
+    // Ensure the input file is valid and parse the JSON
+    def adapter = ""
+    try {
+        def json = new groovy.json.JsonSlurper().parseText(json_file.text)
+        adapter = json?.adapter_cutting?.read1_adapter_sequence ?: ""
+    } catch (Exception ex) {
+        // Log the exception or handle it as needed
+        println "Error parsing JSON or retrieving adapter sequence: ${ex.message}"
     }
+
     return adapter
 }
 
@@ -47,9 +51,20 @@ workflow FASTQ_FASTQC_UMITOOLS_FASTP {
     min_trimmed_reads // integer: > 0
 
     main:
-    ch_versions = Channel.empty()
-    fastqc_raw_html = Channel.empty()
-    fastqc_raw_zip  = Channel.empty()
+    ch_versions       = Channel.empty()
+    fastqc_raw_html   = Channel.empty()
+    fastqc_raw_zip    = Channel.empty()
+    umi_log           = Channel.empty()
+    trim_json         = Channel.empty()
+    trim_html         = Channel.empty()
+    trim_log          = Channel.empty()
+    trim_reads_fail   = Channel.empty()
+    trim_reads_merged = Channel.empty()
+    fastqc_trim_html  = Channel.empty()
+    fastqc_trim_zip   = Channel.empty()
+    trim_read_count   = Channel.empty()
+    adapter_seq       = Channel.empty()
+
     if (!skip_fastqc) {
         FASTQC_RAW (
             reads
@@ -60,7 +75,6 @@ workflow FASTQ_FASTQC_UMITOOLS_FASTP {
     }
 
     umi_reads = reads
-    umi_log   = Channel.empty()
     if (with_umi && !skip_umi_extract) {
         UMITOOLS_EXTRACT (
             reads
@@ -75,24 +89,14 @@ workflow FASTQ_FASTQC_UMITOOLS_FASTP {
                 .out
                 .reads
                 .map {
-                    meta, reads ->
-                        meta.single_end ? [ meta, reads ] : [ meta + [single_end: true], reads[umi_discard_read % 2] ]
+                    meta, _reads ->
+                        meta.single_end ? [ meta, _reads ] : [ meta + [single_end: true], _reads[umi_discard_read % 2] ]
                 }
                 .set { umi_reads }
         }
     }
 
-    trim_reads        = umi_reads
-    trim_json         = Channel.empty()
-    trim_html         = Channel.empty()
-    trim_log          = Channel.empty()
-    trim_reads_fail   = Channel.empty()
-    trim_reads_merged = Channel.empty()
-    fastqc_trim_html  = Channel.empty()
-    fastqc_trim_zip   = Channel.empty()
-    trim_read_count   = Channel.empty()
-    adapter_seq       = Channel.empty()
-
+    trim_reads = umi_reads
     if (!skip_trimming) {
         FASTP (
             umi_reads,
@@ -115,16 +119,16 @@ workflow FASTQ_FASTQC_UMITOOLS_FASTP {
             .out
             .reads
             .join(trim_json)
-            .map { meta, reads, json -> [ meta, reads, getFastpReadsAfterFiltering(json, min_trimmed_reads.toLong()) ] }
+            .map { meta, _reads, json -> [ meta, _reads, getFastpReadsAfterFiltering(json, min_trimmed_reads.toLong()) ] }
             .set { ch_num_trimmed_reads }
 
         ch_num_trimmed_reads
-            .filter { meta, reads, num_reads -> num_reads >= min_trimmed_reads.toLong() }
-            .map { meta, reads, num_reads -> [ meta, reads ] }
+            .filter { meta, _reads, num_reads -> num_reads >= min_trimmed_reads.toLong() }
+            .map { meta, _reads, num_reads -> [ meta, _reads ] }
             .set { trim_reads }
 
         ch_num_trimmed_reads
-            .map { meta, reads, num_reads -> [ meta, num_reads ] }
+            .map { meta, _reads, num_reads -> [ meta, num_reads ] }
             .set { trim_read_count }
 
         trim_json
@@ -143,22 +147,17 @@ workflow FASTQ_FASTQC_UMITOOLS_FASTP {
 
     emit:
     reads = trim_reads // channel: [ val(meta), [ reads ] ]
-
     fastqc_raw_html    // channel: [ val(meta), [ html ] ]
     fastqc_raw_zip     // channel: [ val(meta), [ zip ] ]
-
     umi_log            // channel: [ val(meta), [ log ] ]
     adapter_seq        // channel: [ val(meta), [ adapter_seq] ]
-
     trim_json          // channel: [ val(meta), [ json ] ]
     trim_html          // channel: [ val(meta), [ html ] ]
     trim_log           // channel: [ val(meta), [ log ] ]
     trim_reads_fail    // channel: [ val(meta), [ fastq.gz ] ]
     trim_reads_merged  // channel: [ val(meta), [ fastq.gz ] ]
     trim_read_count    // channel: [ val(meta), val(count) ]
-
     fastqc_trim_html   // channel: [ val(meta), [ html ] ]
     fastqc_trim_zip    // channel: [ val(meta), [ zip ] ]
-
     versions = ch_versions // channel: [ versions.yml ]
 }