From 6ba546c2f86dc205aca931b8bb3c92e85d548ada Mon Sep 17 00:00:00 2001 From: Daniel Schmitz Date: Mon, 18 Nov 2024 11:43:13 +0100 Subject: [PATCH 1/5] Added module trgt/merge (#7012) * Imported module merge from branch trgt * Removed dump channels Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> * Fixed indentation Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> * Updated tests to use params * Delete modules/nf-core/trgt/merge/tests/tags.yml don't need this yml anymore --------- Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> --- modules/nf-core/trgt/merge/environment.yml | 6 + modules/nf-core/trgt/merge/main.nf | 58 ++++++ modules/nf-core/trgt/merge/meta.yml | 80 ++++++++ modules/nf-core/trgt/merge/tests/main.nf.test | 193 ++++++++++++++++++ .../trgt/merge/tests/main.nf.test.snap | 80 ++++++++ .../nf-core/trgt/merge/tests/nextflow.config | 15 ++ 6 files changed, 432 insertions(+) create mode 100644 modules/nf-core/trgt/merge/environment.yml create mode 100644 modules/nf-core/trgt/merge/main.nf create mode 100644 modules/nf-core/trgt/merge/meta.yml create mode 100644 modules/nf-core/trgt/merge/tests/main.nf.test create mode 100644 modules/nf-core/trgt/merge/tests/main.nf.test.snap create mode 100644 modules/nf-core/trgt/merge/tests/nextflow.config diff --git a/modules/nf-core/trgt/merge/environment.yml b/modules/nf-core/trgt/merge/environment.yml new file mode 100644 index 00000000000..614dc872b7b --- /dev/null +++ b/modules/nf-core/trgt/merge/environment.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::trgt=1.2.0" diff --git a/modules/nf-core/trgt/merge/main.nf b/modules/nf-core/trgt/merge/main.nf new file mode 100644 index 00000000000..88eb69457a3 --- /dev/null +++ b/modules/nf-core/trgt/merge/main.nf @@ -0,0 +1,58 @@ +process TRGT_MERGE { + tag "$meta.id" + label 'process_medium' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/trgt:1.2.0--h9ee0642_0': + 'biocontainers/trgt:1.2.0--h9ee0642_0' }" + + input: + tuple val(meta) , path(vcfs), path(tbis) + tuple val(meta2), path(fasta) // optional + tuple val(meta3), path(fai) // optional + + output: + tuple val(meta), path("*.{vcf,vcf.gz,bcf,bcf.gz}"), emit: vcf + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def extension = args.contains("--output-type b") || args.contains("-Ob") ? "bcf.gz" : + args.contains("--output-type u") || args.contains("-Ou") ? "bcf" : + args.contains("--output-type z") || args.contains("-Oz") ? "vcf.gz" : + args.contains("--output-type v") || args.contains("-Ov") ? "vcf" : + "vcf" + def output = args.contains("--output ") || args.contains("--output=") || args.contains("-o ") ? "" : "--output ${prefix}.${extension}" + def reference = fasta ? "--genome ${fasta}" : "" + + """ + trgt merge \\ + $args \\ + $reference \\ + $output \\ + --vcf ${vcfs} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + trgt: \$(trgt --version |& sed '1!d ; s/trgt //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + touch ${prefix}.vcf + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + trgt: \$(trgt --version |& sed '1!d ; s/trgt //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/trgt/merge/meta.yml b/modules/nf-core/trgt/merge/meta.yml new file mode 100644 index 00000000000..e8b79cb2da7 --- /dev/null +++ b/modules/nf-core/trgt/merge/meta.yml @@ -0,0 +1,80 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "trgt_merge" +description: Merge TRGT VCFs from multiple samples +keywords: + - trgt + - repeat expansion + - pacbio + - genomics +tools: + - "trgt": + description: "Tandem repeat genotyping and visualization from PacBio HiFi data" + homepage: "https://github.com/PacificBiosciences/trgt" + documentation: "https://github.com/PacificBiosciences/trgt/blob/main/docs/tutorial.md" + tool_dev_url: "https://github.com/PacificBiosciences/trgt" + doi: "10.1038/s41587-023-02057-3" + licence: + ["Pacific Biosciences Software License (https://github.com/PacificBiosciences/trgt/blob/main/LICENSE.md)"] + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - vcfs: + type: file + description: | + List containing VCF files from TRGT + Must contain at least 2 elements unless `--force-single` is given + Samples in each VCf must be pairwise disjoint + - tbis: + type: file + description: | + List containing indexes of VCF files from TRGT + Must contain at least 2 elements unless `--force-single` is given + - - meta2: + type: map + description: | + Groovy map containing reference information + e.g. `[ id: 'genome' ]` + - fasta: + type: file + description: | + FASTA reference file (optional) + Required if VCFs were generated with TRGT pre 1.0 + pattern: "*.{fasta,fa,fna}" + - - meta3: + type: map + description: | + Groovy map containing reference information + e.g. `[ id: 'genome' ]` + - fai: + type: file + description: | + Index for FASTA file (optional) + Required if VCFs were generated with TRGT pre 1.0 + pattern: "*.fai" + +output: + - vcf: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.{vcf,vcf.gz,bcf,bcf.gz}": + type: file + description: "Merged output file" + pattern: "*.{vcf,vcf.gz,bcf,bcf.gz}" + - versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@Schmytzi" +maintainers: + - "@Schmytzi" diff --git a/modules/nf-core/trgt/merge/tests/main.nf.test b/modules/nf-core/trgt/merge/tests/main.nf.test new file mode 100644 index 00000000000..dfccd4f9ec2 --- /dev/null +++ b/modules/nf-core/trgt/merge/tests/main.nf.test @@ -0,0 +1,193 @@ +// nf-core modules test trgt/merge +nextflow_process { + + name "Test Process TRGT_MERGE" + script "../main.nf" + process "TRGT_MERGE" + config "./nextflow.config" + + tag "modules" + tag "modules_nfcore" + tag "trgt" + tag "trgt/merge" + tag "trgt/genotype" + tag "samtools/faidx" + tag "samtools/sort" + tag "samtools/index" + tag "bcftools/sort" + tag "gunzip" + + setup { + run("GUNZIP"){ + script "../../../gunzip/main.nf" + process { + """ + input[0] = [ + [ id : 'chr22' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr22_chr22_KI270734v1_random/sequence/genome.fa.gz', checkIfExists: true) + ] + """ + } + } + run("SAMTOOLS_FAIDX"){ + script "../../../samtools/faidx/main.nf" + process { + """ + input[0] = GUNZIP.out.gunzip + input[1] = [[],[]] + """ + } + } + run("TRGT_GENOTYPE"){ + script "../../genotype/main.nf" + process { + """ + input[0] = Channel.fromList([ + tuple( + [ id:'test1' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam.bai', checkIfExists: true), + 'XX' + ), + tuple( + [ id:'test2' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam.bai', checkIfExists: true), + 'XX' + ) + ]) + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + .first() + """ + } + } + run("BCFTOOLS_SORT"){ + script "../../../bcftools/sort/main.nf" + process { + """ + input[0] = TRGT_GENOTYPE.out.vcf + """ + } + } + } + test("homo sapiens - 2 VCFs") { + when { + process { + """ + input[0] = BCFTOOLS_SORT.out.vcf + .join(BCFTOOLS_SORT.out.tbi) + .map { meta, vcf, tbi -> [ [ id : 'test' ], vcf, tbi ] } + .groupTuple() + input[1] = [[],[]] + input[2] = [[],[]] + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + path(process.out.vcf.get(0).get(1)).vcf.variantsMD5, + path(process.out.vcf.get(0).get(1)).vcf.summary, + path(process.out.vcf.get(0).get(1)).vcf.sampleCount, + process.out.versions + ).match() } + ) + } + } + test("homo sapiens - 2 VCFs - reference") { + when { + process { + """ + input[0] = BCFTOOLS_SORT.out.vcf + .join(BCFTOOLS_SORT.out.tbi) + .map { meta, vcf, tbi -> [ [ id : 'test' ], vcf, tbi ] } + .groupTuple() + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + path(process.out.vcf.get(0).get(1)).vcf.variantsMD5, + path(process.out.vcf.get(0).get(1)).vcf.summary, + path(process.out.vcf.get(0).get(1)).vcf.sampleCount, + process.out.versions + ).match() } + ) + } + } + test("homo sapiens - 1 VCF") { + when { + process { + """ + input[0] = BCFTOOLS_SORT.out.vcf + .join(BCFTOOLS_SORT.out.tbi) + .first() + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + """ + } + } + then { + assertAll( + { assert !process.success } + ) + } + } + test("homo sapiens - 1 VCF - --force-single") { + when { + params { + trgt_merge_args = "--force-single" + } + process { + """ + input[0] = BCFTOOLS_SORT.out.vcf + .join(BCFTOOLS_SORT.out.tbi) + .first() + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + path(process.out.vcf.get(0).get(1)).vcf.variantsMD5, + path(process.out.vcf.get(0).get(1)).vcf.summary, + path(process.out.vcf.get(0).get(1)).vcf.sampleCount, + process.out.versions + ).match() } + ) + } + } + test("homo sapiens - 2 VCFs - stub") { + options "-stub" + when { + process { + """ + input[0] = BCFTOOLS_SORT.out.vcf + .join(BCFTOOLS_SORT.out.tbi) + .map { meta, vcf, tbi -> [ [ id : 'test' ], vcf, tbi ] } + .groupTuple() + input[1] = [[],[]] + input[2] = [[],[]] + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/nf-core/trgt/merge/tests/main.nf.test.snap b/modules/nf-core/trgt/merge/tests/main.nf.test.snap new file mode 100644 index 00000000000..73f306b0ebf --- /dev/null +++ b/modules/nf-core/trgt/merge/tests/main.nf.test.snap @@ -0,0 +1,80 @@ +{ + "homo sapiens - 2 VCFs - reference": { + "content": [ + "75acbf6205a44ca44d6fc644dda57d82", + "VcfFile [chromosomes=[chr22], sampleCount=2, variantCount=1, phased=false, phasedAutodetect=false]", + 2, + [ + "versions.yml:md5,ac3a6dca80725cee1bf724975e54c067" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T14:52:26.329527707" + }, + "homo sapiens - 2 VCFs": { + "content": [ + "75acbf6205a44ca44d6fc644dda57d82", + "VcfFile [chromosomes=[chr22], sampleCount=2, variantCount=1, phased=false, phasedAutodetect=false]", + 2, + [ + "versions.yml:md5,ac3a6dca80725cee1bf724975e54c067" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T14:39:54.475134386" + }, + "homo sapiens - 2 VCFs - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,ac3a6dca80725cee1bf724975e54c067" + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,ac3a6dca80725cee1bf724975e54c067" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T15:06:23.198648143" + }, + "homo sapiens - 1 VCF - --force-single": { + "content": [ + "6e09773892c37f17e01c9771e8ef2171", + "VcfFile [chromosomes=[chr22], sampleCount=1, variantCount=1, phased=false, phasedAutodetect=false]", + 1, + [ + "versions.yml:md5,ac3a6dca80725cee1bf724975e54c067" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T15:01:59.765895584" + } +} \ No newline at end of file diff --git a/modules/nf-core/trgt/merge/tests/nextflow.config b/modules/nf-core/trgt/merge/tests/nextflow.config new file mode 100644 index 00000000000..b5caec9f71a --- /dev/null +++ b/modules/nf-core/trgt/merge/tests/nextflow.config @@ -0,0 +1,15 @@ +params { + trgt_merge_args = "" +} +process { + withName: "BCFTOOLS_SORT" { + ext.args = "--write-index=tbi --output-type z" + ext.prefix = { "sorted_${meta.id}" } + } + withName: "TRGT_GENOTYPE" { + ext.args = { "--sample-name ${meta.id}"} + } + withName: "TRGT_MERGE" { + ext.args = params.trgt_merge_args + } +} From c3fe6e90faa4c02f6ce451e7418a87b883ff4ea1 Mon Sep 17 00:00:00 2001 From: Daniel Schmitz Date: Mon, 18 Nov 2024 14:18:02 +0100 Subject: [PATCH 2/5] Added module trgt/plot (#7003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Imported module from branch trgt * Changed test snapshots because plot is unstable between environments --------- Co-authored-by: Matthias Hörtenhuber --- modules/nf-core/trgt/plot/environment.yml | 6 + modules/nf-core/trgt/plot/main.nf | 56 +++++++ modules/nf-core/trgt/plot/meta.yml | 94 ++++++++++++ modules/nf-core/trgt/plot/tests/main.nf.test | 142 ++++++++++++++++++ .../nf-core/trgt/plot/tests/main.nf.test.snap | 28 ++++ .../nf-core/trgt/plot/tests/nextflow.config | 6 + 6 files changed, 332 insertions(+) create mode 100644 modules/nf-core/trgt/plot/environment.yml create mode 100644 modules/nf-core/trgt/plot/main.nf create mode 100644 modules/nf-core/trgt/plot/meta.yml create mode 100644 modules/nf-core/trgt/plot/tests/main.nf.test create mode 100644 modules/nf-core/trgt/plot/tests/main.nf.test.snap create mode 100644 modules/nf-core/trgt/plot/tests/nextflow.config diff --git a/modules/nf-core/trgt/plot/environment.yml b/modules/nf-core/trgt/plot/environment.yml new file mode 100644 index 00000000000..614dc872b7b --- /dev/null +++ b/modules/nf-core/trgt/plot/environment.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::trgt=1.2.0" diff --git a/modules/nf-core/trgt/plot/main.nf b/modules/nf-core/trgt/plot/main.nf new file mode 100644 index 00000000000..f60fd8a150c --- /dev/null +++ b/modules/nf-core/trgt/plot/main.nf @@ -0,0 +1,56 @@ +process TRGT_PLOT { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/trgt:1.2.0--h9ee0642_0': + 'biocontainers/trgt:1.2.0--h9ee0642_0' }" + + input: + tuple val(meta) , path(bam), path(bai), path(vcf), path(tbi), val(repeat_id) + tuple val(meta2), path(fasta) + tuple val(meta3), path(fai) + tuple val(meta4), path(repeats) + + output: + tuple val(meta), path("*.{png,pdf,svg}"), emit: plot + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}_${repeat_id}" + // If user didn't specify an output path, plot to PNG + def output_arg = args.contains("--image") || args.contains("-o") ? "" : "--image ${prefix}.png" + + """ + trgt plot \\ + $args \\ + --genome ${fasta} \\ + --repeats ${repeats} \\ + --spanning-reads ${bam} \\ + --vcf ${vcf} \\ + --repeat-id ${repeat_id} \\ + $output_arg + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + trgt: \$(trgt --version |& sed '1!d ; s/trgt //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.png + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + trgt: \$(trgt --version |& sed '1!d ; s/trgt //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/trgt/plot/meta.yml b/modules/nf-core/trgt/plot/meta.yml new file mode 100644 index 00000000000..b1052d8c742 --- /dev/null +++ b/modules/nf-core/trgt/plot/meta.yml @@ -0,0 +1,94 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "trgt_plot" +description: Visualize tandem repeats genotyped by TRGT +keywords: + - trgt + - repeat expansion + - plotting + - pacbio + - genomics +tools: + - "trgt": + description: "Tandem repeat genotyping and visualization from PacBio HiFi data" + homepage: "https://github.com/PacificBiosciences/trgt" + documentation: "https://github.com/PacificBiosciences/trgt/blob/main/docs/tutorial.md" + tool_dev_url: "https://github.com/PacificBiosciences/trgt" + doi: "10.1038/s41587-023-02057-3" + licence: + ["Pacific Biosciences Software License (https://github.com/PacificBiosciences/trgt/blob/main/LICENSE.md)"] + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - bam: + type: file + description: "Sorted reads spanning tandem repeat from TRGT output" + pattern: "*.bam" + - bai: + type: file + description: "Index for reads" + pattern: "*.bai" + - vcf: + type: file + description: "Sorted tandem repeat genotypes called by TRGT" + pattern: "*.vcf.gz" + - tbi: + type: file + description: "Index for genotypes" + pattern: "*.{tbi,csi}" + - repeat_id: + type: string + description: "ID of tandem repeat to plot" + - - meta2: + type: map + description: | + Groovy map containing reference information + e.g. `[ id: 'genome' ]` + - fasta: + type: file + description: "FASTA reference file" + pattern: "*.{fasta,fa,fna}" + - - meta3: + type: map + description: | + Groovy map containing reference information + e.g. `[ id: 'genome' ]` + - fai: + type: file + description: "Index for FASTA file" + pattern: "*.fai" + - - meta4: + type: map + description: | + Groovy map containing repeat information + e.g. `[ id: 'repeats' ]` + - repeats: + type: file + description: "BED file with repeat coordinates and structure" + pattern: "*.bed" + +output: + - plot: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.{png,pdf,svg}": + type: file + description: "Plot of region and reads spanning tandem repeat" + pattern: "*.{png,pdf,svg}" + - versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@Schmytzi" +maintainers: + - "@Schmytzi" diff --git a/modules/nf-core/trgt/plot/tests/main.nf.test b/modules/nf-core/trgt/plot/tests/main.nf.test new file mode 100644 index 00000000000..dcf871df9f8 --- /dev/null +++ b/modules/nf-core/trgt/plot/tests/main.nf.test @@ -0,0 +1,142 @@ +// nf-core modules test trgt/plot +nextflow_process { + + name "Test Process TRGT_PLOT" + script "../main.nf" + process "TRGT_PLOT" + config "./nextflow.config" + + tag "modules" + tag "modules_nfcore" + tag "trgt" + tag "trgt/plot" + tag "trgt/genotype" + tag "samtools/faidx" + tag "samtools/sort" + tag "samtools/index" + tag "bcftools/sort" + tag "gunzip" + + setup { + run("GUNZIP"){ + script "../../../gunzip/main.nf" + process { + """ + input[0] = [ + [ id : 'chr22' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr22_chr22_KI270734v1_random/sequence/genome.fa.gz', checkIfExists: true) + ] + """ + } + } + run("SAMTOOLS_FAIDX"){ + script "../../../samtools/faidx/main.nf" + process { + """ + input[0] = GUNZIP.out.gunzip + input[1] = [[],[]] + """ + } + } + run("TRGT_GENOTYPE"){ + script "../../genotype/main.nf" + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam.bai', checkIfExists: true), + 'XX' + ] + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + """ + } + } + run("SAMTOOLS_SORT"){ + script "../../../samtools/sort/main.nf" + process { + """ + input[0] = TRGT_GENOTYPE.out.bam + input[1] = [[],[]] + """ + } + } + run("SAMTOOLS_INDEX"){ + script "../../../samtools/index/main.nf" + process { + """ + input[0] = SAMTOOLS_SORT.out.bam + """ + } + } + run("BCFTOOLS_SORT"){ + script "../../../bcftools/sort/main.nf" + process { + """ + input[0] = TRGT_GENOTYPE.out.vcf + """ + } + } + } + + test("homo sapiens - [bam, bai, vcf, TEST], fasta, fai, bed") { + when { + process { + """ + input[0] = SAMTOOLS_SORT.out.bam + .join(SAMTOOLS_INDEX.out.bai) + .join(BCFTOOLS_SORT.out.vcf) + .join(BCFTOOLS_SORT.out.tbi) + .map { tuple -> tuple + 'TEST' } + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + file(process.out.plot.get(0).get(1)).name, //unstable + process.out.versions + ).match() } + ) + } + } + + test("homo sapiens - [bam, bai, vcf, TEST], fasta, fai, bed - stub") { + options "-stub" + when { + process { + """ + input[0] = SAMTOOLS_SORT.out.bam + .join(SAMTOOLS_INDEX.out.bai) + .join(BCFTOOLS_SORT.out.vcf) + .join(BCFTOOLS_SORT.out.tbi) + .map { tuple -> tuple + 'TEST' } + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + file(process.out.plot.get(0).get(1)).name, //unstable + process.out.versions + ).match() } + ) + } + } +} diff --git a/modules/nf-core/trgt/plot/tests/main.nf.test.snap b/modules/nf-core/trgt/plot/tests/main.nf.test.snap new file mode 100644 index 00000000000..e8614fde091 --- /dev/null +++ b/modules/nf-core/trgt/plot/tests/main.nf.test.snap @@ -0,0 +1,28 @@ +{ + "homo sapiens - [bam, bai, vcf, TEST], fasta, fai, bed - stub": { + "content": [ + "test.png", + [ + "versions.yml:md5,24cc939a0b88728bae8a2102feaf542c" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-18T13:58:28.083155724" + }, + "homo sapiens - [bam, bai, vcf, TEST], fasta, fai, bed": { + "content": [ + "test_TEST.png", + [ + "versions.yml:md5,24cc939a0b88728bae8a2102feaf542c" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-18T13:57:59.291774232" + } +} \ No newline at end of file diff --git a/modules/nf-core/trgt/plot/tests/nextflow.config b/modules/nf-core/trgt/plot/tests/nextflow.config new file mode 100644 index 00000000000..7b9314db3e7 --- /dev/null +++ b/modules/nf-core/trgt/plot/tests/nextflow.config @@ -0,0 +1,6 @@ +process { + withName: "BCFTOOLS_SORT" { + ext.args = "--write-index=tbi --output-type z" + ext.prefix = "sorted_" + } +} From af84433c2e619cf28e026d4140b1b8763ae3690b Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Mon, 18 Nov 2024 19:38:51 +0530 Subject: [PATCH 3/5] improve workflow logic for `fastq_align_dedup_bismark` (#7005) * improve workflow logic * add paired-end tests * update meta.yml * minor formatting fix * apply suggestions from code review --- .../nf-core/fastq_align_dedup_bismark/main.nf | 127 +++--- .../fastq_align_dedup_bismark/meta.yml | 10 +- .../tests/main.nf.test | 148 ++++++- .../tests/main.nf.test.snap | 394 +++++++++++++----- .../tests/nextflow.config | 2 +- 5 files changed, 491 insertions(+), 190 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/main.nf b/subworkflows/nf-core/fastq_align_dedup_bismark/main.nf index 1497e5adda8..e5f2ffe5af2 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bismark/main.nf +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/main.nf @@ -1,11 +1,11 @@ -include { BISMARK_ALIGN } from '../../../modules/nf-core/bismark/align/main' -include { BISMARK_DEDUPLICATE } from '../../../modules/nf-core/bismark/deduplicate/main' -include { BISMARK_METHYLATIONEXTRACTOR } from '../../../modules/nf-core/bismark/methylationextractor/main' -include { BISMARK_COVERAGE2CYTOSINE } from '../../../modules/nf-core/bismark/coverage2cytosine/main' -include { BISMARK_REPORT } from '../../../modules/nf-core/bismark/report/main' -include { BISMARK_SUMMARY } from '../../../modules/nf-core/bismark/summary/main' -include { SAMTOOLS_SORT as SAMTOOLS_SORT_DEDUPLICATED } from '../../../modules/nf-core/samtools/sort/main' -include { SAMTOOLS_INDEX as SAMTOOLS_INDEX_DEDUPLICATED } from '../../../modules/nf-core/samtools/index/main' +include { BISMARK_ALIGN } from '../../../modules/nf-core/bismark/align/main' +include { BISMARK_DEDUPLICATE } from '../../../modules/nf-core/bismark/deduplicate/main' +include { SAMTOOLS_SORT } from '../../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX } from '../../../modules/nf-core/samtools/index/main' +include { BISMARK_METHYLATIONEXTRACTOR } from '../../../modules/nf-core/bismark/methylationextractor/main' +include { BISMARK_COVERAGE2CYTOSINE } from '../../../modules/nf-core/bismark/coverage2cytosine/main' +include { BISMARK_REPORT } from '../../../modules/nf-core/bismark/report/main' +include { BISMARK_SUMMARY } from '../../../modules/nf-core/bismark/summary/main' workflow FASTQ_ALIGN_DEDUP_BISMARK { @@ -17,18 +17,20 @@ workflow FASTQ_ALIGN_DEDUP_BISMARK { cytosine_report // boolean: whether the run coverage2cytosine main: - ch_versions = Channel.empty() - ch_coverage2cytosine_coverage = Channel.empty() - ch_coverage2cytosine_report = Channel.empty() - ch_coverage2cytosine_summary = Channel.empty() + ch_alignments = Channel.empty() + ch_alignment_reports = Channel.empty() ch_methylation_bedgraph = Channel.empty() ch_methylation_calls = Channel.empty() ch_methylation_coverage = Channel.empty() ch_methylation_report = Channel.empty() ch_methylation_mbias = Channel.empty() + ch_coverage2cytosine_coverage = Channel.empty() + ch_coverage2cytosine_report = Channel.empty() + ch_coverage2cytosine_summary = Channel.empty() ch_bismark_report = Channel.empty() ch_bismark_summary = Channel.empty() ch_multiqc_files = Channel.empty() + ch_versions = Channel.empty() /* * Align with bismark @@ -38,28 +40,44 @@ workflow FASTQ_ALIGN_DEDUP_BISMARK { ch_fasta, ch_bismark_index ) + ch_alignments = BISMARK_ALIGN.out.bam + ch_alignment_reports = BISMARK_ALIGN.out.report.map{ meta, report -> [ meta, report, [] ] } ch_versions = ch_versions.mix(BISMARK_ALIGN.out.versions) - if (skip_deduplication) { - alignments = BISMARK_ALIGN.out.bam - alignment_reports = BISMARK_ALIGN.out.report.map{ meta, report -> [ meta, report, [] ] } - } else { + if (!skip_deduplication) { /* * Run deduplicate_bismark */ BISMARK_DEDUPLICATE ( BISMARK_ALIGN.out.bam ) - alignments = BISMARK_DEDUPLICATE.out.bam - alignment_reports = BISMARK_ALIGN.out.report.join(BISMARK_DEDUPLICATE.out.report) - ch_versions = ch_versions.mix(BISMARK_DEDUPLICATE.out.versions) + ch_alignments = BISMARK_DEDUPLICATE.out.bam + ch_alignment_reports = BISMARK_ALIGN.out.report.join(BISMARK_DEDUPLICATE.out.report) + ch_versions = ch_versions.mix(BISMARK_DEDUPLICATE.out.versions) } + /* + * MODULE: Run samtools sort on aligned or deduplicated bam + */ + SAMTOOLS_SORT ( + ch_alignments, + [[:],[]] // [ [meta], [fasta]] + ) + ch_versions = ch_versions.mix(SAMTOOLS_SORT.out.versions) + + /* + * MODULE: Run samtools index on aligned or deduplicated bam + */ + SAMTOOLS_INDEX ( + SAMTOOLS_SORT.out.bam + ) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions) + /* * Run bismark_methylation_extractor */ BISMARK_METHYLATIONEXTRACTOR ( - alignments, + ch_alignments, ch_bismark_index ) ch_methylation_bedgraph = BISMARK_METHYLATIONEXTRACTOR.out.bedgraph @@ -74,7 +92,7 @@ workflow FASTQ_ALIGN_DEDUP_BISMARK { */ if (cytosine_report) { BISMARK_COVERAGE2CYTOSINE ( - BISMARK_METHYLATIONEXTRACTOR.out.coverage, + ch_methylation_coverage, ch_fasta, ch_bismark_index ) @@ -88,7 +106,7 @@ workflow FASTQ_ALIGN_DEDUP_BISMARK { * Generate bismark sample reports */ BISMARK_REPORT ( - alignment_reports + ch_alignment_reports .join(ch_methylation_report) .join(ch_methylation_mbias) ) @@ -100,54 +118,37 @@ workflow FASTQ_ALIGN_DEDUP_BISMARK { */ BISMARK_SUMMARY ( BISMARK_ALIGN.out.bam.collect{ meta, bam -> bam.name }, - alignment_reports.collect{ meta, align_report, dedup_report -> align_report }, - alignment_reports.collect{ meta, align_report, dedup_report -> dedup_report }.ifEmpty([]), - BISMARK_METHYLATIONEXTRACTOR.out.report.collect{ meta, report -> report }, - BISMARK_METHYLATIONEXTRACTOR.out.mbias.collect{ meta, mbias -> mbias } + ch_alignment_reports.collect{ meta, align_report, dedup_report -> align_report }, + ch_alignment_reports.collect{ meta, align_report, dedup_report -> dedup_report }.ifEmpty([]), + ch_methylation_report.collect{ meta, report -> report }, + ch_methylation_mbias.collect{ meta, mbias -> mbias } ) ch_bismark_summary = BISMARK_SUMMARY.out.summary ch_versions = ch_versions.mix(BISMARK_SUMMARY.out.versions) - /* - * MODULE: Run samtools sort on dedup bam - */ - SAMTOOLS_SORT_DEDUPLICATED ( - alignments, - [[:],[]] // [ [meta], [fasta]] - ) - ch_versions = ch_versions.mix(SAMTOOLS_SORT_DEDUPLICATED.out.versions) - - /* - * MODULE: Run samtools index on dedup bam - */ - SAMTOOLS_INDEX_DEDUPLICATED ( - SAMTOOLS_SORT_DEDUPLICATED.out.bam - ) - ch_versions = ch_versions.mix(SAMTOOLS_INDEX_DEDUPLICATED.out.versions) - /* * Collect MultiQC inputs */ - ch_multiqc_files = BISMARK_SUMMARY.out.summary - .mix(alignment_reports.collect{ meta, align_report, dedup_report -> align_report }) - .mix(alignment_reports.collect{ meta, align_report, dedup_report -> dedup_report }) - .mix(BISMARK_METHYLATIONEXTRACTOR.out.report.collect{ meta, report -> report }) - .mix(BISMARK_METHYLATIONEXTRACTOR.out.mbias.collect{ meta, mbias -> mbias }) - .mix(BISMARK_REPORT.out.report.collect{ meta, report -> report }) + ch_multiqc_files = ch_bismark_summary + .mix(ch_alignment_reports.collect{ meta, align_report, dedup_report -> align_report }) + .mix(ch_alignment_reports.collect{ meta, align_report, dedup_report -> dedup_report }) + .mix(ch_methylation_report.collect{ meta, report -> report }) + .mix(ch_methylation_mbias.collect{ meta, mbias -> mbias }) + .mix(ch_bismark_report.collect{ meta, report -> report }) emit: - dedup_bam = SAMTOOLS_SORT_DEDUPLICATED.out.bam // channel: [ val(meta), [ bam ] ] - dedup_bam_index = SAMTOOLS_INDEX_DEDUPLICATED.out.bai // channel: [ val(meta), [ bai ] ] - coverage2cytosine_coverage = ch_coverage2cytosine_coverage // channel: [ val(meta), [ coverage ] ] - coverage2cytosine_report = ch_coverage2cytosine_report // channel: [ val(meta), [ report ] ] - coverage2cytosine_summary = ch_coverage2cytosine_summary // channel: [ val(meta), [ summary ] ] - methylation_bedgraph = ch_methylation_bedgraph // channel: [ val(meta), [ bedgraph ] ] - methylation_calls = ch_methylation_calls // channel: [ val(meta), [ methylation_calls ] ] - methylation_coverage = ch_methylation_coverage // channel: [ val(meta), [ coverage ] ] - methylation_report = ch_methylation_report // channel: [ val(meta), [ report ] ] - methylation_mbias = ch_methylation_mbias // channel: [ val(meta), [ mbias ] ] - bismark_report = ch_bismark_report // channel: [ val(meta), [ report ] ] - bismark_summary = ch_bismark_summary // channel: [ val(meta), [ summary ] ] - multiqc = ch_multiqc_files // path: *{html,txt} - versions = ch_versions // path: *.version.txt + bam = SAMTOOLS_SORT.out.bam // channel: [ val(meta), [ bam ] ] + bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), [ bai ] ] + coverage2cytosine_coverage = ch_coverage2cytosine_coverage // channel: [ val(meta), [ coverage ] ] + coverage2cytosine_report = ch_coverage2cytosine_report // channel: [ val(meta), [ report ] ] + coverage2cytosine_summary = ch_coverage2cytosine_summary // channel: [ val(meta), [ summary ] ] + methylation_bedgraph = ch_methylation_bedgraph // channel: [ val(meta), [ bedgraph ] ] + methylation_calls = ch_methylation_calls // channel: [ val(meta), [ methylation_calls ] ] + methylation_coverage = ch_methylation_coverage // channel: [ val(meta), [ coverage ] ] + methylation_report = ch_methylation_report // channel: [ val(meta), [ report ] ] + methylation_mbias = ch_methylation_mbias // channel: [ val(meta), [ mbias ] ] + bismark_report = ch_bismark_report // channel: [ val(meta), [ report ] ] + bismark_summary = ch_bismark_summary // channel: [ val(meta), [ summary ] ] + multiqc = ch_multiqc_files // path: *{html,txt} + versions = ch_versions // path: *.version.txt } diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml b/subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml index 4f3aa80b946..7622b98e892 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml @@ -32,7 +32,7 @@ input: description: | Structure: [ val(meta), path(fasta) ] pattern: "*.{fa,fa.gz}" - - ch_index: + - ch_bismark_index: description: | Bismark genome index files Structure: [ val(meta), path(index) ] @@ -46,16 +46,16 @@ input: description: | Produce coverage2cytosine reports that relates methylation calls back to genomic cytosine contexts output: - - dedup_bam: + - bam: type: file description: | - Channel containing deduplicated BAM files. + Channel containing aligned or deduplicated BAM files. Structure: [ val(meta), path(bam) ] pattern: "*.bam" - - dedup_bam_index: + - bai: type: file description: | - Channel containing deduplicated BAM index files. + Channel containing aligned or deduplicated BAM index files. Structure: [ val(meta), path(bam.bai) ] pattern: "*.bai" - coverage2cytosine_coverage: diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test index f30c4344449..27bf692afdd 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test @@ -44,7 +44,7 @@ nextflow_workflow { } } - test("Params: bismark | default") { + test("Params: bismark single-end | default") { when { params { @@ -74,8 +74,8 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, - workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, workflow.out.coverage2cytosine_coverage, workflow.out.coverage2cytosine_report, workflow.out.coverage2cytosine_summary, @@ -93,7 +93,57 @@ nextflow_workflow { } } - test("Params: bismark | skip_deduplication") { + test("Params: bismark paired-end | default") { + + when { + params { + aligner = "bismark" + cytosine_report = false + skip_deduplication = false + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = BOWTIE2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bismark paired-end | skip_deduplication") { when { params { @@ -105,7 +155,8 @@ nextflow_workflow { """ input[0] = Channel.of([ [ id:'test', single_end:true ], - file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) ]) input[1] = Channel.of([ [:], @@ -122,8 +173,8 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, - workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, workflow.out.coverage2cytosine_coverage, workflow.out.coverage2cytosine_report, workflow.out.coverage2cytosine_summary, @@ -141,7 +192,7 @@ nextflow_workflow { } } - test("Params: bismark | cytosine_report") { + test("Params: bismark paired-end | cytosine_report") { when { params { @@ -153,7 +204,8 @@ nextflow_workflow { """ input[0] = Channel.of([ [ id:'test', single_end:true ], - file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) ]) input[1] = Channel.of([ [:], @@ -170,8 +222,8 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, - workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, workflow.out.coverage2cytosine_coverage, workflow.out.coverage2cytosine_report, workflow.out.coverage2cytosine_summary, @@ -189,7 +241,7 @@ nextflow_workflow { } } - test("Params: bismark_hisat | default") { + test("Params: bismark_hisat single-end | default") { when { params { @@ -219,8 +271,8 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, - workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, workflow.out.coverage2cytosine_coverage, workflow.out.coverage2cytosine_report, workflow.out.coverage2cytosine_summary, @@ -238,7 +290,57 @@ nextflow_workflow { } } - test("Params: bismark_hisat | skip_deduplication") { + test("Params: bismark_hisat paired-end | default") { + + when { + params { + aligner = "bismark_hisat" + cytosine_report = false + skip_deduplication = false + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = HISAT2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bismark_hisat paired-end | skip_deduplication") { when { params { @@ -250,7 +352,8 @@ nextflow_workflow { """ input[0] = Channel.of([ [ id:'test', single_end:true ], - file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) ]) input[1] = Channel.of([ [:], @@ -267,8 +370,8 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, - workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, workflow.out.coverage2cytosine_coverage, workflow.out.coverage2cytosine_report, workflow.out.coverage2cytosine_summary, @@ -286,7 +389,7 @@ nextflow_workflow { } } - test("Params: bismark_hisat | cytosine_report") { + test("Params: bismark_hisat paired-end | cytosine_report") { when { params { @@ -298,7 +401,8 @@ nextflow_workflow { """ input[0] = Channel.of([ [ id:'test', single_end:true ], - file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz', checkIfExists: true), + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz', checkIfExists: true) ]) input[1] = Channel.of([ [:], @@ -315,8 +419,8 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, - workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, workflow.out.coverage2cytosine_coverage, workflow.out.coverage2cytosine_report, workflow.out.coverage2cytosine_summary, diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap index b8f199fb126..a4d82189455 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap @@ -1,8 +1,8 @@ { - "Params: bismark | skip_deduplication": { + "Params: bismark_hisat paired-end | default": { "content": [ [ - + "4bf31467bec7b5d5669aa4ac16f6f93f" ], [ "test.sorted.bam.bai" @@ -22,7 +22,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2.bedGraph.gz:md5,6896f135c70e2f4944cb43260a687c6c" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.bedGraph.gz:md5,036f675959865a43f49899108d53b546" ] ], [ @@ -32,12 +32,12 @@ "single_end": true }, [ - "CHG_OB_SRR389222_sub1_bismark_bt2.txt.gz:md5,11ae58ebe6513375468ebe63a521d6e3", - "CHG_OT_SRR389222_sub1_bismark_bt2.txt.gz:md5,d0754aef3f7066881bdee1adf72b4bd1", - "CHH_OB_SRR389222_sub1_bismark_bt2.txt.gz:md5,12e9e4ebf59e6b23fb52f4872f534251", - "CHH_OT_SRR389222_sub1_bismark_bt2.txt.gz:md5,b800b59b0987650ac6eb4d2eb25451ad", - "CpG_OB_SRR389222_sub1_bismark_bt2.txt.gz:md5,21d614f45ecf703117550cb2eebfa3d5", - "CpG_OT_SRR389222_sub1_bismark_bt2.txt.gz:md5,8bbe9103623f502908e81cc573754524" + "CHG_OB_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,2c1285905b97b35a9088f811e2bda70e", + "CHG_OT_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,473499e02bab8a14f8c804746bc98486", + "CHH_OB_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,dda9a584b2a9bc7b84d39fe6210f8bfc", + "CHH_OT_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,265670d72805a8689afe35ace9a7148a", + "CpG_OB_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,d2912083fda7c2b4ad5916fe9cd8acf7", + "CpG_OT_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,3f1d464af4efb77956f06546eabf8f1a" ] ] ], @@ -47,7 +47,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2.bismark.cov.gz:md5,4e31b9a432731c75a604c89cb9d471ad" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.bismark.cov.gz:md5,63511c46275713088957950285acd653" ] ], [ @@ -56,7 +56,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2_splitting_report.txt:md5,d1bc9da2811d0f20981d01c3b1f4c925" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated_splitting_report.txt:md5,5c5711d5cff9b9fb2cbb30617bf7400d" ] ], [ @@ -65,40 +65,42 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2.M-bias.txt:md5,d1083f305a444487fe977362a0174159" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.M-bias.txt:md5,857b28589351f2c63f587a2c040db748" ] ], [ - "SRR389222_sub1_bismark_bt2_SE_report.html" + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.html" ], - "bismark_summary_report.txt:md5,13f7305cf7003f54e0eba22f3590889a", + "bismark_summary_report.txt:md5,9f62c10d005c39d3e5ee7961b6b0bf6b", [ - "SRR389222_sub1_bismark_bt2.M-bias.txt", - "SRR389222_sub1_bismark_bt2_SE_report.html", - "SRR389222_sub1_bismark_bt2_SE_report.txt", - "SRR389222_sub1_bismark_bt2_splitting_report.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.M-bias.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated_splitting_report.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplication_report.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.html", + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.txt", "bismark_summary_report.html", "bismark_summary_report.txt" ], [ "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", - "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", - "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.0" }, - "timestamp": "2024-11-15T15:19:32.095795684" + "timestamp": "2024-11-17T05:13:17.213711049" }, - "Params: bismark | default": { + "Params: bismark single-end | default": { "content": [ [ - + "35d55bee1ca653931716e40dac4aba30" ], [ "test.sorted.bam.bai" @@ -179,24 +181,24 @@ ], [ "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", - "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", - "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.0" }, - "timestamp": "2024-11-15T15:18:51.250132036" + "timestamp": "2024-11-17T05:10:13.435134357" }, - "Params: bismark | cytosine_report": { + "Params: bismark paired-end | cytosine_report": { "content": [ [ - + "4bf31467bec7b5d5669aa4ac16f6f93f" ], [ "test.sorted.bam.bai" @@ -210,7 +212,7 @@ "id": "test", "single_end": true }, - "test.CpG_report.txt.gz:md5,33af7ef6af01da4f1e12457ff1d782c6" + "test.CpG_report.txt.gz:md5,48cfcbd77d450c55fc64c8e6d939fa03" ] ], [ @@ -219,7 +221,7 @@ "id": "test", "single_end": true }, - "test.cytosine_context_summary.txt:md5,0e8c65559470bc64c42dbd0278a531de" + "test.cytosine_context_summary.txt:md5,d9df9696701e5799a06a18af4e49d648" ] ], [ @@ -228,7 +230,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2.deduplicated.bedGraph.gz:md5,5d7312489e51ca88e1ce1f689db93a64" + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.bedGraph.gz:md5,036f675959865a43f49899108d53b546" ] ], [ @@ -238,12 +240,12 @@ "single_end": true }, [ - "CHG_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,5c177e6f00dfc6e2335a55c6b70efa0c", - "CHG_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,5362d479358f644fe2c17cfe02f310f8", - "CHH_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,9469ac28e4369032b855ce1a794a56a4", - "CHH_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,d7628ac941d3fea97297a8dfcfb7aed1", - "CpG_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,f0404fc1fc18c7ff58e8766b405239ed", - "CpG_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,67d04d843e9e1abc8643bc25b27a9ff5" + "CHG_OB_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,2c1285905b97b35a9088f811e2bda70e", + "CHG_OT_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,473499e02bab8a14f8c804746bc98486", + "CHH_OB_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,dda9a584b2a9bc7b84d39fe6210f8bfc", + "CHH_OT_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,265670d72805a8689afe35ace9a7148a", + "CpG_OB_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,d2912083fda7c2b4ad5916fe9cd8acf7", + "CpG_OT_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,3f1d464af4efb77956f06546eabf8f1a" ] ] ], @@ -253,7 +255,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2.deduplicated.bismark.cov.gz:md5,eb0d3f96ed97c5d12afeeb49c234849d" + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.bismark.cov.gz:md5,63511c46275713088957950285acd653" ] ], [ @@ -262,7 +264,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2.deduplicated_splitting_report.txt:md5,ec268cc572b8bcfea0959b47c7510f05" + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated_splitting_report.txt:md5,d994b435696e984419ef01ea7dc51b2c" ] ], [ @@ -271,30 +273,30 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_bt2.deduplicated.M-bias.txt:md5,290ebfeca4da7809cf9adfe60bab1b29" + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.M-bias.txt:md5,857b28589351f2c63f587a2c040db748" ] ], [ - "SRR389222_sub1_bismark_bt2_SE_report.html" + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.html" ], - "bismark_summary_report.txt:md5,46f54eec58516c027e20bc9bdcaaae14", + "bismark_summary_report.txt:md5,855dfc95b1dbb0876aff2831380bba99", [ - "SRR389222_sub1_bismark_bt2.deduplicated.M-bias.txt", - "SRR389222_sub1_bismark_bt2.deduplicated_splitting_report.txt", - "SRR389222_sub1_bismark_bt2.deduplication_report.txt", - "SRR389222_sub1_bismark_bt2_SE_report.html", - "SRR389222_sub1_bismark_bt2_SE_report.txt", + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.M-bias.txt", + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated_splitting_report.txt", + "Ecoli_10K_methylated_R1_bismark_bt2.deduplication_report.txt", + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.html", + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.txt", "bismark_summary_report.html", "bismark_summary_report.txt" ], [ "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", - "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", - "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272", + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c", "versions.yml:md5,eb23ca81cb3427b8c5a8a9a93def4882" ] ], @@ -302,12 +304,12 @@ "nf-test": "0.9.2", "nextflow": "24.10.0" }, - "timestamp": "2024-11-15T15:20:14.052464546" + "timestamp": "2024-11-17T05:11:57.080233925" }, - "Params: bismark_hisat | cytosine_report": { + "Params: bismark_hisat paired-end | cytosine_report": { "content": [ [ - + "4bf31467bec7b5d5669aa4ac16f6f93f" ], [ "test.sorted.bam.bai" @@ -321,7 +323,7 @@ "id": "test", "single_end": true }, - "test.CpG_report.txt.gz:md5,d05837f07e4866f007d1f25166376b10" + "test.CpG_report.txt.gz:md5,48cfcbd77d450c55fc64c8e6d939fa03" ] ], [ @@ -330,7 +332,7 @@ "id": "test", "single_end": true }, - "test.cytosine_context_summary.txt:md5,c2e99c47feb351c6666990bec7d1a9bd" + "test.cytosine_context_summary.txt:md5,d9df9696701e5799a06a18af4e49d648" ] ], [ @@ -339,7 +341,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2.deduplicated.bedGraph.gz:md5,1f1b14f7be535e6098ece13ea1406bfc" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.bedGraph.gz:md5,036f675959865a43f49899108d53b546" ] ], [ @@ -349,12 +351,12 @@ "single_end": true }, [ - "CHG_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,5c177e6f00dfc6e2335a55c6b70efa0c", - "CHG_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,138bccceb9ae71ecdae6828fd8fc327f", - "CHH_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,9469ac28e4369032b855ce1a794a56a4", - "CHH_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,f1acfc10c9f9ffb80f2a7a947a210ad7", - "CpG_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,f0404fc1fc18c7ff58e8766b405239ed", - "CpG_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,75b9acaf689411154f906bcfa7264abf" + "CHG_OB_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,2c1285905b97b35a9088f811e2bda70e", + "CHG_OT_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,473499e02bab8a14f8c804746bc98486", + "CHH_OB_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,dda9a584b2a9bc7b84d39fe6210f8bfc", + "CHH_OT_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,265670d72805a8689afe35ace9a7148a", + "CpG_OB_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,d2912083fda7c2b4ad5916fe9cd8acf7", + "CpG_OT_Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.txt.gz:md5,3f1d464af4efb77956f06546eabf8f1a" ] ] ], @@ -364,7 +366,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2.deduplicated.bismark.cov.gz:md5,ec4a2575b47822ef478b3bda35c70590" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.bismark.cov.gz:md5,63511c46275713088957950285acd653" ] ], [ @@ -373,7 +375,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2.deduplicated_splitting_report.txt:md5,eaee004ea3fffb9b8aaf968f41f9f903" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated_splitting_report.txt:md5,5c5711d5cff9b9fb2cbb30617bf7400d" ] ], [ @@ -382,30 +384,30 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2.deduplicated.M-bias.txt:md5,82b760a6824f55f6732297374d77a6ec" + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.M-bias.txt:md5,857b28589351f2c63f587a2c040db748" ] ], [ - "SRR389222_sub1_bismark_hisat2_SE_report.html" + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.html" ], - "bismark_summary_report.txt:md5,762c303acee29bfffef35b3d9e04d153", + "bismark_summary_report.txt:md5,9f62c10d005c39d3e5ee7961b6b0bf6b", [ - "SRR389222_sub1_bismark_hisat2.deduplicated.M-bias.txt", - "SRR389222_sub1_bismark_hisat2.deduplicated_splitting_report.txt", - "SRR389222_sub1_bismark_hisat2.deduplication_report.txt", - "SRR389222_sub1_bismark_hisat2_SE_report.html", - "SRR389222_sub1_bismark_hisat2_SE_report.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated.M-bias.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplicated_splitting_report.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2.deduplication_report.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.html", + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.txt", "bismark_summary_report.html", "bismark_summary_report.txt" ], [ "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", - "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", - "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272", + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c", "versions.yml:md5,eb23ca81cb3427b8c5a8a9a93def4882" ] ], @@ -413,12 +415,12 @@ "nf-test": "0.9.2", "nextflow": "24.10.0" }, - "timestamp": "2024-11-15T15:22:14.454171707" + "timestamp": "2024-11-17T05:14:44.263768701" }, - "Params: bismark_hisat | default": { + "Params: bismark_hisat single-end | default": { "content": [ [ - + "dae445da532c973dc9149f3ceba2d2c5" ], [ "test.sorted.bam.bai" @@ -499,24 +501,218 @@ ], [ "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", - "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-17T05:12:39.424047426" + }, + "Params: bismark paired-end | default": { + "content": [ + [ + "4bf31467bec7b5d5669aa4ac16f6f93f" + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.bedGraph.gz:md5,036f675959865a43f49899108d53b546" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,2c1285905b97b35a9088f811e2bda70e", + "CHG_OT_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,473499e02bab8a14f8c804746bc98486", + "CHH_OB_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,dda9a584b2a9bc7b84d39fe6210f8bfc", + "CHH_OT_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,265670d72805a8689afe35ace9a7148a", + "CpG_OB_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,d2912083fda7c2b4ad5916fe9cd8acf7", + "CpG_OT_Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.txt.gz:md5,3f1d464af4efb77956f06546eabf8f1a" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.bismark.cov.gz:md5,63511c46275713088957950285acd653" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated_splitting_report.txt:md5,d994b435696e984419ef01ea7dc51b2c" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.M-bias.txt:md5,857b28589351f2c63f587a2c040db748" + ] + ], + [ + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.html" + ], + "bismark_summary_report.txt:md5,855dfc95b1dbb0876aff2831380bba99", + [ + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated.M-bias.txt", + "Ecoli_10K_methylated_R1_bismark_bt2.deduplicated_splitting_report.txt", + "Ecoli_10K_methylated_R1_bismark_bt2.deduplication_report.txt", + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.html", + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", - "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.0" }, - "timestamp": "2024-11-15T15:20:53.556708293" + "timestamp": "2024-11-17T05:10:47.730967756" }, - "Params: bismark_hisat | skip_deduplication": { + "Params: bismark_hisat paired-end | skip_deduplication": { "content": [ + [ + "4bf31467bec7b5d5669aa4ac16f6f93f" + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + + ], [ + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_hisat2.bedGraph.gz:md5,036f675959865a43f49899108d53b546" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_Ecoli_10K_methylated_R1_bismark_hisat2.txt.gz:md5,2c1285905b97b35a9088f811e2bda70e", + "CHG_OT_Ecoli_10K_methylated_R1_bismark_hisat2.txt.gz:md5,473499e02bab8a14f8c804746bc98486", + "CHH_OB_Ecoli_10K_methylated_R1_bismark_hisat2.txt.gz:md5,dda9a584b2a9bc7b84d39fe6210f8bfc", + "CHH_OT_Ecoli_10K_methylated_R1_bismark_hisat2.txt.gz:md5,265670d72805a8689afe35ace9a7148a", + "CpG_OB_Ecoli_10K_methylated_R1_bismark_hisat2.txt.gz:md5,d2912083fda7c2b4ad5916fe9cd8acf7", + "CpG_OT_Ecoli_10K_methylated_R1_bismark_hisat2.txt.gz:md5,3f1d464af4efb77956f06546eabf8f1a" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_hisat2.bismark.cov.gz:md5,63511c46275713088957950285acd653" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_hisat2_splitting_report.txt:md5,7767e575a343a9b077dcd03e594550f1" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "Ecoli_10K_methylated_R1_bismark_hisat2.M-bias.txt:md5,857b28589351f2c63f587a2c040db748" + ] + ], + [ + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.html" + ], + "bismark_summary_report.txt:md5,dd73bb37e3116c25480990bd37f3f99f", + [ + "Ecoli_10K_methylated_R1_bismark_hisat2.M-bias.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.html", + "Ecoli_10K_methylated_R1_bismark_hisat2_SE_report.txt", + "Ecoli_10K_methylated_R1_bismark_hisat2_splitting_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-17T05:13:57.892659443" + }, + "Params: bismark paired-end | skip_deduplication": { + "content": [ + [ + "4bf31467bec7b5d5669aa4ac16f6f93f" ], [ "test.sorted.bam.bai" @@ -536,7 +732,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2.bedGraph.gz:md5,75be0d8be0134a48fd55e15a49568d3f" + "Ecoli_10K_methylated_R1_bismark_bt2.bedGraph.gz:md5,036f675959865a43f49899108d53b546" ] ], [ @@ -546,12 +742,12 @@ "single_end": true }, [ - "CHG_OB_SRR389222_sub1_bismark_hisat2.txt.gz:md5,11ae58ebe6513375468ebe63a521d6e3", - "CHG_OT_SRR389222_sub1_bismark_hisat2.txt.gz:md5,2963a2702be660068818073a2989c790", - "CHH_OB_SRR389222_sub1_bismark_hisat2.txt.gz:md5,12e9e4ebf59e6b23fb52f4872f534251", - "CHH_OT_SRR389222_sub1_bismark_hisat2.txt.gz:md5,e6815f62702d06dc01d12ea18dbd9a03", - "CpG_OB_SRR389222_sub1_bismark_hisat2.txt.gz:md5,21d614f45ecf703117550cb2eebfa3d5", - "CpG_OT_SRR389222_sub1_bismark_hisat2.txt.gz:md5,1a07c82f278b4ed637d33833df1e9749" + "CHG_OB_Ecoli_10K_methylated_R1_bismark_bt2.txt.gz:md5,2c1285905b97b35a9088f811e2bda70e", + "CHG_OT_Ecoli_10K_methylated_R1_bismark_bt2.txt.gz:md5,473499e02bab8a14f8c804746bc98486", + "CHH_OB_Ecoli_10K_methylated_R1_bismark_bt2.txt.gz:md5,dda9a584b2a9bc7b84d39fe6210f8bfc", + "CHH_OT_Ecoli_10K_methylated_R1_bismark_bt2.txt.gz:md5,265670d72805a8689afe35ace9a7148a", + "CpG_OB_Ecoli_10K_methylated_R1_bismark_bt2.txt.gz:md5,d2912083fda7c2b4ad5916fe9cd8acf7", + "CpG_OT_Ecoli_10K_methylated_R1_bismark_bt2.txt.gz:md5,3f1d464af4efb77956f06546eabf8f1a" ] ] ], @@ -561,7 +757,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2.bismark.cov.gz:md5,ab610e006112a52e5ecb230a1704f236" + "Ecoli_10K_methylated_R1_bismark_bt2.bismark.cov.gz:md5,63511c46275713088957950285acd653" ] ], [ @@ -570,7 +766,7 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2_splitting_report.txt:md5,e6b01d28d7248361750dbd23eef667df" + "Ecoli_10K_methylated_R1_bismark_bt2_splitting_report.txt:md5,39c7721863c121645d68a4827e57ec77" ] ], [ @@ -579,34 +775,34 @@ "id": "test", "single_end": true }, - "SRR389222_sub1_bismark_hisat2.M-bias.txt:md5,bbfe49763c571a55a50d1e46c0fb4e46" + "Ecoli_10K_methylated_R1_bismark_bt2.M-bias.txt:md5,857b28589351f2c63f587a2c040db748" ] ], [ - "SRR389222_sub1_bismark_hisat2_SE_report.html" + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.html" ], - "bismark_summary_report.txt:md5,7a466655c11944b6035901910378e559", + "bismark_summary_report.txt:md5,9f639cca6fe43c0461d2da88d931a036", [ - "SRR389222_sub1_bismark_hisat2.M-bias.txt", - "SRR389222_sub1_bismark_hisat2_SE_report.html", - "SRR389222_sub1_bismark_hisat2_SE_report.txt", - "SRR389222_sub1_bismark_hisat2_splitting_report.txt", + "Ecoli_10K_methylated_R1_bismark_bt2.M-bias.txt", + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.html", + "Ecoli_10K_methylated_R1_bismark_bt2_SE_report.txt", + "Ecoli_10K_methylated_R1_bismark_bt2_splitting_report.txt", "bismark_summary_report.html", "bismark_summary_report.txt" ], [ "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", - "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6b5bf82c070f4410ead76cc9a363529c", "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", - "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + "versions.yml:md5,c2438ca96ec6728b06c8706639eb233c" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.0" }, - "timestamp": "2024-11-15T15:21:33.809590706" + "timestamp": "2024-11-17T05:11:19.884389766" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config index 903bb19fa88..8d94d8ff37b 100644 --- a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config @@ -9,7 +9,7 @@ process { ext.args = { params.aligner == 'bismark_hisat' ? ' --hisat2' : ' --bowtie2' } } - withName: 'SAMTOOLS_SORT_DEDUPLICATED' { + withName: 'SAMTOOLS_SORT' { ext.prefix = { "${meta.id}.sorted" } } } From 3a897454edabb27d21debfec644e40f7bcfa4c43 Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:47:23 -0600 Subject: [PATCH 4/5] Use new nf-test features (#6286) * ci: Attempt to split everything out Don't hate me Adam * ci: Add changed since, sharding, and ci * ci: Add filter to try to get jobs split up * ci: Switch to only-changed * ci: See if follow-dependencies works without "related-tests" * ci: Remove skipped tests Idk what we're going to do about these... * ci: Actually use the nf-test version * ci: module => process * ci: Clean up job names * dummy: Make a change * ci: Skip test.tap * ci: Add fetch-depth * ci: Clean up name * ci: Lint everything * ci: Get the job names clean * ci: Add hide-progress on linting * ci: Add psuedocode for conda-fail.yml https://nfcore.slack.com/archives/CJRH30T6V/p1724406283145319 * test: Snapshot the versions contents, not the hash https://github.com/nf-core/modules/issues/6505 * ci: Keep running nf-core lint the way it was Blocked by https://github.com/nf-core/tools/issues/3140 * ci: Move conda skips out * ci: Address a comment * style: Move prettier and editorconfig to pre-commit https://github.com/nf-core/modules/pull/4554#issuecomment-1845488824 * ci: Add note about nf-core lint pre-commit * chore: Copy over conda skips https://github.com/nf-core/modules/commit/a004c86eea62d4eed15768c0edb3adcc3f2c1424 https://github.com/nf-core/modules/commit/2da71b7a52a7c6609cb8f29d316ed3fd90d0f7cc * ci: only-changed => changed-since * Add confirm-pass * ci: dynamically set shards * ci: Run 3 process jobs per CI run as an example * install nf-test * fix install nf-test * fix install nf-test * ci: Remove dynamic number of shards, and combine process and workflow filters * ci: Remove variable Nextflow versions * test: Update bowtie versions * ci: Fix indention * ci: We're not testing multiple python versions * style: Clean up job names * build: Remove invisible characters? * test(bowtie): Bump snapshots * style: Remove stray comment * ci: Add GPU tests in their own workflows These will only get trigger on the paths listed * ci(gpu): Clean up triggers and make a note about how to make the includes smarter * ci: pip install cryptography * ci: Try removing path * ci: Add fetch-depth and remove duplicate checkout * test: Remove the second gpu profile * ci: Add path back in * ci(gpu): Remove changed-since * ci: Fix missed dorny/paths-filter update Co-authored-by: mashehu * Add changes from codereview Co-authored-by: mashehu * style: Remove example TODO comments https://github.com/nf-core/modules/pull/6286/files#r1846856369 Co-authored-by: mashehu * refactor: Try tags for GPU CI https://github.com/askimed/nf-test/issues/272 * style: Set NFT_WORKDIR as an ENV variable * style: Run lsp format on a parabricks module For science * test: Add gpu tags to parabricks * ci: Split tags out of matrix * Revert "style: Run lsp format on a parabricks module" This reverts commit d134e78a0aa14fef469e30c8d90006092c9f3a6b. * ci: Fix name --------- Co-authored-by: Edmund Miller Co-authored-by: maxulysse Co-authored-by: mashehu --- .github/actions/nf-test-action/action.yml | 29 +- .github/conda_skip.yml | 184 +++++ .github/workflows/gpu-tests.yml | 71 ++ .github/workflows/lint.yml | 168 ++++ .github/workflows/nf-test.yml | 71 ++ .github/workflows/pytest-workflow.yml | 342 ++++++++ .github/workflows/test.yml | 745 ------------------ .pre-commit-config.yaml | 5 + modules/nf-core/bowtie/align/main.nf | 3 +- .../bowtie/align/tests/main.nf.test.snap | 36 +- modules/nf-core/bowtie/build/main.nf | 2 +- .../nf-core/bowtie/build/tests/main.nf.test | 3 +- .../bowtie/build/tests/main.nf.test.snap | 34 +- .../parabricks/applybqsr/tests/main.nf.test | 3 +- .../parabricks/fq2bam/tests/main.nf.test | 3 +- .../parabricks/fq2bammeth/tests/main.nf.test | 3 +- tests/config/nf-test.config | 6 - 17 files changed, 916 insertions(+), 792 deletions(-) create mode 100644 .github/conda_skip.yml create mode 100644 .github/workflows/gpu-tests.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/nf-test.yml create mode 100644 .github/workflows/pytest-workflow.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index e1e1f4bc57d..5b87e1e3611 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -1,18 +1,22 @@ name: "nf-test Action" description: "Runs nf-test with common setup steps" inputs: - path: - description: "Path to test" - required: true profile: description: "Profile to use" required: true + shard: + description: "Shard number for this CI job" + required: true + total_shards: + description: "Total number of test shards(NOT the total number of matrix jobs)" + required: true + tags: + description: "Tags to test (`[,...]`)" + required: false runs: using: "composite" steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 with: distribution: "temurin" @@ -56,20 +60,31 @@ runs: if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' shell: bash run: | + python -m pip install cryptography nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + # TODO Skip failing conda tests and document their failures + # https://github.com/nf-core/modules/issues/7017 - name: Run nf-test shell: bash env: SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" + TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }} + NFT_WORKDIR: "~" run: | - NFT_WORKDIR=~ \ nf-test test \ --profile=${{ inputs.profile }} \ --tap=test.tap \ --verbose \ - ${{ inputs.path }} + --ci \ + --changed-since HEAD^ \ + --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ + --filter process,workflow \ + --follow-dependencies \ + ${{ env.TAGS }} + + # TODO If no test.tap, then make one to spoof? - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 if: ${{ inputs.path != '' }} with: diff --git a/.github/conda_skip.yml b/.github/conda_skip.yml new file mode 100644 index 00000000000..2ef7216da3e --- /dev/null +++ b/.github/conda_skip.yml @@ -0,0 +1,184 @@ +# TODO What do we do with these? +# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now +exclude: + - path: modules/nf-core/nf-test + - profile: conda + path: modules/nf-core/angsd/gl + - profile: conda + path: modules/nf-core/annotsv/installannotations + - profile: conda + path: modules/nf-core/happy/sompy + - profile: conda + path: modules/nf-core/backsub + - profile: conda + path: modules/nf-core/bakta/bakta + - profile: conda + path: modules/nf-core/bakta/baktadbdownload + - profile: conda + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/bcl2fastq + - profile: conda + path: modules/nf-core/bclconvert + - profile: conda + path: modules/nf-core/celesta + - profile: conda + path: modules/nf-core/cellpose + - profile: conda + path: modules/nf-core/cellranger/count + - profile: conda + path: modules/nf-core/cellranger/mkfastq + - profile: conda + path: modules/nf-core/cellranger/mkgtf + - profile: conda + path: modules/nf-core/cellranger/mkref + - profile: conda + path: modules/nf-core/cellranger/mkvdjref + - profile: conda + path: modules/nf-core/cellranger/multi + - profile: conda + path: modules/nf-core/cellranger/vdj + - profile: conda + path: modules/nf-core/checkqc + - profile: conda + path: modules/nf-core/custom/dumpsoftwareversions + - profile: conda + path: modules/nf-core/deepcell/mesmer + - profile: conda + path: modules/nf-core/deepsomatic + - profile: singularity + path: modules/nf-core/deepsomatic + - profile: conda + path: modules/nf-core/deepvariant + - profile: conda + path: modules/nf-core/deepvariant/callvariants + - profile: conda + path: modules/nf-core/deepvariant/makeexamples + - profile: conda + path: modules/nf-core/deepvariant/postprocessvariants + - profile: conda + path: modules/nf-core/deepvariant/rundeepvariant + - profile: conda + path: modules/nf-core/deepvariant/vcfstatsreport + - profile: conda + path: modules/nf-core/doubletdetection + - profile: conda + path: modules/nf-core/ensemblvep/vep + - profile: conda + path: modules/nf-core/fastk/fastk + - profile: conda + path: modules/nf-core/cellrangerarc/mkgtf + - profile: conda + path: modules/nf-core/fastk/histex + - profile: conda + path: modules/nf-core/fastk/merge + - profile: conda + path: modules/nf-core/fcs/fcsadaptor + - profile: conda + path: modules/nf-core/fcs/fcsgx + - profile: conda + path: modules/nf-core/ganon/buildcustom + - profile: conda + path: modules/nf-core/ganon/classify + - profile: conda + path: modules/nf-core/ganon/report + - profile: conda + path: modules/nf-core/ganon/table + - profile: conda + path: modules/nf-core/gatk4/cnnscorevariants + - profile: conda + path: modules/nf-core/gatk4/determinegermlinecontigploidy + - profile: conda + path: modules/nf-core/genescopefk + - profile: conda + path: modules/nf-core/ilastik/multicut + - profile: conda + path: modules/nf-core/ilastik/pixelclassification + - profile: conda + path: modules/nf-core/imputeme/vcftoprs + - profile: conda + path: modules/nf-core/mcstaging/imc2mc + - profile: conda + path: modules/nf-core/mcquant + - profile: conda + path: modules/nf-core/mcstaging/phenoimager2mc + - profile: conda + path: modules/nf-core/merquryfk/katcomp + - profile: conda + path: modules/nf-core/merquryfk/katgc + - profile: conda + path: modules/nf-core/merquryfk/merquryfk + - profile: conda + path: modules/nf-core/merquryfk/ploidyplot + - profile: conda + path: modules/nf-core/molkartgarage/clahe + - profile: conda + path: modules/nf-core/quartonotebook + - profile: conda + path: modules/nf-core/scimap/spatiallda + - profile: conda + path: modules/nf-core/sentieon/bwaindex + - profile: conda + path: modules/nf-core/sentieon/bwamem + - profile: conda + path: modules/nf-core/sentieon/datametrics + - profile: conda + path: modules/nf-core/sentieon/dedup + - profile: conda + path: modules/nf-core/sentieon/qualcal + - profile: conda + path: modules/nf-core/spaceranger/mkgtf + - profile: conda + path: modules/nf-core/spaceranger/mkref + - profile: conda + path: modules/nf-core/spaceranger/count + - profile: conda + path: modules/nf-core/spotiflow + - profile: conda + path: modules/nf-core/svanalyzer/svbenchmark + - profile: conda + path: modules/nf-core/universc + - profile: singularity + path: modules/nf-core/universc + - profile: conda + path: modules/nf-core/vt/decompose + - profile: singularity + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/wittyer + - profile: conda + path: modules/nf-core/islandpath + - profile: conda + path: modules/nf-core/scimap/mcmicro + - profile: conda + path: modules/nf-core/parabricks/fq2bammeth + - profile: docker_self_hosted + path: modules/nf-core/parabricks/fq2bammeth + - profile: singularity + path: modules/nf-core/parabricks/fq2bammeth + - profile: conda + path: modules/nf-core/parabricks/fq2bam + - profile: docker_self_hosted + path: modules/nf-core/parabricks/fq2bam + - profile: singularity + path: modules/nf-core/parabricks/fq2bam + - profile: conda + path: subworkflows/nf-core/vcf_annotate_ensemblvep + - profile: conda + path: subworkflows/nf-core/bcl_demultiplex + - profile: conda + path: subworkflows/nf-core/deepvariant + - profile: conda + path: subworkflows/nf-core/fastq_align_bamcmp_bwa + - profile: conda + path: subworkflows/nf-core/fastq_align_bwa + - profile: conda + path: subworkflows/nf-core/fasta_newick_epang_gappa + - profile: conda + path: modules/nf-core/xeniumranger/relabel + - profile: conda + path: modules/nf-core/xeniumranger/rename + - profile: conda + path: modules/nf-core/xeniumranger/resegment + - profile: conda + path: modules/nf-core/xeniumranger/import-segmentation diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml new file mode 100644 index 00000000000..dfabddf07ca --- /dev/null +++ b/.github/workflows/gpu-tests.yml @@ -0,0 +1,71 @@ +name: Run GPU nf-tests +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + paths: + - ".github/workflows/gpu-tests.yml" + - "modules/nf-core/parabricks/**" + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NFT_VER: "0.9.2" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + NXF_VER: "24.04.4" + +jobs: + nf-test-gpu: + runs-on: "gpu" + name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}" + strategy: + fail-fast: false + matrix: + shard: [1, 2] + profile: [docker_self_hosted, singularity] # conda? + # TODO Pass these in from GitHub PR trigger events + tags: + - parabricks/applybqsr + - parabricks/fq2bam + - parabricks/fq2bammeth + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: 2 + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action + env: + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + with: + profile: ${{ matrix.profile }},gpu + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} + tags: ${{matrix.tags}},gpu diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..039e5069a7f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,168 @@ +name: Run Linting +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + with: + extra_args: "" + + ################### + # nf-core linting # + ################### + # TODO Move these to pre-commit + # https://github.com/nf-core/tools/pull/3141 + nf-core-changes: + name: nf-core-changes + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 2 # To retrieve the preceding commit. + + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + token: "" + + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + + nf-core-lint-modules: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: nf-core lint modules + needs: nf-core-changes + if: ${{ (needs.nf-core-changes.outputs.modules != '[]') }} + strategy: + fail-fast: false + matrix: + tags: "${{ fromJson(needs.nf-core-changes.outputs.modules) }}" + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 + with: + python-version: "3.11" + + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + id: cache-pip + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + restore-keys: | + ${{ runner.os }}-pip + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} + + nf-core-lint-subworkflows: + runs-on: ubuntu-latest + name: nf-core lint subworkflows + needs: nf-core-changes + if: ${{ (needs.nf-core-changes.outputs.subworkflows != '[]') }} + strategy: + fail-fast: false + matrix: + tags: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows) }}" + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 + with: + python-version: "3.11" + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core subworkflows lint ${{ matrix.tags }} + + confirm-pass: + runs-on: ubuntu-latest + needs: [nf-core-lint-modules, nf-core-lint-subworkflows] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 00000000000..38ebea1eff9 --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,71 @@ +name: Run nf-test +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NFT_VER: "0.9.2" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + NXF_VER: "24.04.4" + +jobs: + nf-test: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + # NOTE I think this is the cleanest way to get them organized + # process | conda | 1 + # process | conda | 2 + # process | conda | 3 + # process | docker_self_hosted | 1 + # ... + # workflow | singularity | 3 + name: "${{ matrix.profile }} | ${{ matrix.shard }}" + # TODO + # needs: get-number-of-shards + # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }} + strategy: + fail-fast: false + matrix: + # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now + # NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}" + # filter: [process, workflow] + profile: [conda, docker_self_hosted, singularity] + shard: [1, 2, 3, 4, 5] + env: + # FIXME Bumping them up to make the transition smooth, then we can throttle them back + TOTAL_SHARDS: 5 + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action + with: + profile: ${{ matrix.profile }} + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml new file mode 100644 index 00000000000..a772dfc1db1 --- /dev/null +++ b/.github/workflows/pytest-workflow.yml @@ -0,0 +1,342 @@ +name: Run pytest-workflow +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + pytest-changes: + name: pytest-changes + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 2 # To retrieve the preceding commit. + + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + token: "" + + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + + pytest: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: pytest + needs: [pytest-changes] + if: needs.pytest-changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] + profile: [conda, docker, singularity] + exclude: + - tags: nf-test + - profile: conda + tags: backsub + - profile: conda + tags: bases2fastq + - profile: singularity + tags: bases2fastq + - profile: conda + tags: basicpy + - profile: conda + tags: bcl2fastq + - profile: conda + tags: bclconvert + - profile: conda + tags: bwa/aln + - profile: conda + tags: bwa/index + - profile: conda + tags: bwa/mem + - profile: conda + tags: bwa/sampe + - profile: conda + tags: bwa/samse + - profile: conda + tags: cellpose + - profile: conda + tags: cellrangerarc/count + - profile: conda + tags: cellrangerarc/mkfastq + - profile: conda + tags: cellrangerarc/mkref + - profile: conda + tags: cellrangeratac/count + - profile: conda + tags: cellrangeratac/mkfastq + - profile: conda + tags: cellrangeratac/mkref + - profile: conda + tags: checkm2/databasedownload + - profile: conda + tags: checkm2/predict + - profile: conda + tags: controlfreec/makegraph2 + - profile: conda + tags: coreograph + - profile: conda + tags: deepcell/mesmer + - profile: conda + tags: deepvariant + - profile: conda + tags: fastani + - profile: conda + tags: fastk/fastk + - profile: conda + tags: fastk/histex + - profile: conda + tags: fastk/merge + - profile: conda + tags: fcs/fcsadaptor + - profile: conda + tags: fcs/fcsgx + - profile: conda + tags: gatk4/cnnscorevariants + - profile: conda + tags: gatk4/determinegermlinecontigploidy + - profile: singularity + tags: gatk4/determinegermlinecontigploidy + - profile: conda + tags: gatk4/germlinecnvcaller + - profile: conda + tags: gatk4/postprocessgermlinecnvcalls + - profile: conda + tags: genescopefk + - profile: conda + tags: happy/sompy + - profile: conda + tags: hlala/preparegraph + - profile: conda + tags: ilastik/multicut + - profile: conda + tags: ilastik/pixelclassification + - profile: conda + tags: imputeme/vcftoprs + - profile: conda + tags: islandpath + - profile: conda + tags: manta/convertinversion + - profile: conda + tags: mcstaging/imc2mc + - profile: conda + tags: mcquant + - profile: conda + tags: medaka + - profile: conda + tags: merquryfk/katcomp + - profile: conda + tags: merquryfk/katgc + - profile: conda + tags: merquryfk/merquryfk + - profile: conda + tags: merquryfk/ploidyplot + - profile: conda + tags: minimap2/align + - profile: conda + tags: mitohifi/findmitoreference + - profile: conda + tags: mitohifi/mitohifi + - profile: conda + tags: nanoplot + - profile: conda + tags: ncbitools/vecscreen + - profile: conda + tags: parabricks/applybqsr + - profile: conda + tags: parabricks/dbsnp + - profile: conda + tags: parabricks/deepvariant + - profile: conda + tags: parabricks/fq2bam + - profile: conda + tags: parabricks/genotypegvcf + - profile: conda + tags: parabricks/haplotypecaller + - profile: conda + tags: parabricks/indexgvcf + - profile: conda + tags: parabricks/mutectcaller + - profile: conda + tags: picard/collecthsmetrics + - profile: conda + tags: picard/collectwgsmetrics + - profile: conda + tags: sentieon/applyvarcal + - profile: conda + tags: sentieon/datametrics + - profile: conda + tags: sentieon/dnamodelapply + - profile: conda + tags: sentieon/dnascope + - profile: conda + tags: sentieon/readwriter + - profile: conda + tags: sentieon/tnfilter + - profile: conda + tags: sentieon/tnhaplotyper2 + - profile: conda + tags: sentieon/tnscope + - profile: conda + tags: sentieon/varcal + - profile: conda + tags: sentieon/wgsmetrics + - profile: conda + tags: subworkflows/bam_qc_picard + - profile: conda + tags: subworkflows/bcl_demultiplex + - profile: conda + tags: subworkflows/fasta_clean_fcs + - profile: conda + tags: svanalyzer/svbenchmark + - profile: conda + tags: universc + - profile: singularity + tags: universc + - profile: conda + tags: vt/decompose + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 + id: cache-pip-pytest + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-pytest + restore-keys: | + ${{ runner.os }}-pip-pytest + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow cryptography + + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up miniconda + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 + with: + miniconda-version: "latest" + channels: conda-forge,bioconda + python-version: "3.11" + + - name: Conda setup + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt-get update > /dev/null + sudo apt-get install bat > /dev/null + batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} + + - name: Setting global variables + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: parsed + with: + script: | + return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') + result-encoding: string + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 + with: + name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} + path: | + /home/ubuntu/pytest_workflow_*/*/.nextflow.log + /home/ubuntu/pytest_workflow_*/*/log.out + /home/ubuntu/pytest_workflow_*/*/log.err + /home/ubuntu/pytest_workflow_*/*/work + !/home/ubuntu/pytest_workflow_*/*/work/conda + !/home/ubuntu/pytest_workflow_*/*/work/singularity + !${{ github.workspace }}/.singularity + + confirm-pass: + runs-on: ubuntu-latest + needs: [pytest-changes, pytest] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 736b2c9eae9..00000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,745 +0,0 @@ -name: Run tests -on: - push: - branches: - # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging - - "renovate/**" # branches Renovate creates - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity - NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - # FIXME Flip this off once we get to less than a couple hundred. Adding - # this so it will only run against changed files. It'll make it much - # easier to fix these as they come up rather than everything at once. - with: - extra_args: "--all-files" - - prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Install NodeJS - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 - with: - node-version: "20" - - - name: Install Prettier - run: npm install -g prettier@3.2.5 - - - name: Run Prettier --check - run: prettier --check . - - editorconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 - with: - node-version: "20" - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) - - pytest-changes: - name: pytest-changes - runs-on: ubuntu-latest - outputs: - tags: ${{ steps.filter.outputs.changes }} - modules: ${{ steps.tags.outputs.modules }} - subworkflows: ${{ steps.tags.outputs.subworkflows }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - fetch-depth: 2 # To retrieve the preceding commit. - - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - token: "" - - - name: Fetch module tags - id: tags - run: | - echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.tags.outputs.modules }} - echo ${{ steps.tags.outputs.subworkflows }} - - nf-test-changes: - name: nf-test-changes - runs-on: ubuntu-latest - outputs: - # Expose detected tags as 'modules' and 'workflows' output variables - paths: ${{ steps.list.outputs.components }} - modules: ${{ steps.outputs.outputs.modules }} - subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - fetch-depth: 0 - - - name: List nf-test files - id: list - uses: adamrtalbot/detect-nf-test-changes@7c8be3ffd0d6538312b363c8c949dbbf5f26c3dd # v0.0.4 - with: - head: ${{ github.sha }} - base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - n_parents: 2 - - - name: Separate modules and subworkflows - id: outputs - run: | - echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.filter.outputs.components }} - echo ${{ steps.outputs.outputs.modules }} - echo ${{ steps.outputs.outputs.subworkflows }} - - nf-core-lint-modules: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.modules) }}", - "${{ fromJson(needs.nf-test-changes.outputs.modules) }}", - ] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - restore-keys: | - ${{ runner.os }}-pip - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core modules lint ${{ matrix.tags }} - - nf-core-lint-subworkflows: - runs-on: ubuntu-latest - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", - "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", - ] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core subworkflows lint ${{ matrix.tags }} - - pytest: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} - name: pytest - needs: [pytest-changes] - if: needs.pytest-changes.outputs.tags != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] - profile: [conda, docker, singularity] - exclude: - - tags: nf-test - - profile: conda - tags: backsub - - profile: conda - tags: bases2fastq - - profile: singularity - tags: bases2fastq - - profile: conda - tags: basicpy - - profile: conda - tags: bcl2fastq - - profile: conda - tags: bclconvert - - profile: conda - tags: bwa/aln - - profile: conda - tags: bwa/index - - profile: conda - tags: bwa/mem - - profile: conda - tags: bwa/sampe - - profile: conda - tags: bwa/samse - - profile: conda - tags: cellpose - - profile: conda - tags: cellrangerarc/count - - profile: conda - tags: cellrangerarc/mkfastq - - profile: conda - tags: cellrangerarc/mkref - - profile: conda - tags: cellrangeratac/count - - profile: conda - tags: cellrangeratac/mkfastq - - profile: conda - tags: cellrangeratac/mkref - - profile: conda - tags: checkm2/databasedownload - - profile: conda - tags: checkm2/predict - - profile: conda - tags: controlfreec/makegraph2 - - profile: conda - tags: coreograph - - profile: conda - tags: deepcell/mesmer - - profile: conda - tags: deepvariant - - profile: conda - tags: fastani - - profile: conda - tags: fastk/fastk - - profile: conda - tags: fastk/histex - - profile: conda - tags: fastk/merge - - profile: conda - tags: fcs/fcsadaptor - - profile: conda - tags: fcs/fcsgx - - profile: conda - tags: gatk4/cnnscorevariants - - profile: conda - tags: gatk4/determinegermlinecontigploidy - - profile: singularity - tags: gatk4/determinegermlinecontigploidy - - profile: conda - tags: gatk4/germlinecnvcaller - - profile: conda - tags: gatk4/postprocessgermlinecnvcalls - - profile: conda - tags: genescopefk - - profile: conda - tags: happy/sompy - - profile: conda - tags: hlala/preparegraph - - profile: conda - tags: ilastik/multicut - - profile: conda - tags: ilastik/pixelclassification - - profile: conda - tags: imputeme/vcftoprs - - profile: conda - tags: islandpath - - profile: conda - tags: manta/convertinversion - - profile: conda - tags: mcstaging/imc2mc - - profile: conda - tags: mcquant - - profile: conda - tags: medaka - - profile: conda - tags: merquryfk/katcomp - - profile: conda - tags: merquryfk/katgc - - profile: conda - tags: merquryfk/merquryfk - - profile: conda - tags: merquryfk/ploidyplot - - profile: conda - tags: minimap2/align - - profile: conda - tags: mitohifi/findmitoreference - - profile: conda - tags: mitohifi/mitohifi - - profile: conda - tags: nanoplot - - profile: conda - tags: ncbitools/vecscreen - - profile: conda - tags: parabricks/dbsnp - - profile: conda - tags: parabricks/deepvariant - - profile: conda - tags: parabricks/genotypegvcf - - profile: conda - tags: parabricks/haplotypecaller - - profile: conda - tags: parabricks/indexgvcf - - profile: conda - tags: parabricks/mutectcaller - - profile: conda - tags: picard/collecthsmetrics - - profile: conda - tags: picard/collectwgsmetrics - - profile: conda - tags: sentieon/applyvarcal - - profile: conda - tags: sentieon/datametrics - - profile: conda - tags: sentieon/dnamodelapply - - profile: conda - tags: sentieon/dnascope - - profile: conda - tags: sentieon/readwriter - - profile: conda - tags: sentieon/tnfilter - - profile: conda - tags: sentieon/tnhaplotyper2 - - profile: conda - tags: sentieon/tnscope - - profile: conda - tags: sentieon/varcal - - profile: conda - tags: sentieon/wgsmetrics - - profile: conda - tags: subworkflows/bam_qc_picard - - profile: conda - tags: subworkflows/bcl_demultiplex - - profile: conda - tags: subworkflows/fasta_clean_fcs - - profile: conda - tags: svanalyzer/svbenchmark - - profile: conda - tags: universc - - profile: singularity - tags: universc - - profile: conda - tags: vt/decompose - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip-pytest - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pytest - restore-keys: | - ${{ runner.os }}-pip-pytest - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow ${{ matrix.NXF_VER }} - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up miniconda - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 - with: - miniconda-version: "latest" - channels: conda-forge,bioconda - python-version: ${{ matrix.python-version }} - conda-remove-defaults: true - - - name: Conda setup - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get update > /dev/null - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} - - - name: Setting global variables - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - id: parsed - with: - script: | - return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') - result-encoding: string - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 - with: - name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} - path: | - /home/ubuntu/pytest_workflow_*/*/.nextflow.log - /home/ubuntu/pytest_workflow_*/*/log.out - /home/ubuntu/pytest_workflow_*/*/log.err - /home/ubuntu/pytest_workflow_*/*/work - !/home/ubuntu/pytest_workflow_*/*/work/conda - !/home/ubuntu/pytest_workflow_*/*/work/singularity - !${{ github.workspace }}/.singularity - - nf-test: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-test - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) - strategy: - fail-fast: false - matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] - profile: [conda, docker_self_hosted, singularity] - exclude: - - path: modules/nf-core/nf-test - - profile: conda - path: modules/nf-core/angsd/gl - - profile: conda - path: modules/nf-core/annotsv/installannotations - - profile: conda - path: modules/nf-core/happy/sompy - - profile: conda - path: modules/nf-core/backsub - - profile: conda - path: modules/nf-core/bakta/bakta - - profile: conda - path: modules/nf-core/bakta/baktadbdownload - - profile: conda - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/bcl2fastq - - profile: conda - path: modules/nf-core/bclconvert - - profile: conda - path: modules/nf-core/celesta - - profile: conda - path: modules/nf-core/cellpose - - profile: conda - path: modules/nf-core/cellranger/count - - profile: conda - path: modules/nf-core/cellranger/mkfastq - - profile: conda - path: modules/nf-core/cellranger/mkgtf - - profile: conda - path: modules/nf-core/cellranger/mkref - - profile: conda - path: modules/nf-core/cellranger/mkvdjref - - profile: conda - path: modules/nf-core/cellranger/multi - - profile: conda - path: modules/nf-core/cellranger/vdj - - profile: conda - path: modules/nf-core/checkqc - - profile: conda - path: modules/nf-core/custom/dumpsoftwareversions - - profile: conda - path: modules/nf-core/deepcell/mesmer - - profile: conda - path: modules/nf-core/deepsomatic - - profile: singularity - path: modules/nf-core/deepsomatic - - profile: conda - path: modules/nf-core/deepvariant - - profile: conda - path: modules/nf-core/deepvariant/callvariants - - profile: conda - path: modules/nf-core/deepvariant/makeexamples - - profile: conda - path: modules/nf-core/deepvariant/postprocessvariants - - profile: conda - path: modules/nf-core/deepvariant/rundeepvariant - - profile: conda - path: modules/nf-core/deepvariant/vcfstatsreport - - profile: conda - path: modules/nf-core/doubletdetection - - profile: conda - path: modules/nf-core/ensemblvep/vep - - profile: conda - path: modules/nf-core/fastk/fastk - - profile: conda - path: modules/nf-core/cellrangerarc/mkgtf - - profile: conda - path: modules/nf-core/fastk/histex - - profile: conda - path: modules/nf-core/fastk/merge - - profile: conda - path: modules/nf-core/fcs/fcsadaptor - - profile: conda - path: modules/nf-core/fcs/fcsgx - - profile: conda - path: modules/nf-core/ganon/buildcustom - - profile: conda - path: modules/nf-core/ganon/classify - - profile: conda - path: modules/nf-core/ganon/report - - profile: conda - path: modules/nf-core/ganon/table - - profile: conda - path: modules/nf-core/gatk4/cnnscorevariants - - profile: conda - path: modules/nf-core/gatk4/determinegermlinecontigploidy - - profile: conda - path: modules/nf-core/genescopefk - - profile: conda - path: modules/nf-core/ilastik/multicut - - profile: conda - path: modules/nf-core/ilastik/pixelclassification - - profile: conda - path: modules/nf-core/imputeme/vcftoprs - - profile: conda - path: modules/nf-core/mcstaging/imc2mc - - profile: conda - path: modules/nf-core/mcquant - - profile: conda - path: modules/nf-core/mcstaging/phenoimager2mc - - profile: conda - path: modules/nf-core/merquryfk/katcomp - - profile: conda - path: modules/nf-core/merquryfk/katgc - - profile: conda - path: modules/nf-core/merquryfk/merquryfk - - profile: conda - path: modules/nf-core/merquryfk/ploidyplot - - profile: conda - path: modules/nf-core/molkartgarage/clahe - - profile: conda - path: modules/nf-core/quartonotebook - - profile: conda - path: modules/nf-core/scimap/spatiallda - - profile: conda - path: modules/nf-core/sentieon/bwaindex - - profile: conda - path: modules/nf-core/sentieon/bwamem - - profile: conda - path: modules/nf-core/sentieon/datametrics - - profile: conda - path: modules/nf-core/sentieon/dedup - - profile: conda - path: modules/nf-core/sentieon/qualcal - - profile: conda - path: modules/nf-core/spaceranger/mkgtf - - profile: conda - path: modules/nf-core/spaceranger/mkref - - profile: conda - path: modules/nf-core/spaceranger/count - - profile: conda - path: modules/nf-core/spotiflow - - profile: conda - path: modules/nf-core/svanalyzer/svbenchmark - - profile: conda - path: modules/nf-core/universc - - profile: singularity - path: modules/nf-core/universc - - profile: conda - path: modules/nf-core/vt/decompose - - profile: singularity - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/wittyer - - profile: conda - path: modules/nf-core/islandpath - - profile: conda - path: modules/nf-core/scimap/mcmicro - - profile: conda - path: subworkflows/nf-core/vcf_annotate_ensemblvep - - profile: conda - path: subworkflows/nf-core/bcl_demultiplex - - profile: conda - path: subworkflows/nf-core/deepvariant - - profile: conda - path: subworkflows/nf-core/fastq_align_bamcmp_bwa - - profile: conda - path: subworkflows/nf-core/fastq_align_bwa - - profile: conda - path: subworkflows/nf-core/fasta_newick_epang_gappa - - path: modules/nf-core/parabricks/fq2bammeth - - path: modules/nf-core/parabricks/fq2bam - - path: modules/nf-core/parabricks/applybqsr - - profile: conda - path: modules/nf-core/xeniumranger/relabel - - profile: conda - path: modules/nf-core/xeniumranger/rename - - profile: conda - path: modules/nf-core/xeniumranger/resegment - - profile: conda - path: modules/nf-core/xeniumranger/import-segmentation - - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - - name: Run nf-test Action - uses: ./.github/actions/nf-test-action - with: - path: ${{ matrix.path }} - profile: ${{ matrix.profile }} - - nf-test-gpu: - runs-on: "gpu" - name: nf-test-gpu - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') ) - strategy: - fail-fast: false - matrix: - include: - - path: modules/nf-core/parabricks/applybqsr - profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bam - profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bammeth - profile: [docker_self_hosted, singularity] - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - - name: Run nf-test Action - uses: ./.github/actions/nf-test-action - env: - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" - with: - path: ${{ matrix.path }} - profile: ${{ matrix.profile }},gpu - - confirm-pass: - runs-on: ubuntu-latest - needs: - [ - prettier, - editorconfig, - pytest-changes, - nf-core-lint-modules, - nf-core-lint-subworkflows, - pytest, - nf-test-changes, - nf-test, - nf-test-gpu, - ] - if: always() - steps: - - name: All tests ok - if: ${{ success() || !contains(needs.*.result, 'failure') }} - run: exit 0 - - name: One or more tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 - - - name: debug-print - if: always() - run: | - echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b97fd79f3a8..b6517edd742 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,3 +25,8 @@ repos: files: \.py$ args: [--fix, --exit-non-zero-on-fix, "--select", "I,E1,E4,E7,E9,F,UP,N"] # sort imports and fix (rules taken from nf-core/tools) - id: ruff-format # formatter + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: "3.0.3" + hooks: + - id: editorconfig-checker + alias: ec diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 3dee85ffb4d..e00acce210f 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -5,7 +5,7 @@ process BOWTIE_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : - 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" + 'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(reads) @@ -29,6 +29,7 @@ process BOWTIE_ALIGN { def endedness = meta.single_end ? "$reads" : "-1 ${reads[0]} -2 ${reads[1]}" """ INDEX=\$(find -L ./ -name "*.3.ebwt" | sed 's/\\.3.ebwt\$//') + bowtie \\ --threads $task.cpus \\ --sam \\ diff --git a/modules/nf-core/bowtie/align/tests/main.nf.test.snap b/modules/nf-core/bowtie/align/tests/main.nf.test.snap index de95bb81531..e24eff79e62 100644 --- a/modules/nf-core/bowtie/align/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/align/tests/main.nf.test.snap @@ -2,7 +2,7 @@ "sarscov2 - single_end": { "content": [ [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], [ "7bdcfc6f54ae6e8f4570395cc85db9a3" @@ -27,10 +27,10 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-26T09:25:24.60746041" + "timestamp": "2024-11-14T10:55:41.588212" }, "sarscov2 - single_end - stub": { "content": [ @@ -63,7 +63,7 @@ ] ], "3": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], "bam": [ [ @@ -93,20 +93,20 @@ ] ], "versions": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-25T10:00:28.666281812" + "timestamp": "2024-11-14T10:55:46.311312" }, "sarscov2 - paired_end": { "content": [ [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], [ [ @@ -125,10 +125,10 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-26T11:57:56.604464368" + "timestamp": "2024-11-14T10:55:51.790156" }, "sarscov2 - paired_end - stub": { "content": [ @@ -155,7 +155,7 @@ ], "3": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], "bam": [ [ @@ -179,14 +179,14 @@ ] ], "versions": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-25T10:01:02.043164876" + "timestamp": "2024-11-14T10:55:56.594761" } } \ No newline at end of file diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 0f6b9d4d1bf..953e12f6875 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -5,7 +5,7 @@ process BOWTIE_BUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : - 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" + 'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test b/modules/nf-core/bowtie/build/tests/main.nf.test index 81209a7b1fc..1811013d5ae 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test +++ b/modules/nf-core/bowtie/build/tests/main.nf.test @@ -34,6 +34,7 @@ nextflow_process { test("sarscov2 - fasta - stub") { options "-stub" + tag "version" tag "stub" when { @@ -56,4 +57,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test.snap b/modules/nf-core/bowtie/build/tests/main.nf.test.snap index e8061756ba2..e71a80d922f 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/build/tests/main.nf.test.snap @@ -18,7 +18,7 @@ ] ], "1": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ], "index": [ [ @@ -36,15 +36,29 @@ ] ], "versions": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-18T08:38:14.852528155" + "timestamp": "2024-11-14T10:56:03.634167" + }, + "versions": { + "content": [ + { + "BOWTIE_BUILD": { + "bowtie": "1.3.1" + } + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-09-10T11:48:00.007395903" }, "sarscov2 - fasta": { "content": [ @@ -65,7 +79,7 @@ ] ], "1": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ], "index": [ [ @@ -83,14 +97,14 @@ ] ], "versions": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-18T08:37:53.65689025" + "timestamp": "2024-11-14T10:56:00.181902" } } \ No newline at end of file diff --git a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test index c85fd447b3d..9cecef89358 100644 --- a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test +++ b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test @@ -8,6 +8,7 @@ nextflow_process { tag "modules_nfcore" tag "parabricks" tag "parabricks/applybqsr" + tag "gpu" test("sarscov2 - bam - pe") { @@ -210,4 +211,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 8e4adb2966c..c2273e762ba 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "parabricks/fq2bam" tag "modules_nfcore" tag "parabricks" + tag "gpu" setup { run("BWA_INDEX") { @@ -171,4 +172,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test index 42e436a3ba3..459eb6317db 100644 --- a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "parabricks" tag "parabricks/fq2bammeth" tag "modules_nfcore" + tag "gpu" setup { run("BWAMETH_INDEX") { @@ -119,4 +120,4 @@ nextflow_process { ) } } -} +} \ No newline at end of file diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index f651ff4d89d..6a534a8f7c7 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -32,12 +32,6 @@ profiles { docker.enabled = true docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } - gpu { - docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' - apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' - singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' - use_gpu = true - } docker_self_hosted { docker.enabled = true docker.fixOwnership = true From c0ef8c30bbb68f2437f02f4f53400df26a15ad5a Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Mon, 18 Nov 2024 21:13:46 -0600 Subject: [PATCH 5/5] Bump Nextflow Versions (#7019) * ci(renovate): Add custom VERSION updates in GitHub actions https://docs.renovatebot.com/presets-customManagers/#custommanagersgithubactionsversions * ci(#6664): Bump all Nextflow versions to 24.10.1 * ci: Add renovate comments to version in actions --- .github/actions/nf-test-action/action.yml | 4 +++- .github/renovate.json5 | 20 ++++++++++++++++++-- .github/workflows/gpu-tests.yml | 6 ++++-- .github/workflows/lint.yml | 2 ++ .github/workflows/nf-test.yml | 6 ++++-- .github/workflows/pytest-workflow.yml | 4 ++++ .github/workflows/wave.yml | 18 +++++++++++------- tests/config/nextflow.config | 2 +- tests/config/nf-test.config | 2 +- 9 files changed, 48 insertions(+), 16 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 5b87e1e3611..b45e04d467a 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -24,6 +24,8 @@ runs: - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 + with: + version: "${{ env.NXF_VERSION }}" - name: Set up Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 @@ -33,7 +35,7 @@ runs: - name: Install nf-test uses: nf-core/setup-nf-test@v1 with: - version: "0.9.2" + version: "${{ env.NFTEST_VERSION }}" install-pdiff: true - name: Setup apptainer diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 46af7821e36..eacaed1524b 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -7,8 +7,24 @@ "prHeader": "## WARNING\n - [ ] I have made sure to update the singularity and conda images", "packageRules": [ { - "matchDatasources": ["conda", "pypi", "docker"], + "matchDatasources": [ + "conda", + "pypi", + "docker" + ], "automerge": false } - ] + ], + "customManagers": [ + { + "customType": "regex", + "fileMatch": [ + "(^|/)(workflow-templates|\\.(?:github|gitea|forgejo)/(?:workflows|actions))/.+\\.ya?ml$", + "(^|/)action\\.ya?ml$" + ], + "matchStrings": [ + "# renovate: datasource=(?[a-z-.]+?) depName=(?[^\\s]+?)(?: (?:lookupName|packageName)=(?[^\\s]+?))?(?: versioning=(?[^\\s]+?))?(?: extractVersion=(?[^\\s]+?))?\\s+[A-Za-z0-9_]+?_VERSION\\s*:\\s*[\"']?(?.+?)[\"']?\\s" + ] + } + ] } diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index dfabddf07ca..92188ed9702 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -29,11 +29,13 @@ concurrency: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NFT_VER: "0.9.2" + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFTEST_VERSION: "0.9.2" NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - NXF_VER: "24.04.4" + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VERSION: "24.10.1" jobs: nf-test-gpu: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 039e5069a7f..756894a27b0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,6 +28,8 @@ env: NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + NXF_VER: "24.10.1" jobs: pre-commit: diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 38ebea1eff9..6eb3525404a 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -26,11 +26,13 @@ concurrency: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NFT_VER: "0.9.2" + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFTEST_VERSION: "0.9.2" NXF_ANSI_LOG: false NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - NXF_VER: "24.04.4" + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + NXF_VERSION: "24.10.1" jobs: nf-test: diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index a772dfc1db1..4d4a96d0f38 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -28,6 +28,8 @@ env: NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VERSION: "24.10.1" jobs: pytest-changes: @@ -263,6 +265,8 @@ jobs: - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 + with: + version: ${{ env.NXF_VERSION }} - name: Setup apptainer if: matrix.profile == 'singularity' diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index a86fd9bb10a..61d0d06a509 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -10,8 +10,12 @@ on: # TODO Skip testing CI if any changes to environment.yml env: - WAVE_VER: "1.5.0" - NFTEST_VER: "0.9.1" + # renovate: datasource=github-releases depName=seqeralabs/wave-cli versioning=semver + WAVE_VERSION: "1.5.0" + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFTEST_VERSION: "0.9.1" + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + NXF_VERSION: "24.10.1" jobs: generate-matrix: @@ -62,8 +66,8 @@ jobs: - name: Install wave-cli run: | - wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 - sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VERSION}/wave-${WAVE_VERSION}-linux-x86_64 + sudo mv wave-${WAVE_VERSION}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave - name: Build ${{ matrix.profile }} container @@ -98,8 +102,8 @@ jobs: - name: Install wave-cli run: | - wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 - sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VERSION}/wave-${WAVE_VERSION}-linux-x86_64 + sudo mv wave-${WAVE_VERSION}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave - name: Create a registry name @@ -139,7 +143,7 @@ jobs: # - uses: nf-core/setup-nf-test@v1 # with: - # version: ${{ env.NFTEST_VER }} + # version: ${{ env.NFTEST_VERSION }} # - name: Bump Snapshot Versions # env: diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c86fb134d75..f90c24c2bc8 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -43,5 +43,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=24.10.1' } diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 6a534a8f7c7..fa21cb2edd6 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -56,5 +56,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=24.10.1' }