diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index e1e1f4bc57d..b45e04d467a 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -1,18 +1,22 @@ name: "nf-test Action" description: "Runs nf-test with common setup steps" inputs: - path: - description: "Path to test" - required: true profile: description: "Profile to use" required: true + shard: + description: "Shard number for this CI job" + required: true + total_shards: + description: "Total number of test shards(NOT the total number of matrix jobs)" + required: true + tags: + description: "Tags to test (`[,...]`)" + required: false runs: using: "composite" steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 with: distribution: "temurin" @@ -20,6 +24,8 @@ runs: - name: Setup Nextflow uses: nf-core/setup-nextflow@v2 + with: + version: "${{ env.NXF_VERSION }}" - name: Set up Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 @@ -29,7 +35,7 @@ runs: - name: Install nf-test uses: nf-core/setup-nf-test@v1 with: - version: "0.9.2" + version: "${{ env.NFTEST_VERSION }}" install-pdiff: true - name: Setup apptainer @@ -56,20 +62,31 @@ runs: if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' shell: bash run: | + python -m pip install cryptography nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + # TODO Skip failing conda tests and document their failures + # https://github.com/nf-core/modules/issues/7017 - name: Run nf-test shell: bash env: SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" + TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }} + NFT_WORKDIR: "~" run: | - NFT_WORKDIR=~ \ nf-test test \ --profile=${{ inputs.profile }} \ --tap=test.tap \ --verbose \ - ${{ inputs.path }} + --ci \ + --changed-since HEAD^ \ + --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \ + --filter process,workflow \ + --follow-dependencies \ + ${{ env.TAGS }} + + # TODO If no test.tap, then make one to spoof? - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 if: ${{ inputs.path != '' }} with: diff --git a/.github/conda_skip.yml b/.github/conda_skip.yml new file mode 100644 index 00000000000..2ef7216da3e --- /dev/null +++ b/.github/conda_skip.yml @@ -0,0 +1,184 @@ +# TODO What do we do with these? +# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now +exclude: + - path: modules/nf-core/nf-test + - profile: conda + path: modules/nf-core/angsd/gl + - profile: conda + path: modules/nf-core/annotsv/installannotations + - profile: conda + path: modules/nf-core/happy/sompy + - profile: conda + path: modules/nf-core/backsub + - profile: conda + path: modules/nf-core/bakta/bakta + - profile: conda + path: modules/nf-core/bakta/baktadbdownload + - profile: conda + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/bcl2fastq + - profile: conda + path: modules/nf-core/bclconvert + - profile: conda + path: modules/nf-core/celesta + - profile: conda + path: modules/nf-core/cellpose + - profile: conda + path: modules/nf-core/cellranger/count + - profile: conda + path: modules/nf-core/cellranger/mkfastq + - profile: conda + path: modules/nf-core/cellranger/mkgtf + - profile: conda + path: modules/nf-core/cellranger/mkref + - profile: conda + path: modules/nf-core/cellranger/mkvdjref + - profile: conda + path: modules/nf-core/cellranger/multi + - profile: conda + path: modules/nf-core/cellranger/vdj + - profile: conda + path: modules/nf-core/checkqc + - profile: conda + path: modules/nf-core/custom/dumpsoftwareversions + - profile: conda + path: modules/nf-core/deepcell/mesmer + - profile: conda + path: modules/nf-core/deepsomatic + - profile: singularity + path: modules/nf-core/deepsomatic + - profile: conda + path: modules/nf-core/deepvariant + - profile: conda + path: modules/nf-core/deepvariant/callvariants + - profile: conda + path: modules/nf-core/deepvariant/makeexamples + - profile: conda + path: modules/nf-core/deepvariant/postprocessvariants + - profile: conda + path: modules/nf-core/deepvariant/rundeepvariant + - profile: conda + path: modules/nf-core/deepvariant/vcfstatsreport + - profile: conda + path: modules/nf-core/doubletdetection + - profile: conda + path: modules/nf-core/ensemblvep/vep + - profile: conda + path: modules/nf-core/fastk/fastk + - profile: conda + path: modules/nf-core/cellrangerarc/mkgtf + - profile: conda + path: modules/nf-core/fastk/histex + - profile: conda + path: modules/nf-core/fastk/merge + - profile: conda + path: modules/nf-core/fcs/fcsadaptor + - profile: conda + path: modules/nf-core/fcs/fcsgx + - profile: conda + path: modules/nf-core/ganon/buildcustom + - profile: conda + path: modules/nf-core/ganon/classify + - profile: conda + path: modules/nf-core/ganon/report + - profile: conda + path: modules/nf-core/ganon/table + - profile: conda + path: modules/nf-core/gatk4/cnnscorevariants + - profile: conda + path: modules/nf-core/gatk4/determinegermlinecontigploidy + - profile: conda + path: modules/nf-core/genescopefk + - profile: conda + path: modules/nf-core/ilastik/multicut + - profile: conda + path: modules/nf-core/ilastik/pixelclassification + - profile: conda + path: modules/nf-core/imputeme/vcftoprs + - profile: conda + path: modules/nf-core/mcstaging/imc2mc + - profile: conda + path: modules/nf-core/mcquant + - profile: conda + path: modules/nf-core/mcstaging/phenoimager2mc + - profile: conda + path: modules/nf-core/merquryfk/katcomp + - profile: conda + path: modules/nf-core/merquryfk/katgc + - profile: conda + path: modules/nf-core/merquryfk/merquryfk + - profile: conda + path: modules/nf-core/merquryfk/ploidyplot + - profile: conda + path: modules/nf-core/molkartgarage/clahe + - profile: conda + path: modules/nf-core/quartonotebook + - profile: conda + path: modules/nf-core/scimap/spatiallda + - profile: conda + path: modules/nf-core/sentieon/bwaindex + - profile: conda + path: modules/nf-core/sentieon/bwamem + - profile: conda + path: modules/nf-core/sentieon/datametrics + - profile: conda + path: modules/nf-core/sentieon/dedup + - profile: conda + path: modules/nf-core/sentieon/qualcal + - profile: conda + path: modules/nf-core/spaceranger/mkgtf + - profile: conda + path: modules/nf-core/spaceranger/mkref + - profile: conda + path: modules/nf-core/spaceranger/count + - profile: conda + path: modules/nf-core/spotiflow + - profile: conda + path: modules/nf-core/svanalyzer/svbenchmark + - profile: conda + path: modules/nf-core/universc + - profile: singularity + path: modules/nf-core/universc + - profile: conda + path: modules/nf-core/vt/decompose + - profile: singularity + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/wittyer + - profile: conda + path: modules/nf-core/islandpath + - profile: conda + path: modules/nf-core/scimap/mcmicro + - profile: conda + path: modules/nf-core/parabricks/fq2bammeth + - profile: docker_self_hosted + path: modules/nf-core/parabricks/fq2bammeth + - profile: singularity + path: modules/nf-core/parabricks/fq2bammeth + - profile: conda + path: modules/nf-core/parabricks/fq2bam + - profile: docker_self_hosted + path: modules/nf-core/parabricks/fq2bam + - profile: singularity + path: modules/nf-core/parabricks/fq2bam + - profile: conda + path: subworkflows/nf-core/vcf_annotate_ensemblvep + - profile: conda + path: subworkflows/nf-core/bcl_demultiplex + - profile: conda + path: subworkflows/nf-core/deepvariant + - profile: conda + path: subworkflows/nf-core/fastq_align_bamcmp_bwa + - profile: conda + path: subworkflows/nf-core/fastq_align_bwa + - profile: conda + path: subworkflows/nf-core/fasta_newick_epang_gappa + - profile: conda + path: modules/nf-core/xeniumranger/relabel + - profile: conda + path: modules/nf-core/xeniumranger/rename + - profile: conda + path: modules/nf-core/xeniumranger/resegment + - profile: conda + path: modules/nf-core/xeniumranger/import-segmentation diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 46af7821e36..eacaed1524b 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -7,8 +7,24 @@ "prHeader": "## WARNING\n - [ ] I have made sure to update the singularity and conda images", "packageRules": [ { - "matchDatasources": ["conda", "pypi", "docker"], + "matchDatasources": [ + "conda", + "pypi", + "docker" + ], "automerge": false } - ] + ], + "customManagers": [ + { + "customType": "regex", + "fileMatch": [ + "(^|/)(workflow-templates|\\.(?:github|gitea|forgejo)/(?:workflows|actions))/.+\\.ya?ml$", + "(^|/)action\\.ya?ml$" + ], + "matchStrings": [ + "# renovate: datasource=(?[a-z-.]+?) depName=(?[^\\s]+?)(?: (?:lookupName|packageName)=(?[^\\s]+?))?(?: versioning=(?[^\\s]+?))?(?: extractVersion=(?[^\\s]+?))?\\s+[A-Za-z0-9_]+?_VERSION\\s*:\\s*[\"']?(?.+?)[\"']?\\s" + ] + } + ] } diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml new file mode 100644 index 00000000000..92188ed9702 --- /dev/null +++ b/.github/workflows/gpu-tests.yml @@ -0,0 +1,73 @@ +name: Run GPU nf-tests +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + paths: + - ".github/workflows/gpu-tests.yml" + - "modules/nf-core/parabricks/**" + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFTEST_VERSION: "0.9.2" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VERSION: "24.10.1" + +jobs: + nf-test-gpu: + runs-on: "gpu" + name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}" + strategy: + fail-fast: false + matrix: + shard: [1, 2] + profile: [docker_self_hosted, singularity] # conda? + # TODO Pass these in from GitHub PR trigger events + tags: + - parabricks/applybqsr + - parabricks/fq2bam + - parabricks/fq2bammeth + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: 2 + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action + env: + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + with: + profile: ${{ matrix.profile }},gpu + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} + tags: ${{matrix.tags}},gpu diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..756894a27b0 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,170 @@ +name: Run Linting +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + NXF_VER: "24.10.1" + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + with: + extra_args: "" + + ################### + # nf-core linting # + ################### + # TODO Move these to pre-commit + # https://github.com/nf-core/tools/pull/3141 + nf-core-changes: + name: nf-core-changes + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 2 # To retrieve the preceding commit. + + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + token: "" + + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + + nf-core-lint-modules: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: nf-core lint modules + needs: nf-core-changes + if: ${{ (needs.nf-core-changes.outputs.modules != '[]') }} + strategy: + fail-fast: false + matrix: + tags: "${{ fromJson(needs.nf-core-changes.outputs.modules) }}" + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 + with: + python-version: "3.11" + + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + id: cache-pip + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + restore-keys: | + ${{ runner.os }}-pip + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} + + nf-core-lint-subworkflows: + runs-on: ubuntu-latest + name: nf-core lint subworkflows + needs: nf-core-changes + if: ${{ (needs.nf-core-changes.outputs.subworkflows != '[]') }} + strategy: + fail-fast: false + matrix: + tags: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows) }}" + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + + - name: Set up Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5 + with: + python-version: "3.11" + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core subworkflows lint ${{ matrix.tags }} + + confirm-pass: + runs-on: ubuntu-latest + needs: [nf-core-lint-modules, nf-core-lint-subworkflows] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 00000000000..6eb3525404a --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,73 @@ +name: Run nf-test +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFTEST_VERSION: "0.9.2" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + NXF_VERSION: "24.10.1" + +jobs: + nf-test: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + # NOTE I think this is the cleanest way to get them organized + # process | conda | 1 + # process | conda | 2 + # process | conda | 3 + # process | docker_self_hosted | 1 + # ... + # workflow | singularity | 3 + name: "${{ matrix.profile }} | ${{ matrix.shard }}" + # TODO + # needs: get-number-of-shards + # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }} + strategy: + fail-fast: false + matrix: + # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now + # NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}" + # filter: [process, workflow] + profile: [conda, docker_self_hosted, singularity] + shard: [1, 2, 3, 4, 5] + env: + # FIXME Bumping them up to make the transition smooth, then we can throttle them back + TOTAL_SHARDS: 5 + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action + with: + profile: ${{ matrix.profile }} + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml new file mode 100644 index 00000000000..4d4a96d0f38 --- /dev/null +++ b/.github/workflows/pytest-workflow.yml @@ -0,0 +1,346 @@ +name: Run pytest-workflow +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver + NXF_VERSION: "24.10.1" + +jobs: + pytest-changes: + name: pytest-changes + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + with: + fetch-depth: 2 # To retrieve the preceding commit. + + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + token: "" + + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + + pytest: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: pytest + needs: [pytest-changes] + if: needs.pytest-changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] + profile: [conda, docker, singularity] + exclude: + - tags: nf-test + - profile: conda + tags: backsub + - profile: conda + tags: bases2fastq + - profile: singularity + tags: bases2fastq + - profile: conda + tags: basicpy + - profile: conda + tags: bcl2fastq + - profile: conda + tags: bclconvert + - profile: conda + tags: bwa/aln + - profile: conda + tags: bwa/index + - profile: conda + tags: bwa/mem + - profile: conda + tags: bwa/sampe + - profile: conda + tags: bwa/samse + - profile: conda + tags: cellpose + - profile: conda + tags: cellrangerarc/count + - profile: conda + tags: cellrangerarc/mkfastq + - profile: conda + tags: cellrangerarc/mkref + - profile: conda + tags: cellrangeratac/count + - profile: conda + tags: cellrangeratac/mkfastq + - profile: conda + tags: cellrangeratac/mkref + - profile: conda + tags: checkm2/databasedownload + - profile: conda + tags: checkm2/predict + - profile: conda + tags: controlfreec/makegraph2 + - profile: conda + tags: coreograph + - profile: conda + tags: deepcell/mesmer + - profile: conda + tags: deepvariant + - profile: conda + tags: fastani + - profile: conda + tags: fastk/fastk + - profile: conda + tags: fastk/histex + - profile: conda + tags: fastk/merge + - profile: conda + tags: fcs/fcsadaptor + - profile: conda + tags: fcs/fcsgx + - profile: conda + tags: gatk4/cnnscorevariants + - profile: conda + tags: gatk4/determinegermlinecontigploidy + - profile: singularity + tags: gatk4/determinegermlinecontigploidy + - profile: conda + tags: gatk4/germlinecnvcaller + - profile: conda + tags: gatk4/postprocessgermlinecnvcalls + - profile: conda + tags: genescopefk + - profile: conda + tags: happy/sompy + - profile: conda + tags: hlala/preparegraph + - profile: conda + tags: ilastik/multicut + - profile: conda + tags: ilastik/pixelclassification + - profile: conda + tags: imputeme/vcftoprs + - profile: conda + tags: islandpath + - profile: conda + tags: manta/convertinversion + - profile: conda + tags: mcstaging/imc2mc + - profile: conda + tags: mcquant + - profile: conda + tags: medaka + - profile: conda + tags: merquryfk/katcomp + - profile: conda + tags: merquryfk/katgc + - profile: conda + tags: merquryfk/merquryfk + - profile: conda + tags: merquryfk/ploidyplot + - profile: conda + tags: minimap2/align + - profile: conda + tags: mitohifi/findmitoreference + - profile: conda + tags: mitohifi/mitohifi + - profile: conda + tags: nanoplot + - profile: conda + tags: ncbitools/vecscreen + - profile: conda + tags: parabricks/applybqsr + - profile: conda + tags: parabricks/dbsnp + - profile: conda + tags: parabricks/deepvariant + - profile: conda + tags: parabricks/fq2bam + - profile: conda + tags: parabricks/genotypegvcf + - profile: conda + tags: parabricks/haplotypecaller + - profile: conda + tags: parabricks/indexgvcf + - profile: conda + tags: parabricks/mutectcaller + - profile: conda + tags: picard/collecthsmetrics + - profile: conda + tags: picard/collectwgsmetrics + - profile: conda + tags: sentieon/applyvarcal + - profile: conda + tags: sentieon/datametrics + - profile: conda + tags: sentieon/dnamodelapply + - profile: conda + tags: sentieon/dnascope + - profile: conda + tags: sentieon/readwriter + - profile: conda + tags: sentieon/tnfilter + - profile: conda + tags: sentieon/tnhaplotyper2 + - profile: conda + tags: sentieon/tnscope + - profile: conda + tags: sentieon/varcal + - profile: conda + tags: sentieon/wgsmetrics + - profile: conda + tags: subworkflows/bam_qc_picard + - profile: conda + tags: subworkflows/bcl_demultiplex + - profile: conda + tags: subworkflows/fasta_clean_fcs + - profile: conda + tags: svanalyzer/svbenchmark + - profile: conda + tags: universc + - profile: singularity + tags: universc + - profile: conda + tags: vt/decompose + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 + id: cache-pip-pytest + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-pytest + restore-keys: | + ${{ runner.os }}-pip-pytest + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow cryptography + + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + with: + version: ${{ env.NXF_VERSION }} + + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up miniconda + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 + with: + miniconda-version: "latest" + channels: conda-forge,bioconda + python-version: "3.11" + + - name: Conda setup + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt-get update > /dev/null + sudo apt-get install bat > /dev/null + batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} + + - name: Setting global variables + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: parsed + with: + script: | + return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') + result-encoding: string + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 + with: + name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} + path: | + /home/ubuntu/pytest_workflow_*/*/.nextflow.log + /home/ubuntu/pytest_workflow_*/*/log.out + /home/ubuntu/pytest_workflow_*/*/log.err + /home/ubuntu/pytest_workflow_*/*/work + !/home/ubuntu/pytest_workflow_*/*/work/conda + !/home/ubuntu/pytest_workflow_*/*/work/singularity + !${{ github.workspace }}/.singularity + + confirm-pass: + runs-on: ubuntu-latest + needs: [pytest-changes, pytest] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 736b2c9eae9..00000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,745 +0,0 @@ -name: Run tests -on: - push: - branches: - # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging - - "renovate/**" # branches Renovate creates - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity - NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - # FIXME Flip this off once we get to less than a couple hundred. Adding - # this so it will only run against changed files. It'll make it much - # easier to fix these as they come up rather than everything at once. - with: - extra_args: "--all-files" - - prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Install NodeJS - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 - with: - node-version: "20" - - - name: Install Prettier - run: npm install -g prettier@3.2.5 - - - name: Run Prettier --check - run: prettier --check . - - editorconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 - with: - node-version: "20" - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) - - pytest-changes: - name: pytest-changes - runs-on: ubuntu-latest - outputs: - tags: ${{ steps.filter.outputs.changes }} - modules: ${{ steps.tags.outputs.modules }} - subworkflows: ${{ steps.tags.outputs.subworkflows }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - fetch-depth: 2 # To retrieve the preceding commit. - - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - token: "" - - - name: Fetch module tags - id: tags - run: | - echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.tags.outputs.modules }} - echo ${{ steps.tags.outputs.subworkflows }} - - nf-test-changes: - name: nf-test-changes - runs-on: ubuntu-latest - outputs: - # Expose detected tags as 'modules' and 'workflows' output variables - paths: ${{ steps.list.outputs.components }} - modules: ${{ steps.outputs.outputs.modules }} - subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - fetch-depth: 0 - - - name: List nf-test files - id: list - uses: adamrtalbot/detect-nf-test-changes@7c8be3ffd0d6538312b363c8c949dbbf5f26c3dd # v0.0.4 - with: - head: ${{ github.sha }} - base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - n_parents: 2 - - - name: Separate modules and subworkflows - id: outputs - run: | - echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.filter.outputs.components }} - echo ${{ steps.outputs.outputs.modules }} - echo ${{ steps.outputs.outputs.subworkflows }} - - nf-core-lint-modules: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.modules) }}", - "${{ fromJson(needs.nf-test-changes.outputs.modules) }}", - ] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - restore-keys: | - ${{ runner.os }}-pip - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core modules lint ${{ matrix.tags }} - - nf-core-lint-subworkflows: - runs-on: ubuntu-latest - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", - "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", - ] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core subworkflows lint ${{ matrix.tags }} - - pytest: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} - name: pytest - needs: [pytest-changes] - if: needs.pytest-changes.outputs.tags != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] - profile: [conda, docker, singularity] - exclude: - - tags: nf-test - - profile: conda - tags: backsub - - profile: conda - tags: bases2fastq - - profile: singularity - tags: bases2fastq - - profile: conda - tags: basicpy - - profile: conda - tags: bcl2fastq - - profile: conda - tags: bclconvert - - profile: conda - tags: bwa/aln - - profile: conda - tags: bwa/index - - profile: conda - tags: bwa/mem - - profile: conda - tags: bwa/sampe - - profile: conda - tags: bwa/samse - - profile: conda - tags: cellpose - - profile: conda - tags: cellrangerarc/count - - profile: conda - tags: cellrangerarc/mkfastq - - profile: conda - tags: cellrangerarc/mkref - - profile: conda - tags: cellrangeratac/count - - profile: conda - tags: cellrangeratac/mkfastq - - profile: conda - tags: cellrangeratac/mkref - - profile: conda - tags: checkm2/databasedownload - - profile: conda - tags: checkm2/predict - - profile: conda - tags: controlfreec/makegraph2 - - profile: conda - tags: coreograph - - profile: conda - tags: deepcell/mesmer - - profile: conda - tags: deepvariant - - profile: conda - tags: fastani - - profile: conda - tags: fastk/fastk - - profile: conda - tags: fastk/histex - - profile: conda - tags: fastk/merge - - profile: conda - tags: fcs/fcsadaptor - - profile: conda - tags: fcs/fcsgx - - profile: conda - tags: gatk4/cnnscorevariants - - profile: conda - tags: gatk4/determinegermlinecontigploidy - - profile: singularity - tags: gatk4/determinegermlinecontigploidy - - profile: conda - tags: gatk4/germlinecnvcaller - - profile: conda - tags: gatk4/postprocessgermlinecnvcalls - - profile: conda - tags: genescopefk - - profile: conda - tags: happy/sompy - - profile: conda - tags: hlala/preparegraph - - profile: conda - tags: ilastik/multicut - - profile: conda - tags: ilastik/pixelclassification - - profile: conda - tags: imputeme/vcftoprs - - profile: conda - tags: islandpath - - profile: conda - tags: manta/convertinversion - - profile: conda - tags: mcstaging/imc2mc - - profile: conda - tags: mcquant - - profile: conda - tags: medaka - - profile: conda - tags: merquryfk/katcomp - - profile: conda - tags: merquryfk/katgc - - profile: conda - tags: merquryfk/merquryfk - - profile: conda - tags: merquryfk/ploidyplot - - profile: conda - tags: minimap2/align - - profile: conda - tags: mitohifi/findmitoreference - - profile: conda - tags: mitohifi/mitohifi - - profile: conda - tags: nanoplot - - profile: conda - tags: ncbitools/vecscreen - - profile: conda - tags: parabricks/dbsnp - - profile: conda - tags: parabricks/deepvariant - - profile: conda - tags: parabricks/genotypegvcf - - profile: conda - tags: parabricks/haplotypecaller - - profile: conda - tags: parabricks/indexgvcf - - profile: conda - tags: parabricks/mutectcaller - - profile: conda - tags: picard/collecthsmetrics - - profile: conda - tags: picard/collectwgsmetrics - - profile: conda - tags: sentieon/applyvarcal - - profile: conda - tags: sentieon/datametrics - - profile: conda - tags: sentieon/dnamodelapply - - profile: conda - tags: sentieon/dnascope - - profile: conda - tags: sentieon/readwriter - - profile: conda - tags: sentieon/tnfilter - - profile: conda - tags: sentieon/tnhaplotyper2 - - profile: conda - tags: sentieon/tnscope - - profile: conda - tags: sentieon/varcal - - profile: conda - tags: sentieon/wgsmetrics - - profile: conda - tags: subworkflows/bam_qc_picard - - profile: conda - tags: subworkflows/bcl_demultiplex - - profile: conda - tags: subworkflows/fasta_clean_fcs - - profile: conda - tags: svanalyzer/svbenchmark - - profile: conda - tags: universc - - profile: singularity - tags: universc - - profile: conda - tags: vt/decompose - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip-pytest - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pytest - restore-keys: | - ${{ runner.os }}-pip-pytest - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow ${{ matrix.NXF_VER }} - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up miniconda - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 - with: - miniconda-version: "latest" - channels: conda-forge,bioconda - python-version: ${{ matrix.python-version }} - conda-remove-defaults: true - - - name: Conda setup - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get update > /dev/null - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} - - - name: Setting global variables - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - id: parsed - with: - script: | - return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') - result-encoding: string - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 - with: - name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} - path: | - /home/ubuntu/pytest_workflow_*/*/.nextflow.log - /home/ubuntu/pytest_workflow_*/*/log.out - /home/ubuntu/pytest_workflow_*/*/log.err - /home/ubuntu/pytest_workflow_*/*/work - !/home/ubuntu/pytest_workflow_*/*/work/conda - !/home/ubuntu/pytest_workflow_*/*/work/singularity - !${{ github.workspace }}/.singularity - - nf-test: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-test - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) - strategy: - fail-fast: false - matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] - profile: [conda, docker_self_hosted, singularity] - exclude: - - path: modules/nf-core/nf-test - - profile: conda - path: modules/nf-core/angsd/gl - - profile: conda - path: modules/nf-core/annotsv/installannotations - - profile: conda - path: modules/nf-core/happy/sompy - - profile: conda - path: modules/nf-core/backsub - - profile: conda - path: modules/nf-core/bakta/bakta - - profile: conda - path: modules/nf-core/bakta/baktadbdownload - - profile: conda - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/bcl2fastq - - profile: conda - path: modules/nf-core/bclconvert - - profile: conda - path: modules/nf-core/celesta - - profile: conda - path: modules/nf-core/cellpose - - profile: conda - path: modules/nf-core/cellranger/count - - profile: conda - path: modules/nf-core/cellranger/mkfastq - - profile: conda - path: modules/nf-core/cellranger/mkgtf - - profile: conda - path: modules/nf-core/cellranger/mkref - - profile: conda - path: modules/nf-core/cellranger/mkvdjref - - profile: conda - path: modules/nf-core/cellranger/multi - - profile: conda - path: modules/nf-core/cellranger/vdj - - profile: conda - path: modules/nf-core/checkqc - - profile: conda - path: modules/nf-core/custom/dumpsoftwareversions - - profile: conda - path: modules/nf-core/deepcell/mesmer - - profile: conda - path: modules/nf-core/deepsomatic - - profile: singularity - path: modules/nf-core/deepsomatic - - profile: conda - path: modules/nf-core/deepvariant - - profile: conda - path: modules/nf-core/deepvariant/callvariants - - profile: conda - path: modules/nf-core/deepvariant/makeexamples - - profile: conda - path: modules/nf-core/deepvariant/postprocessvariants - - profile: conda - path: modules/nf-core/deepvariant/rundeepvariant - - profile: conda - path: modules/nf-core/deepvariant/vcfstatsreport - - profile: conda - path: modules/nf-core/doubletdetection - - profile: conda - path: modules/nf-core/ensemblvep/vep - - profile: conda - path: modules/nf-core/fastk/fastk - - profile: conda - path: modules/nf-core/cellrangerarc/mkgtf - - profile: conda - path: modules/nf-core/fastk/histex - - profile: conda - path: modules/nf-core/fastk/merge - - profile: conda - path: modules/nf-core/fcs/fcsadaptor - - profile: conda - path: modules/nf-core/fcs/fcsgx - - profile: conda - path: modules/nf-core/ganon/buildcustom - - profile: conda - path: modules/nf-core/ganon/classify - - profile: conda - path: modules/nf-core/ganon/report - - profile: conda - path: modules/nf-core/ganon/table - - profile: conda - path: modules/nf-core/gatk4/cnnscorevariants - - profile: conda - path: modules/nf-core/gatk4/determinegermlinecontigploidy - - profile: conda - path: modules/nf-core/genescopefk - - profile: conda - path: modules/nf-core/ilastik/multicut - - profile: conda - path: modules/nf-core/ilastik/pixelclassification - - profile: conda - path: modules/nf-core/imputeme/vcftoprs - - profile: conda - path: modules/nf-core/mcstaging/imc2mc - - profile: conda - path: modules/nf-core/mcquant - - profile: conda - path: modules/nf-core/mcstaging/phenoimager2mc - - profile: conda - path: modules/nf-core/merquryfk/katcomp - - profile: conda - path: modules/nf-core/merquryfk/katgc - - profile: conda - path: modules/nf-core/merquryfk/merquryfk - - profile: conda - path: modules/nf-core/merquryfk/ploidyplot - - profile: conda - path: modules/nf-core/molkartgarage/clahe - - profile: conda - path: modules/nf-core/quartonotebook - - profile: conda - path: modules/nf-core/scimap/spatiallda - - profile: conda - path: modules/nf-core/sentieon/bwaindex - - profile: conda - path: modules/nf-core/sentieon/bwamem - - profile: conda - path: modules/nf-core/sentieon/datametrics - - profile: conda - path: modules/nf-core/sentieon/dedup - - profile: conda - path: modules/nf-core/sentieon/qualcal - - profile: conda - path: modules/nf-core/spaceranger/mkgtf - - profile: conda - path: modules/nf-core/spaceranger/mkref - - profile: conda - path: modules/nf-core/spaceranger/count - - profile: conda - path: modules/nf-core/spotiflow - - profile: conda - path: modules/nf-core/svanalyzer/svbenchmark - - profile: conda - path: modules/nf-core/universc - - profile: singularity - path: modules/nf-core/universc - - profile: conda - path: modules/nf-core/vt/decompose - - profile: singularity - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/wittyer - - profile: conda - path: modules/nf-core/islandpath - - profile: conda - path: modules/nf-core/scimap/mcmicro - - profile: conda - path: subworkflows/nf-core/vcf_annotate_ensemblvep - - profile: conda - path: subworkflows/nf-core/bcl_demultiplex - - profile: conda - path: subworkflows/nf-core/deepvariant - - profile: conda - path: subworkflows/nf-core/fastq_align_bamcmp_bwa - - profile: conda - path: subworkflows/nf-core/fastq_align_bwa - - profile: conda - path: subworkflows/nf-core/fasta_newick_epang_gappa - - path: modules/nf-core/parabricks/fq2bammeth - - path: modules/nf-core/parabricks/fq2bam - - path: modules/nf-core/parabricks/applybqsr - - profile: conda - path: modules/nf-core/xeniumranger/relabel - - profile: conda - path: modules/nf-core/xeniumranger/rename - - profile: conda - path: modules/nf-core/xeniumranger/resegment - - profile: conda - path: modules/nf-core/xeniumranger/import-segmentation - - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - - name: Run nf-test Action - uses: ./.github/actions/nf-test-action - with: - path: ${{ matrix.path }} - profile: ${{ matrix.profile }} - - nf-test-gpu: - runs-on: "gpu" - name: nf-test-gpu - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') ) - strategy: - fail-fast: false - matrix: - include: - - path: modules/nf-core/parabricks/applybqsr - profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bam - profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bammeth - profile: [docker_self_hosted, singularity] - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - - name: Run nf-test Action - uses: ./.github/actions/nf-test-action - env: - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" - with: - path: ${{ matrix.path }} - profile: ${{ matrix.profile }},gpu - - confirm-pass: - runs-on: ubuntu-latest - needs: - [ - prettier, - editorconfig, - pytest-changes, - nf-core-lint-modules, - nf-core-lint-subworkflows, - pytest, - nf-test-changes, - nf-test, - nf-test-gpu, - ] - if: always() - steps: - - name: All tests ok - if: ${{ success() || !contains(needs.*.result, 'failure') }} - run: exit 0 - - name: One or more tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 - - - name: debug-print - if: always() - run: | - echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index a86fd9bb10a..61d0d06a509 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -10,8 +10,12 @@ on: # TODO Skip testing CI if any changes to environment.yml env: - WAVE_VER: "1.5.0" - NFTEST_VER: "0.9.1" + # renovate: datasource=github-releases depName=seqeralabs/wave-cli versioning=semver + WAVE_VERSION: "1.5.0" + # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver + NFTEST_VERSION: "0.9.1" + # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver + NXF_VERSION: "24.10.1" jobs: generate-matrix: @@ -62,8 +66,8 @@ jobs: - name: Install wave-cli run: | - wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 - sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VERSION}/wave-${WAVE_VERSION}-linux-x86_64 + sudo mv wave-${WAVE_VERSION}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave - name: Build ${{ matrix.profile }} container @@ -98,8 +102,8 @@ jobs: - name: Install wave-cli run: | - wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 - sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VERSION}/wave-${WAVE_VERSION}-linux-x86_64 + sudo mv wave-${WAVE_VERSION}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave - name: Create a registry name @@ -139,7 +143,7 @@ jobs: # - uses: nf-core/setup-nf-test@v1 # with: - # version: ${{ env.NFTEST_VER }} + # version: ${{ env.NFTEST_VERSION }} # - name: Bump Snapshot Versions # env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b97fd79f3a8..b6517edd742 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,3 +25,8 @@ repos: files: \.py$ args: [--fix, --exit-non-zero-on-fix, "--select", "I,E1,E4,E7,E9,F,UP,N"] # sort imports and fix (rules taken from nf-core/tools) - id: ruff-format # formatter + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: "3.0.3" + hooks: + - id: editorconfig-checker + alias: ec diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf index 3dee85ffb4d..e00acce210f 100644 --- a/modules/nf-core/bowtie/align/main.nf +++ b/modules/nf-core/bowtie/align/main.nf @@ -5,7 +5,7 @@ process BOWTIE_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : - 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" + 'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(reads) @@ -29,6 +29,7 @@ process BOWTIE_ALIGN { def endedness = meta.single_end ? "$reads" : "-1 ${reads[0]} -2 ${reads[1]}" """ INDEX=\$(find -L ./ -name "*.3.ebwt" | sed 's/\\.3.ebwt\$//') + bowtie \\ --threads $task.cpus \\ --sam \\ diff --git a/modules/nf-core/bowtie/align/tests/main.nf.test.snap b/modules/nf-core/bowtie/align/tests/main.nf.test.snap index de95bb81531..e24eff79e62 100644 --- a/modules/nf-core/bowtie/align/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/align/tests/main.nf.test.snap @@ -2,7 +2,7 @@ "sarscov2 - single_end": { "content": [ [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], [ "7bdcfc6f54ae6e8f4570395cc85db9a3" @@ -27,10 +27,10 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-26T09:25:24.60746041" + "timestamp": "2024-11-14T10:55:41.588212" }, "sarscov2 - single_end - stub": { "content": [ @@ -63,7 +63,7 @@ ] ], "3": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], "bam": [ [ @@ -93,20 +93,20 @@ ] ], "versions": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-25T10:00:28.666281812" + "timestamp": "2024-11-14T10:55:46.311312" }, "sarscov2 - paired_end": { "content": [ [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], [ [ @@ -125,10 +125,10 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-26T11:57:56.604464368" + "timestamp": "2024-11-14T10:55:51.790156" }, "sarscov2 - paired_end - stub": { "content": [ @@ -155,7 +155,7 @@ ], "3": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ], "bam": [ [ @@ -179,14 +179,14 @@ ] ], "versions": [ - "versions.yml:md5,96e36b0b99c80da0be8239d03db30ecc" + "versions.yml:md5,40a84d909e40d3b39ab6ed522cc75145" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-25T10:01:02.043164876" + "timestamp": "2024-11-14T10:55:56.594761" } } \ No newline at end of file diff --git a/modules/nf-core/bowtie/build/main.nf b/modules/nf-core/bowtie/build/main.nf index 0f6b9d4d1bf..953e12f6875 100644 --- a/modules/nf-core/bowtie/build/main.nf +++ b/modules/nf-core/bowtie/build/main.nf @@ -5,7 +5,7 @@ process BOWTIE_BUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8c0819a9b1f520c49c933e667ae50de2a0730ece4c8b9efe79ac5e403963a9f/data' : - 'community​.wave​.seqera​.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" + 'community.wave.seqera.io/library/bowtie_samtools:e1a14e1ce4e0170d' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test b/modules/nf-core/bowtie/build/tests/main.nf.test index 81209a7b1fc..1811013d5ae 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test +++ b/modules/nf-core/bowtie/build/tests/main.nf.test @@ -34,6 +34,7 @@ nextflow_process { test("sarscov2 - fasta - stub") { options "-stub" + tag "version" tag "stub" when { @@ -56,4 +57,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/bowtie/build/tests/main.nf.test.snap b/modules/nf-core/bowtie/build/tests/main.nf.test.snap index e8061756ba2..e71a80d922f 100644 --- a/modules/nf-core/bowtie/build/tests/main.nf.test.snap +++ b/modules/nf-core/bowtie/build/tests/main.nf.test.snap @@ -18,7 +18,7 @@ ] ], "1": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ], "index": [ [ @@ -36,15 +36,29 @@ ] ], "versions": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-18T08:38:14.852528155" + "timestamp": "2024-11-14T10:56:03.634167" + }, + "versions": { + "content": [ + { + "BOWTIE_BUILD": { + "bowtie": "1.3.1" + } + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-09-10T11:48:00.007395903" }, "sarscov2 - fasta": { "content": [ @@ -65,7 +79,7 @@ ] ], "1": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ], "index": [ [ @@ -83,14 +97,14 @@ ] ], "versions": [ - "versions.yml:md5,afbd066e1dd5ae4a30b21c49149ea09a" + "versions.yml:md5,01c0299b236b1052f80f4be6aacfae6b" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-18T08:37:53.65689025" + "timestamp": "2024-11-14T10:56:00.181902" } } \ No newline at end of file diff --git a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test index c85fd447b3d..9cecef89358 100644 --- a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test +++ b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test @@ -8,6 +8,7 @@ nextflow_process { tag "modules_nfcore" tag "parabricks" tag "parabricks/applybqsr" + tag "gpu" test("sarscov2 - bam - pe") { @@ -210,4 +211,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 8e4adb2966c..c2273e762ba 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "parabricks/fq2bam" tag "modules_nfcore" tag "parabricks" + tag "gpu" setup { run("BWA_INDEX") { @@ -171,4 +172,4 @@ nextflow_process { } -} +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test index 42e436a3ba3..459eb6317db 100644 --- a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test @@ -10,6 +10,7 @@ nextflow_process { tag "parabricks" tag "parabricks/fq2bammeth" tag "modules_nfcore" + tag "gpu" setup { run("BWAMETH_INDEX") { @@ -119,4 +120,4 @@ nextflow_process { ) } } -} +} \ No newline at end of file diff --git a/modules/nf-core/vcf2db/environment.yml b/modules/nf-core/vcf2db/environment.yml index 58c477f9591..037e15309fd 100644 --- a/modules/nf-core/vcf2db/environment.yml +++ b/modules/nf-core/vcf2db/environment.yml @@ -1,5 +1,12 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json channels: - conda-forge - bioconda dependencies: + # renovate: datasource=conda depName=conda-forge/python + - conda-forge::python=2.7 + # renovate: datasource=conda depName=bioconda/cyvcf2 + - bioconda::cyvcf2=0.20.9 + # renovate: datasource=conda depName=bioconda/vcf2db - bioconda::vcf2db=2020.02.24 diff --git a/modules/nf-core/vcf2db/main.nf b/modules/nf-core/vcf2db/main.nf index 56c26bb6c5f..9e050793e2d 100644 --- a/modules/nf-core/vcf2db/main.nf +++ b/modules/nf-core/vcf2db/main.nf @@ -5,8 +5,8 @@ process VCF2DB { // WARN: Version information not provided by tool on CLI. Please update version string below when bumping container versions. conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/vcf2db:2020.02.24--pl5321hdfd78af_3': - 'biocontainers/vcf2db:2020.02.24--pl5321hdfd78af_3' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/14/14d1257c98f789e23a888e2961673b5b98d89e4d03e6a3efba2b1134ed439f61/data': + 'community.wave.seqera.io/library/python_vcf2db:91f604106ada5cf2' }" input: tuple val(meta), path(vcf), path(ped) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c86fb134d75..f90c24c2bc8 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -43,5 +43,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=24.10.1' } diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index f651ff4d89d..fa21cb2edd6 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -32,12 +32,6 @@ profiles { docker.enabled = true docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } - gpu { - docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' - apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' - singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' - use_gpu = true - } docker_self_hosted { docker.enabled = true docker.fixOwnership = true @@ -62,5 +56,5 @@ conda { createTimeout = "120 min" } includeConfig 'test_data.config' manifest { - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=24.10.1' }