Merge branch 'master' into bcftools_pluginsplit

.github/actions/nf-test-action/action.yml

@@ -1,25 +1,31 @@
 name: "nf-test Action"
 description: "Runs nf-test with common setup steps"
 inputs:
-  path:
-    description: "Path to test"
-    required: true
   profile:
     description: "Profile to use"
     required: true
+  shard:
+    description: "Shard number for this CI job"
+    required: true
+  total_shards:
+    description: "Total number of test shards(NOT the total number of matrix jobs)"
+    required: true
+  tags:
+    description: "Tags to test (`<tags>[,<tags>...]`)"
+    required: false
 
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
-
     - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
       with:
         distribution: "temurin"
         java-version: "17"
 
     - name: Setup Nextflow
       uses: nf-core/setup-nextflow@v2
+      with:
+        version: "${{ env.NXF_VERSION }}"
 
     - name: Set up Python
       uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
@@ -29,7 +35,7 @@ runs:
     - name: Install nf-test
       uses: nf-core/setup-nf-test@v1
       with:
-        version: "0.9.2"
+        version: "${{ env.NFTEST_VERSION }}"
         install-pdiff: true
 
     - name: Setup apptainer
@@ -56,20 +62,31 @@ runs:
       if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != ''
       shell: bash
       run: |
+        python -m pip install cryptography
         nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
 
+      # TODO Skip failing conda tests and document their failures
+      # https://github.com/nf-core/modules/issues/7017
     - name: Run nf-test
       shell: bash
       env:
         SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
         SENTIEON_AUTH_MECH: "GitHub Actions - token"
+        TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }}
+        NFT_WORKDIR: "~"
       run: |
-        NFT_WORKDIR=~ \
         nf-test test \
           --profile=${{ inputs.profile }} \
           --tap=test.tap \
           --verbose \
-          ${{ inputs.path }}
+          --ci \
+          --changed-since HEAD^ \
+          --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
+          --filter process,workflow \
+          --follow-dependencies \
+          ${{ env.TAGS }}
+
+      # TODO If no test.tap, then make one to spoof?
     - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
       if: ${{ inputs.path != '' }}
       with:

.github/conda_skip.yml

@@ -0,0 +1,184 @@
+# TODO What do we do with these?
+# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now
+exclude:
+  - path: modules/nf-core/nf-test
+  - profile: conda
+    path: modules/nf-core/angsd/gl
+  - profile: conda
+    path: modules/nf-core/annotsv/installannotations
+  - profile: conda
+    path: modules/nf-core/happy/sompy
+  - profile: conda
+    path: modules/nf-core/backsub
+  - profile: conda
+    path: modules/nf-core/bakta/bakta
+  - profile: conda
+    path: modules/nf-core/bakta/baktadbdownload
+  - profile: conda
+    path: modules/nf-core/bases2fastq
+  - profile: conda
+    path: modules/nf-core/bcl2fastq
+  - profile: conda
+    path: modules/nf-core/bclconvert
+  - profile: conda
+    path: modules/nf-core/celesta
+  - profile: conda
+    path: modules/nf-core/cellpose
+  - profile: conda
+    path: modules/nf-core/cellranger/count
+  - profile: conda
+    path: modules/nf-core/cellranger/mkfastq
+  - profile: conda
+    path: modules/nf-core/cellranger/mkgtf
+  - profile: conda
+    path: modules/nf-core/cellranger/mkref
+  - profile: conda
+    path: modules/nf-core/cellranger/mkvdjref
+  - profile: conda
+    path: modules/nf-core/cellranger/multi
+  - profile: conda
+    path: modules/nf-core/cellranger/vdj
+  - profile: conda
+    path: modules/nf-core/checkqc
+  - profile: conda
+    path: modules/nf-core/custom/dumpsoftwareversions
+  - profile: conda
+    path: modules/nf-core/deepcell/mesmer
+  - profile: conda
+    path: modules/nf-core/deepsomatic
+  - profile: singularity
+    path: modules/nf-core/deepsomatic
+  - profile: conda
+    path: modules/nf-core/deepvariant
+  - profile: conda
+    path: modules/nf-core/deepvariant/callvariants
+  - profile: conda
+    path: modules/nf-core/deepvariant/makeexamples
+  - profile: conda
+    path: modules/nf-core/deepvariant/postprocessvariants
+  - profile: conda
+    path: modules/nf-core/deepvariant/rundeepvariant
+  - profile: conda
+    path: modules/nf-core/deepvariant/vcfstatsreport
+  - profile: conda
+    path: modules/nf-core/doubletdetection
+  - profile: conda
+    path: modules/nf-core/ensemblvep/vep
+  - profile: conda
+    path: modules/nf-core/fastk/fastk
+  - profile: conda
+    path: modules/nf-core/cellrangerarc/mkgtf
+  - profile: conda
+    path: modules/nf-core/fastk/histex
+  - profile: conda
+    path: modules/nf-core/fastk/merge
+  - profile: conda
+    path: modules/nf-core/fcs/fcsadaptor
+  - profile: conda
+    path: modules/nf-core/fcs/fcsgx
+  - profile: conda
+    path: modules/nf-core/ganon/buildcustom
+  - profile: conda
+    path: modules/nf-core/ganon/classify
+  - profile: conda
+    path: modules/nf-core/ganon/report
+  - profile: conda
+    path: modules/nf-core/ganon/table
+  - profile: conda
+    path: modules/nf-core/gatk4/cnnscorevariants
+  - profile: conda
+    path: modules/nf-core/gatk4/determinegermlinecontigploidy
+  - profile: conda
+    path: modules/nf-core/genescopefk
+  - profile: conda
+    path: modules/nf-core/ilastik/multicut
+  - profile: conda
+    path: modules/nf-core/ilastik/pixelclassification
+  - profile: conda
+    path: modules/nf-core/imputeme/vcftoprs
+  - profile: conda
+    path: modules/nf-core/mcstaging/imc2mc
+  - profile: conda
+    path: modules/nf-core/mcquant
+  - profile: conda
+    path: modules/nf-core/mcstaging/phenoimager2mc
+  - profile: conda
+    path: modules/nf-core/merquryfk/katcomp
+  - profile: conda
+    path: modules/nf-core/merquryfk/katgc
+  - profile: conda
+    path: modules/nf-core/merquryfk/merquryfk
+  - profile: conda
+    path: modules/nf-core/merquryfk/ploidyplot
+  - profile: conda
+    path: modules/nf-core/molkartgarage/clahe
+  - profile: conda
+    path: modules/nf-core/quartonotebook
+  - profile: conda
+    path: modules/nf-core/scimap/spatiallda
+  - profile: conda
+    path: modules/nf-core/sentieon/bwaindex
+  - profile: conda
+    path: modules/nf-core/sentieon/bwamem
+  - profile: conda
+    path: modules/nf-core/sentieon/datametrics
+  - profile: conda
+    path: modules/nf-core/sentieon/dedup
+  - profile: conda
+    path: modules/nf-core/sentieon/qualcal
+  - profile: conda
+    path: modules/nf-core/spaceranger/mkgtf
+  - profile: conda
+    path: modules/nf-core/spaceranger/mkref
+  - profile: conda
+    path: modules/nf-core/spaceranger/count
+  - profile: conda
+    path: modules/nf-core/spotiflow
+  - profile: conda
+    path: modules/nf-core/svanalyzer/svbenchmark
+  - profile: conda
+    path: modules/nf-core/universc
+  - profile: singularity
+    path: modules/nf-core/universc
+  - profile: conda
+    path: modules/nf-core/vt/decompose
+  - profile: singularity
+    path: modules/nf-core/bases2fastq
+  - profile: conda
+    path: modules/nf-core/wittyer
+  - profile: conda
+    path: modules/nf-core/islandpath
+  - profile: conda
+    path: modules/nf-core/scimap/mcmicro
+  - profile: conda
+    path: modules/nf-core/parabricks/fq2bammeth
+  - profile: docker_self_hosted
+    path: modules/nf-core/parabricks/fq2bammeth
+  - profile: singularity
+    path: modules/nf-core/parabricks/fq2bammeth
+  - profile: conda
+    path: modules/nf-core/parabricks/fq2bam
+  - profile: docker_self_hosted
+    path: modules/nf-core/parabricks/fq2bam
+  - profile: singularity
+    path: modules/nf-core/parabricks/fq2bam
+  - profile: conda
+    path: subworkflows/nf-core/vcf_annotate_ensemblvep
+  - profile: conda
+    path: subworkflows/nf-core/bcl_demultiplex
+  - profile: conda
+    path: subworkflows/nf-core/deepvariant
+  - profile: conda
+    path: subworkflows/nf-core/fastq_align_bamcmp_bwa
+  - profile: conda
+    path: subworkflows/nf-core/fastq_align_bwa
+  - profile: conda
+    path: subworkflows/nf-core/fasta_newick_epang_gappa
+  - profile: conda
+    path: modules/nf-core/xeniumranger/relabel
+  - profile: conda
+    path: modules/nf-core/xeniumranger/rename
+  - profile: conda
+    path: modules/nf-core/xeniumranger/resegment
+  - profile: conda
+    path: modules/nf-core/xeniumranger/import-segmentation

.github/renovate.json5

@@ -7,8 +7,24 @@
     "prHeader": "## WARNING\n - [ ] I have made sure to update the singularity and conda images",
     "packageRules": [
         {
-            "matchDatasources": ["conda", "pypi", "docker"],
+            "matchDatasources": [
+                "conda",
+                "pypi",
+                "docker"
+            ],
             "automerge": false
         }
-    ]
+    ],
+        "customManagers": [
+            {
+                "customType": "regex",
+                "fileMatch": [
+                    "(^|/)(workflow-templates|\\.(?:github|gitea|forgejo)/(?:workflows|actions))/.+\\.ya?ml$",
+                    "(^|/)action\\.ya?ml$"
+                ],
+                "matchStrings": [
+                    "# renovate: datasource=(?<datasource>[a-z-.]+?) depName=(?<depName>[^\\s]+?)(?: (?:lookupName|packageName)=(?<packageName>[^\\s]+?))?(?: versioning=(?<versioning>[^\\s]+?))?(?: extractVersion=(?<extractVersion>[^\\s]+?))?\\s+[A-Za-z0-9_]+?_VERSION\\s*:\\s*[\"']?(?<currentValue>.+?)[\"']?\\s"
+                ]
+            }
+        ]
 }

.github/workflows/gpu-tests.yml

@@ -0,0 +1,73 @@
+name: Run GPU nf-tests
+on:
+  push:
+    branches:
+      # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging
+      - "renovate/**" # branches Renovate creates
+  pull_request:
+    branches: [master]
+    paths:
+      - ".github/workflows/gpu-tests.yml"
+      - "modules/nf-core/parabricks/**"
+  merge_group:
+    types: [checks_requested]
+    branches: [master]
+  workflow_dispatch:
+    inputs:
+      runners:
+        description: "Runners to test on"
+        type: choice
+        options:
+          - "ubuntu-latest"
+          - "self-hosted"
+        default: "self-hosted"
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver
+  NFTEST_VERSION: "0.9.2"
+  NXF_ANSI_LOG: false
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  # renovate: datasource=github-releases depName=nextflow-io/nextflow versioning=semver
+  NXF_VERSION: "24.10.1"
+
+jobs:
+  nf-test-gpu:
+    runs-on: "gpu"
+    name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2]
+        profile: [docker_self_hosted, singularity] # conda?
+        # TODO Pass these in from GitHub PR trigger events
+        tags:
+          - parabricks/applybqsr
+          - parabricks/fq2bam
+          - parabricks/fq2bammeth
+    env:
+      NXF_ANSI_LOG: false
+      TOTAL_SHARDS: 2
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+      - name: Run nf-test Action
+        uses: ./.github/actions/nf-test-action
+        env:
+          SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+          SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
+          SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+          SENTIEON_AUTH_MECH: "GitHub Actions - token"
+        with:
+          profile: ${{ matrix.profile }},gpu
+          shard: ${{ matrix.shard }}
+          total_shards: ${{ env.TOTAL_SHARDS }}
+          tags: ${{matrix.tags}},gpu