Use new nf-test features

.github/conda_skip.yml

@@ -0,0 +1,184 @@
+# TODO What do we do with these?
+# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now
+exclude:
+  - path: modules/nf-core/nf-test
+  - profile: conda
+    path: modules/nf-core/angsd/gl
+  - profile: conda
+    path: modules/nf-core/annotsv/installannotations
+  - profile: conda
+    path: modules/nf-core/happy/sompy
+  - profile: conda
+    path: modules/nf-core/backsub
+  - profile: conda
+    path: modules/nf-core/bakta/bakta
+  - profile: conda
+    path: modules/nf-core/bakta/baktadbdownload
+  - profile: conda
+    path: modules/nf-core/bases2fastq
+  - profile: conda
+    path: modules/nf-core/bcl2fastq
+  - profile: conda
+    path: modules/nf-core/bclconvert
+  - profile: conda
+    path: modules/nf-core/celesta
+  - profile: conda
+    path: modules/nf-core/cellpose
+  - profile: conda
+    path: modules/nf-core/cellranger/count
+  - profile: conda
+    path: modules/nf-core/cellranger/mkfastq
+  - profile: conda
+    path: modules/nf-core/cellranger/mkgtf
+  - profile: conda
+    path: modules/nf-core/cellranger/mkref
+  - profile: conda
+    path: modules/nf-core/cellranger/mkvdjref
+  - profile: conda
+    path: modules/nf-core/cellranger/multi
+  - profile: conda
+    path: modules/nf-core/cellranger/vdj
+  - profile: conda
+    path: modules/nf-core/checkqc
+  - profile: conda
+    path: modules/nf-core/custom/dumpsoftwareversions
+  - profile: conda
+    path: modules/nf-core/deepcell/mesmer
+  - profile: conda
+    path: modules/nf-core/deepsomatic
+  - profile: singularity
+    path: modules/nf-core/deepsomatic
+  - profile: conda
+    path: modules/nf-core/deepvariant
+  - profile: conda
+    path: modules/nf-core/deepvariant/callvariants
+  - profile: conda
+    path: modules/nf-core/deepvariant/makeexamples
+  - profile: conda
+    path: modules/nf-core/deepvariant/postprocessvariants
+  - profile: conda
+    path: modules/nf-core/deepvariant/rundeepvariant
+  - profile: conda
+    path: modules/nf-core/deepvariant/vcfstatsreport
+  - profile: conda
+    path: modules/nf-core/doubletdetection
+  - profile: conda
+    path: modules/nf-core/ensemblvep/vep
+  - profile: conda
+    path: modules/nf-core/fastk/fastk
+  - profile: conda
+    path: modules/nf-core/cellrangerarc/mkgtf
+  - profile: conda
+    path: modules/nf-core/fastk/histex
+  - profile: conda
+    path: modules/nf-core/fastk/merge
+  - profile: conda
+    path: modules/nf-core/fcs/fcsadaptor
+  - profile: conda
+    path: modules/nf-core/fcs/fcsgx
+  - profile: conda
+    path: modules/nf-core/ganon/buildcustom
+  - profile: conda
+    path: modules/nf-core/ganon/classify
+  - profile: conda
+    path: modules/nf-core/ganon/report
+  - profile: conda
+    path: modules/nf-core/ganon/table
+  - profile: conda
+    path: modules/nf-core/gatk4/cnnscorevariants
+  - profile: conda
+    path: modules/nf-core/gatk4/determinegermlinecontigploidy
+  - profile: conda
+    path: modules/nf-core/genescopefk
+  - profile: conda
+    path: modules/nf-core/ilastik/multicut
+  - profile: conda
+    path: modules/nf-core/ilastik/pixelclassification
+  - profile: conda
+    path: modules/nf-core/imputeme/vcftoprs
+  - profile: conda
+    path: modules/nf-core/mcstaging/imc2mc
+  - profile: conda
+    path: modules/nf-core/mcquant
+  - profile: conda
+    path: modules/nf-core/mcstaging/phenoimager2mc
+  - profile: conda
+    path: modules/nf-core/merquryfk/katcomp
+  - profile: conda
+    path: modules/nf-core/merquryfk/katgc
+  - profile: conda
+    path: modules/nf-core/merquryfk/merquryfk
+  - profile: conda
+    path: modules/nf-core/merquryfk/ploidyplot
+  - profile: conda
+    path: modules/nf-core/molkartgarage/clahe
+  - profile: conda
+    path: modules/nf-core/quartonotebook
+  - profile: conda
+    path: modules/nf-core/scimap/spatiallda
+  - profile: conda
+    path: modules/nf-core/sentieon/bwaindex
+  - profile: conda
+    path: modules/nf-core/sentieon/bwamem
+  - profile: conda
+    path: modules/nf-core/sentieon/datametrics
+  - profile: conda
+    path: modules/nf-core/sentieon/dedup
+  - profile: conda
+    path: modules/nf-core/sentieon/qualcal
+  - profile: conda
+    path: modules/nf-core/spaceranger/mkgtf
+  - profile: conda
+    path: modules/nf-core/spaceranger/mkref
+  - profile: conda
+    path: modules/nf-core/spaceranger/count
+  - profile: conda
+    path: modules/nf-core/spotiflow
+  - profile: conda
+    path: modules/nf-core/svanalyzer/svbenchmark
+  - profile: conda
+    path: modules/nf-core/universc
+  - profile: singularity
+    path: modules/nf-core/universc
+  - profile: conda
+    path: modules/nf-core/vt/decompose
+  - profile: singularity
+    path: modules/nf-core/bases2fastq
+  - profile: conda
+    path: modules/nf-core/wittyer
+  - profile: conda
+    path: modules/nf-core/islandpath
+  - profile: conda
+    path: modules/nf-core/scimap/mcmicro
+  - profile: conda
+    path: modules/nf-core/parabricks/fq2bammeth
+  - profile: docker_self_hosted
+    path: modules/nf-core/parabricks/fq2bammeth
+  - profile: singularity
+    path: modules/nf-core/parabricks/fq2bammeth
+  - profile: conda
+    path: modules/nf-core/parabricks/fq2bam
+  - profile: docker_self_hosted
+    path: modules/nf-core/parabricks/fq2bam
+  - profile: singularity
+    path: modules/nf-core/parabricks/fq2bam
+  - profile: conda
+    path: subworkflows/nf-core/vcf_annotate_ensemblvep
+  - profile: conda
+    path: subworkflows/nf-core/bcl_demultiplex
+  - profile: conda
+    path: subworkflows/nf-core/deepvariant
+  - profile: conda
+    path: subworkflows/nf-core/fastq_align_bamcmp_bwa
+  - profile: conda
+    path: subworkflows/nf-core/fastq_align_bwa
+  - profile: conda
+    path: subworkflows/nf-core/fasta_newick_epang_gappa
+  - profile: conda
+    path: modules/nf-core/xeniumranger/relabel
+  - profile: conda
+    path: modules/nf-core/xeniumranger/rename
+  - profile: conda
+    path: modules/nf-core/xeniumranger/resegment
+  - profile: conda
+    path: modules/nf-core/xeniumranger/import-segmentation

.github/workflows/lint.yml

@@ -0,0 +1,172 @@
+name: Run Linting
+on:
+  push:
+    branches:
+      # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging
+      - "renovate/**" # branches Renovate creates
+  pull_request:
+    branches: [master]
+  merge_group:
+    types: [checks_requested]
+    branches: [master]
+  workflow_dispatch:
+    inputs:
+      runners:
+        description: "Runners to test on"
+        type: choice
+        options:
+          - "ubuntu-latest"
+          - "self-hosted"
+        default: "self-hosted"
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+      - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
+        # FIXME Flip this off once we get to less than a couple hundred. Adding
+        # this so it will only run against changed files. It'll make it much
+        # easier to fix these as they come up rather than everything at once.
+        with:
+          extra_args: ""
+
+  ###################
+  # nf-core linting #
+  ###################
+  # TODO Move these to pre-commit
+  # https://github.com/nf-core/tools/pull/3141
+  nf-core-changes:
+    name: nf-core-changes
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.filter.outputs.changes }}
+      modules: ${{ steps.tags.outputs.modules }}
+      subworkflows: ${{ steps.tags.outputs.subworkflows }}
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 2 # To retrieve the preceding commit.
+
+      # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
+      - uses: mirpedrol/paths-filter@main
+        id: filter
+        with:
+          filters: "tests/config/pytest_modules.yml"
+          token: ""
+
+      - name: Fetch module tags
+        id: tags
+        run: |
+          echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT
+          echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT
+
+      - name: debug
+        run: |
+          echo ${{ steps.tags.outputs.modules }}
+          echo ${{ steps.tags.outputs.subworkflows }}
+
+  nf-core-lint-modules:
+    runs-on: ${{ github.event.inputs.runners  || 'self-hosted' }}
+    name: nf-core lint modules
+    needs: nf-core-changes
+    if: ${{ (needs.nf-core-changes.outputs.modules != '[]') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: "${{ fromJson(needs.nf-core-changes.outputs.modules) }}"
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - name: Set up Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+        with:
+          python-version: "3.11"
+
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
+        id: cache-pip
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+          restore-keys: |
+            ${{ runner.os }}-pip
+
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+
+      - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@v2
+
+      - name: Install nf-core tools development version
+        run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
+
+      - name: Lint module ${{ matrix.tags }}
+        run: nf-core modules lint ${{ matrix.tags }}
+
+  nf-core-lint-subworkflows:
+    runs-on: ubuntu-latest
+    name: nf-core lint subworkflows
+    needs: nf-core-changes
+    if: ${{ (needs.nf-core-changes.outputs.subworkflows != '[]') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: "${{ fromJson(needs.nf-core-changes.outputs.subworkflows) }}"
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+
+      - name: Set up Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+        with:
+          python-version: "3.11"
+
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+
+      - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2
+
+      - name: Install nf-core tools development version
+        run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
+
+      - name: Lint module ${{ matrix.tags }}
+        run: nf-core subworkflows lint ${{ matrix.tags }}
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-core-lint-modules, nf-core-lint-subworkflows]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"