Merge pull request nf-core#1321 from nf-core/dev

ACTUAL Release 3.4

.github/CODEOWNERS

@@ -1,2 +1,3 @@
-*   @FriederikeHanssen
-*   @maxulysse
+* @FriederikeHanssen @maxulysse
+*.nf.test* @nf-core/nf-test
+.github/workflows/ @nf-core/a-team

.github/workflows/ci.yml

@@ -11,34 +11,40 @@ on:
       - master
       - dev
 
+env:
+  NXF_ANSI_LOG: false
+  NFTEST_VER: "0.8.1"
+
 # Cancel if a newer run is started
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  changes:
-    name: Check for changes
+  pytest-changes:
+    name: Check for changes (pytest)
     runs-on: ubuntu-latest
     outputs:
       # Expose matched filters as job 'tags' output variable
       tags: ${{ steps.filter.outputs.changes }}
     steps:
       - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
+
+      - uses: frouioui/paths-filter@main
         id: filter
         with:
-          filters: "tests/config/tags.yml"
+          filters: "tests/config/pytesttags.yml"
+          token: ""
 
-  test:
+  pytest:
     name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
     runs-on: ubuntu-latest
-    needs: changes
-    if: needs.changes.outputs.tags != '[]'
+    needs: pytest-changes
+    if: needs.pytest-changes.outputs.tags != '[]'
     strategy:
       fail-fast: false
       matrix:
-        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
+        tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"]
         profile: ["docker"]
         # profile: ["docker", "singularity", "conda"]
         TEST_DATA_BASE:
@@ -50,6 +56,7 @@ jobs:
       NXF_ANSI_LOG: false
       TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
       SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v3
@@ -154,10 +161,136 @@ jobs:
             !/home/runner/pytest_workflow_*/*/work/conda
             !/home/runner/pytest_workflow_*/*/work/singularity
 
+  nftest-changes:
+    name: Check for changes (nf-test)
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.filter.outputs.changes }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Combine all tags.yml files
+        id: get_tags
+        run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
+
+      - name: debug
+        run: cat .github/tags.yml
+
+      - uses: frouioui/paths-filter@main
+        id: filter
+        with:
+          filters: ".github/tags.yml"
+          token: ""
+
+  nftest:
+    name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
+    runs-on: ubuntu-latest
+    needs: nftest-changes
+    if: needs.nftest-changes.outputs.tags != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["${{ fromJson(needs.nftest-changes.outputs.tags) }}"]
+        profile: ["docker"]
+        # profile: ["docker", "singularity", "conda"]
+        TEST_DATA_BASE:
+          - "test-datasets/data"
+        NXF_VER:
+          - "23.04.0"
+          - "latest-everything"
+        exclude:
+          - tags: "bwa/index"
+          - tags: "bwa/mem"
+          - tags: "cat/cat"
+          - tags: "cat/fastq"
+          - tags: "custom/dumpsoftwareversions"
+          - tags: "fastp"
+          - tags: "fastqc"
+          - tags: "samtools/stats"
+          - tags: "untar"
+    env:
+      NXF_ANSI_LOG: false
+      TEST_DATA_BASE: "${{ github.workspace }}/test-datasets"
+      SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: "temurin"
+          java-version: "17"
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@v1
+
+      - name: Cache nf-test installation
+        id: cache-software
+        uses: actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
+
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Setup apptainer
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-apptainer@main
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        run: |
+          mkdir -p $NXF_SINGULARITY_CACHEDIR
+          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+      - name: Set up miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: "latest"
+          auto-update-conda: true
+          channels: conda-forge,bioconda,defaults
+          python-version: ${{ matrix.python-version }}
+
+      - name: Conda setup
+        run: |
+          conda clean -a
+          conda install -n base conda-libmamba-solver
+          conda config --set solver libmamba
+          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
+          echo $(realpath python) >> $GITHUB_PATH
+
+      # Set up secrets
+      - name: Set up nextflow secrets
+        if: env.SENTIEON_LICENSE_BASE64 != null
+        run: |
+          nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+          nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
+          SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
+          SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
+          SENTIEON_AUTH_DATA=$(python3 bin/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
+          SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
+          nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
+
+      # Test the module
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --profile=${{ matrix.profile }} \
+            --tag ${{ matrix.tags }} \
+            --tap=test.tap \
+            --verbose
+
   confirm-pass:
     runs-on: ubuntu-latest
     needs:
-      - test
+      - pytest
+      - nftest
     if: always()
     steps:
       - name: All tests ok

.gitignore

@@ -1,10 +1,12 @@
 *.code-workspace
+*.pyc
 .nextflow*
-work/
-data/
-results/
 .DS_Store
-testing/
+.nf-test*
+.nf-test/
+data/
 testing*
-*.pyc
+testing/
 test-datasets/
+results/
+work/

.nf-core.yml

@@ -5,6 +5,7 @@ lint:
     - .github/workflows/awsfulltest.yml
     - conf/modules.config
   files_unchanged:
+    - .gitignore
     - assets/nf-core-sarek_logo_light.png
     - docs/images/nf-core-sarek_logo_dark.png
     - docs/images/nf-core-sarek_logo_light.png

CHANGELOG.md

@@ -5,6 +5,65 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.4.0](https://github.com/nf-core/sarek/releases/tag/3.4.0) - Pårtetjåkko
+
+Pårtetjåkko is a mountain in the south of the park.
+
+### Added
+
+- [#1113](https://github.com/nf-core/sarek/pull/1113) - Adding CNVkit genemetrics module
+- [#1193](https://github.com/nf-core/sarek/pull/1193) - Adding support for Sentieon's DnaScope for germline variant-calling including joint-germline
+- [#1244](https://github.com/nf-core/sarek/pull/1244) - Add bcf annotate module
+- [#1252](https://github.com/nf-core/sarek/pull/1252) - Added NGSCheckMate tool for checking that samples come from the same individual
+- [#1271](https://github.com/nf-core/sarek/pull/1271) - Back to dev
+- [#1288](https://github.com/nf-core/sarek/pull/1288) - Add nf-test continuous integration (but no tests)
+- [#1290](https://github.com/nf-core/sarek/pull/1290) - Add nf-test for whole pipeline
+
+### Changed
+
+- [#1278](https://github.com/nf-core/sarek/pull/1278) - Hide sentieon parameters similar to other variant callers
+- [#1280](https://github.com/nf-core/sarek/pull/1280) - Replacing link to `SentieonDNAscopeModel1.1.model` in Sentieon's S3 with link to same file in igenomes' S3
+- [#1303](https://github.com/nf-core/sarek/pull/1303) - Ressurect vep_version params and changed its scope to pipeline to enable usage for vep loftee plugin
+- [#1304](https://github.com/nf-core/sarek/pull/1304) - Update modules
+- [#1311](https://github.com/nf-core/sarek/pull/1311) - Update local modules with an `environment.yml` file
+- [#1317](https://github.com/nf-core/sarek/pull/1317) - Add new tools to subway map
+- [#1325](https://github.com/nf-core/sarek/pull/1325) - Move `sentieon_dnascope_model` params into `igenomes.config`
+- [#1325](https://github.com/nf-core/sarek/pull/1325) - Refactor config files
+- [#1327](https://github.com/nf-core/sarek/pull/1327) - Update modules to have an conda environment name
+
+### Fixed
+
+- [#1277](https://github.com/nf-core/sarek/pull/1277) - Fix null value issue for Mutect2 joint calling
+- [#1287](https://github.com/nf-core/sarek/pull/1287) - Adding label `process_single` to local modules
+- [#1298](https://github.com/nf-core/sarek/pull/1298) - Fix annotation cache usage
+- [#1301](https://github.com/nf-core/sarek/pull/1301) - Fix nf-prov usage
+- [#1315](https://github.com/nf-core/sarek/pull/1315) - Avoid clash of configs of `FILTERVARIANTTRANCHES` in the Sentieon-Haplotyper and GATK-Haplotypecaller subworkflows
+- [#1318](https://github.com/nf-core/sarek/pull/1218) - Fix writing of params.json on S3
+- [#1324](https://github.com/nf-core/sarek/pull/1324) - Fix various typos & code formatting
+- [#1325](https://github.com/nf-core/sarek/pull/1325) - Update bcfannotate tests and related config files
+- [#1328](https://github.com/nf-core/sarek/pull/1328) - Fix links to docs in `nextflow_schema.json` and `docs/output.md`
+- [#1328](https://github.com/nf-core/sarek/pull/1328) - Add missing icons in `nextflow_schema.json`
+- [#1330](https://github.com/nf-core/sarek/pull/1330) - Add SnpEff to full sized tests
+
+### Removed
+
+- [#1298](https://github.com/nf-core/sarek/pull/1298) - Remove `--use_annotation_cache_keys` params
+
+### Dependencies
+
+| Dependency | Old version | New version |
+| ---------- | ----------- | ----------- |
+| fastqc     | 0.11.9      | 0.12.1      |
+| multiqc    | 1.15        | 1.17        |
+
+### Modules / Subworkflows
+
+| script                        | Old name                      | New name                      |
+| ----------------------------- | ----------------------------- | ----------------------------- |
+| `gatk4spark/applybqsr`        | `GATK4_APPLYBQSRSPARK`        | `GATK4SPARK_APPLYBQSR`        |
+| `gatk4spark/baserecalibrator` | `GATK4_BASERECALIBRATORSPARK` | `GATK4SPARK_BASERECALIBRATOR` |
+| `gatk4spark/markduplicates`   | `GATK4_MARKDUPLICATESSPARK`   | `GATK4SPARK_MARKDUPLICATES`   |
+
 ## [3.3.2](https://github.com/nf-core/sarek/releases/tag/3.3.2) - Ráhpajávvre
 
 Ráhpajávvre is the Lule Sámi spelling of Rapaselet.
@@ -13,6 +72,7 @@ Ráhpajávvre is the Lule Sámi spelling of Rapaselet.
 
 - [#1246](https://github.com/nf-core/sarek/pull/1246) - Back to dev
 - [#1259](https://github.com/nf-core/sarek/pull/1259) - nf-prov plugin
+- [#1288](https://github.com/nf-core/sarek/pull/1288) - Add nf-test continuous integration.
 
 ### Changed
 

README.md

@@ -50,7 +50,7 @@ Depending on the options and samples provided, the pipeline can currently perfor
   - `Sentieon Haplotyper`
   - `Strelka2`
   - `TIDDIT`
-- Variant filtering and annotation (`SnpEff`, `Ensembl VEP`)
+- Variant filtering and annotation (`SnpEff`, `Ensembl VEP`, `BCFtools annotate`)
 - Summarise and represent QC (`MultiQC`)
 
 <p align="center">
@@ -131,6 +131,7 @@ We thank the following people for their extensive assistance in the development
 - [Francesco Lescai](https://github.com/lescai)
 - [Gavin Mackenzie](https://github.com/GCJMackenzie)
 - [Gisela Gabernet](https://github.com/ggabernet)
+- [Grant Neilson](https://github.com/grantn5)
 - [gulfshores](https://github.com/gulfshores)
 - [Harshil Patel](https://github.com/drpatelh)
 - [James A. Fellows Yates](https://github.com/jfy133)

assets/multiqc_config.yml

@@ -3,9 +3,9 @@ custom_logo_url: https://github.com/nf-core/sarek/
 custom_logo_title: "nf-core/sarek"
 
 report_comment: >
-  This report has been generated by the <a href="https://github.com/nf-core/sarek/releases/tag/3.3.2" target="_blank">nf-core/sarek</a>
+  This report has been generated by the <a href="https://github.com/nf-core/sarek/releases/tag/3.4.0" target="_blank">nf-core/sarek</a>
   analysis pipeline. For information about how to interpret these results, please see the
-  <a href="https://nf-co.re/sarek/3.3.2/docs/output" target="_blank">documentation</a>.
+  <a href="https://nf-co.re/sarek/3.4.0/docs/output" target="_blank">documentation</a>.
 report_section_order:
   "nf-core-sarek-methods-description":
     order: -1000