Merge branch 'dev' into nf-core-template-merge-3.1.1

nf-core · Dec 20, 2024 · c2bc59d · c2bc59d
2 parents 7c333f5 + 3f4406a
commit c2bc59d
Show file tree

Hide file tree

Showing 297 changed files with 24,572 additions and 192 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -26,7 +26,7 @@ If you're not used to this workflow with git, you can start with some [docs from
 
 ## Tests
 
-You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command:
+You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command, BUT DON'T FORGET TO ADD THE PARAMETERS cosmic_username AND cosmic_passwd in tests/main.nf.test.
 
 ```bash
 nf-test test --profile debug,test,docker --verbose

diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml
@@ -45,9 +45,6 @@ jobs:
 
       - name: Launch workflow via Seqera Platform
         uses: seqeralabs/action-tower-launch@v2
-        # TODO nf-core: You can customise AWS full pipeline tests as required
-        # Add full size test data (but still relatively small datasets for few samples)
-        # on the `test_full.config` test runs with only one set of parameters
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
@@ -57,13 +54,33 @@ jobs:
           parameters: |
             {
               "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}"
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}",
+              "genomes_base": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}/references",
+              "cosmic_username": "${{ secrets.cosmic_username }}",
+              "cosmic_passwd": "${{ secrets.cosmic_passwd }}",
+              "all": true,
+              "build_references": true
             }
-          profiles: test_full
-
+          profiles: test_full,aws_tower
       - uses: actions/upload-artifact@v4
         with:
           name: Seqera Platform debug log file
           path: |
             seqera_platform_action_*.log
             seqera_platform_action_*.json
+      - name: Launch run workflow via tower
+        uses: seqeralabs/action-tower-launch@v2
+        with:
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/rnafusion/work-${{ github.sha }}
+          parameters: |
+            {
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}",
+              "genomes_base": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}/references",
+              "cosmic_username": "${{ secrets.cosmic_username }}",
+              "cosmic_passwd": "${{ secrets.cosmic_passwd }}",
+              "all": true,
+            }
+          profiles: test_full,aws_tower
diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml
@@ -21,10 +21,39 @@ jobs:
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/rnafusion/work-${{ github.sha }}
           parameters: |
             {
-              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-test-${{ github.sha }}"
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}",
+              "genomes_base": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}/references",
+              "cosmic_username": "${{ secrets.cosmic_username }}",
+              "cosmic_passwd": "${{ secrets.cosmic_passwd }}",
+              "all": true,
+              "stub": true,
+              "build_references": true
             }
-          profiles: test
+          profiles: test,aws_tower
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Tower debug log file
+          path: |
+            tower_action_*.log
+            tower_action_*.json
 
+      - name: Launch workflow via tower
+        uses: seqeralabs/action-tower-launch@v2
+        with:
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/rnafusion/work-${{ github.sha }}
+          parameters: |
+            {
+              "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}",
+              "genomes_base": "s3://${{ secrets.AWS_S3_BUCKET }}/rnafusion/results-${{ github.sha }}/references",
+              "cosmic_username": "${{ secrets.cosmic_username }}",
+              "cosmic_passwd": "${{ secrets.cosmic_passwd }}",
+              "all": true,
+              "stub": true
+            }
+          profiles: test,aws_tower
       - uses: actions/upload-artifact@v4
         with:
           name: Seqera Platform debug log file

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,11 +5,18 @@ on:
     branches:
       - dev
   pull_request:
+    branches:
+      - dev
+      - master
   release:
     types: [published]
   workflow_dispatch:
 
 env:
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
+  NFT_VER: "0.9.2"
+  NFT_WORKDIR: "~"
   NXF_ANSI_LOG: false
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
@@ -20,29 +27,31 @@ concurrency:
 
 jobs:
   test:
-    name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})"
+    name: "${{ matrix.NXF_VER }} | ${{ matrix.test_profile }} | ${{ matrix.compute_profile }}"
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/rnafusion') }}"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "24.04.2"
-          - "latest-everything"
-        profile:
-          - "conda"
+          - "latest-stable"
+        test_profile:
+          - "test_stub"
+          - "test_build"
+        compute_profile:
           - "docker"
           - "singularity"
-        test_name:
-          - "test"
+          - "conda"
         isMaster:
           - ${{ github.base_ref == 'master' }}
         # Exclude conda and singularity on dev
         exclude:
           - isMaster: false
-            profile: "conda"
+            compute_profile: "conda"
           - isMaster: false
-            profile: "singularity"
+            compute_profile: "singularity"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -53,17 +62,17 @@ jobs:
           version: "${{ matrix.NXF_VER }}"
 
       - name: Set up Apptainer
-        if: matrix.profile == 'singularity'
+        if: matrix.compute_profile == 'singularity'
         uses: eWaterCycle/setup-apptainer@main
 
       - name: Set up Singularity
-        if: matrix.profile == 'singularity'
+        if: matrix.compute_profile == 'singularity'
         run: |
           mkdir -p $NXF_SINGULARITY_CACHEDIR
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
       - name: Set up Miniconda
-        if: matrix.profile == 'conda'
+        if: matrix.compute_profile == 'conda'
         uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
         with:
           miniconda-version: "latest"
@@ -72,14 +81,40 @@ jobs:
           channels: conda-forge,bioconda
 
       - name: Set up Conda
-        if: matrix.profile == 'conda'
+        if: matrix.compute_profile == 'conda'
         run: |
           echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
           echo $(realpath python) >> $GITHUB_PATH
 
       - name: Clean up Disk space
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+
+      - name: Install pdiff to see diff between nf-test snapshots
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results
+          python -m pip install --upgrade pip
+          pip install pdiff
+
+      - uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFT_VER }}
+
+      - name: Run Tests (${{matrix.NXF_VER}} | ${{matrix.test_profile}} | ${{matrix.compute_profile}})
+        run: |
+          nf-test test \
+              --ci \
+              --tag ${{matrix.test_profile}} \
+              --profile "+${{ matrix.compute_profile }}" \
+              --junitxml=test.xml \
+              --debug --verbose
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml
+          annotate_only: true
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,4 @@ testing/
 testing*
 *.pyc
 null/
+.nf-test*
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,3 +7,4 @@ testing/ @@
     testing*
     *.pyc
     null/
+    .nf-test*