Fastq align dedup bwameth (#7007)

* init bwameth subworkflow * update output channels * bwameth single, paired-end default, skip_dedup tests * add GPU tests * separate GPU tests * add subworkflow GPU test path to test.yml include * add subworkflow GPU test path to test.yml include * add subworkflow GPU test path to test.yml include * use more descriptive collect variables than it * rename file to be explicit * separate profile exclusions * add gpu tag * rm old test.yml * add gpu test path to gpu-tests.yml:ci * Added log_level: DEBUG * Setup CI for debug * add more debug steps * usman's fix for exclude tags * Updated fail condition * ci sync to master * fix filtering by usman * Removed --changed-since when tags are supplied * Enabled more checks and added a non-gpu module * Two fixes * Now using paths * Fixed paths * Fixed typo * Now pass all paths instead of matrix * Added confirm pass, cleaned dispatch and removed DEBUG * Renamed all passes to confirm-pass * Now using adamrtalbot/detect-nf-test-changes * small ch format fix --------- Co-authored-by: Simon Pearce <[email protected]> Co-authored-by: Usman Rashid <[email protected]>
nf-core · Nov 30, 2024 · bd5f75c · bd5f75c
1 parent c1a6b17
commit bd5f75c
Show file tree

Hide file tree

Showing 11 changed files with 1,078 additions and 47 deletions.
diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml
@@ -10,9 +10,9 @@ inputs:
   total_shards:
     description: "Total number of test shards(NOT the total number of matrix jobs)"
     required: true
-  tags:
-    description: "Tags to test (`<tags>[,<tags>...]`)"
-    required: false
+  paths:
+    description: "Test paths"
+    required: true
 
 runs:
   using: "composite"
@@ -72,19 +72,16 @@ runs:
       env:
         SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }}
         SENTIEON_AUTH_MECH: "GitHub Actions - token"
-        TAGS: ${{ inputs.tags && format('--tag {0}', inputs.tags) || '' }}
       run: |
         NFT_WORKDIR=~ \
         nf-test test \
           --profile=${{ inputs.profile }} \
           --tap=test.tap \
           --verbose \
           --ci \
-          --changed-since HEAD^ \
           --shard ${{ inputs.shard }}/${{ inputs.total_shards }} \
           --filter process,workflow \
-          --follow-dependencies \
-          ${{ env.TAGS }}
+          ${{ inputs.paths }}
 
       # TODO If no test.tap, then make one to spoof?
     - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1

diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml
@@ -6,21 +6,15 @@ on:
       - "renovate/**" # branches Renovate creates
   pull_request:
     branches: [master]
-    paths:
-      - ".github/workflows/gpu-tests.yml"
-      - "modules/nf-core/parabricks/**"
   merge_group:
     types: [checks_requested]
     branches: [master]
   workflow_dispatch:
     inputs:
       runners:
         description: "Runners to test on"
-        type: choice
-        options:
-          - "ubuntu-latest"
-          - "self-hosted"
-        default: "self-hosted"
+        type: string
+        default: "gpu"
 
 # Cancel if a newer run is started
 concurrency:
@@ -43,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       # Expose detected tags as 'modules' and 'workflows' output variables
-      paths: ${{ steps.outputs.outputs.components }}
+      paths: ${{ steps.list.outputs.components }}
       modules: ${{ steps.outputs.outputs.modules }}
       subworkflows: ${{ steps.outputs.outputs.subworkflows}}
       # Prod for version bumping
@@ -60,11 +54,11 @@ jobs:
 
       - name: List nf-test files
         id: list
-        uses: adamrtalbot/detect-nf-test-changes@6bf6fd9fe0fb63a0362fb0e09de5acb6d055a754 # v0.0.5
+        uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
         with:
           head: ${{ github.sha }}
           base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
-          n_parents: 2
+          n_parents: 0
           tags: "gpu"
 
       - name: Separate modules and subworkflows
@@ -74,21 +68,20 @@ jobs:
           echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
       - name: debug
         run: |
-          echo ${{ steps.outputs.outputs.components }}
+          echo ${{ steps.list.outputs.components }}
           echo ${{ steps.outputs.outputs.modules }}
           echo ${{ steps.outputs.outputs.subworkflows }}
 
   nf-test-gpu:
     runs-on: "gpu"
+    name: "GPU | ${{ matrix.profile }} | ${{ matrix.shard }}"
     needs: nf-test-changes
-    if: ${{ fromJSON(needs.nf-test-changes.outputs.paths) != '[]' || needs.nf-test-changes.outputs.paths != '' }}
-    name: "GPU | ${{ matrix.tags}} | ${{ matrix.profile }} | ${{ matrix.shard }}"
+    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:
       fail-fast: false
       matrix:
         shard: [1, 2]
         profile: [docker_self_hosted, singularity] # conda?
-        tags: ${{ fromJSON(needs.nf-test-changes.outputs.modules) && fromJSON(needs.nf-test-changes.outputs.subworkflows) }}
     env:
       NXF_ANSI_LOG: false
       TOTAL_SHARDS: 2
@@ -108,4 +101,22 @@ jobs:
           profile: ${{ matrix.profile }},gpu
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
-          tags: ${{matrix.tags}},gpu
+          paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test-gpu]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml
@@ -35,45 +35,90 @@ env:
   NXF_VER: "24.10.1"
 
 jobs:
+  nf-test-changes:
+    name: nf-test-changes
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose detected tags as 'modules' and 'workflows' output variables
+      paths: ${{ steps.list.outputs.components }}
+      modules: ${{ steps.outputs.outputs.modules }}
+      subworkflows: ${{ steps.outputs.outputs.subworkflows}}
+      # Prod for version bumping
+    steps:
+      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
+        run: |
+          ls -la ./
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+          ls -la ./
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6
+        with:
+          head: ${{ github.sha }}
+          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+          n_parents: 0
+          exclude_tags: "gpu"
+
+      - name: Separate modules and subworkflows
+        id: outputs
+        run: |
+          echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
+          echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
+      - name: debug
+        run: |
+          echo ${{ steps.list.outputs.components }}
+          echo ${{ steps.outputs.outputs.modules }}
+          echo ${{ steps.outputs.outputs.subworkflows }}
   nf-test:
     runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
-    # NOTE I think this is the cleanest way to get them organized
-    # process | conda | 1
-    # process | conda | 2
-    # process | conda | 3
-    # process | docker_self_hosted | 1
-    # ...
-    # workflow | singularity | 3
     name: "${{ matrix.profile }} | ${{ matrix.shard }}"
-    # TODO
-    # needs: get-number-of-shards
-    # if: ${{ fromJSON(needs.get-number-of-shards.outputs.shards) != fromJSON('["1", "0"]') }}
+    needs: nf-test-changes
+    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
     strategy:
       fail-fast: false
       matrix:
-        # NOTE We could split these, but there's probably going to be more process tests than workflow tests, so we're just going to combine them all and bump up the shards for now
-        # NOTE The name of the test would be name: "${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}"
-        # filter: [process, workflow]
-        profile: [conda, docker_self_hosted, singularity]
         shard: [1, 2, 3, 4, 5]
+        profile: [conda, docker_self_hosted, singularity]
     env:
-      # FIXME Bumping them up to make the transition smooth, then we can throttle them back
+      NXF_ANSI_LOG: false
       TOTAL_SHARDS: 5
-      SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
-      SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+
     steps:
-      - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
-        run: |
-          ls -la ./
-          rm -rf ./* || true
-          rm -rf ./.??* || true
-          ls -la ./
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
         with:
           fetch-depth: 0
       - name: Run nf-test Action
         uses: ./.github/actions/nf-test-action
+        env:
+          SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+          SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
+          SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+          SENTIEON_AUTH_MECH: "GitHub Actions - token"
         with:
           profile: ${{ matrix.profile }}
           shard: ${{ matrix.shard }}
           total_shards: ${{ env.TOTAL_SHARDS }}
+          paths: "${{ join(fromJson(needs.nf-test-changes.outputs.paths), ' ') }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [nf-test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
diff --git a/modules/nf-core/gffread/tests/main.nf.test b/modules/nf-core/gffread/tests/main.nf.test
@@ -23,6 +23,7 @@ nextflow_process {
                     file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.gff3", checkIfExists: true)
                 ]
                 input[1] = []
+
                 """
             }
         }
@@ -220,4 +221,4 @@ nextflow_process {
 
     }
 
-}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,6 +23,7 @@ nextflow_process { @@
                         file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.gff3", checkIfExists: true)
                     ]
                     input[1] = []
                     """
                 }
             }
@@ Expand Down Expand Up / @@ -220,4 +221,4 @@ nextflow_process { @@
         }
-    }
+    }