Merge pull request #52 from kbestak/todo_module_update

Module update + CI + final TODOs

.devcontainer/devcontainer.json

@@ -18,11 +18,11 @@
                 "python.linting.flake8Path": "/opt/conda/bin/flake8",
                 "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
                 "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
-                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+                "python.linting.pylintPath": "/opt/conda/bin/pylint",
             },
 
             // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
-        }
-    }
+            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"],
+        },
+    },
 }

.github/workflows/awsfulltest.yml

@@ -15,9 +15,6 @@ jobs:
     steps:
       - name: Launch workflow via tower
         uses: seqeralabs/action-tower-launch@v2
-        # TODO nf-core: You can customise AWS full pipeline tests as required
-        # Add full size test data (but still relatively small datasets for few samples)
-        # on the `test_full.config` test runs with only one set of parameters
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}

.github/workflows/ci.yml

@@ -1,43 +1,108 @@
-name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+name: nf-core CI
 on:
   push:
     branches:
-      - dev
+      - "dev"
   pull_request:
+    branches:
+      - "dev"
+      - "master"
   release:
     types: [published]
 
 env:
   NXF_ANSI_LOG: false
+  NFTEST_VER: "0.8.2"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
+  list:
+    name: List Tests
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/molkart') }}"
+    outputs:
+      # Expose matched filters as job 'modules' output variable
+      tests: ${{ steps.list.outputs.tests }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@v1
+
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: List nf-test
+        id: list
+        run: |
+          echo "tests=$(nf-test list --silent --format=json)" >> "$GITHUB_OUTPUT"
+
   test:
-    name: Run pipeline with test data
+    name: ${{ matrix.tags }} (${{ matrix.profile }}-${{ matrix.NXF_VER }})
     # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/molkart') }}"
+    if: needs.list.outputs.tests != '[]'
+    needs: [list]
     runs-on: ubuntu-latest
     strategy:
       matrix:
         NXF_VER:
           - "23.04.0"
           - "latest-everything"
+        profile: ["docker"] #TODO add "singularity"
+        test: ["${{ fromJson(needs.list.outputs.tests) }}"]
+
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v4
 
+      - name: Cache Nextflow installation
+        id: cache-software
+        uses: actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: molkart-${{ runner.os }}-${{ matrix.NXF_VER }}
+
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Run pipeline with test data
-        # TODO nf-core: You can customise CI pipeline run tests as required
-        # For example: adding multiple test runs with different parameters
-        # Remember that you can parallelise this by using strategy.matrix
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --profile="test,${{ matrix.profile }}" \
+            ${{ matrix.test }} \
+            --junitxml=${{ matrix.test }}-${{ matrix.profile }}.xml
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: success() || failure() # run this step even if previous step failed
+        with:
+          path: ${{ matrix.test }}-${{ matrix.profile }}.xml
+
+      - name: Output log on failure
+        if: failure()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/output/pipeline_info/software_versions.yml

CHANGELOG.md

@@ -3,7 +3,17 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## v1.0.1dev - [2024.01.08
+## v1.0.1dev - [2024.01.16]
+
+### `Added`
+
+- mindagap_boxsize, mindagap_loopnum, mindagap_edges parameters
+- updated mindagap version and adapted molkart.nf and modules.config to match updated mindagap module version
+- CI testing with nf-test
+- updated base.config to account for ilastik's high memory use
+- updated metromap
+
+## v1.0.1dev - [2024.01.08]
 
 ### `Fixed`
 

conf/base.config

@@ -9,8 +9,6 @@
 */
 
 process {
-
-    // TODO nf-core: Check the defaults for all processes
     cpus   = { check_max( 1    * task.attempt, 'cpus'   ) }
     memory = { check_max( 6.GB * task.attempt, 'memory' ) }
     time   = { check_max( 4.h  * task.attempt, 'time'   ) }
@@ -24,8 +22,6 @@ process {
     //        These labels are used and recognised by default in DSL2 files hosted on nf-core/modules.
     //        If possible, it would be nice to keep the same label naming convention when
     //        adding in your local modules too.
-    // TODO nf-core: Customise requirements for specific processes.
-    // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors
     withLabel:process_single {
         cpus   = { check_max( 1                  , 'cpus'    ) }
         memory = { check_max( 6.GB * task.attempt, 'memory'  ) }
@@ -57,9 +53,13 @@ process {
     }
     withLabel:error_retry {
         errorStrategy = 'retry'
-        maxRetries    = 2
+        maxRetries    = 3
     }
     withName:CUSTOM_DUMPSOFTWAREVERSIONS {
         cache = false
     }
+
+    withName:'ILASTIK_PIXELCLASSIFICATION|ILASTIK_MULTICUT' {
+        label = "process_high"
+    }
 }

conf/modules.config

@@ -79,6 +79,8 @@ process {
 
     withName: "MINDAGAP_MINDAGAP" {
         ext.args = [ "",
+            params.mindagap_boxsize ? "${params.mindagap_boxsize}" : "",
+            params.mindagap_loopnum ? "${params.mindagap_loopnum}" : "",
             params.mindagap_tilesize ? "-xt ${params.mindagap_tilesize}" : ""
         ].join(" ").trim()
         publishDir = [

conf/test.config

@@ -22,6 +22,8 @@ params {
     // Input data
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/molkart/test_data/samplesheets/samplesheet_membrane.csv'
     mindagap_tilesize = 90
+    mindagap_boxsize = 7
+    mindagap_loopnum = 100
     clahe_pyramid_tile = 368
-    segmentation_method = "cellpose,mesmer"
+    segmentation_method = "mesmer,cellpose"
 }

conf/test_full.config

@@ -17,5 +17,7 @@ params {
     input = 'https://raw.githubusercontent.com/nf-core/test-datasets/molkart/test_data/samplesheets/samplesheet_full_test.csv'
 
     segmentation_method = "mesmer,cellpose"
+    mindagap_boxsize = 3
+    mindagap_loopnum = 40
     cellpose_pretrained_model = "nuclei"
 }

docs/usage.md

@@ -98,6 +98,9 @@ mesmer_compartment: "whole-cell"
 ilastik_pixel_project: null
 ilastik_multicut_project: null
 mindagap_tilesize: 2144
+mindagap_boxsize: 3
+mindagap_loopnum: 40
+mindagap_edges: false
 skip_clahe: false
 clahe_cliplimit: 0.01
 clahe_nbins: 256

modules.json

@@ -12,7 +12,7 @@
                     },
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "bba7e362e4afead70653f84d8700588ea28d0f9e",
+                        "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93",
                         "installed_by": ["modules"]
                     },
                     "deepcell/mesmer": {
@@ -37,12 +37,12 @@
                     },
                     "mindagap/mindagap": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "97dbec551f491ee562721008cdecd1e7a9a56b14",
                         "installed_by": ["modules"]
                     },
                     "multiqc": {
                         "branch": "master",
-                        "git_sha": "4ab13872435962dadc239979554d13709e20bf29",
+                        "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93",
                         "installed_by": ["modules"]
                     }
                 }