Cellpose fix - custom model and multiple samples

CHANGELOG.md

@@ -3,6 +3,19 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v1.0.1dev - [2023.11.15]
+
+Upgraded workflow, fixed multisample cellpose segmentation with custom model. Added options necessary to make testing work on small images.
+
+### `Added`
+
+- white background in metromap
+- clahe_skip_pyramid parameter to skip pyramid generation in the clahe step - necessary for smaller data
+
+### `Fixed`
+
+- Cellpose custom model functions with multiple samples now.
+
 ## v1.0.1dev - [2023.11.13]
 
 Added documentation - usage.md and output.md

bin/apply_clahe.dask.py

@@ -55,6 +55,13 @@ def get_args():
         "-g", "--nbins", dest="nbins", action="store", required=False, default=256, help="Number of bins for CLAHE"
     )
     inputs.add_argument("-p", "--pixel-size", dest="pixel_size", action="store", required=True, help="Image pixel size")
+    inputs.add_argument(
+        "--skip_pyramid",
+        dest="skip_pyramid",
+        action="store_true",
+        default=False,
+        help="Should pyramid creation be skipped",
+    )
 
     arg = parser.parse_args()
 
@@ -125,53 +132,58 @@ def main(args):
     # construct levels
     tile_size = 1024
     scale = 2
-
     pixel_size = args.pixel_size
-    dtype = img_dask.dtype
-    base_shape = img_dask[0].shape
-    num_channels = img_dask.shape[0]
-    num_levels = (np.ceil(np.log2(max(base_shape) / tile_size)) + 1).astype(int)
-    factors = 2 ** np.arange(num_levels)
-    shapes = (np.ceil(np.array(base_shape) / factors[:, None])).astype(int)
-
-    print("Pyramid level sizes: ")
-    for i, shape in enumerate(shapes):
-        print(f"   level {i+1}: {format_shape(shape)}", end="")
-        if i == 0:
-            print("(original size)", end="")
+
+    if args.skip_pyramid:
+        with tifffile.TiffWriter(args.output, ome=True, bigtiff=True) as tiff:
+            tiff.write(
+                data=img_dask,
+                shape=img_dask.shape,
+                dtype=img_dask.dtype,
+                resolution=(10000 / pixel_size, 10000 / pixel_size, "centimeter"),
+            )
+    else:
+        dtype = img_dask.dtype
+        base_shape = img_dask[0].shape
+        num_channels = img_dask.shape[0]
+        num_levels = (np.ceil(np.log2(max(base_shape) / tile_size)) + 1).astype(int)
+        factors = 2 ** np.arange(num_levels)
+        shapes = (np.ceil(np.array(base_shape) / factors[:, None])).astype(int)
+
+        print("Pyramid level sizes: ")
+        for i, shape in enumerate(shapes):
+            print(f"   level {i+1}: {format_shape(shape)}", end="")
+            if i == 0:
+                print("(original size)", end="")
+            print()
         print()
-    print()
-    print(shapes)
-
-    level_full_shapes = []
-    for shape in shapes:
-        level_full_shapes.append((num_channels, shape[0], shape[1]))
-    level_shapes = shapes
-    tip_level = np.argmax(np.all(level_shapes < tile_size, axis=1))
-    tile_shapes = [(tile_size, tile_size) if i <= tip_level else None for i in range(len(level_shapes))]
-
-    # write pyramid
-    with tifffile.TiffWriter(args.output, ome=True, bigtiff=True) as tiff:
-        tiff.write(
-            data=img_dask,
-            shape=level_full_shapes[0],
-            subifds=int(num_levels - 1),
-            dtype=dtype,
-            resolution=(10000 / pixel_size, 10000 / pixel_size, "centimeter"),
-            tile=tile_shapes[0],
-        )
-        for level, (shape, tile_shape) in enumerate(zip(level_full_shapes[1:], tile_shapes[1:]), 1):
+        print(shapes)
+
+        level_full_shapes = []
+        for shape in shapes:
+            level_full_shapes.append((num_channels, shape[0], shape[1]))
+        level_shapes = shapes
+        tip_level = np.argmax(np.all(level_shapes < tile_size, axis=1))
+        tile_shapes = [(tile_size, tile_size) if i <= tip_level else None for i in range(len(level_shapes))]
+
+        # write pyramid
+        with tifffile.TiffWriter(args.output, ome=True, bigtiff=True) as tiff:
             tiff.write(
-                data=subres_tiles(level, level_full_shapes, tile_shapes, args.output, scale),
-                shape=shape,
-                subfiletype=1,
+                data=img_dask,
+                shape=level_full_shapes[0],
+                subifds=int(num_levels - 1),
                 dtype=dtype,
-                tile=tile_shape,
+                resolution=(10000 / pixel_size, 10000 / pixel_size, "centimeter"),
+                tile=tile_shapes[0],
             )
-
-    # note about metadata: the channels, planes etc were adjusted not to include the removed channels, however
-    # the channel ids have stayed the same as before removal. E.g if channels 1 and 2 are removed,
-    # the channel ids in the metadata will skip indices 1 and 2 (channel_id:0, channel_id:3, channel_id:4 ...)
+            for level, (shape, tile_shape) in enumerate(zip(level_full_shapes[1:], tile_shapes[1:]), 1):
+                tiff.write(
+                    data=subres_tiles(level, level_full_shapes, tile_shapes, args.output, scale),
+                    shape=shape,
+                    subfiletype=1,
+                    dtype=dtype,
+                    tile=tile_shape,
+                )
     # tifffile.tiffcomment(args.output, to_xml(metadata))
     print()
 

conf/modules.config

@@ -108,10 +108,11 @@ process {
         ext.when  = { !params.skip_clahe }
         ext.args  = [ "",
             "--channel 0",
-            params.clahe_cliplimit  ? "--cliplimit ${params.clahe_cliplimit}"   : "",
-            params.clahe_nbins      ? "--nbins ${params.clahe_nbins}"           : "",
-            params.clahe_pixel_size ? "--pixel-size ${params.clahe_pixel_size}" : "",
-            params.clahe_kernel     ? "--kernel ${params.clahe_kernel}"         : ""
+            params.clahe_cliplimit    ? "--cliplimit ${params.clahe_cliplimit}"   : "",
+            params.clahe_nbins        ? "--nbins ${params.clahe_nbins}"           : "",
+            params.clahe_pixel_size   ? "--pixel-size ${params.clahe_pixel_size}" : "",
+            params.clahe_kernel       ? "--kernel ${params.clahe_kernel}"         : "",
+            params.clahe_skip_pyramid ? "--skip_pyramid"                          : ""
         ].join(" ").trim()
     }
 
@@ -124,7 +125,6 @@ process {
             params.mesmer_image_mpp   ? "--image-mpp ${params.mesmer_image_mpp}"     : "",
             "--nuclear-channel 0"
         ].join(" ").trim()
-        containerOptions =  '--entrypoint ""'
         publishDir = [
             path: "${params.outdir}/segmentation/mesmer",
             pattern: "*.tif",
@@ -133,6 +133,7 @@ process {
     }
 
     withName: "CELLPOSE" {
+        singularity.runOptions = "--bind $HOME:$HOME"
         ext.when  = { params.segmentation_method.split(',').contains('cellpose') }
         memory    = "16GB"
         cpus      = 8
@@ -147,7 +148,6 @@ process {
             params.cellpose_flow_threshold   ? "--flow_threshold ${params.cellpose_flow_threshold}"     : "",
             params.cellpose_edge_exclude     ? "--exclude_on_edges"                                     : ""
         ].join(" ").trim()
-        containerOptions =  '--entrypoint ""'
         publishDir = [
             path: "${params.outdir}/segmentation/cellpose",
             pattern: "*_cp_masks.tif",

docs/usage.md

@@ -101,6 +101,7 @@ clahe_cliplimit: 0.01
 clahe_nbins: 256
 clahe_pixel_size: 0.138
 clahe_kernel: 25
+clahe_skip_pyramid: false
 create_training_subset: false
 crop_amount: 4
 crop_nonzero_fraction: 0.4

modules.json

@@ -7,7 +7,7 @@
                 "nf-core": {
                     "cellpose": {
                         "branch": "master",
-                        "git_sha": "516189e968feb4ebdd9921806988b4c12b4ac2dc",
+                        "git_sha": "0975c63a8ce4488c3259f595270b3f0d419abafe",
                         "installed_by": ["modules"]
                     },
                     "custom/dumpsoftwareversions": {

nextflow.config

@@ -35,6 +35,7 @@ params {
     clahe_nbins                = 256
     clahe_pixel_size           = 0.138
     clahe_kernel               = 25
+    clahe_skip_pyramid         = false
 
     // Training subset command line
     create_training_subset     = false
@@ -136,6 +137,7 @@ profiles {
         shifter.enabled        = false
         charliecloud.enabled   = false
         apptainer.enabled      = false
+        docker.runOptions      = '--entrypoint ""'
     }
     arm {
         docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64'

nextflow_schema.json

@@ -72,7 +72,8 @@
                 "cellpose_save_flows": {
                     "type": "boolean",
                     "description": "Should flow fields from Cellpose be saved?",
-                    "fa_icon": "fas fa-arrows-alt"
+                    "fa_icon": "fas fa-arrows-alt",
+                    "hidden": true
                 },
                 "ilastik_pixel_project": {
                     "type": "string",
@@ -131,6 +132,12 @@
                     "default": 2144,
                     "description": "Tile size (distance between gridlines)",
                     "fa_icon": "fas fa-th"
+                },
+                "clahe_skip_pyramid": {
+                    "type": "boolean",
+                    "description": "Should pyramid generation be skipped for CLAHE",
+                    "fa_icon": "fas fa-cubes",
+                    "hidden": true
                 }
             }
         },

workflows/molkart.nf

@@ -185,10 +185,13 @@ workflow MOLKART {
     }
     //
     // MODULE: Cellpose segmentation
-    // TODO: check if there is a problem with resume for Cellpose.
-    cellpose_custom_model = params.cellpose_custom_model ? Channel.fromPath(params.cellpose_custom_model) : []
+    //
+    cellpose_custom_model = params.cellpose_custom_model ? stack_mix.combine(Channel.fromPath(params.cellpose_custom_model)) : []
     if (params.segmentation_method.split(',').contains('cellpose')) {
-        CELLPOSE(stack_mix, cellpose_custom_model)
+        CELLPOSE(
+            stack_mix,
+            cellpose_custom_model ? cellpose_custom_model.map{it[2]} : []
+            )
         ch_versions = ch_versions.mix(CELLPOSE.out.versions)
         segmentation_masks = segmentation_masks
             .mix(CELLPOSE.out.mask