From a191f8ed710569fdb4b5e219bfccbfd9797660bb Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 21 Jun 2022 13:44:57 +0100 Subject: [PATCH 01/11] add --allow_overlaps arg to cli --- src/omero_zarr/cli.py | 5 +++++ src/omero_zarr/masks.py | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 5d94f72..c00e327 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -272,6 +272,11 @@ def _configure(self, parser: Parser) -> None: subcommand.add_argument( "--output", type=str, default="", help="The output directory" ) + for subcommand in (polygons, masks): + subcommand.add_argument( + "--allow_overlaps", action="store_true", + help="Allow overlapping shapes. Output labels will add values in overlapping regions" + ) @gateway_required def masks(self, args: argparse.Namespace) -> None: diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index f238679..7ed313e 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -152,9 +152,10 @@ def image_shapes_to_zarr( print("Boolean type makes no sense for labeled. Using 64") dtype = MASK_DTYPE_SIZE[64] + check_overlaps = (not args.allow_overlaps) if masks: saver = MaskSaver( - None, image, dtype, args.label_path, args.style, args.source_image + None, image, dtype, args.label_path, args.style, args.source_image, check_overlaps ) if args.style == "split": @@ -186,6 +187,7 @@ def __init__( path: str = "labels", style: str = "labeled", source: str = "..", + check_overlaps: bool = True, ) -> None: self.dtype = dtype self.path = path @@ -193,6 +195,7 @@ def __init__( self.source_image = source self.plate = plate self.plate_path = Optional[str] + self.check_overlaps = check_overlaps if image: self.image = image self.size_t = image.getSizeT() @@ -302,7 +305,7 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: masks, mask_shape, ignored_dimensions, - check_overlaps=True, + check_overlaps=self.check_overlaps, ) axes = marshal_axes(self.image) From f0df696859eb57cf02cbc60608da5358f4bd4b12 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 12:51:06 +0000 Subject: [PATCH 02/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/omero_zarr/cli.py | 5 +++-- src/omero_zarr/masks.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index c00e327..6dc4ccb 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -274,8 +274,9 @@ def _configure(self, parser: Parser) -> None: ) for subcommand in (polygons, masks): subcommand.add_argument( - "--allow_overlaps", action="store_true", - help="Allow overlapping shapes. Output labels will add values in overlapping regions" + "--allow_overlaps", + action="store_true", + help="Allow overlapping shapes. Output labels will add values in overlapping regions", ) @gateway_required diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 7ed313e..21e9da7 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -152,10 +152,16 @@ def image_shapes_to_zarr( print("Boolean type makes no sense for labeled. Using 64") dtype = MASK_DTYPE_SIZE[64] - check_overlaps = (not args.allow_overlaps) + check_overlaps = not args.allow_overlaps if masks: saver = MaskSaver( - None, image, dtype, args.label_path, args.style, args.source_image, check_overlaps + None, + image, + dtype, + args.label_path, + args.style, + args.source_image, + check_overlaps, ) if args.style == "split": From 829626641a03b43d62dda9f3ba81fa569f174e0a Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 21 Jun 2022 15:54:48 +0100 Subject: [PATCH 03/11] Add polygons and --allow_overlaps to README --- README.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index a1624b7..19ff78a 100644 --- a/README.rst +++ b/README.rst @@ -59,10 +59,10 @@ To export images via bioformats2raw we use the ```--bf``` flag:: $ omero zarr --output /home/user/zarr_files export 1 --bf Image exported to /home/user/zarr_files/2chZT.lsm -Masks -^^^^^ +Masks and Polygons +^^^^^^^^^^^^^^^^^^ -To export Masks for an Image or Plate:: +To export Masks or Polygons for an Image or Plate, use the `masks` or `polygons` command:: # Saved under 1.zarr/labels/0 - 1.zarr/ must already exist $ omero zarr masks Image:1 @@ -78,12 +78,16 @@ To export Masks for an Image or Plate:: # e.g. Export to 1.zarr/labels/A $ omero zarr masks Image:1 --label-name=A -The default behaviour is to export all masks on the Image to a single 5D -"labeled" zarr array, with a different value for each mask Shape. -An exception will be thrown if any of the masks overlap. + # Allow overlapping masks or polygons (overlap will be sum of each label) + $ omero zarr polygons Image:1 --allow_overlaps -To handle overlapping masks, split masks into non-overlapping zarr groups -using a "label-map" which is a csv file of that specifies the name of +The default behaviour is to export all masks or polygons on the Image to a single 5D +"labeled" zarr array, with a different value for each Shape. +An exception will be thrown if any of the masks overlap, unless the `--allow_overlaps` +flag is used as above. + +An alternative to handle overlapping masks is to split masks into non-overlapping zarr +groups using a "label-map" which is a csv file that specifies the name of the zarr group for each ROI on the Image. Columns are ID, NAME, ROI_ID. For example, to create a group from the `textValue` of each Shape, From be29ce7111309c0d6ee6091b62597edfd4e03758 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 21 Jun 2022 16:06:54 +0100 Subject: [PATCH 04/11] flake8 fix --- src/omero_zarr/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 6dc4ccb..ff90f96 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -276,7 +276,8 @@ def _configure(self, parser: Parser) -> None: subcommand.add_argument( "--allow_overlaps", action="store_true", - help="Allow overlapping shapes. Output labels will add values in overlapping regions", + help="Allow overlapping shapes." + " Output labels will add values in overlapping regions", ) @gateway_required From 4d79d1e6009f734509132b7dc79dcec5d1b2f956 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 24 Jun 2022 11:38:05 +0100 Subject: [PATCH 05/11] Overlapping regions set to max(dtype) --- src/omero_zarr/cli.py | 4 ++-- src/omero_zarr/masks.py | 28 ++++++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index ff90f96..debd21a 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -276,8 +276,8 @@ def _configure(self, parser: Parser) -> None: subcommand.add_argument( "--allow_overlaps", action="store_true", - help="Allow overlapping shapes." - " Output labels will add values in overlapping regions", + help="Allow overlapping shapes. All overlapping regions" + " will be set to the max value for the dtype", ) @gateway_required diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 21e9da7..5d29ca9 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -521,22 +521,26 @@ def masks_to_labels( for i_z in self._get_indices( ignored_dimensions, "Z", z, size_z ): - if check_overlaps and np.any( - np.logical_and( - labels[ - i_t, i_c, i_z, y : (y + h), x : (x + w) - ].astype(np.bool), - binim_yx, - ) - ): - raise Exception( - f"Mask {shape.roi.id.val} overlaps " - "with existing labels" - ) + overlap = np.logical_and( + labels[ + i_t, i_c, i_z, y : (y + h), x : (x + w) + ].astype(np.bool), + binim_yx, + ) # ADD to the array, so zeros in our binarray don't # wipe out previous shapes labels[i_t, i_c, i_z, y : (y + h), x : (x + w)] += ( binim_yx * shape_value ) + if np.any(overlap): + if check_overlaps: + raise Exception( + f"Shape {shape.roi.id.val} overlaps " + "with existing labels" + ) + else: + # set overlapping region to max(dtype) + labels[i_t, i_c, i_z, y : (y + h), x : (x + w)][ + overlap] = np.iinfo(labels_dtype).max return labels, fillColors, properties From df01b44e4f9db3cc9127d81a327023d9599a05e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 10:38:23 +0000 Subject: [PATCH 06/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/omero_zarr/masks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 5d29ca9..d646fdb 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -522,9 +522,9 @@ def masks_to_labels( ignored_dimensions, "Z", z, size_z ): overlap = np.logical_and( - labels[ - i_t, i_c, i_z, y : (y + h), x : (x + w) - ].astype(np.bool), + labels[i_t, i_c, i_z, y : (y + h), x : (x + w)].astype( + np.bool + ), binim_yx, ) # ADD to the array, so zeros in our binarray don't @@ -542,5 +542,6 @@ def masks_to_labels( else: # set overlapping region to max(dtype) labels[i_t, i_c, i_z, y : (y + h), x : (x + w)][ - overlap] = np.iinfo(labels_dtype).max + overlap + ] = np.iinfo(labels_dtype).max return labels, fillColors, properties From a88a130011a6946665a7131029a723a85a49ecff Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 24 Jun 2022 14:23:45 +0100 Subject: [PATCH 07/11] Use --overlaps=dtype_max. Also pass agrument for Plates --- README.rst | 8 ++++---- src/omero_zarr/cli.py | 8 ++++---- src/omero_zarr/masks.py | 9 +++++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 19ff78a..38d5fc5 100644 --- a/README.rst +++ b/README.rst @@ -78,13 +78,13 @@ To export Masks or Polygons for an Image or Plate, use the `masks` or `polygons` # e.g. Export to 1.zarr/labels/A $ omero zarr masks Image:1 --label-name=A - # Allow overlapping masks or polygons (overlap will be sum of each label) - $ omero zarr polygons Image:1 --allow_overlaps + # Allow overlapping masks or polygons (overlap will be maximum value of the dtype) + $ omero zarr polygons Image:1 --overlaps=dtype_max The default behaviour is to export all masks or polygons on the Image to a single 5D "labeled" zarr array, with a different value for each Shape. -An exception will be thrown if any of the masks overlap, unless the `--allow_overlaps` -flag is used as above. +An exception will be thrown if any of the masks overlap, unless the `--overlaps` +option is used as above. An alternative to handle overlapping masks is to split masks into non-overlapping zarr groups using a "label-map" which is a csv file that specifies the name of diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index debd21a..67813f1 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -274,10 +274,10 @@ def _configure(self, parser: Parser) -> None: ) for subcommand in (polygons, masks): subcommand.add_argument( - "--allow_overlaps", - action="store_true", - help="Allow overlapping shapes. All overlapping regions" - " will be set to the max value for the dtype", + "--overlaps", type=str, choices=["dtype_max"], + help="To allow overlapping shapes, use 'dtype_max':" + " All overlapping regions will be set to the" + " max value for the dtype", ) @gateway_required diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index d646fdb..f2546d1 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -60,9 +60,13 @@ def plate_shapes_to_zarr( n_fields = plate.getNumberOfFields() total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1) + # If overlaps isn't 'dtype_max', an exception is thrown if any overlaps exist + check_overlaps = args.overlaps != "dtype_max" + dtype = MASK_DTYPE_SIZE[int(args.label_bits)] saver = MaskSaver( - plate, None, dtype, args.label_path, args.style, args.source_image + plate, None, dtype, args.label_path, args.style, args.source_image, + check_overlaps ) count = 0 @@ -152,7 +156,8 @@ def image_shapes_to_zarr( print("Boolean type makes no sense for labeled. Using 64") dtype = MASK_DTYPE_SIZE[64] - check_overlaps = not args.allow_overlaps + # If overlaps isn't 'dtype_max', an exception is thrown if any overlaps exist + check_overlaps = args.overlaps != "dtype_max" if masks: saver = MaskSaver( None, From c2e7e346ef60ed6321536d8e94855b4b2862864c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 13:25:03 +0000 Subject: [PATCH 08/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/omero_zarr/cli.py | 4 +++- src/omero_zarr/masks.py | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 67813f1..22bdcd1 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -274,7 +274,9 @@ def _configure(self, parser: Parser) -> None: ) for subcommand in (polygons, masks): subcommand.add_argument( - "--overlaps", type=str, choices=["dtype_max"], + "--overlaps", + type=str, + choices=["dtype_max"], help="To allow overlapping shapes, use 'dtype_max':" " All overlapping regions will be set to the" " max value for the dtype", diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index f2546d1..278b323 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -65,8 +65,13 @@ def plate_shapes_to_zarr( dtype = MASK_DTYPE_SIZE[int(args.label_bits)] saver = MaskSaver( - plate, None, dtype, args.label_path, args.style, args.source_image, - check_overlaps + plate, + None, + dtype, + args.label_path, + args.style, + args.source_image, + check_overlaps, ) count = 0 From 05bce566586515ac6ffaac9dfcb9fced3e8123ef Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 27 Jun 2022 19:38:52 +0100 Subject: [PATCH 09/11] Forward cli args to MaskSaver [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci flake8 and mypy fixes Fix mypy --- src/omero_zarr/cli.py | 10 ++++++++-- src/omero_zarr/masks.py | 26 +++++++++++++++----------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/omero_zarr/cli.py b/src/omero_zarr/cli.py index 22bdcd1..a94500f 100644 --- a/src/omero_zarr/cli.py +++ b/src/omero_zarr/cli.py @@ -11,7 +11,12 @@ from zarr.hierarchy import open_group from zarr.storage import FSStore -from .masks import MASK_DTYPE_SIZE, image_shapes_to_zarr, plate_shapes_to_zarr +from .masks import ( + MASK_DTYPE_SIZE, + MaskSaver, + image_shapes_to_zarr, + plate_shapes_to_zarr, +) from .raw_pixels import ( add_omero_metadata, add_toplevel_metadata, @@ -276,7 +281,8 @@ def _configure(self, parser: Parser) -> None: subcommand.add_argument( "--overlaps", type=str, - choices=["dtype_max"], + default=MaskSaver.OVERLAPS[0], + choices=MaskSaver.OVERLAPS, help="To allow overlapping shapes, use 'dtype_max':" " All overlapping regions will be set to the" " max value for the dtype", diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 278b323..535b3ed 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -60,9 +60,6 @@ def plate_shapes_to_zarr( n_fields = plate.getNumberOfFields() total = n_rows * n_cols * (n_fields[1] - n_fields[0] + 1) - # If overlaps isn't 'dtype_max', an exception is thrown if any overlaps exist - check_overlaps = args.overlaps != "dtype_max" - dtype = MASK_DTYPE_SIZE[int(args.label_bits)] saver = MaskSaver( plate, @@ -71,7 +68,7 @@ def plate_shapes_to_zarr( args.label_path, args.style, args.source_image, - check_overlaps, + args=args, ) count = 0 @@ -161,8 +158,6 @@ def image_shapes_to_zarr( print("Boolean type makes no sense for labeled. Using 64") dtype = MASK_DTYPE_SIZE[64] - # If overlaps isn't 'dtype_max', an exception is thrown if any overlaps exist - check_overlaps = args.overlaps != "dtype_max" if masks: saver = MaskSaver( None, @@ -171,7 +166,7 @@ def image_shapes_to_zarr( args.label_path, args.style, args.source_image, - check_overlaps, + args=args, ) if args.style == "split": @@ -195,6 +190,8 @@ class MaskSaver: masks to zarr groups/arrays. """ + OVERLAPS = ["error", "dtype_max"] + def __init__( self, plate: Optional[omero.gateway.PlateWrapper], @@ -203,7 +200,7 @@ def __init__( path: str = "labels", style: str = "labeled", source: str = "..", - check_overlaps: bool = True, + args: argparse.Namespace = None, ) -> None: self.dtype = dtype self.path = path @@ -211,7 +208,10 @@ def __init__( self.source_image = source self.plate = plate self.plate_path = Optional[str] - self.check_overlaps = check_overlaps + if args is not None and args.overlaps is not None: + self.overlaps = args.overlaps + else: + self.overlaps = "error" if image: self.image = image self.size_t = image.getSizeT() @@ -321,7 +321,6 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None: masks, mask_shape, ignored_dimensions, - check_overlaps=self.check_overlaps, ) axes = marshal_axes(self.image) @@ -452,7 +451,7 @@ def masks_to_labels( masks: List[omero.model.Mask], mask_shape: Tuple[int, ...], ignored_dimensions: Set[str] = None, - check_overlaps: bool = True, + check_overlaps: bool = None, ) -> Tuple[np.ndarray, Dict[int, str], Dict[int, Dict]]: """ :param masks [MaskI]: Iterable container of OMERO masks @@ -483,6 +482,11 @@ def masks_to_labels( sorted_roi_ids = list(set(roi_ids)) sorted_roi_ids.sort() + if check_overlaps is None: + # If overlaps isn't 'dtype_max', an exception is thrown + # if any overlaps exist + check_overlaps = self.overlaps != "dtype_max" + # label values are 1...n max_value = len(sorted_roi_ids) # find most suitable dtype... From b2cb5ed412e318a3185571df9f0d20fa7a299d7f Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 30 Jun 2022 15:30:27 +0100 Subject: [PATCH 10/11] Don't pass cli args to MaskSaver --- src/omero_zarr/masks.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 535b3ed..9d60e84 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -68,7 +68,7 @@ def plate_shapes_to_zarr( args.label_path, args.style, args.source_image, - args=args, + args.overlaps, ) count = 0 @@ -166,7 +166,7 @@ def image_shapes_to_zarr( args.label_path, args.style, args.source_image, - args=args, + args.overlaps, ) if args.style == "split": @@ -200,7 +200,7 @@ def __init__( path: str = "labels", style: str = "labeled", source: str = "..", - args: argparse.Namespace = None, + overlaps: str = "error", ) -> None: self.dtype = dtype self.path = path @@ -208,10 +208,7 @@ def __init__( self.source_image = source self.plate = plate self.plate_path = Optional[str] - if args is not None and args.overlaps is not None: - self.overlaps = args.overlaps - else: - self.overlaps = "error" + self.overlaps = overlaps if image: self.image = image self.size_t = image.getSizeT() From 2a0c9ce39c949cac65b2207bfe2a41dbfccc894a Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 6 Jul 2022 11:33:48 +0100 Subject: [PATCH 11/11] Use tuple for OVERLAPS options --- src/omero_zarr/masks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/omero_zarr/masks.py b/src/omero_zarr/masks.py index 9d60e84..26020aa 100644 --- a/src/omero_zarr/masks.py +++ b/src/omero_zarr/masks.py @@ -190,7 +190,7 @@ class MaskSaver: masks to zarr groups/arrays. """ - OVERLAPS = ["error", "dtype_max"] + OVERLAPS = ("error", "dtype_max") def __init__( self,