From 85c968d766eed5d9bb7ec4e7d9fad00b288156b8 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Wed, 25 Sep 2024 17:34:43 +0200 Subject: [PATCH 01/15] update dependency versions --- tools/idr_download/idr_download_by_ids.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.xml b/tools/idr_download/idr_download_by_ids.xml index 26af1ddf947..5e3395043dc 100644 --- a/tools/idr_download/idr_download_by_ids.xml +++ b/tools/idr_download/idr_download_by_ids.xml @@ -1,5 +1,5 @@ - + @@ -19,8 +19,8 @@ operation_3443 - omero-py - pylibtiff + omero-py + pylibtiff Date: Wed, 25 Sep 2024 17:36:17 +0200 Subject: [PATCH 02/15] enable download of the full image (hyperstack) --- tools/idr_download/idr_download_by_ids.py | 148 ++++++++++++++------- tools/idr_download/idr_download_by_ids.xml | 101 ++++++++++---- 2 files changed, 177 insertions(+), 72 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 6c2adddf345..a399cc699af 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -5,7 +5,10 @@ import tarfile from contextlib import ExitStack from tempfile import TemporaryDirectory +from itertools import product +import numpy +from tifffile import imwrite from libtiff import TIFF from omero.cli import cli_login from omero.gateway import BlitzGateway # noqa @@ -110,9 +113,27 @@ def get_image_array(image, tile, z, c, t): return selection +def get_full_image_array(image): + # The goal is to get the image in TZCYX order + pixels = image.getPrimaryPixels() + tzclist = list(product(range(image.getSizeT()), range(image.getSizeZ()), range(image.getSizeC()))) + zctlist = [(z, c, t) for (t, z, c) in tzclist] + try: + all_planes = numpy.array(list(pixels.getPlanes(zctlist))) + all_planes_reshaped = all_planes.reshape(image.getSizeT(), image.getSizeZ(), image.getSizeC(), all_planes.shape[-2], all_planes.shape[-1]) + except Exception as e: + warning = '{0} (ID: {1})'.format(image.getName(), + image.getId()) + warn('Could not download the requested region', warning) + warn(e.msg) + return + + return all_planes_reshaped + + def download_image_data( image_ids_or_dataset_id, dataset=False, - download_original=False, + download_original=False, download_full=False, channel=None, z_stack=0, frame=0, coord=(0, 0), width=0, height=0, region_spec='rectangle', skip_failed=False, download_tar=False, omero_host='idr.openmicroscopy.org', omero_secured=False, config_file=None @@ -131,7 +152,7 @@ def download_image_data( omero_username = 'public' omero_password = 'public' - if not download_original and region_spec not in ['rectangle', 'center']: + if not download_original and not download_full and region_spec not in ['rectangle', 'center']: raise ValueError( 'Got unknown value "{0}" as region_spec argument' .format(region_spec) @@ -226,16 +247,79 @@ def download_image_data( 'database. Aborting!' .format(image_warning_id) ) - if not download_original: + try: + # try to extract image name + # if anything goes wrong here skip the image + # or abort. + image_name = os.path.splitext(image.getName())[0] + image_warning_id = '{0} (ID: {1})'.format( + image_name, image_id + ) + except Exception as e: + # respect skip_failed on unexpected errors + if skip_failed: + warn(str(e), image_warning_id, warn_skip=True) + continue + else: + raise + if download_full: + fname = '__'.join([image_name.replace(' ', '_'), str(image_id), "full"]) + '.tiff' + # download and save the region as TIFF + try: + im_array = get_full_image_array(image) + + if download_tar: + fname = os.path.join(tempdir, fname) + + imwrite(fname, im_array, imagej=True) + # move image into tarball + if download_tar: + archive.add(fname, os.path.basename(fname)) + os.remove(fname) + except Exception as e: + if skip_failed: + # respect skip_failed on unexpected errors + warn(str(e), image_warning_id, warn_skip=True) + continue + else: + raise + + elif download_original: + try: + # try to extract image properties + # if anything goes wrong here skip the image + # or abort. + original_image_name = image.getFileset().listFiles()[0].getName() + fname = image_name + "__" + str(image_id) + os.path.splitext(original_image_name)[1] + fname = fname.replace(' ', '_') + fname = fname.replace('/', '_') + download_directory = "./" + if download_tar: + download_directory = tempdir + with cli_login("-u", omero_username, "-s", omero_host, "-w", omero_password) as cli: + cli.invoke(["download", f"Image:{image_id}", download_directory]) + if cli.rv != 0: + raise Exception("Download failed.") + # This will download to download_directory/original_image_name + os.rename(os.path.join(download_directory, original_image_name), + os.path.join(download_directory, fname)) + # move image into tarball + if download_tar: + archive.add(os.path.join(download_directory, fname), + os.path.basename(fname)) + os.remove(os.path.join(download_directory, fname)) + except Exception as e: + # respect skip_failed on unexpected errors + if skip_failed: + warn(str(e), image_warning_id, warn_skip=True) + continue + else: + raise + else: try: # try to extract image properties # if anything goes wrong here skip the image # or abort. - image_name = os.path.splitext(image.getName())[0] - image_warning_id = '{0} (ID: {1})'.format( - image_name, image_id - ) - if region_spec == 'rectangle': tile = get_clipping_region(image, *coord, width, height) elif region_spec == 'center': @@ -314,16 +398,13 @@ def download_image_data( .format(channel, image_warning_id) ) - # download and save the region as TIFF fname = '__'.join( [image_name, str(image_id)] + [str(x) for x in tile] ) + fname += '.tiff' + fname = fname.replace(' ', '_') + # download and save the region as TIFF try: - if fname[-5:] != '.tiff': - fname += '.tiff' - - fname = fname.replace(' ', '_') - im_array = get_image_array(image, tile, z_stack, channel_index, frame) if download_tar: @@ -344,41 +425,6 @@ def download_image_data( continue else: raise - else: - try: - # try to extract image properties - # if anything goes wrong here skip the image - # or abort. - image_name = os.path.splitext(image.getName())[0] - image_warning_id = '{0} (ID: {1})'.format( - image_name, image_id - ) - original_image_name = image.getFileset().listFiles()[0].getName() - fname = image_name + "__" + str(image_id) + os.path.splitext(original_image_name)[1] - fname = fname.replace(' ', '_') - fname = fname.replace('/', '_') - download_directory = "./" - if download_tar: - download_directory = tempdir - with cli_login("-u", omero_username, "-s", omero_host, "-w", omero_password) as cli: - cli.invoke(["download", f"Image:{image_id}", download_directory]) - if cli.rv != 0: - raise Exception("Download failed.") - # This will download to download_directory/original_image_name - os.rename(os.path.join(download_directory, original_image_name), - os.path.join(download_directory, fname)) - # move image into tarball - if download_tar: - archive.add(os.path.join(download_directory, fname), - os.path.basename(fname)) - os.remove(os.path.join(download_directory, fname)) - except Exception as e: - # respect skip_failed on unexpected errors - if skip_failed: - warn(str(e), image_warning_id, warn_skip=True) - continue - else: - raise def _center_to_ul(center_x, center_y, width, height): @@ -409,6 +455,10 @@ def _center_to_ul(center_x, center_y, width, height): '--download-original', dest='download_original', action='store_true', help="download the original file uploaded to omero" ) + p.add_argument( + '--download-full', dest='download_full', action='store_true', + help="download the full image on omero" + ) p.add_argument( '-c', '--channel', help='name of the channel to retrieve data for ' diff --git a/tools/idr_download/idr_download_by_ids.xml b/tools/idr_download/idr_download_by_ids.xml index 5e3395043dc..f42af81814b 100644 --- a/tools/idr_download/idr_download_by_ids.xml +++ b/tools/idr_download/idr_download_by_ids.xml @@ -21,6 +21,7 @@ omero-py pylibtiff + tifffile - + + + - + @@ -202,10 +207,10 @@ - + - + @@ -225,10 +230,10 @@ - + - + @@ -248,10 +253,10 @@ - + - + @@ -270,7 +275,11 @@ - + + + + + @@ -299,7 +308,7 @@ - + @@ -321,7 +330,11 @@ - + + + + + @@ -333,7 +346,7 @@ - + @@ -354,17 +367,25 @@ - + - + + + + + - + + + + + - + @@ -387,7 +408,7 @@ - + @@ -415,7 +436,7 @@ - + @@ -426,27 +447,43 @@ - + + + + + - + + + + + - + + + + + - + + + + + - + @@ -462,6 +499,24 @@ + + + + + + + + + + + + + + + + + + Date: Wed, 25 Sep 2024 17:36:47 +0200 Subject: [PATCH 03/15] run isort --- tools/idr_download/idr_download_by_ids.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index a399cc699af..f630b1feb29 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -4,15 +4,15 @@ import sys import tarfile from contextlib import ExitStack -from tempfile import TemporaryDirectory from itertools import product -import numpy +from tempfile import TemporaryDirectory -from tifffile import imwrite +import numpy from libtiff import TIFF from omero.cli import cli_login -from omero.gateway import BlitzGateway # noqa from omero.constants.namespaces import NSBULKANNOTATIONS # noqa +from omero.gateway import BlitzGateway # noqa +from tifffile import imwrite def warn(message, image_identifier, warn_skip=False): From 62767452d5453019198cdf1d466765fbe3411d20 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Wed, 25 Sep 2024 17:36:59 +0200 Subject: [PATCH 04/15] run black --- tools/idr_download/idr_download_by_ids.py | 344 ++++++++++++---------- 1 file changed, 181 insertions(+), 163 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index f630b1feb29..62e599f9373 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -18,20 +18,15 @@ def warn(message, image_identifier, warn_skip=False): message = message.rstrip() if warn_skip: - if message[-1] in ['.', '!', '?']: - skip_msg = ' Skipping download!' + if message[-1] in [".", "!", "?"]: + skip_msg = " Skipping download!" else: - skip_msg = '. Skipping download!' + skip_msg = ". Skipping download!" else: - skip_msg = '' + skip_msg = "" print( - 'ImageSpecWarning for {0}: {1}{2}' - .format( - image_identifier, - message, - skip_msg - ), - file=sys.stderr + "ImageSpecWarning for {0}: {1}{2}".format(image_identifier, message, skip_msg), + file=sys.stderr, ) @@ -47,7 +42,7 @@ def find_channel_index(image, channel_name): if p[0] == "Channels": channels = p[1].replace(" ", "").split(";") for n, c in enumerate(channels): - for value in c.split(':'): + for value in c.split(":"): if channel_name == value.lower(): return n return -1 @@ -61,16 +56,16 @@ def get_clipping_region(image, x, y, w, h): # It may be better to abort in this situation. if x < 0 or y < 0: raise ValueError( - 'Too small upper left coordinate ({0}, {1}) for clipping region.' - .format(x, y) + "Too small upper left coordinate ({0}, {1}) for clipping region.".format( + x, y + ) ) size_x = image.getSizeX() size_y = image.getSizeY() if x >= size_x or y >= size_y: raise ValueError( - 'Upper left coordinate ({0}, {1}) of clipping region lies ' - 'outside of image.' - .format(x, y) + "Upper left coordinate ({0}, {1}) of clipping region lies " + "outside of image.".format(x, y) ) # adjust width and height to the image dimensions if w <= 0 or x + w > size_x: @@ -105,9 +100,8 @@ def get_image_array(image, tile, z, c, t): try: selection = pixels.getTile(theZ=z, theT=t, theC=c, tile=tile) except Exception: - warning = '{0} (ID: {1})'.format(image.getName(), - image.getId()) - warn('Could not download the requested region', warning) + warning = "{0} (ID: {1})".format(image.getName(), image.getId()) + warn("Could not download the requested region", warning) return return selection @@ -116,15 +110,24 @@ def get_image_array(image, tile, z, c, t): def get_full_image_array(image): # The goal is to get the image in TZCYX order pixels = image.getPrimaryPixels() - tzclist = list(product(range(image.getSizeT()), range(image.getSizeZ()), range(image.getSizeC()))) + tzclist = list( + product( + range(image.getSizeT()), range(image.getSizeZ()), range(image.getSizeC()) + ) + ) zctlist = [(z, c, t) for (t, z, c) in tzclist] try: all_planes = numpy.array(list(pixels.getPlanes(zctlist))) - all_planes_reshaped = all_planes.reshape(image.getSizeT(), image.getSizeZ(), image.getSizeC(), all_planes.shape[-2], all_planes.shape[-1]) + all_planes_reshaped = all_planes.reshape( + image.getSizeT(), + image.getSizeZ(), + image.getSizeC(), + all_planes.shape[-2], + all_planes.shape[-1], + ) except Exception as e: - warning = '{0} (ID: {1})'.format(image.getName(), - image.getId()) - warn('Could not download the requested region', warning) + warning = "{0} (ID: {1})".format(image.getName(), image.getId()) + warn("Could not download the requested region", warning) warn(e.msg) return @@ -132,51 +135,59 @@ def get_full_image_array(image): def download_image_data( - image_ids_or_dataset_id, dataset=False, - download_original=False, download_full=False, - channel=None, z_stack=0, frame=0, - coord=(0, 0), width=0, height=0, region_spec='rectangle', - skip_failed=False, download_tar=False, omero_host='idr.openmicroscopy.org', omero_secured=False, config_file=None + image_ids_or_dataset_id, + dataset=False, + download_original=False, + download_full=False, + channel=None, + z_stack=0, + frame=0, + coord=(0, 0), + width=0, + height=0, + region_spec="rectangle", + skip_failed=False, + download_tar=False, + omero_host="idr.openmicroscopy.org", + omero_secured=False, + config_file=None, ): if config_file is None: # IDR connection - omero_username = 'public' - omero_password = 'public' + omero_username = "public" + omero_password = "public" else: # other omero instance with open(config_file) as f: cfg = json.load(f) - omero_username = cfg['username'] - omero_password = cfg['password'] + omero_username = cfg["username"] + omero_password = cfg["password"] if omero_username == "" or omero_password == "": - omero_username = 'public' - omero_password = 'public' - - if not download_original and not download_full and region_spec not in ['rectangle', 'center']: + omero_username = "public" + omero_password = "public" + + if ( + not download_original + and not download_full + and region_spec not in ["rectangle", "center"] + ): raise ValueError( - 'Got unknown value "{0}" as region_spec argument' - .format(region_spec) + 'Got unknown value "{0}" as region_spec argument'.format(region_spec) ) with ExitStack() as exit_stack: conn = exit_stack.enter_context( BlitzGateway( - omero_username, omero_password, - host=omero_host, - secure=omero_secured + omero_username, omero_password, host=omero_host, secure=omero_secured ) ) # exit_stack.callback(conn.connect().close) if download_tar: # create an archive file to write images to - archive = exit_stack.enter_context( - tarfile.open('images.tar', mode='w') - ) - tempdir = exit_stack.enter_context( - TemporaryDirectory() - ) + archive = exit_stack.enter_context(tarfile.open("images.tar", mode="w")) + tempdir = exit_stack.enter_context(TemporaryDirectory()) if dataset: - dataset_warning_id = 'Dataset-ID: {0}'.format(image_ids_or_dataset_id[0]) + dataset_warning_id = "Dataset-ID: {0}".format(image_ids_or_dataset_id[0]) try: dataset_id = int(image_ids_or_dataset_id[0]) except ValueError: @@ -196,28 +207,26 @@ def download_image_data( if image_ids is None: if skip_failed: warn( - 'Unable to find a dataset with this ID in the ' - 'database.', + "Unable to find a dataset with this ID in the " "database.", dataset_warning_id, - warn_skip=True + warn_skip=True, ) else: raise ValueError( - '{0}: Unable to find a dataset with this ID in the ' - 'database. Aborting!' - .format(dataset_warning_id) + "{0}: Unable to find a dataset with this ID in the " + "database. Aborting!".format(dataset_warning_id) ) else: # basic argument sanity checks and adjustments - prefix = 'image-' + prefix = "image-" # normalize image ids by stripping off prefix if it exists image_ids = [ - iid[len(prefix):] if iid[:len(prefix)] == prefix else iid + iid[len(prefix) :] if iid[: len(prefix)] == prefix else iid for iid in image_ids_or_dataset_id ] for image_id in image_ids: - image_warning_id = 'Image-ID: {0}'.format(image_id) + image_warning_id = "Image-ID: {0}".format(image_id) try: image_id = int(image_id) except ValueError: @@ -236,25 +245,21 @@ def download_image_data( if image is None: if skip_failed: warn( - 'Unable to find an image with this ID in the ' - 'database.', + "Unable to find an image with this ID in the " "database.", image_warning_id, - warn_skip=True + warn_skip=True, ) continue raise ValueError( - '{0}: Unable to find an image with this ID in the ' - 'database. Aborting!' - .format(image_warning_id) + "{0}: Unable to find an image with this ID in the " + "database. Aborting!".format(image_warning_id) ) try: # try to extract image name # if anything goes wrong here skip the image # or abort. image_name = os.path.splitext(image.getName())[0] - image_warning_id = '{0} (ID: {1})'.format( - image_name, image_id - ) + image_warning_id = "{0} (ID: {1})".format(image_name, image_id) except Exception as e: # respect skip_failed on unexpected errors if skip_failed: @@ -263,7 +268,10 @@ def download_image_data( else: raise if download_full: - fname = '__'.join([image_name.replace(' ', '_'), str(image_id), "full"]) + '.tiff' + fname = ( + "__".join([image_name.replace(" ", "_"), str(image_id), "full"]) + + ".tiff" + ) # download and save the region as TIFF try: im_array = get_full_image_array(image) @@ -290,23 +298,36 @@ def download_image_data( # if anything goes wrong here skip the image # or abort. original_image_name = image.getFileset().listFiles()[0].getName() - fname = image_name + "__" + str(image_id) + os.path.splitext(original_image_name)[1] - fname = fname.replace(' ', '_') - fname = fname.replace('/', '_') + fname = ( + image_name + + "__" + + str(image_id) + + os.path.splitext(original_image_name)[1] + ) + fname = fname.replace(" ", "_") + fname = fname.replace("/", "_") download_directory = "./" if download_tar: download_directory = tempdir - with cli_login("-u", omero_username, "-s", omero_host, "-w", omero_password) as cli: - cli.invoke(["download", f"Image:{image_id}", download_directory]) + with cli_login( + "-u", omero_username, "-s", omero_host, "-w", omero_password + ) as cli: + cli.invoke( + ["download", f"Image:{image_id}", download_directory] + ) if cli.rv != 0: raise Exception("Download failed.") # This will download to download_directory/original_image_name - os.rename(os.path.join(download_directory, original_image_name), - os.path.join(download_directory, fname)) + os.rename( + os.path.join(download_directory, original_image_name), + os.path.join(download_directory, fname), + ) # move image into tarball if download_tar: - archive.add(os.path.join(download_directory, fname), - os.path.basename(fname)) + archive.add( + os.path.join(download_directory, fname), + os.path.basename(fname), + ) os.remove(os.path.join(download_directory, fname)) except Exception as e: # respect skip_failed on unexpected errors @@ -320,12 +341,11 @@ def download_image_data( # try to extract image properties # if anything goes wrong here skip the image # or abort. - if region_spec == 'rectangle': + if region_spec == "rectangle": tile = get_clipping_region(image, *coord, width, height) - elif region_spec == 'center': + elif region_spec == "center": tile = get_clipping_region( - image, - *_center_to_ul(*coord, width, height) + image, *_center_to_ul(*coord, width, height) ) ori_z, z_stack = z_stack, confine_plane(image, z_stack) @@ -348,69 +368,66 @@ def download_image_data( # The downloaded image region will have smaller dimensions # than the specified width x height. warn( - 'Downloaded image dimensions ({0} x {1}) will be smaller ' - 'than the specified width and height ({2} x {3}).' - .format(tile[2], tile[3], width, height), - image_warning_id + "Downloaded image dimensions ({0} x {1}) will be smaller " + "than the specified width and height ({2} x {3}).".format( + tile[2], tile[3], width, height + ), + image_warning_id, ) # z-stack sanity checks and warnings if z_stack != ori_z: warn( - 'Specified image plane ({0}) is out of bounds. Using {1} ' - 'instead.' - .format(ori_z, z_stack), - image_warning_id + "Specified image plane ({0}) is out of bounds. Using {1} " + "instead.".format(ori_z, z_stack), + image_warning_id, ) # frame sanity checks and warnings if frame != ori_frame: warn( - 'Specified image frame ({0}) is out of bounds. Using ' - 'frame {1} instead.' - .format(ori_frame, frame), - image_warning_id + "Specified image frame ({0}) is out of bounds. Using " + "frame {1} instead.".format(ori_frame, frame), + image_warning_id, ) # channel index sanity checks and warnings if channel is None: if num_channels > 1: warn( - 'No specific channel selected for multi-channel ' - 'image. Using first of {0} channels.' - .format(num_channels), - image_warning_id + "No specific channel selected for multi-channel " + "image. Using first of {0} channels.".format(num_channels), + image_warning_id, ) else: if channel_index == -1 or channel_index >= num_channels: if skip_failed: warn( str(channel) - + ' is not a known channel name for this image.', + + " is not a known channel name for this image.", image_warning_id, - warn_skip=True + warn_skip=True, ) continue else: raise ValueError( '"{0}" is not a known channel name for image {1}. ' - 'Aborting!' - .format(channel, image_warning_id) + "Aborting!".format(channel, image_warning_id) ) - fname = '__'.join( - [image_name, str(image_id)] + [str(x) for x in tile] - ) - fname += '.tiff' - fname = fname.replace(' ', '_') + fname = "__".join([image_name, str(image_id)] + [str(x) for x in tile]) + fname += ".tiff" + fname = fname.replace(" ", "_") # download and save the region as TIFF try: - im_array = get_image_array(image, tile, z_stack, channel_index, frame) + im_array = get_image_array( + image, tile, z_stack, channel_index, frame + ) if download_tar: fname = os.path.join(tempdir, fname) try: - tiff = TIFF.open(fname, mode='w') + tiff = TIFF.open(fname, mode="w") tiff.write_image(im_array) finally: tiff.close() @@ -446,81 +463,82 @@ def _center_to_ul(center_x, center_y, width, height): if __name__ == "__main__": p = argparse.ArgumentParser() p.add_argument( - 'image_ids_or_dataset_id', nargs='*', default=[], - help='one or more IDR image ids or a single dataset id' - 'for which to retrieve data (default: ' - 'read ids from stdin).' + "image_ids_or_dataset_id", + nargs="*", + default=[], + help="one or more IDR image ids or a single dataset id" + "for which to retrieve data (default: " + "read ids from stdin).", ) p.add_argument( - '--download-original', dest='download_original', action='store_true', - help="download the original file uploaded to omero" + "--download-original", + dest="download_original", + action="store_true", + help="download the original file uploaded to omero", ) p.add_argument( - '--download-full', dest='download_full', action='store_true', - help="download the full image on omero" + "--download-full", + dest="download_full", + action="store_true", + help="download the full image on omero", ) p.add_argument( - '-c', '--channel', - help='name of the channel to retrieve data for ' - '(note: the first channel of each image will be downloaded if ' - 'left unspecified)' + "-c", + "--channel", + help="name of the channel to retrieve data for " + "(note: the first channel of each image will be downloaded if " + "left unspecified)", ) region = p.add_mutually_exclusive_group() region.add_argument( - '--rectangle', nargs=4, type=int, default=argparse.SUPPRESS, - help='specify a clipping region for the image as x y width height, ' - 'where x and y give the upper left coordinate of the rectangle ' - 'to clip to. Set width and height to 0 to extend the rectangle ' - 'to the actual size of the image.' + "--rectangle", + nargs=4, + type=int, + default=argparse.SUPPRESS, + help="specify a clipping region for the image as x y width height, " + "where x and y give the upper left coordinate of the rectangle " + "to clip to. Set width and height to 0 to extend the rectangle " + "to the actual size of the image.", ) region.add_argument( - '--center', nargs=4, type=int, default=argparse.SUPPRESS, - help='specify a clipping region for the image as x y width height, ' - 'where x and y define the center of a width x height rectangle. ' - 'Set either width or height to 0 to extend the region to the ' - 'actual size of the image along the x- or y-axis.\n' - 'Note: Even values for width and height will be rounded down to ' - 'the nearest odd number.' - ) - p.add_argument( - '-f', '--frame', type=int, default=0 - ) - p.add_argument( - '-z', '--z-stack', type=int, default=0 - ) - p.add_argument( - '--skip-failed', action='store_true' - ) - p.add_argument( - '--download-tar', action='store_true' - ) - p.add_argument( - '-oh', '--omero-host', type=str, default="idr.openmicroscopy.org" - ) - p.add_argument( - '--omero-secured', action='store_true', default=True - ) - p.add_argument( - '-cf', '--config-file', dest='config_file', default=None - ) - p.add_argument( - '--dataset', action='store_true' + "--center", + nargs=4, + type=int, + default=argparse.SUPPRESS, + help="specify a clipping region for the image as x y width height, " + "where x and y define the center of a width x height rectangle. " + "Set either width or height to 0 to extend the region to the " + "actual size of the image along the x- or y-axis.\n" + "Note: Even values for width and height will be rounded down to " + "the nearest odd number.", ) + p.add_argument("-f", "--frame", type=int, default=0) + p.add_argument("-z", "--z-stack", type=int, default=0) + p.add_argument("--skip-failed", action="store_true") + p.add_argument("--download-tar", action="store_true") + p.add_argument("-oh", "--omero-host", type=str, default="idr.openmicroscopy.org") + p.add_argument("--omero-secured", action="store_true", default=True) + p.add_argument("-cf", "--config-file", dest="config_file", default=None) + p.add_argument("--dataset", action="store_true") args = p.parse_args() if not args.image_ids_or_dataset_id: args.image_ids_or_dataset_id = sys.stdin.read().split() if args.dataset and len(args.image_ids_or_dataset_id) > 1: warn("Multiple dataset ids provided. Only the first one will be used.") - if 'center' in args: + if "center" in args: args.coord, args.width, args.height = ( - args.center[:2], args.center[2], args.center[3] + args.center[:2], + args.center[2], + args.center[3], ) - args.region_spec = 'center' + args.region_spec = "center" del args.center - elif 'rectangle' in args: + elif "rectangle" in args: args.coord, args.width, args.height = ( - args.rectangle[:2], args.rectangle[2], args.rectangle[3] + args.rectangle[:2], + args.rectangle[2], + args.rectangle[3], ) - args.region_spec = 'rectangle' + args.region_spec = "rectangle" del args.rectangle download_image_data(**vars(args)) From b7862310784a2088945d84a3df9942355e4beebc Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Wed, 25 Sep 2024 17:37:29 +0200 Subject: [PATCH 05/15] check flake8 --- tools/idr_download/idr_download_by_ids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 62e599f9373..787e9faaab3 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -222,7 +222,7 @@ def download_image_data( prefix = "image-" # normalize image ids by stripping off prefix if it exists image_ids = [ - iid[len(prefix) :] if iid[: len(prefix)] == prefix else iid + iid[len(prefix):] if iid[:len(prefix)] == prefix else iid for iid in image_ids_or_dataset_id ] for image_id in image_ids: From fe2d7396b32572f248d42606bb0caf44088befe3 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 09:44:40 +0200 Subject: [PATCH 06/15] Apply suggestions from @kostrykin Co-authored-by: Leonid Kostrykin --- tools/idr_download/idr_download_by_ids.py | 4 +--- tools/idr_download/idr_download_by_ids.xml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 787e9faaab3..d1a60fed189 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -180,7 +180,6 @@ def download_image_data( omero_username, omero_password, host=omero_host, secure=omero_secured ) ) - # exit_stack.callback(conn.connect().close) if download_tar: # create an archive file to write images to archive = exit_stack.enter_context(tarfile.open("images.tar", mode="w")) @@ -245,7 +244,7 @@ def download_image_data( if image is None: if skip_failed: warn( - "Unable to find an image with this ID in the " "database.", + "Unable to find an image with this ID in the database.", image_warning_id, warn_skip=True, ) @@ -291,7 +290,6 @@ def download_image_data( continue else: raise - elif download_original: try: # try to extract image properties diff --git a/tools/idr_download/idr_download_by_ids.xml b/tools/idr_download/idr_download_by_ids.xml index f42af81814b..a56d4f9684d 100644 --- a/tools/idr_download/idr_download_by_ids.xml +++ b/tools/idr_download/idr_download_by_ids.xml @@ -138,7 +138,7 @@ - + From b6f985894293b503ea4a2139f886ba79b20787c1 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 09:46:23 +0200 Subject: [PATCH 07/15] remove noqa --- tools/idr_download/idr_download_by_ids.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index d1a60fed189..d4d0004b16a 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -10,8 +10,8 @@ import numpy from libtiff import TIFF from omero.cli import cli_login -from omero.constants.namespaces import NSBULKANNOTATIONS # noqa -from omero.gateway import BlitzGateway # noqa +from omero.constants.namespaces import NSBULKANNOTATIONS +from omero.gateway import BlitzGateway from tifffile import imwrite From ee4f66c602b458badbbd84e1fb11ea385f847da9 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 10:22:52 +0200 Subject: [PATCH 08/15] add docstrings to all functions --- tools/idr_download/idr_download_by_ids.py | 119 ++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index d4d0004b16a..84017a2f183 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -16,6 +16,15 @@ def warn(message, image_identifier, warn_skip=False): + """Print an error message to stderr and + - prefix with the image identifier + - suffix with 'Skipping download!' if warn_skip is True + + Args: + message (string): Message to print to stderr + image_identifier (string): Image identifier + warn_skip (bool, optional): Whether 'skipping download' should be suffix to the message. Defaults to False. + """ message = message.rstrip() if warn_skip: if message[-1] in [".", "!", "?"]: @@ -31,6 +40,15 @@ def warn(message, image_identifier, warn_skip=False): def find_channel_index(image, channel_name): + """Identify the channel index from the image and the channel name + + Args: + image (omero.gateway._ImageWrapper): image wrapper on which the channel should be identified + channel_name (string): name of the channel to look for + + Returns: + int: Index of the channel or -1 if was not found. + """ channel_name = channel_name.lower() for n, channel in enumerate(image.getChannelLabels()): if channel_name == channel.lower(): @@ -49,6 +67,22 @@ def find_channel_index(image, channel_name): def get_clipping_region(image, x, y, w, h): + """Check x,y and adjust w,h to image size to be able to crop the image with these coordinates + + Args: + image (omero.gateway._ImageWrapper): image wrapper on which region want to be cropped + x (int): left x coordinate + y (int): top y coordinate + w (int): width + h (int): height + + Raises: + ValueError: if the x or y coordinates are negative. + ValueError: if the x or y coordinates are larger than the width or height of the image. + + Returns: + list of int: new x, y, width, height adjusted to the image + """ # If the (x, y) coordinate falls outside the image boundaries, we # cannot just shift it because that would render the meaning of # w and h undefined (should width and height be decreased or the whole @@ -76,6 +110,15 @@ def get_clipping_region(image, x, y, w, h): def confine_plane(image, z): + """Adjust/Confine z to be among the possible z for the image + + Args: + image (omero.gateway._ImageWrapper): image wrapper for which the z is adjusted + z (int): plane index that need to be confined + + Returns: + int: confined z + """ if z < 0: z = 0 else: @@ -86,6 +129,15 @@ def confine_plane(image, z): def confine_frame(image, t): + """Adjust/Confine t to be among the possible t for the image + + Args: + image (omero.gateway._ImageWrapper): image wrapper for which the t is adjusted + t (int): frame index that need to be confined + + Returns: + int: confined t + """ if t < 0: t = 0 else: @@ -96,6 +148,18 @@ def confine_frame(image, t): def get_image_array(image, tile, z, c, t): + """Get a 2D numpy array from an image wrapper for a given tile, z, c, t + + Args: + image (omero.gateway._ImageWrapper): image wrapper from which values are taken + tile (list of int): [x, y, width, height] where x,y is the top left coordinate of the region to crop + z (int): plane index + c (int): channel index + t (int): frame index + + Returns: + numpy array of 2 dimensions: image values of the selected area + """ pixels = image.getPrimaryPixels() try: selection = pixels.getTile(theZ=z, theT=t, theC=c, tile=tile) @@ -108,13 +172,24 @@ def get_image_array(image, tile, z, c, t): def get_full_image_array(image): + """Get a 5D numpy array with all values from an image wrapper + + Args: + image (omero.gateway._ImageWrapper): image wrapper from which values are taken + + Returns: + numpy array of 5 dimensions: image values in the TZCYX order + """ # The goal is to get the image in TZCYX order pixels = image.getPrimaryPixels() + # Get the final tzclist in the order that will make the numpy reshape work tzclist = list( product( range(image.getSizeT()), range(image.getSizeZ()), range(image.getSizeC()) ) ) + # As getPlanes requires the indices in the zct order + # We keep the final order but switch indices zctlist = [(z, c, t) for (t, z, c) in tzclist] try: all_planes = numpy.array(list(pixels.getPlanes(zctlist))) @@ -152,6 +227,39 @@ def download_image_data( omero_secured=False, config_file=None, ): + """Download the image data of + either a list of image ids or all images from a dataset. + The image data can be: + - a 2D cropped region or + - a hyperstack written in a tiff file + - the original image uploaded in omero + Optionally, th final file can be in a tar + + Args: + image_ids_or_dataset_id (list of string): Can be either a list with a single id (int) of a dataset or a list with images ids (int) or images ids prefixed by 'image-' + dataset (bool, optional): Whether the image_ids_or_dataset_id is a dataset id and all images from this dataset should be retrieved (true) or image_ids_or_dataset_id are individual image ids (false). Defaults to False. + download_original (bool, optional): Whether the original file uploded to omero should be downloaded. Defaults to False. + download_full (bool, optional): Whether the full image (hyperstack) on omero should be written to TIFF. Defaults to False. + channel (string, optional): Channel name. Defaults to None. + z_stack (int, optional): Z stack (plane) index. Defaults to 0. + frame (int, optional): T frame index. Defaults to 0. + coord (tuple of int, optional): Coordinates of the top left or center of the region to crop. Defaults to (0, 0). + width (int, optional): Width of the region to crop. Defaults to 0. + height (int, optional): Height of the region to crop. Defaults to 0. + region_spec (str, optional): How the region is specified ('rectangle' = coord is top left or 'center' = the region is center). Defaults to "rectangle". + skip_failed (bool, optional): Do not stop the downloads if one fails. Defaults to False. + download_tar (bool, optional): Put all downloaded images into a tar file. Defaults to False. + omero_host (str, optional): omero host url. Defaults to "idr.openmicroscopy.org". + omero_secured (bool, optional): Whether the omero connects with secure connection. Defaults to False. + config_file (string, optional): File path with config file with credentials to connect to OMERO. Defaults to None. + + Raises: + ValueError: If the region_spec is not 'rectangle' nor 'center' and a cropped region is wanted. + ValueError: If there is no dataset with this number in OMERO + ValueError: If there is no image with this number in OMERO + Exception: If the command to download the original image fails + ValueError: If the channel name could not be identified + """ if config_file is None: # IDR connection omero_username = "public" @@ -443,6 +551,17 @@ def download_image_data( def _center_to_ul(center_x, center_y, width, height): + """Convert the center coordinates, width, height to upper left coordinates, width, height + + Args: + center_x (int): x coordinate of center + center_y (int): y coordinate of center + width (int): width + height (int): height + + Returns: + list of 4 int: x, y, width, height where x,y are the upper left coordinates + """ if width > 0: ext_x = (width - 1) // 2 ul_x = max([center_x - ext_x, 0]) From 0d9e4ec16d5f64ac6afae5a09f6b190fa74e05ae Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 10:48:55 +0200 Subject: [PATCH 09/15] clarify the mutually exclusion of rectangle/center/download_full/download_original and the fact that channel/frame/z_stack are ignored if download_full or download_orignal is used. --- tools/idr_download/idr_download_by_ids.py | 73 ++++++++++++++--------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 84017a2f183..bde548ac799 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -238,15 +238,15 @@ def download_image_data( Args: image_ids_or_dataset_id (list of string): Can be either a list with a single id (int) of a dataset or a list with images ids (int) or images ids prefixed by 'image-' dataset (bool, optional): Whether the image_ids_or_dataset_id is a dataset id and all images from this dataset should be retrieved (true) or image_ids_or_dataset_id are individual image ids (false). Defaults to False. - download_original (bool, optional): Whether the original file uploded to omero should be downloaded. Defaults to False. + download_original (bool, optional): Whether the original file uploded to omero should be downloaded (ignored if `download_full` is set to True). Defaults to False. download_full (bool, optional): Whether the full image (hyperstack) on omero should be written to TIFF. Defaults to False. - channel (string, optional): Channel name. Defaults to None. - z_stack (int, optional): Z stack (plane) index. Defaults to 0. - frame (int, optional): T frame index. Defaults to 0. - coord (tuple of int, optional): Coordinates of the top left or center of the region to crop. Defaults to (0, 0). - width (int, optional): Width of the region to crop. Defaults to 0. - height (int, optional): Height of the region to crop. Defaults to 0. - region_spec (str, optional): How the region is specified ('rectangle' = coord is top left or 'center' = the region is center). Defaults to "rectangle". + channel (string, optional): Channel name (ignored if `download_full` or `download_original` is set to True). Defaults to None. + z_stack (int, optional): Z stack (plane) index (ignored if `download_full` or `download_original` is set to True). Defaults to 0. + frame (int, optional): T frame index (ignored if `download_full` or `download_original` is set to True). Defaults to 0. + coord (tuple of int, optional): Coordinates of the top left or center of the region to crop (ignored if `download_full` or `download_original` is set to True). Defaults to (0, 0). + width (int, optional): Width of the region to crop (ignored if `download_full` or `download_original` is set to True). Defaults to 0. + height (int, optional): Height of the region to crop (ignored if `download_full` or `download_original` is set to True). Defaults to 0. + region_spec (str, optional): How the region is specified ('rectangle' = coord is top left or 'center' = the region is center, ignored if `download_full` or `download_original` is set to True). Defaults to "rectangle". skip_failed (bool, optional): Do not stop the downloads if one fails. Defaults to False. download_tar (bool, optional): Put all downloaded images into a tar file. Defaults to False. omero_host (str, optional): omero host url. Defaults to "idr.openmicroscopy.org". @@ -587,25 +587,6 @@ def _center_to_ul(center_x, center_y, width, height): "for which to retrieve data (default: " "read ids from stdin).", ) - p.add_argument( - "--download-original", - dest="download_original", - action="store_true", - help="download the original file uploaded to omero", - ) - p.add_argument( - "--download-full", - dest="download_full", - action="store_true", - help="download the full image on omero", - ) - p.add_argument( - "-c", - "--channel", - help="name of the channel to retrieve data for " - "(note: the first channel of each image will be downloaded if " - "left unspecified)", - ) region = p.add_mutually_exclusive_group() region.add_argument( "--rectangle", @@ -629,8 +610,42 @@ def _center_to_ul(center_x, center_y, width, height): "Note: Even values for width and height will be rounded down to " "the nearest odd number.", ) - p.add_argument("-f", "--frame", type=int, default=0) - p.add_argument("-z", "--z-stack", type=int, default=0) + region.add_argument( + "--download-original", + dest="download_original", + action="store_true", + help="download the original file uploaded to omero", + ) + region.add_argument( + "--download-full", + dest="download_full", + action="store_true", + help="download the full image on omero", + ) + p.add_argument( + "-c", + "--channel", + help="name of the channel to retrieve data for " + "(note: the first channel of each image will be downloaded if " + "left unspecified), ignored with `--download-original` and " + "`--download-full`", + ) + p.add_argument( + "-f", + "--frame", + type=int, + default=0, + help="index of the frame to retrive data for (first frame is 0)," + " ignored with `--download-original` and `--download-full`", + ) + p.add_argument( + "-z", + "--z-stack", + type=int, + default=0, + help="index of the slice to retrive data for (first slice is 0)," + " ignored with `--download-original` and `--download-full`", + ) p.add_argument("--skip-failed", action="store_true") p.add_argument("--download-tar", action="store_true") p.add_argument("-oh", "--omero-host", type=str, default="idr.openmicroscopy.org") From c463311a8056f1adeb6599a4a44d4236981096fc Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 10:51:46 +0200 Subject: [PATCH 10/15] remove libtiff dependency --- tools/idr_download/idr_download_by_ids.py | 7 +------ tools/idr_download/idr_download_by_ids.xml | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index bde548ac799..bd7d42e6264 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -8,7 +8,6 @@ from tempfile import TemporaryDirectory import numpy -from libtiff import TIFF from omero.cli import cli_login from omero.constants.namespaces import NSBULKANNOTATIONS from omero.gateway import BlitzGateway @@ -532,11 +531,7 @@ def download_image_data( if download_tar: fname = os.path.join(tempdir, fname) - try: - tiff = TIFF.open(fname, mode="w") - tiff.write_image(im_array) - finally: - tiff.close() + imwrite(fname, im_array) # move image into tarball if download_tar: archive.add(fname, os.path.basename(fname)) diff --git a/tools/idr_download/idr_download_by_ids.xml b/tools/idr_download/idr_download_by_ids.xml index a56d4f9684d..31fe96c3514 100644 --- a/tools/idr_download/idr_download_by_ids.xml +++ b/tools/idr_download/idr_download_by_ids.xml @@ -20,7 +20,6 @@ omero-py - pylibtiff tifffile Date: Thu, 26 Sep 2024 13:13:23 +0200 Subject: [PATCH 11/15] improve docstrings --- tools/idr_download/idr_download_by_ids.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index bd7d42e6264..465f045fb24 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -15,9 +15,9 @@ def warn(message, image_identifier, warn_skip=False): - """Print an error message to stderr and - - prefix with the image identifier - - suffix with 'Skipping download!' if warn_skip is True + """Print an error `message` to stderr and + - prefix with the `image_identifier` + - suffix with 'Skipping download!' if `warn_skip` is True Args: message (string): Message to print to stderr @@ -39,7 +39,7 @@ def warn(message, image_identifier, warn_skip=False): def find_channel_index(image, channel_name): - """Identify the channel index from the image and the channel name + """Identify the channel index from the `image` and the `channel_name` Args: image (omero.gateway._ImageWrapper): image wrapper on which the channel should be identified @@ -66,7 +66,7 @@ def find_channel_index(image, channel_name): def get_clipping_region(image, x, y, w, h): - """Check x,y and adjust w,h to image size to be able to crop the image with these coordinates + """Check `x`, `y` and adjust `w`, `h` to image size to be able to crop the `image` with these coordinates Args: image (omero.gateway._ImageWrapper): image wrapper on which region want to be cropped @@ -109,7 +109,7 @@ def get_clipping_region(image, x, y, w, h): def confine_plane(image, z): - """Adjust/Confine z to be among the possible z for the image + """Adjust/Confine `z` to be among the possible z for the `image` Args: image (omero.gateway._ImageWrapper): image wrapper for which the z is adjusted @@ -128,7 +128,7 @@ def confine_plane(image, z): def confine_frame(image, t): - """Adjust/Confine t to be among the possible t for the image + """Adjust/Confine `t` to be among the possible t for the `image` Args: image (omero.gateway._ImageWrapper): image wrapper for which the t is adjusted @@ -147,7 +147,7 @@ def confine_frame(image, t): def get_image_array(image, tile, z, c, t): - """Get a 2D numpy array from an image wrapper for a given tile, z, c, t + """Get a 2D numpy array from an `image` wrapper for a given `tile`, `z`, `c`, `t` Args: image (omero.gateway._ImageWrapper): image wrapper from which values are taken @@ -171,7 +171,7 @@ def get_image_array(image, tile, z, c, t): def get_full_image_array(image): - """Get a 5D numpy array with all values from an image wrapper + """Get a 5D numpy array with all values from an `image` wrapper Args: image (omero.gateway._ImageWrapper): image wrapper from which values are taken @@ -232,7 +232,7 @@ def download_image_data( - a 2D cropped region or - a hyperstack written in a tiff file - the original image uploaded in omero - Optionally, th final file can be in a tar + Optionally, the final file can be in a tar Args: image_ids_or_dataset_id (list of string): Can be either a list with a single id (int) of a dataset or a list with images ids (int) or images ids prefixed by 'image-' @@ -546,7 +546,7 @@ def download_image_data( def _center_to_ul(center_x, center_y, width, height): - """Convert the center coordinates, width, height to upper left coordinates, width, height + """Convert the center coordinates (`center_x`, `center_y`), `width`, `height` to upper left coordinates, width, height Args: center_x (int): x coordinate of center From f3b73864fc34568226f758ef617ff124162692a0 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 13:34:23 +0200 Subject: [PATCH 12/15] add typing --- tools/idr_download/idr_download_by_ids.py | 78 +++++++++++------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 465f045fb24..963dc0e94b0 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -10,11 +10,11 @@ import numpy from omero.cli import cli_login from omero.constants.namespaces import NSBULKANNOTATIONS -from omero.gateway import BlitzGateway +from omero.gateway import BlitzGateway, _ImageWrapper from tifffile import imwrite -def warn(message, image_identifier, warn_skip=False): +def warn(message: str, image_identifier: str, warn_skip: bool = False) -> None: """Print an error `message` to stderr and - prefix with the `image_identifier` - suffix with 'Skipping download!' if `warn_skip` is True @@ -38,11 +38,11 @@ def warn(message, image_identifier, warn_skip=False): ) -def find_channel_index(image, channel_name): +def find_channel_index(image: _ImageWrapper, channel_name: str) -> int: """Identify the channel index from the `image` and the `channel_name` Args: - image (omero.gateway._ImageWrapper): image wrapper on which the channel should be identified + image (_ImageWrapper): image wrapper on which the channel should be identified channel_name (string): name of the channel to look for Returns: @@ -65,11 +65,11 @@ def find_channel_index(image, channel_name): return -1 -def get_clipping_region(image, x, y, w, h): +def get_clipping_region(image: _ImageWrapper, x: int, y: int, w: int, h: int) -> list[int]: """Check `x`, `y` and adjust `w`, `h` to image size to be able to crop the `image` with these coordinates Args: - image (omero.gateway._ImageWrapper): image wrapper on which region want to be cropped + image (_ImageWrapper): image wrapper on which region want to be cropped x (int): left x coordinate y (int): top y coordinate w (int): width @@ -80,7 +80,7 @@ def get_clipping_region(image, x, y, w, h): ValueError: if the x or y coordinates are larger than the width or height of the image. Returns: - list of int: new x, y, width, height adjusted to the image + list[int]: new [x, y, width, height] adjusted to the image """ # If the (x, y) coordinate falls outside the image boundaries, we # cannot just shift it because that would render the meaning of @@ -108,11 +108,11 @@ def get_clipping_region(image, x, y, w, h): return [x, y, w, h] -def confine_plane(image, z): +def confine_plane(image: _ImageWrapper, z: int) -> int: """Adjust/Confine `z` to be among the possible z for the `image` Args: - image (omero.gateway._ImageWrapper): image wrapper for which the z is adjusted + image (_ImageWrapper): image wrapper for which the z is adjusted z (int): plane index that need to be confined Returns: @@ -127,11 +127,11 @@ def confine_plane(image, z): return z -def confine_frame(image, t): +def confine_frame(image: _ImageWrapper, t: int) -> int: """Adjust/Confine `t` to be among the possible t for the `image` Args: - image (omero.gateway._ImageWrapper): image wrapper for which the t is adjusted + image (_ImageWrapper): image wrapper for which the t is adjusted t (int): frame index that need to be confined Returns: @@ -146,18 +146,18 @@ def confine_frame(image, t): return t -def get_image_array(image, tile, z, c, t): +def get_image_array(image: _ImageWrapper, tile: list[int], z: int, c: int, t: int) -> numpy.ndarray: """Get a 2D numpy array from an `image` wrapper for a given `tile`, `z`, `c`, `t` Args: - image (omero.gateway._ImageWrapper): image wrapper from which values are taken - tile (list of int): [x, y, width, height] where x,y is the top left coordinate of the region to crop + image (_ImageWrapper): image wrapper from which values are taken + tile (list[int]): [x, y, width, height] where x,y is the top left coordinate of the region to crop z (int): plane index c (int): channel index t (int): frame index Returns: - numpy array of 2 dimensions: image values of the selected area + numpy.ndarray: image values of the selected area (2 dimensions) """ pixels = image.getPrimaryPixels() try: @@ -170,14 +170,14 @@ def get_image_array(image, tile, z, c, t): return selection -def get_full_image_array(image): +def get_full_image_array(image: _ImageWrapper) -> numpy.ndarray: """Get a 5D numpy array with all values from an `image` wrapper Args: - image (omero.gateway._ImageWrapper): image wrapper from which values are taken + image (_ImageWrapper): image wrapper from which values are taken Returns: - numpy array of 5 dimensions: image values in the TZCYX order + numpy.ndarray: image values in the TZCYX order (5 dimensions) """ # The goal is to get the image in TZCYX order pixels = image.getPrimaryPixels() @@ -209,23 +209,23 @@ def get_full_image_array(image): def download_image_data( - image_ids_or_dataset_id, - dataset=False, - download_original=False, - download_full=False, - channel=None, - z_stack=0, - frame=0, - coord=(0, 0), - width=0, - height=0, - region_spec="rectangle", - skip_failed=False, - download_tar=False, - omero_host="idr.openmicroscopy.org", - omero_secured=False, - config_file=None, -): + image_ids_or_dataset_id: str, + dataset: bool = False, + download_original: bool = False, + download_full: bool = False, + channel: str = None, + z_stack: int = 0, + frame: int = 0, + coord: tuple[int, int] = (0, 0), + width: int = 0, + height: int = 0, + region_spec: str = "rectangle", + skip_failed: bool = False, + download_tar: bool = False, + omero_host: str = "idr.openmicroscopy.org", + omero_secured: bool = False, + config_file: str = None, +) -> None: """Download the image data of either a list of image ids or all images from a dataset. The image data can be: @@ -235,14 +235,14 @@ def download_image_data( Optionally, the final file can be in a tar Args: - image_ids_or_dataset_id (list of string): Can be either a list with a single id (int) of a dataset or a list with images ids (int) or images ids prefixed by 'image-' + image_ids_or_dataset_id (list[str]): Can be either a list with a single id (int) of a dataset or a list with images ids (int) or images ids prefixed by 'image-' dataset (bool, optional): Whether the image_ids_or_dataset_id is a dataset id and all images from this dataset should be retrieved (true) or image_ids_or_dataset_id are individual image ids (false). Defaults to False. download_original (bool, optional): Whether the original file uploded to omero should be downloaded (ignored if `download_full` is set to True). Defaults to False. download_full (bool, optional): Whether the full image (hyperstack) on omero should be written to TIFF. Defaults to False. channel (string, optional): Channel name (ignored if `download_full` or `download_original` is set to True). Defaults to None. z_stack (int, optional): Z stack (plane) index (ignored if `download_full` or `download_original` is set to True). Defaults to 0. frame (int, optional): T frame index (ignored if `download_full` or `download_original` is set to True). Defaults to 0. - coord (tuple of int, optional): Coordinates of the top left or center of the region to crop (ignored if `download_full` or `download_original` is set to True). Defaults to (0, 0). + coord (tuple[int, int], optional): Coordinates of the top left or center of the region to crop (ignored if `download_full` or `download_original` is set to True). Defaults to (0, 0). width (int, optional): Width of the region to crop (ignored if `download_full` or `download_original` is set to True). Defaults to 0. height (int, optional): Height of the region to crop (ignored if `download_full` or `download_original` is set to True). Defaults to 0. region_spec (str, optional): How the region is specified ('rectangle' = coord is top left or 'center' = the region is center, ignored if `download_full` or `download_original` is set to True). Defaults to "rectangle". @@ -545,7 +545,7 @@ def download_image_data( raise -def _center_to_ul(center_x, center_y, width, height): +def _center_to_ul(center_x: int, center_y: int, width: int, height: int) -> list[int]: """Convert the center coordinates (`center_x`, `center_y`), `width`, `height` to upper left coordinates, width, height Args: @@ -555,7 +555,7 @@ def _center_to_ul(center_x, center_y, width, height): height (int): height Returns: - list of 4 int: x, y, width, height where x,y are the upper left coordinates + list[int]: [x, y, width, height] where x,y are the upper left coordinates """ if width > 0: ext_x = (width - 1) // 2 From 902d6f92b38836abf76b5a8a6f7cc5b8fa47c3cf Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 13:35:19 +0200 Subject: [PATCH 13/15] run black --- tools/idr_download/idr_download_by_ids.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 963dc0e94b0..2892df4ebe8 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -65,7 +65,9 @@ def find_channel_index(image: _ImageWrapper, channel_name: str) -> int: return -1 -def get_clipping_region(image: _ImageWrapper, x: int, y: int, w: int, h: int) -> list[int]: +def get_clipping_region( + image: _ImageWrapper, x: int, y: int, w: int, h: int +) -> list[int]: """Check `x`, `y` and adjust `w`, `h` to image size to be able to crop the `image` with these coordinates Args: @@ -146,7 +148,9 @@ def confine_frame(image: _ImageWrapper, t: int) -> int: return t -def get_image_array(image: _ImageWrapper, tile: list[int], z: int, c: int, t: int) -> numpy.ndarray: +def get_image_array( + image: _ImageWrapper, tile: list[int], z: int, c: int, t: int +) -> numpy.ndarray: """Get a 2D numpy array from an `image` wrapper for a given `tile`, `z`, `c`, `t` Args: From c255347182d5877e32997c8189a86479a08aee30 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 13:36:24 +0200 Subject: [PATCH 14/15] fix warn I introduced --- tools/idr_download/idr_download_by_ids.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 2892df4ebe8..70b7c8ff9c3 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -205,8 +205,7 @@ def get_full_image_array(image: _ImageWrapper) -> numpy.ndarray: ) except Exception as e: warning = "{0} (ID: {1})".format(image.getName(), image.getId()) - warn("Could not download the requested region", warning) - warn(e.msg) + warn(f"Could not download the full image \n {e.msg}", warning) return return all_planes_reshaped From edceaaff2891854d44bb42344533b3c5d07e3338 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Thu, 26 Sep 2024 13:41:14 +0200 Subject: [PATCH 15/15] fix linting --- tools/idr_download/idr_download_by_ids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/idr_download/idr_download_by_ids.py b/tools/idr_download/idr_download_by_ids.py index 70b7c8ff9c3..bae7da3a5ec 100644 --- a/tools/idr_download/idr_download_by_ids.py +++ b/tools/idr_download/idr_download_by_ids.py @@ -10,7 +10,7 @@ import numpy from omero.cli import cli_login from omero.constants.namespaces import NSBULKANNOTATIONS -from omero.gateway import BlitzGateway, _ImageWrapper +from omero.gateway import _ImageWrapper, BlitzGateway from tifffile import imwrite