diff --git a/.omeroci/py-setup b/.omeroci/py-setup index cfcd089..9cf7446 100755 --- a/.omeroci/py-setup +++ b/.omeroci/py-setup @@ -18,7 +18,7 @@ source /tmp/miniconda/bin/activate conda init conda create -n omero python=3.9 conda activate omero -conda install -c conda-forge zeroc-ice=3.6.5 +pip install https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20231130/zeroc_ice-3.6.5-cp39-cp39-manylinux_2_28_x86_64.whl conda install -c bioconda bftools cd $TARGET diff --git a/README.md b/README.md index f7714dd..27a5644 100755 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ about the files (name, mimetype). `--plugin` allows you to export omero data to a desired format by using an external plugin. See for example the [arc plugin](https://github.com/cmohl2013/omero-arc), which exports omero projects to ARC repositories. + `--binaries` allows to specify whether to archive binary data (e.g images, ROIs, FileAnnotations) or only create the transfer.xml. Default is `all` and will create the archive. With `none`, only the `transfer.xml` diff --git a/schemas/provenancemetadata.xsd b/schemas/provenancemetadata.xsd new file mode 100644 index 0000000..de711b9 --- /dev/null +++ b/schemas/provenancemetadata.xsd @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/schemas/serverpath.xsd b/schemas/serverpath.xsd new file mode 100644 index 0000000..acd6e0f --- /dev/null +++ b/schemas/serverpath.xsd @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/setup.py b/setup.py index 41c6870..60b8b65 100755 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def read(fname): packages=['', 'omero.plugins'], package_dir={"": "src"}, name="omero-cli-transfer", - version='0.8.0', + version='1.0.0', maintainer="Erick Ratamero", maintainer_email="erick.ratamero@jax.org", description=("A set of utilities for exporting a transfer" @@ -95,8 +95,8 @@ def read(fname): long_description_content_type="text/markdown", url="https://github.com/TheJacksonLaboratory/omero-cli-transfer", install_requires=[ - 'ezomero==2.0.0', - 'ome-types==0.4.2' + 'ezomero>=2.1.0, <3.0.0', + 'ome-types>=0.4.5,<0.5.0' ], extras_require={ "rocrate": ["rocrate==0.7.0"], diff --git a/src/generate_omero_objects.py b/src/generate_omero_objects.py index 6c3fdaa..5ca5c98 100644 --- a/src/generate_omero_objects.py +++ b/src/generate_omero_objects.py @@ -4,14 +4,15 @@ # Use is subject to license terms supplied in LICENSE. import ezomero -from typing import List, Tuple +from ome_types import to_xml +from typing import List, Tuple, Union from omero.model import DatasetI, IObject, PlateI, WellI, WellSampleI, ImageI from omero.gateway import DatasetWrapper from ome_types.model import TagAnnotation, MapAnnotation, FileAnnotation, ROI from ome_types.model import CommentAnnotation, LongAnnotation, Annotation from ome_types.model import Line, Point, Rectangle, Ellipse, Polygon, Shape from ome_types.model import Polyline, Label, Project, Screen, Dataset, OME -from ome_types.model import Image, Plate +from ome_types.model import Image, Plate, XMLAnnotation, AnnotationRef from ome_types.model.simple_types import Marker from omero.gateway import TagAnnotationWrapper, MapAnnotationWrapper from omero.gateway import CommentAnnotationWrapper, LongAnnotationWrapper @@ -21,6 +22,7 @@ from omero.rtypes import rstring, RStringI, rint from ezomero import rois from pathlib import Path +import xml.etree.cElementTree as ETree import os import copy @@ -165,32 +167,7 @@ def create_annotations(ans: List[Annotation], conn: BlitzGateway, hash: str, map_ann.setNs(namespace) key_value_data = [] for v in an.value.ms: - if int(an.id.split(":")[-1]) < 0: - if not metadata: - key_value_data.append(['empty_metadata', "True"]) - break - if v.k == "md5" and "md5" in metadata: - key_value_data.append(['zip_file_md5', hash]) - if v.k == "origin_image_id" and "img_id" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "origin_plate_id" and "plate_id" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "packing_timestamp" and "timestamp" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "software" and "software" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "version" and "version" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "origin_hostname" and "hostname" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "original_user" and "orig_user" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "original_group" and "orig_group" in metadata: - key_value_data.append([v.k, v.value]) - if v.k == "database_id" and "db_id" in metadata: - key_value_data.append([v.k, v.value]) - else: - key_value_data.append([v.k, v.value]) + key_value_data.append([v.k, v.value]) map_ann.setValue(key_value_data) map_ann.save() ann_map[an.id] = map_ann.getId() @@ -220,30 +197,101 @@ def create_annotations(ans: List[Annotation], conn: BlitzGateway, hash: str, file_ann.setFile(original_file) file_ann.save() ann_map[an.id] = file_ann.getId() + elif isinstance(an, XMLAnnotation): + # pass if path, use if provenance metadata + tree = ETree.fromstring(to_xml(an.value, + canonicalize=True)) + is_metadata = False + for el in tree: + if el.tag.rpartition('}')[2] == "CLITransferMetadata": + is_metadata = True + if is_metadata: + map_ann = MapAnnotationWrapper(conn) + namespace = an.namespace + map_ann.setNs(namespace) + key_value_data = [] + if not metadata: + key_value_data.append(['empty_metadata', "True"]) + else: + key_value_data = parse_xml_metadata(an, metadata, hash) + map_ann.setValue(key_value_data) + map_ann.save() + ann_map[an.id] = map_ann.getId() return ann_map +def parse_xml_metadata(ann: XMLAnnotation, + metadata: List[str], + hash: str) -> List[List[str]]: + kv_data = [] + tree = ETree.fromstring(to_xml(ann.value, canonicalize=True)) + for el in tree: + if el.tag.rpartition('}')[2] == "CLITransferMetadata": + for el2 in el: + item = el2.tag.rpartition('}')[2] + val = el2.text + if item == "md5" and "md5" in metadata: + kv_data.append(['md5', hash]) + if item == "origin_image_id" and "img_id" in metadata: + kv_data.append([item, val]) + if item == "origin_plate_id" and "plate_id" in metadata: + kv_data.append([item, val]) + if item == "packing_timestamp" and "timestamp" in metadata: + kv_data.append([item, val]) + if item == "software" and "software" in metadata: + kv_data.append([item, val]) + if item == "version" and "version" in metadata: + kv_data.append([item, val]) + if item == "origin_hostname" and "hostname" in metadata: + kv_data.append([item, val]) + if item == "original_user" and "orig_user" in metadata: + kv_data.append([item, val]) + if item == "original_group" and "orig_group" in metadata: + kv_data.append([item, val]) + if item == "database_id" and "db_id" in metadata: + kv_data.append([item, val]) + return kv_data + + +def get_server_path(anrefs: List[AnnotationRef], + ans: List[Annotation]) -> Union[str, None]: + fpath = None + xml_ids = [] + for an in anrefs: + for an_loop in ans: + if an.id == an_loop.id: + if isinstance(an_loop, XMLAnnotation): + xml_ids.append(an_loop.id) + else: + continue + for an_loop in ans: + if an_loop.id in xml_ids: + if not fpath: + tree = ETree.fromstring(to_xml(an_loop.value, + canonicalize=True)) + for el in tree: + if el.tag.rpartition('}')[2] == "CLITransferServerPath": + for el2 in el: + if el2.tag.rpartition('}')[2] == "Path": + fpath = el2.text + return fpath + + def update_figure_refs(ann: FileAnnotation, ans: List[Annotation], img_map: dict, folder: str): curr_folder = str(Path('.').resolve()) - for an in ann.annotation_refs: - clean_id = int(an.id.split(":")[-1]) - if clean_id < 0: - cmnt_id = an.id - for an_loop in ans: - if an_loop.id == cmnt_id and isinstance(an_loop, CommentAnnotation): - fpath = str(an_loop.value) - dest_path = str(os.path.join(curr_folder, folder, '.', fpath)) - with open(dest_path, 'r') as file: - filedata = file.read() - for src_id, dest_id in img_map.items(): - clean_id = int(src_id.split(":")[-1]) - src_str = f"\"imageId\": {clean_id}" - dest_str = f"\"imageId\": {dest_id}" - print(src_str, dest_str) - filedata = filedata.replace(src_str, dest_str) - with open(dest_path, 'w') as file: - file.write(filedata) + fpath = get_server_path(ann.annotation_refs, ans) + if fpath: + dest_path = str(os.path.join(curr_folder, folder, '.', fpath)) + with open(dest_path, 'r') as file: + filedata = file.read() + for src_id, dest_id in img_map.items(): + clean_id = int(src_id.split(":")[-1]) + src_str = f"\"imageId\": {clean_id}" + dest_str = f"\"imageId\": {dest_id}" + filedata = filedata.replace(src_str, dest_str) + with open(dest_path, 'w') as file: + file.write(filedata) return @@ -251,13 +299,7 @@ def create_original_file(ann: FileAnnotation, ans: List[Annotation], conn: BlitzGateway, folder: str ) -> OriginalFileWrapper: curr_folder = str(Path('.').resolve()) - for an in ann.annotation_refs: - clean_id = int(an.id.split(":")[-1]) - if clean_id < 0: - cmnt_id = an.id - for an_loop in ans: - if an_loop.id == cmnt_id and isinstance(an_loop, CommentAnnotation): - fpath = str(an_loop.value) + fpath = get_server_path(ann.annotation_refs, ans) dest_path = str(os.path.join(curr_folder, folder, '.', fpath)) ofile = conn.createOriginalFileFromLocalFile(dest_path) return ofile @@ -270,19 +312,30 @@ def create_plate_map(ome: OME, img_map: dict, conn: BlitzGateway map_ref_ids = [] for plate in ome.plates: ann_ids = [i.id for i in plate.annotation_refs] - file_path = "" for ann in ome.structured_annotations: if (ann.id in ann_ids and - isinstance(ann, CommentAnnotation) and - int(ann.id.split(":")[-1]) < 0): - newome.structured_annotations.remove(ann) - map_ref_ids.append(ann.id) - file_path = ann.value + isinstance(ann, XMLAnnotation)): + tree = ETree.fromstring(to_xml(ann.value, + canonicalize=True)) + is_metadata = False + for el in tree: + if el.tag.rpartition('}')[2] == "CLITransferMetadata": + is_metadata = True + if not is_metadata: + newome.structured_annotations.remove(ann) + map_ref_ids.append(ann.id) + file_path = get_server_path(plate.annotation_refs, + ome.structured_annotations) + annref = next(filter(lambda x: x.id == ann.id, + plate.annotation_refs)) + newplate = next(filter(lambda x: x.id == plate.id, + newome.plates)) + newplate.annotation_refs.remove(annref) q = conn.getQueryService() params = Parameters() if not file_path: raise ValueError(f"Plate ID {plate.id} does not have a \ - CommentAnnotation with a file path!") + XMLAnnotation with a file path!") path_query = str(file_path).strip('/') if path_query.endswith('mock_folder'): path_query = path_query.rstrip("mock_folder") @@ -584,6 +637,8 @@ def link_one_annotation(obj: IObject, ann: Annotation, ann_map: dict, ann_obj = conn.getObject("LongAnnotation", ann_id) elif isinstance(ann, FileAnnotation): ann_obj = conn.getObject("FileAnnotation", ann_id) + elif isinstance(ann, XMLAnnotation): + ann_obj = conn.getObject("MapAnnotation", ann_id) else: ann_obj = None if ann_obj: diff --git a/src/generate_xml.py b/src/generate_xml.py index 5c2fb2d..1444c29 100644 --- a/src/generate_xml.py +++ b/src/generate_xml.py @@ -11,7 +11,7 @@ from ome_types.model import Plate from ome_types.model import Dataset, DatasetRef from ome_types.model import Image, ImageRef, Pixels -from ome_types.model import TagAnnotation, MapAnnotation, ROI +from ome_types.model import TagAnnotation, MapAnnotation, ROI, XMLAnnotation from ome_types.model import FileAnnotation, BinaryFile, BinData from ome_types.model import AnnotationRef, ROIRef, Map from ome_types.model import CommentAnnotation, LongAnnotation @@ -28,6 +28,8 @@ from omero.cli import CLI from typing import Tuple, List, Optional, Union, Any, Dict, TextIO from subprocess import PIPE, DEVNULL +from generate_omero_objects import get_server_path +import xml.etree.cElementTree as ETree from os import PathLike import pkg_resources import ezomero @@ -40,6 +42,8 @@ import shutil import copy +ann_count = 0 + def create_proj_and_ref(**kwargs) -> Tuple[Project, ProjectRef]: proj = Project(**kwargs) @@ -104,6 +108,12 @@ def create_kv_and_ref(**kwargs) -> Tuple[MapAnnotation, AnnotationRef]: return kv, kvref +def create_xml_and_ref(**kwargs) -> Tuple[XMLAnnotation, AnnotationRef]: + xml = XMLAnnotation(**kwargs) + xmlref = AnnotationRef(id=xml.id) + return xml, xmlref + + def create_long_and_ref(**kwargs) -> Tuple[LongAnnotation, AnnotationRef]: long = LongAnnotation(**kwargs) longref = AnnotationRef(id=long.id) @@ -353,13 +363,14 @@ def create_filepath_annotations(id: str, conn: BlitzGateway, plate_path: Optional[str] = None, ds: Optional[str] = None, proj: Optional[str] = None, - ) -> Tuple[List[CommentAnnotation], + ) -> Tuple[List[XMLAnnotation], List[AnnotationRef]]: - ns = id + global ann_count + ns = 'openmicroscopy.org/cli/transfer' anns = [] anrefs = [] - fp_type = ns.split(":")[0] - clean_id = int(ns.split(":")[-1]) + fp_type = id.split(":")[0] + clean_id = int(id.split(":")[-1]) if not ds: ds = "" if not proj: @@ -377,12 +388,12 @@ def create_filepath_annotations(id: str, conn: BlitzGateway, common_root = "./" common_root = Path(common_root) / proj / ds path = os.path.join(common_root, 'mock_folder') - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=ns, - value=str(path) - ) + xml = create_path_xml(path) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) anns.append(an) + ann_count += 1 anref = AnnotationRef(id=an.id) anrefs.append(anref) else: @@ -393,70 +404,70 @@ def create_filepath_annotations(id: str, conn: BlitzGateway, if simple: filename = Path(f).name f = Path(common_root) / proj / ds / filename - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=ns, - value=str(f) - ) + xml = create_path_xml(str(f)) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) anns.append(an) + ann_count += 1 anref = AnnotationRef(id=an.id) anrefs.append(anref) else: + f = f'pixel_images/{clean_id}.tiff' if simple: f = f'{clean_id}.tiff' f = Path(common_root) / proj / ds / f - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=ns, - value=str(f) - ) + xml = create_path_xml(str(f)) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) anns.append(an) + ann_count += 1 anref = AnnotationRef(id=an.id) anrefs.append(anref) - f = f'pixel_images/{clean_id}.tiff' - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=ns, - value=str(f) - ) + xml = create_path_xml(str(f)) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) anns.append(an) + ann_count += 1 anref = AnnotationRef(id=an.id) anrefs.append(anref) elif fp_type == "Annotation": filename = str(Path(filename).name) f = f'file_annotations/{clean_id}/{filename}' - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=ns, - value=f - ) + xml = create_path_xml(str(f)) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) anns.append(an) + ann_count += 1 anref = AnnotationRef(id=an.id) anrefs.append(anref) elif fp_type == "Plate": - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=ns, - value=plate_path - ) + xml = create_path_xml(plate_path) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) anns.append(an) + ann_count += 1 anref = AnnotationRef(id=an.id) anrefs.append(anref) return anns, anrefs -def create_figure_annotations(id: str) -> Tuple[CommentAnnotation, +def create_figure_annotations(id: str) -> Tuple[XMLAnnotation, AnnotationRef]: ns = id + global ann_count clean_id = int(ns.split(":")[-1]) f = f'figures/Figure_{clean_id}.json' - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=ns, - value=f - ) - anref = AnnotationRef(id=an.id) + xml = create_path_xml(str(f)) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) + ann_count += 1 return (an, anref) @@ -465,6 +476,7 @@ def create_provenance_metadata(conn: BlitzGateway, img_id: int, metadata: Union[List[str], None], plate: bool ) -> Union[Tuple[MapAnnotation, AnnotationRef], Tuple[None, None]]: + global ann_count if not metadata: return None, None software = "omero-cli-transfer" @@ -473,7 +485,6 @@ def create_provenance_metadata(conn: BlitzGateway, img_id: int, ns = 'openmicroscopy.org/cli/transfer' curr_user = conn.getUser().getName() curr_group = conn.getGroupFromContext().getName() - id = (-1) * uuid4().int db_id = conn.getConfigService().getDatabaseUuid() md_dict: Dict[str, Any] = {} @@ -499,17 +510,12 @@ def create_provenance_metadata(conn: BlitzGateway, img_id: int, md_dict['original_group'] = curr_group if "db_id" in metadata: md_dict['database_id'] = db_id - - mmap = [] - for _key, _value in md_dict.items(): - if _value: - mmap.append(M(k=_key, value=str(_value))) - else: - mmap.append(M(k=_key, value='')) - kv, ref = create_kv_and_ref(id=id, - namespace=ns, - value=Map(ms=mmap)) - return kv, ref + xml = create_metadata_xml(md_dict) + an, anref = create_xml_and_ref(id=ann_count, + namespace=ns, + value=xml) + ann_count += 1 + return an, anref def create_objects(folder, filelist): @@ -553,6 +559,7 @@ def create_objects(folder, filelist): annotations = [] counter_imgs = 1 counter_pls = 1 + counter_anns = 1 for target in targets: if filelist: folder = par_folder @@ -562,12 +569,13 @@ def create_objects(folder, filelist): if filelist: folder = par_folder imgs, pls, anns = parse_showinf(res, counter_imgs, counter_pls, - target, folder) + counter_anns, target, folder) images.extend(imgs) counter_imgs = counter_imgs + len(imgs) plates.extend(pls) counter_pls = counter_pls + len(pls) annotations.extend(anns) + counter_anns = counter_anns + len(anns) return images, plates, annotations @@ -590,35 +598,43 @@ def parse_files_import(text, folder): return clean_targets -def parse_showinf(text, counter_imgs, counter_plates, target, folder): +def parse_showinf(text, counter_imgs, counter_plates, counter_ann, + target, folder): ome = from_xml(text) images = [] plates = [] annotations = [] img_id = counter_imgs pl_id = counter_plates + ann_id = counter_ann img_ref = {} for image in ome.images: img_id_str = f"Image:{str(img_id)}" img_ref[image.id] = img_id_str pix = create_empty_pixels(image, img_id) if len(ome.images) > 1: # differentiating names + if image.name == "": + image_name = "0" + else: + image_name = image.name filename = Path(target).name - img = Image(id=img_id_str, name=filename + " [" + image.name + "]", + img = Image(id=img_id_str, name=filename + " [" + image_name + "]", pixels=pix) else: img = Image(id=img_id_str, name=image.name, pixels=pix) img_id += 1 - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=img_id_str, - value=target - ) + xml = create_path_xml(target) + ns = 'openmicroscopy.org/cli/transfer' + an, anref = create_xml_and_ref(id=ann_id, + namespace=ns, + value=xml) annotations.append(an) + ann_id += 1 anref = AnnotationRef(id=an.id) img.annotation_refs.append(anref) - an, anref = create_prepare_metadata() + an, anref = create_prepare_metadata(ann_id) annotations.append(an) + ann_id += 1 img.annotation_refs.append(anref) images.append(img) for plate in ome.plates: @@ -628,11 +644,10 @@ def parse_showinf(text, counter_imgs, counter_plates, target, folder): for ws in w.well_samples: ws.image_ref.id = img_ref[ws.image_ref.id] pl_id += 1 - uid = (-1) * uuid4().int - an = CommentAnnotation(id=uid, - namespace=pl_id_str, - value=target - ) + xml = create_path_xml(target) + an, anref = create_xml_and_ref(id=ann_id, + namespace=ns, + value=xml) annotations.append(an) anref = AnnotationRef(id=an.id) pl.annotation_refs.append(anref) @@ -640,26 +655,41 @@ def parse_showinf(text, counter_imgs, counter_plates, target, folder): return images, plates, annotations -def create_prepare_metadata(): +def create_path_xml(target): + base = ETree.Element("CLITransferServerPath", attrib={ + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": + "https://raw.githubusercontent.com/ome/omero-cli-transfer/" + "main/schemas/serverpath.xsd"}) + ETree.SubElement(base, "Path").text = target + return ETree.tostring(base, encoding='unicode') + + +def create_prepare_metadata(ann_id): software = "omero-cli-transfer" version = pkg_resources.get_distribution(software).version date_time = datetime.now().strftime("%d/%m/%Y, %H:%M:%S") ns = 'openmicroscopy.org/cli/transfer/prepare' - id = (-1) * uuid4().int md_dict: Dict[str, Any] = {} md_dict['software'] = software md_dict['version'] = version md_dict['packing_timestamp'] = date_time - mmap = [] - for _key, _value in md_dict.items(): - if _value: - mmap.append(M(k=_key, value=str(_value))) - else: - mmap.append(M(k=_key, value='')) - kv, ref = create_kv_and_ref(id=id, - namespace=ns, - value=Map(ms=mmap)) - return kv, ref + xml = create_metadata_xml(md_dict) + xml_ann, ref = create_xml_and_ref(id=ann_id, + namespace=ns, + value=xml) + return xml_ann, ref + + +def create_metadata_xml(metadata): + base = ETree.Element("CLITransferMetadata", attrib={ + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": + "https://raw.githubusercontent.com/ome/omero-cli-transfer/" + "main/schemas/preparemetadata.xsd"}) + for _key, _value in metadata.items(): + ETree.SubElement(base, _key).text = str(_value) + return ETree.tostring(base, encoding='unicode') def create_empty_pixels(image, id): @@ -826,13 +856,10 @@ def populate_plate(obj: PlateI, ome: OME, conn: BlitzGateway, well_obj = conn.getObject('Well', well.getId()) well_ref = populate_well(well_obj, ome, conn, hostname, metadata) pl.wells.append(well_ref) + + # this will need some changing to tackle XMLs last_image_anns = ome.images[-1].annotation_refs - last_image_anns_ids = [i.id for i in last_image_anns] - for ann in ome.structured_annotations: - if (ann.id in last_image_anns_ids and - isinstance(ann, CommentAnnotation) and - int(ann.id.split(":")[-1]) < 0): - plate_path = ann.value + plate_path = get_server_path(last_image_anns, ome.structured_annotations) filepath_anns, refs = create_filepath_annotations(pl.id, conn, simple=False, plate_path=plate_path) @@ -937,10 +964,15 @@ def add_annotation(obj: Union[Project, Dataset, Image, Plate, Screen, def list_file_ids(ome: OME) -> dict: id_list = {} + for img in ome.images: + path = get_server_path(img.annotation_refs, ome.structured_annotations) + id_list[img.id] = path for ann in ome.structured_annotations: - clean_id = int(ann.id.split(":")[-1]) - if isinstance(ann, CommentAnnotation) and clean_id < 0: - id_list[ann.namespace] = ann.value + if isinstance(ann, FileAnnotation): + if ann.namespace != "omero.web.figure.json": + path = get_server_path(ann.annotation_refs, + ome.structured_annotations) + id_list[ann.id] = path return id_list @@ -948,6 +980,8 @@ def populate_xml(datatype: str, id: int, filepath: str, conn: BlitzGateway, hostname: str, barchive: bool, simple: bool, figure: bool, metadata: List[str]) -> Tuple[OME, dict]: ome = OME() + global ann_count + ann_count = uuid4().int >> 64 obj = conn.getObject(datatype, id) if datatype == 'Project': populate_project(obj, ome, conn, hostname, metadata, simple) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index 1d9fded..f5c55f8 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -20,13 +20,14 @@ import hashlib from zipfile import ZipFile from typing import Callable, List, Any, Dict, Union, Optional, Tuple +import xml.etree.cElementTree as ETree from generate_xml import populate_xml, populate_tsv, populate_rocrate from generate_xml import populate_xml_folder -from generate_omero_objects import populate_omero +from generate_omero_objects import populate_omero, get_server_path import ezomero -from ome_types.model import CommentAnnotation, OME +from ome_types.model import XMLAnnotation, OME from ome_types import from_xml, to_xml from omero.sys import Parameters from omero.rtypes import rstring @@ -323,30 +324,32 @@ def _copy_files(self, id_list: Dict[str, Any], folder: str, for id in id_list: clean_id = int(id.split(":")[-1]) dtype = id.split(":")[0] - if clean_id not in downloaded_ids: - path = id_list[id] - rel_path = path - if dtype == "Image": + if (dtype == "Image"): + if (clean_id not in downloaded_ids): + path = id_list[id] + rel_path = path rel_path = str(Path(rel_path).parent) - subfolder = os.path.join(str(Path(folder)), rel_path) - if dtype == "Image": + subfolder = os.path.join(str(Path(folder)), rel_path) os.makedirs(subfolder, mode=DIR_PERM, exist_ok=True) - else: - ann_folder = str(Path(subfolder).parent) - os.makedirs(ann_folder, mode=DIR_PERM, exist_ok=True) - if dtype == "Annotation": - id = "File" + id - if rel_path == "pixel_images": - filepath = str(Path(subfolder) / (str(clean_id) + ".tiff")) - cli.invoke(['export', '--file', filepath, id]) - downloaded_ids.append(id) - else: - cli.invoke(['download', id, subfolder]) - if dtype == "Image": - obj = conn.getObject("Image", clean_id) - fileset = obj.getFileset() + obj = conn.getObject("Image", clean_id) + fileset = obj.getFileset() + if rel_path == "pixel_images" or fileset is None: + filepath = str(Path(subfolder) / + (str(clean_id) + ".tiff")) + cli.invoke(['export', '--file', filepath, id]) + downloaded_ids.append(id) + else: + cli.invoke(['download', id, subfolder]) for fs_image in fileset.copyImages(): downloaded_ids.append(fs_image.getId()) + else: + path = id_list[id] + rel_path = path + subfolder = os.path.join(str(Path(folder)), rel_path) + ann_folder = str(Path(subfolder).parent) + os.makedirs(ann_folder, mode=DIR_PERM, exist_ok=True) + id = "File" + id + cli.invoke(['download', id, subfolder]) def _package_files(self, tar_path: str, zip: bool, folder: str): if zip: @@ -577,25 +580,31 @@ def _load_from_pack(self, filepath: str, output: Optional[str] = None def _create_image_map(self, ome: OME ) -> Tuple[OME, DefaultDict, List[str]]: - if not (type(ome) is OME): + if not (isinstance(ome, OME)): raise TypeError("XML is not valid OME format") img_map = DefaultDict(list) filelist = [] newome = copy.deepcopy(ome) map_ref_ids = [] - for ann in ome.structured_annotations: - if int(ann.id.split(":")[-1]) < 0 \ - and isinstance(ann, CommentAnnotation) \ - and ann.namespace: - if ann.namespace.split(":")[0] == "Image": - map_ref_ids.append(ann.id) - img_map[ann.value].append(int( - ann.namespace.split(":")[-1])) - if ann.value.endswith('mock_folder'): - filelist.append(ann.value.rstrip("mock_folder")) - else: - filelist.append(ann.value) - newome.structured_annotations.remove(ann) + for img in ome.images: + fpath = get_server_path(img.annotation_refs, + ome.structured_annotations) + img_map[fpath].append(int(img.id.split(":")[-1])) + # use XML path annotation instead + if fpath.endswith('mock_folder'): + filelist.append(fpath.rstrip("mock_folder")) + else: + filelist.append(fpath) + for anref in img.annotation_refs: + for an in newome.structured_annotations: + if anref.id == an.id and isinstance(an, XMLAnnotation): + tree = ETree.fromstring(to_xml(an.value, + canonicalize=True)) + for el in tree: + if el.tag.rpartition('}')[2] == \ + "CLITransferServerPath": + newome.structured_annotations.remove(an) + map_ref_ids.append(an.id) for i in newome.images: for ref in i.annotation_refs: if ref.id in map_ref_ids: diff --git a/test/data/incomplete_project.zip b/test/data/incomplete_project.zip index c180cbc..ed2e286 100644 Binary files a/test/data/incomplete_project.zip and b/test/data/incomplete_project.zip differ diff --git a/test/data/simple_plate.zip b/test/data/simple_plate.zip index 3e59b10..f69c921 100644 Binary files a/test/data/simple_plate.zip and b/test/data/simple_plate.zip differ diff --git a/test/data/simple_screen.zip b/test/data/simple_screen.zip index 52d41da..f1cbcb9 100644 Binary files a/test/data/simple_screen.zip and b/test/data/simple_screen.zip differ diff --git a/test/data/transfer.xml b/test/data/transfer.xml index 9e04406..6ea4e70 100644 --- a/test/data/transfer.xml +++ b/test/data/transfer.xml @@ -10,7 +10,7 @@ - + @@ -23,7 +23,7 @@ - + @@ -37,12 +37,20 @@ simple_tag - - root_0/2022-01/14/18-30-55.264/combined_result.tiff - - - root_0/2022-01/14/18-30-55.264/combined_result.tiff - + + + + root_0/2022-01/14/18-30-55.264/combined_result.tiff + + + + + + + root_0/2022-01/14/18-30-55.264/combined_result.tiff + + + diff --git a/test/data/valid_single_dataset.zip b/test/data/valid_single_dataset.zip index ba09895..8baed9f 100644 Binary files a/test/data/valid_single_dataset.zip and b/test/data/valid_single_dataset.zip differ diff --git a/test/data/valid_single_image.tar b/test/data/valid_single_image.tar index c9f6dfc..f0ed9dc 100644 Binary files a/test/data/valid_single_image.tar and b/test/data/valid_single_image.tar differ diff --git a/test/data/valid_single_image.zip b/test/data/valid_single_image.zip index 82d8de7..8a2cbf4 100644 Binary files a/test/data/valid_single_image.zip and b/test/data/valid_single_image.zip differ diff --git a/test/data/valid_single_image/root_0/2023-12/18/14-52-03.548/combined_result.tiff b/test/data/valid_single_image/root_0/2023-12/18/14-52-03.548/combined_result.tiff new file mode 100644 index 0000000..8508bf6 Binary files /dev/null and b/test/data/valid_single_image/root_0/2023-12/18/14-52-03.548/combined_result.tiff differ diff --git a/test/data/valid_single_image/transfer.xml b/test/data/valid_single_image/transfer.xml index 6398a09..d3901b3 100644 --- a/test/data/valid_single_image/transfer.xml +++ b/test/data/valid_single_image/transfer.xml @@ -1,69 +1,74 @@ - - - - - - - - - - - - - - - - - - - - - this is a test to see if the kv pairs in omero have any length limits. I don't think they do, but I will write something relatively long just so I can double-check whether that is the case or not. - - - - simple_tag - - - - 13 - 26/08/2022, 11:01:27 - omero-cli-transfer - 0.0.0 - localhost - TBC - root - system - 59149c9d-0ff3-430a-9f1a-1d8093b4a98c - - - - root_0/2022-08/26/14-59-42.116/combined_result.tiff - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + this is a test to see if the kv pairs in omero have any length limits. I don't think they do, but I will write something relatively long just so I can double-check whether that is the case or not. + + + + simple_tag + + + + + + + + + + + + + 51 + 18/12/2023, 09:57:59 + omero-cli-transfer + 0.8.0 + localhost + TBC + root + system + + + + + + + root_0/2023-12/18/14-52-03.548/combined_result.tiff + + + + + + + + + + + + + + + + + + + diff --git a/test/data/valid_single_project.zip b/test/data/valid_single_project.zip index 1e36dfe..a7be4c6 100644 Binary files a/test/data/valid_single_project.zip and b/test/data/valid_single_project.zip differ diff --git a/test/integration/test_prepare.py b/test/integration/test_prepare.py index c3eb80e..d3b6628 100644 --- a/test/integration/test_prepare.py +++ b/test/integration/test_prepare.py @@ -15,6 +15,8 @@ from ome_types.model import AnnotationRef, ROI, ROIRef, Rectangle from ome_types.model.screen import PlateRef from ome_types.model.map import M, Map +from uuid import uuid4 + import ezomero import pytest @@ -168,6 +170,7 @@ def run_asserts_edited(self): elif img_name == "edited image name": kvs = ezomero.get_map_annotation_ids(self.gw, "Image", img.getId()) + kvs = sorted(kvs) assert len(kvs) == 2 kv = ezomero.get_map_annotation(self.gw, kvs[-1]) assert len(kv) == 2 @@ -196,10 +199,15 @@ def run_asserts_edited(self): def edit_xml(self, filename): ome = from_xml(filename) + ann_count = uuid4().int >> 64 new_proj = Project(id="Project:1", name="edited project") new_ds = Dataset(id="Dataset:1", name="edited dataset") - newtag1 = TagAnnotation(id="Annotation:1", value="tag for proj") - newtag2 = TagAnnotation(id="Annotation:2", value="tag for img") + newtag1 = TagAnnotation(id=f"Annotation:{ann_count}", + value="tag for proj") + ann_count += 1 + newtag2 = TagAnnotation(id=f"Annotation:{ann_count}", + value="tag for img") + ann_count += 1 new_proj.annotation_refs.append(AnnotationRef(id=newtag1.id)) md_dict = {"key1": "value1", "key2": 2} mmap = [] @@ -208,7 +216,8 @@ def edit_xml(self, filename): mmap.append(M(k=_key, value=str(_value))) else: mmap.append(M(k=_key, value='')) - mapann = MapAnnotation(id="Annotation:3", value=Map(ms=mmap)) + mapann = MapAnnotation(id=f"Annotation:{ann_count}", + value=Map(ms=mmap)) rect = Rectangle(id="Shape:1", x=1, y=2, width=3, height=4) roi = ROI(id="ROI:1", union=[rect]) ome.rois.append(roi) diff --git a/test/integration/test_transfer.py b/test/integration/test_transfer.py index e609055..459baca 100644 --- a/test/integration/test_transfer.py +++ b/test/integration/test_transfer.py @@ -221,8 +221,11 @@ def test_unpack_folder(self, folder_name): map_ann_ids = ezomero.get_map_annotation_ids( self.gw, "Image", im_ids[-1]) assert len(map_ann_ids) == 3 - provenance = ezomero.get_map_annotation(self.gw, map_ann_ids[-1]) - assert len(provenance) == 8 + for annid in map_ann_ids: + ann_obj = self.gw.getObject("MapAnnotation", annid) + ann = ezomero.get_map_annotation(self.gw, annid) + if ann_obj.getNs() == "openmicroscopy.org/cli/transfer": + assert len(ann) == 8 assert len(ezomero.get_tag_ids( self.gw, "Image", im_ids[-1])) == 1 self.delete_all() @@ -236,8 +239,11 @@ def test_unpack_folder(self, folder_name): map_ann_ids = ezomero.get_map_annotation_ids( self.gw, "Image", im_ids[-1]) assert len(map_ann_ids) == 3 - provenance = ezomero.get_map_annotation(self.gw, map_ann_ids[-1]) - assert len(provenance) == 1 + for annid in map_ann_ids: + ann_obj = self.gw.getObject("MapAnnotation", annid) + ann = ezomero.get_map_annotation(self.gw, annid) + if ann_obj.getNs() == "openmicroscopy.org/cli/transfer": + assert len(ann) == 1 self.delete_all() self.args = temp_args + ["--metadata", "orig_user", "db_id"] @@ -248,8 +254,11 @@ def test_unpack_folder(self, folder_name): map_ann_ids = ezomero.get_map_annotation_ids( self.gw, "Image", im_ids[-1]) assert len(map_ann_ids) == 3 - provenance = ezomero.get_map_annotation(self.gw, map_ann_ids[-1]) - assert len(provenance) == 2 + for annid in map_ann_ids: + ann_obj = self.gw.getObject("MapAnnotation", annid) + ann = ezomero.get_map_annotation(self.gw, annid) + if ann_obj.getNs() == "openmicroscopy.org/cli/transfer": + assert len(ann) == 1 self.delete_all() @pytest.mark.parametrize('package_name', TEST_FILES) @@ -298,24 +307,20 @@ def test_unpack(self, package_name): if package_name == "test/data/valid_single_project.zip": ezomero.print_projects(self.gw) - pjs = self.gw.getObjects("Project") - count = 0 - for p in pjs: - pj_id = p.getId() - count += 1 - assert count == 1 + pjs = ezomero.get_project_ids(self.gw) + assert len(pjs) == 1 count = 0 - proj = self.gw.getObject("Project", pj_id) - for d in proj.listChildren(): - ds_id = d.getId() - count += 1 - assert count == 2 - im_ids = ezomero.get_image_ids(self.gw, dataset=ds_id) + ds_ids = ezomero.get_dataset_ids(self.gw, pjs[-1]) + ds_ids.sort() + assert len(ds_ids) == 2 + im_ids = ezomero.get_image_ids(self.gw, dataset=ds_ids[0]) + assert len(im_ids) == 2 + im_ids = ezomero.get_image_ids(self.gw, dataset=ds_ids[1]) assert len(im_ids) == 1 assert len(ezomero.get_map_annotation_ids( - self.gw, "Project", pj_id)) == 1 + self.gw, "Project", pjs[-1])) == 1 assert len(ezomero.get_tag_ids( - self.gw, "Project", pj_id)) == 0 + self.gw, "Project", pjs[-1])) == 0 self.delete_all() if package_name == "test/data/incomplete_project.zip": @@ -398,14 +403,16 @@ def test_unpack_merge(self): assert len(orphan) == 1 scr_args = self.args + ['unpack', "test/data/simple_screen.zip"] self.cli.invoke(scr_args, strict=True) - scr_args += ['--merge'] - self.cli.invoke(scr_args, strict=True) scr_ids = [] for screen in self.gw.getObjects("Screen"): scr_ids.append(screen.getId()) + screen = self.gw.getObject("Screen", scr_ids[0]) + for plate in screen.listChildren(): + print(plate.getId()) + scr_args += ['--merge'] + self.cli.invoke(scr_args, strict=True) assert len(scr_ids) == 1 pl_ids = [] - screen = self.gw.getObject("Screen", scr_ids[0]) for plate in screen.listChildren(): pl_ids.append(plate.getId()) assert len(pl_ids) == 4 diff --git a/test/unit/test_set.py b/test/unit/test_set.py index 726ec21..f454831 100644 --- a/test/unit/test_set.py +++ b/test/unit/test_set.py @@ -74,7 +74,7 @@ def test_load_pack(self): 111, 'test/data/output_folder') hash, ome, folder = self.transfer._load_from_pack( "test/data/valid_single_image.zip", "tmp_folder") - assert hash == "ac050c218f01bf189f9b3bdc9cab4f37" + assert hash == "6bc8b78eb85f8244f86eded682f95feb" assert len(ome.images) == 1 assert str(folder.resolve()) == "/omero-cli-transfer/tmp_folder" hash, ome, folder = self.transfer._load_from_pack(