Native LDF Parser

luxonis_ml/data/parsers/base_parser.py

@@ -7,7 +7,7 @@
 from luxonis_ml.enums.enums import DatasetType
 from luxonis_ml.typing import PathType
 
-ParserOutput = Tuple[DatasetIterator, List[str], Dict[str, Dict], List[str]]
+ParserOutput = Tuple[DatasetIterator, Dict[str, Dict], List[PathType]]
 """Type alias for parser output.
 
 Contains a function to create the annotation generator, list of classes
@@ -90,7 +90,7 @@ def _parse_split(self, **kwargs) -> List[str]:
         @rtype: List[str]
         @return: List of added images.
         """
-        generator, _, skeletons, added_images = self.from_split(**kwargs)
+        generator, skeletons, added_images = self.from_split(**kwargs)
         self.dataset.add(self._add_task(generator))
         if skeletons:
             for skeleton in skeletons.values():

luxonis_ml/data/parsers/classification_directory_parser.py

@@ -85,4 +85,4 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), class_names, {}, added_images
+        return generator(), {}, added_images

luxonis_ml/data/parsers/coco_parser.py

@@ -209,7 +209,6 @@ def from_split(
         coco_categories = annotation_data["categories"]
         categories = {cat["id"]: cat["name"] for cat in coco_categories}
 
-        class_names = list(categories.values())
         skeletons = {}
         for cat in coco_categories:
             if "keypoints" in cat.keys() and "skeleton" in cat.keys():
@@ -312,7 +311,7 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), class_names, skeletons, added_images
+        return generator(), skeletons, added_images
 
 
 def clean_annotations(annotation_path: Path) -> Path:

luxonis_ml/data/parsers/create_ml_parser.py

@@ -89,7 +89,6 @@ def from_split(
         with open(annotation_path) as f:
             annotations_data = json.load(f)
 
-        class_names = set()
         images_annotations = []
         for annotations in annotations_data:
             path = image_dir.absolute().resolve() / annotations["image"]
@@ -103,7 +102,6 @@ def from_split(
             for curr_ann in annotations["annotations"]:
                 class_name = curr_ann["label"]
                 curr_annotations["classes"].append(class_name)
-                class_names.add(class_name)
 
                 bbox_ann = curr_ann["coordinates"]
                 bbox_xywh = [
@@ -134,4 +132,4 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), list(class_names), {}, added_images
+        return generator(), {}, added_images

luxonis_ml/data/parsers/darknet_parser.py

@@ -109,4 +109,4 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), list(class_names.values()), {}, added_images
+        return generator(), {}, added_images

luxonis_ml/data/parsers/luxonis_parser.py

@@ -24,6 +24,7 @@
 from .coco_parser import COCOParser
 from .create_ml_parser import CreateMLParser
 from .darknet_parser import DarknetParser
+from .native_parser import NativeParser
 from .segmentation_mask_directory_parser import SegmentationMaskDirectoryParser
 from .solo_parser import SOLOParser
 from .tensorflow_csv_parser import TensorflowCSVParser
@@ -54,6 +55,7 @@ class LuxonisParser(Generic[T]):
         DatasetType.CLSDIR: ClassificationDirectoryParser,
         DatasetType.SEGMASK: SegmentationMaskDirectoryParser,
         DatasetType.SOLO: SOLOParser,
+        DatasetType.NATIVE: NativeParser,
     }
 
     def __init__(

luxonis_ml/data/parsers/native_parser.py

@@ -0,0 +1,73 @@
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+from luxonis_ml.data import DatasetIterator
+
+from .base_parser import BaseParser, ParserOutput
+
+
+class NativeParser(BaseParser):
+    """Parses directory with native LDF annotations.
+
+    Expected format::
+
+        dataset_dir/
+        ├── train/
+        │   └── annotations.json
+        ├── valid/
+        └── test/
+
+    The annotations are stored in a single JSON file as a list of dictionaries
+    in the same format as the output of the generator function used
+    in L{BaseDataset.add} method.
+    """
+
+    @staticmethod
+    def validate_split(split_path: Path) -> Optional[Dict[str, Any]]:
+        annotation_path = split_path / "annotations.json"
+        if not annotation_path.exists():
+            return None
+        return {"annotation_path": annotation_path}
+
+    @staticmethod
+    def validate(dataset_dir: Path) -> bool:
+        for split in ["train", "valid", "test"]:
+            split_path = dataset_dir / split
+            if NativeParser.validate_split(split_path) is None:
+                return False
+        return True
+
+    def from_dir(
+        self, dataset_dir: Path
+    ) -> Tuple[List[str], List[str], List[str]]:
+        added_train_imgs = self._parse_split(
+            image_dir=dataset_dir / "train",
+            annotation_dir=dataset_dir / "train",
+        )
+        added_val_imgs = self._parse_split(
+            image_dir=dataset_dir / "valid",
+            annotation_dir=dataset_dir / "valid",
+        )
+        added_test_imgs = self._parse_split(
+            image_dir=dataset_dir / "test",
+            annotation_dir=dataset_dir / "test",
+        )
+        return added_train_imgs, added_val_imgs, added_test_imgs
+
+    def from_split(self, annotation_path: Path) -> ParserOutput:
+        """Parses annotations from LDF Format.
+
+        @type annotation_path: C{Path}
+        @param annotation_dir: Path to the JSON file with annotations.
+        @rtype: L{ParserOutput}
+        @return: Annotation generator, list of classes names, skeleton
+            dictionary for keypoints and list of added images.
+        """
+
+        def generator() -> DatasetIterator:
+            yield from json.loads(annotation_path.read_text())
+
+        added_images = self._get_added_images(generator())
+
+        return generator(), {}, added_images

luxonis_ml/data/parsers/segmentation_mask_directory_parser.py

@@ -96,7 +96,8 @@ def from_split(
             dictionary for keypoints and list of added images
         """
 
-        idx_class = " Class"  # NOTE: space prefix included
+        # NOTE: space prefix included
+        idx_class = " Class"
 
         df = pl.read_csv(classes_path).filter(pl.col(idx_class).is_not_null())
         class_names = df[idx_class].to_list()
@@ -124,4 +125,4 @@ def generator() -> DatasetIterator:
                     }
 
         added_images = self._get_added_images(generator())
-        return generator(), class_names, {}, added_images
+        return generator(), {}, added_images

luxonis_ml/data/parsers/solo_parser.py

@@ -260,7 +260,7 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), class_names, skeletons, added_images
+        return generator(), skeletons, added_images
 
     def _get_solo_annotation_types(
         self, annotation_definitions_dict: dict

luxonis_ml/data/parsers/tensorflow_csv_parser.py

@@ -84,7 +84,6 @@ def from_split(
         )
         images_annotations = {}
 
-        class_names = set(df["class"])
         for row in df.rows(named=True):
             path = str(image_dir / str(row["filename"]))
             if path not in images_annotations:
@@ -129,4 +128,4 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), list(class_names), {}, added_images
+        return generator(), {}, added_images

luxonis_ml/data/parsers/voc_parser.py

@@ -80,7 +80,6 @@ def from_split(
             dictionary for keypoints and list of added images.
         """
 
-        class_names = set()
         images_annotations = []
         for anno_xml in annotation_dir.glob("*.xml"):
             annotation_data = ET.parse(anno_xml)
@@ -101,7 +100,6 @@ def from_split(
             for object_item in root.findall("object"):
                 class_name = self._xml_find(object_item, "name")
                 curr_annotations["classes"].append(class_name)
-                class_names.add(class_name)
 
                 bbox_info = object_item.find("bndbox")
                 if bbox_info is not None:
@@ -140,7 +138,7 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), list(class_names), {}, added_images
+        return generator(), {}, added_images
 
     @staticmethod
     def _xml_find(root: ET.Element, tag: str) -> str:

luxonis_ml/data/parsers/yolov4_parser.py

@@ -127,4 +127,4 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), list(class_names.values()), {}, added_images
+        return generator(), {}, added_images

luxonis_ml/data/parsers/yolov6_parser.py

@@ -138,4 +138,4 @@ def generator() -> DatasetIterator:
 
         added_images = self._get_added_images(generator())
 
-        return generator(), list(class_names.values()), {}, added_images
+        return generator(), {}, added_images

luxonis_ml/enums/enums.py

@@ -13,3 +13,4 @@ class DatasetType(str, Enum):
     CLSDIR = "clsdir"
     SEGMASK = "segmask"
     SOLO = "solo"
+    NATIVE = "native"

tests/test_data/test_parsers.py

@@ -39,6 +39,23 @@ def prepare_dir():
             "Thermal_Dogs_and_People.v1-resize-416x416.coco.zip",
             ["boundingbox", "classification"],
         ),
+        (
+            DatasetType.COCO,
+            "roboflow://team-roboflow/coco-128/2/coco",
+            ["boundingbox", "classification"],
+        ),
+        (
+            DatasetType.NATIVE,
+            "ParkingLot.zip",
+            [
+                "boundingbox",
+                "classification",
+                "instance_segmentation",
+                "keypoints",
+                "metadata/brand",
+                "metadata/color",
+            ],
+        ),
         (
             DatasetType.VOC,
             "Thermal_Dogs_and_People.v1-resize-416x416.voc.zip",
@@ -81,13 +98,8 @@ def prepare_dir():
         ),
         (
             DatasetType.SOLO,
-            "D1_ParkingSlot-solo.zip",
-            ["boundingbox", "segmentation", "classification"],
-        ),
-        (
-            DatasetType.COCO,
-            "roboflow://team-roboflow/coco-128/2/coco",
-            ["boundingbox", "classification"],
+            "D2_ParkingLot.zip",
+            ["boundingbox", "segmentation", "classification", "keypoints"],
         ),
     ],
 )