Merge branch 'main' into feature/layout-support

.gitignore

@@ -134,7 +134,7 @@ dmypy.json
 vino/
 models/
 docker/extra_packages/*
-!extra_packages/mo_patch.diff
+!docker/extra_packages/mo_patch.diff
 envs
 results.csv
 
@@ -146,4 +146,4 @@ shared_with_container/*
 shared_with_container/configs/_*
 tests/data/test_packages
 
-.DS_Store
+.DS_Store

docker/bases/Dockerfile.base-hailo

@@ -1,4 +1,4 @@
-FROM hailo_ai_sw_suite_2024-04:1
+FROM hailo_ai_sw_suite_2024-07:1
 USER root
 
 COPY requirements.txt requirements.txt

docker/extra_packages/mo_patch.diff

@@ -0,0 +1,11 @@
+--- convert_impl.py	2023-08-09 14:48:35.300720667 +0000
++++ /usr/local/lib/python3.8/site-packages/openvino/tools/mo/convert_impl.py	2023-08-09 14:50:48.555078595 +0000
+@@ -275,7 +275,7 @@
+
+     mean_values = parse_tuple_pairs(argv.mean_values)
+     scale_values = parse_tuple_pairs(argv.scale_values)
+-    mean_scale = get_mean_scale_dictionary(mean_values, scale_values, argv.input)
++    mean_scale = get_mean_scale_dictionary(mean_values, scale_values, ','.join(argv.inputs_list))
+     argv.mean_scale_values = mean_scale
+     argv.layout_values = get_layout_values(argv.layout, argv.source_layout, argv.target_layout)
+

docker/hailo/Dockerfile.public

@@ -1,4 +1,4 @@
-FROM hailo_ai_sw_suite_2024-04:1
+FROM hailo_ai_sw_suite_2024-07:1
 USER root
 
 COPY requirements.txt .

modelconverter/packages/base_inferer.py

@@ -4,7 +4,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 import numpy as np
 
@@ -29,6 +29,7 @@ class Inferer(ABC):
     out_dtypes: Dict[str, DataType]
     resize_method: Dict[str, ResizeMethod]
     encoding: Dict[str, Encoding]
+    config: Optional[SingleStageConfig] = None
 
     def __post_init__(self):
         if self.dest.exists():
@@ -70,6 +71,7 @@ def from_config(
                 else Encoding.BGR
                 for inp in config.inputs
             },
+            config=config,
         )
 
     @abstractmethod

modelconverter/packages/hailo/exporter.py

@@ -65,6 +65,7 @@ def __init__(self, config: SingleStageConfig, output_dir: Path):
         self.batch_size = config.hailo.batch_size
         self.disable_compilation = config.hailo.disable_compilation
         self._alls: List[str] = []
+        self.hw_arch = config.hailo.hw_arch
         if not tf.config.list_physical_devices("GPU"):
             logger.error(
                 "No GPU found. Setting optimization and compression level to 0."
@@ -89,7 +90,7 @@ def _get_end_nodes(self):
         return end_nodes
 
     def export(self) -> Path:
-        runner = ClientRunner(hw_arch="hailo8")
+        runner = ClientRunner(hw_arch=self.hw_arch)
         start_nodes, net_input_shapes = self._get_start_nodes()
 
         logger.info("Translating model to Hailo IR.")
@@ -129,7 +130,7 @@ def export(self) -> Path:
             )
             return copy_path
 
-        runner = ClientRunner(hw_arch="hailo8", har=quantized_har_path)
+        runner = ClientRunner(hw_arch=self.hw_arch, har=quantized_har_path)
         hef = runner.compile()
 
         hef_path = self.input_model.with_suffix(".hef")
@@ -176,7 +177,7 @@ def _get_calibration_data(
     def _calibrate(self, har_path: Path) -> str:
         logger.info("Calibrating model.")
 
-        runner = ClientRunner(hw_arch="hailo8", har=str(har_path))
+        runner = ClientRunner(hw_arch=self.hw_arch, har=str(har_path))
         alls = self._get_alls(runner)
         logger.info(f"Using the following configuration: {alls}")
 

modelconverter/packages/hailo/inferer.py

@@ -23,7 +23,9 @@
 class HailoInferer(Inferer):
     def setup(self):
         self.runner = ClientRunner(
-            hw_arch="hailo8",
+            hw_arch=self.config.hailo.hw_arch
+            if self.config is not None
+            else "hailo8",
             har=str(self.model_path),
         )
         hn_dict = self.runner.get_hn_dict()

modelconverter/packages/rvc2/exporter.py

@@ -33,6 +33,7 @@ class RVC2Exporter(Exporter):
 
     def __init__(self, config: SingleStageConfig, output_dir: Path):
         super().__init__(config=config, output_dir=output_dir)
+        self.compress_to_fp16 = config.rvc2.compress_to_fp16
         self.number_of_shaves = config.rvc2.number_of_shaves
         self.number_of_cmx_slices = config.rvc2.number_of_cmx_slices
         self.superblob = config.rvc2.superblob
@@ -54,6 +55,8 @@ def _export_openvino_ir(self) -> Path:
         self._add_args(
             args, ["--output", ",".join(name for name in self.outputs)]
         )
+        if self.compress_to_fp16:
+            self._add_args(args, ["--compress_to_fp16"])
 
         if "--input" not in args:
             inp_str = ""

modelconverter/utils/config.py

@@ -270,11 +270,15 @@ class HailoConfig(TargetConfig):
     batch_size: int = 8
     disable_compilation: bool = False
     alls: List[str] = []
+    hw_arch: Literal[
+        "hailo8", "hailo8l", "hailo8r", "hailo10h", "hailo15h", "hailo15m"
+    ] = "hailo8"
 
 
 class BlobBaseConfig(TargetConfig):
     mo_args: List[str] = []
     compile_tool_args: List[str] = []
+    compress_to_fp16: bool = True
 
 
 class RVC2Config(BlobBaseConfig):
@@ -387,7 +391,7 @@ def _validate_model(cls, data: Dict[str, Any]) -> Dict[str, Any]:
 
             if (
                 inp.get("reverse_input_channels") is not None
-                or reverse_input_channels
+                or reverse_input_channels is not None
             ):
                 inp["reverse_input_channels"] = inp.get(
                     "reverse_input_channels"

modelconverter/utils/nn_archive.py

@@ -59,7 +59,6 @@ def process_nn_archive(
         "input_model": str(untar_path / archive_config.model.metadata.path),
         "inputs": [],
         "outputs": [],
-        "calibration": "random",
     }
 
     for inp in archive_config.model.inputs:
@@ -113,7 +112,6 @@ def process_nn_archive(
                 "input_model": str(input_model_path),
                 "inputs": [],
                 "outputs": [],
-                "calibration": "random",
                 "encoding": "NONE",
             }
             config["stages"][input_model_path.stem] = head_stage_config

shared_with_container/configs/defaults.yaml

@@ -131,6 +131,9 @@ stages:
       # Disables calibration.
       disable_calibration: false
 
+      # Hardware architecture to be used.
+      hw_arch: hailo8
+
     # --- RVC2-Specific Arguments ---
     rvc2:
       # Specifies number of shaves.
@@ -150,6 +153,9 @@ stages:
       # Produces .superblob file instead of regular .blob.
       superblob: true
 
+      # If the original model has FP32 weights or biases, they are compressed to FP16. All intermediate data is kept in original precision.
+      compress_to_fp16: true
+
     # --- RVC3-Specific Argument ---
     rvc3:
       # Target device for POT. Can be one of { VPU, ANY }
@@ -163,6 +169,9 @@ stages:
       # List of additional arguments to pass to the compile_tool.
       compile_tool_args: []
 
+      # If the original model has FP32 weights or biases, they are compressed to FP16. All intermediate data is kept in original precision.
+      compress_to_fp16: true
+
     # --- RVC4-Specific Arguments ---
     rvc4:
       # List of additional arguments to pass to SNPE onnx-to-dlc.

tests/test_utils/test_config.py

@@ -35,12 +35,14 @@
         "number_of_shaves": 8,
         "number_of_cmx_slices": 8,
         "disable_calibration": False,
+        "compress_to_fp16": True,
     },
     "rvc3": {
         "mo_args": [],
         "compile_tool_args": [],
         "pot_target_device": PotDevice.VPU,
         "disable_calibration": False,
+        "compress_to_fp16": True,
     },
     "rvc4": {
         "snpe_onnx_to_dlc_args": [],
@@ -57,6 +59,7 @@
         "disable_compilation": False,
         "alls": [],
         "disable_calibration": False,
+        "hw_arch": "hailo8",
     },
 }
 
@@ -254,6 +257,7 @@ def test_correct():
                 "compile_tool_args": [],
                 "pot_target_device": PotDevice.VPU,
                 "disable_calibration": False,
+                "compress_to_fp16": True,
             },
             "rvc4": {**DEFAULT_TARGET_CONFIGS["rvc4"]},
             "hailo": {
@@ -263,6 +267,7 @@ def test_correct():
                 "batch_size": 4,
                 "disable_compilation": False,
                 "alls": [],
+                "hw_arch": "hailo8",
             },
         },
     )