Add yaml log file for evaluation metrics in building validation optim…

…ization

.github/workflows/building_validation_thresholds_optimization.yaml

@@ -43,6 +43,7 @@ jobs:
           echo "test input_las_dir: ${{env.IO_DIR}}/preds-testset/"
           echo "output thresholds file: ${{env.IO_DIR}}/${{env.THRESHOLDS_FILE}}"
           echo "output config file: ${{env.IO_DIR}}/${{env.OUTPUT_CONFIG_FILE}}"
+          echo "evaluation metrics (on test dataset): ${{env.IO_DIR}}/preds-testset/evaluation.yaml"
 
       - name: Checkout branch
         uses: actions/checkout@v4
@@ -80,4 +81,12 @@ jobs:
           building_validation.optimization.paths.input_las_dir=/io_dir/preds-testset/
           building_validation.optimization.paths.results_output_dir=/io_dir/testset-opti-results/
           building_validation.optimization.paths.building_validation_thresholds=/io_dir/${{env.THRESHOLDS_FILE}}
+          building_validation.optimization.paths.evaluation_results_yaml=/io_dir/preds-testset/evaluation.yaml
           hydra.run.dir=/io_dir/testset-opti-results/
+
+      - name: Log evaluation results on test dataset
+        run: |
+          echo "Evaluation results on the test dataset"
+          echo "The most important metric to inspect is: p_auto (automation proportion)
+          echo ""
+          cat ${{env.IO_DIR}}/preds-testset/evaluation.yaml

configs/building_validation/optimization/default.yaml

@@ -13,6 +13,7 @@ paths:
   group_info_pickle_path: ${.results_output_dir}/group_info.pickle
   prepared_las_dir: ${.results_output_dir}/prepared/
   updated_las_dir: ${.results_output_dir}/updated/
+  evaluation_results_yaml: ${.results_output_dir}/evaluation.yaml
   building_validation_thresholds: ${.results_output_dir}/optimized_thresholds.yaml
   output_optimized_config: ${.results_output_dir}/config_with_optimized_thresholds.yaml
 

docs/source/guides/thresholds_optimization.md

@@ -48,7 +48,8 @@ python lidar_prod/run.py \
 building_validation.optimization.todo='prepare+evaluate+update' \
 building_validation.optimization.paths.input_las_dir=[path/to/labelled/test/dataset/] \
 building_validation.optimization.paths.results_output_dir=[path/to/save/results] \
-building_validation.optimization.paths.building_validation_thresholds=[path/to/optimized_thresholds.yaml]
+building_validation.optimization.paths.building_validation_thresholds=[path/to/optimized_thresholds.yaml] \
+building_validation.optimization.paths.evaluation_results_yaml=[path/to/saved/metrics.yaml]
 ```
 
 ### Utils

lidar_prod/tasks/building_validation_optimization.py

@@ -10,6 +10,7 @@
 
 import numpy as np
 import optuna
+import yaml
 from omegaconf import DictConfig, OmegaConf
 from sklearn.metrics import confusion_matrix
 from tqdm import tqdm
@@ -191,6 +192,8 @@ def evaluate(self) -> dict:
         mts_gt = np.array([c.target for c in clusters])
         metrics_dict = self.evaluate_decisions(mts_gt, decisions)
         log.info(f"\n Results:\n{self._get_results_logs_str(metrics_dict)}")
+        self._save_results_to_yaml(metrics_dict)
+
         return metrics_dict
 
     def _set_thresholds_from_file_if_available(self):
@@ -528,6 +531,19 @@ def _get_results_logs_str(self, metrics_dict: dict):
         )
         return results_logs
 
+    def _save_results_to_yaml(self, metrics_dict: dict):
+        out_dict = metrics_dict.copy()
+        for k, v in out_dict.items():
+            print(k, v, type(v))
+            if isinstance(v, np.ndarray):
+                out_dict[k] = v.tolist()
+            elif isinstance(v, np.float64):
+                out_dict[k] = float(v)
+
+        if self.paths.evaluation_results_yaml:
+            with open(self.paths.evaluation_results_yaml, "w") as f:
+                yaml.safe_dump(out_dict, f)
+
     def save_config_with_optimized_thresolds(self, config: DictConfig):
         """Save config the thresholds in the building_validation.application
         part replaced by optimized thresholds"""

tests/lidar_prod/tasks/test_building_validation_optimization.py

@@ -6,6 +6,7 @@
 import hydra
 import numpy as np
 import pytest
+import yaml
 
 from lidar_prod.tasks.building_validation import thresholds
 from lidar_prod.tasks.building_validation_optimization import (
@@ -28,6 +29,14 @@
 TMP_DIR = Path("tmp/lidar_prod/tasks/building_validation_optimization")
 
 
+def setup_module(module):
+    try:
+        shutil.rmtree(TMP_DIR)
+    except FileNotFoundError:
+        pass
+    TMP_DIR.mkdir(parents=True, exist_ok=True)
+
+
 # Small LAS, for which we optimize thresholds and reach perfect validation,
 # to quickly check optimization logic.
 LAS_SUBSET_FILE = "tests/files/870000_6618000.subset.postIA.corrected.las"
@@ -100,6 +109,17 @@ def test_BVOptimization_on_subset(hydra_cfg):
     # prepared data and the threshold from previous run
     metrics_dict = bvo.evaluate()
     print(metrics_dict)
+
+    # Check that metrics are correctly saved to file
+    assert os.path.isfile(bvo.paths.evaluation_results_yaml)
+    with open(bvo.paths.evaluation_results_yaml, "r") as f:
+        saved_metrics_dict = yaml.safe_load(f)
+        for k, v in metrics_dict.items():
+            if isinstance(v, np.ndarray):
+                assert saved_metrics_dict[k] == v.tolist()
+            else:
+                assert saved_metrics_dict[k] == v
+
     # Assert inclusion
     assert SUBSET_EXPECTED_METRICS["exact"].items() <= metrics_dict.items()
     # Assert <= with a relative tolerance