Merge pull request #27 from IGNF/fix-replace-classes

Update pdal to 2.6+
IGNF · Oct 31, 2023 · 3fa6c30 · 3fa6c30
2 parents a5119c3 + 6b1876e
commit 3fa6c30
Show file tree

Hide file tree

Showing 9 changed files with 75 additions and 64 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,7 @@
 # dev
 - count_occurences / replace_value: add copy_and_hack decorator to run on tscan output files
+- Update to pdal 2.6+ to better handle classification values and flags in replace_attribute_in_las
+(was treating values over 31 as {classification under 31 + flag} even when saving to LAS 1.4)
 
 # 1.3.1
 - fix color: ensure that tmp orthoimages are deleted after use by using the namedTemporaryFile properly.

diff --git a/environment.yml b/environment.yml
@@ -3,7 +3,7 @@ channels:
   - conda-forge
 dependencies:
   - python=3.10.*
-  - conda-forge:pdal==2.5.*
+  - conda-forge:pdal>=2.6.*
   - conda-forge:python-pdal==3.2.*
   - requests
   - gdal

diff --git a/test/count_occurences/test_count_occurences_for_attribute.py b/test/count_occurences/test_count_occurences_for_attribute.py
@@ -24,7 +24,7 @@
         "4": 2160,
         "5": 42546,
         "6": 33595,
-        "64": 83,
+        "0": 83,
     }
 )
 

diff --git a/test/count_occurences/test_merge_occurences_counts.py b/test/count_occurences/test_merge_occurences_counts.py
@@ -21,7 +21,7 @@
         "4": 2160,
         "5": 42546,
         "6": 33595,
-        "64": 83,
+        "0": 83,
     }
 )
 

diff --git a/test/data/counts/count_test_data_77050_627755_LA93_IGN69.json b/test/data/counts/count_test_data_77050_627755_LA93_IGN69.json
@@ -5,5 +5,5 @@
   "4": 1227,
   "5": 30392,
   "6": 29447,
-  "64": 13
+  "0": 13
 }
diff --git a/test/data/counts/count_test_data_77050_627760_LA93_IGN69.json b/test/data/counts/count_test_data_77050_627760_LA93_IGN69.json
@@ -5,5 +5,5 @@
   "4": 933,
   "5": 12154,
   "6": 4148,
-  "64": 70
+  "0": 70
 }
diff --git a/test/test_replace_attribute_in_las.py b/test/test_replace_attribute_in_las.py
@@ -10,7 +10,7 @@
 from pdaltools.standardize_format import get_writer_parameters
 import pytest
 import shutil
-from test.utils import get_pdal_infos_summary
+from test.utils import get_pdal_infos_summary, EXPECTED_DIMS_BY_DATAFORMAT
 from typing import Dict
 from test.test_standardize_format import assert_lasinfo_no_warning
 
@@ -28,20 +28,22 @@
         "4": 1227,
         "5": 30392,
         "6": 29447,
-        "64": 13,
+        "0": 13,
     }
 )
+colored_las_params = get_writer_parameters({"dataformat_id": 8})
 
-expected_counts = Counter({"2": 21172, "3": 226, "4": 1227, "5": 30392, "64": 29447, "201": 2047 + 13})
+expected_counts = Counter({"0": 13, "2": 226, "4": 1227, "5": 30392, "65": 29447, "201": 2047 + 21172})
 
 replacement_map_fail = {
     "201": ["1", "64"],
     "6": ["64"],
-}  # has duplicatevalue to replace
+}  # has duplicate value to replace, so it should fail
 
 replacement_map_success = {
-    "201": ["1", "64"],
-    "64": ["6"],
+    "201": ["1", "2"],
+    "2": ["3"],  # check that the replacement is correct when a value is both replaced and to replace
+    "65": ["6"],  # check that values over 31 are interpreted correctly in las 1.4 output
 }
 
 # test replacement map parsing
@@ -59,8 +61,8 @@ def setup_module(module):
     os.mkdir(tmp_path)
 
 
-def test_replace_values():
-    replace_values(input_file, output_file, replacement_map_success, attribute)
+def test_replace_values_ok():
+    replace_values(input_file, output_file, replacement_map_success, attribute, colored_las_params)
     count = compute_count_one_file(output_file, attribute)
 
     assert count == expected_counts
@@ -78,11 +80,11 @@ def test_replace_values_duplicate_input():
 
 
 def check_dimensions(input_file, output_file):
-    input_summary = get_pdal_infos_summary(input_file)
-    input_dimensions = set(input_summary["summary"]["dimensions"])
     output_summary = get_pdal_infos_summary(output_file)
-    output_dimensions = set(output_summary["summary"]["dimensions"])
-    assert input_dimensions == output_dimensions
+    output_dimensions = [s.strip() for s in output_summary["summary"]["dimensions"].split(",")]
+    print(sorted(output_dimensions))
+    print(sorted(EXPECTED_DIMS_BY_DATAFORMAT[8]))
+    assert set(output_dimensions) == set(EXPECTED_DIMS_BY_DATAFORMAT[8])
 
 
 def test_parse_replacement_map_from_path_or_json_string_path_ok():

diff --git a/test/test_standardize_format.py b/test/test_standardize_format.py
@@ -3,7 +3,7 @@
 import shutil
 from pdaltools.standardize_format import rewrite_with_pdal, standardize, exec_las2las
 import logging
-from test.utils import get_pdal_infos_summary
+from test.utils import get_pdal_infos_summary, EXPECTED_DIMS_BY_DATAFORMAT
 import pdal
 import subprocess as sp
 
@@ -21,51 +21,6 @@
     {"dataformat_id": 8, "a_srs": "EPSG:4326"},
 ]
 
-expected_dims = {
-    6: set(
-        [
-            "X",
-            "Y",
-            "Z",
-            "Intensity",
-            "ReturnNumber",
-            "NumberOfReturns",
-            "ClassFlags",
-            "ScanChannel",
-            "ScanDirectionFlag",
-            "EdgeOfFlightLine",
-            "Classification",
-            "UserData",
-            "ScanAngleRank",
-            "PointSourceId",
-            "GpsTime",
-        ]
-    ),
-    8: set(
-        [
-            "X",
-            "Y",
-            "Z",
-            "Intensity",
-            "ReturnNumber",
-            "NumberOfReturns",
-            "ClassFlags",
-            "ScanChannel",
-            "ScanDirectionFlag",
-            "EdgeOfFlightLine",
-            "Classification",
-            "UserData",
-            "ScanAngleRank",
-            "PointSourceId",
-            "GpsTime",
-            "Red",
-            "Green",
-            "Blue",
-            "Infrared",
-        ]
-    ),
-}
-
 
 def setup_module(module):
     try:
@@ -94,7 +49,7 @@ def _test_standardize_format_one_params_set(params):
     assert metadata["dataformat_id"] == params["dataformat_id"]
     # Check that there is no extra dim
     dimensions = set([d.strip() for d in json_info["summary"]["dimensions"].split(",")])
-    assert dimensions == expected_dims[params["dataformat_id"]]
+    assert dimensions == EXPECTED_DIMS_BY_DATAFORMAT[params["dataformat_id"]]
 
     # TODO: Check srs
     # TODO: check precision

diff --git a/test/utils.py b/test/utils.py
@@ -9,3 +9,55 @@ def get_pdal_infos_summary(f: str):
     r = sp.run(["pdal", "info", "--summary", f], stderr=sp.PIPE, stdout=sp.PIPE)
     json_info = json.loads(r.stdout.decode())
     return json_info
+
+
+EXPECTED_DIMS_BY_DATAFORMAT = {
+    6: set(
+        [
+            "X",
+            "Y",
+            "Z",
+            "Intensity",
+            "ReturnNumber",
+            "NumberOfReturns",
+            "ScanChannel",
+            "ScanDirectionFlag",
+            "EdgeOfFlightLine",
+            "Classification",
+            "UserData",
+            "ScanAngleRank",
+            "PointSourceId",
+            "GpsTime",
+            "KeyPoint",
+            "Overlap",
+            "Synthetic",
+            "Withheld",
+        ]
+    ),
+    8: set(
+        [
+            "X",
+            "Y",
+            "Z",
+            "Intensity",
+            "ReturnNumber",
+            "NumberOfReturns",
+            "ScanChannel",
+            "ScanDirectionFlag",
+            "EdgeOfFlightLine",
+            "Classification",
+            "UserData",
+            "ScanAngleRank",
+            "PointSourceId",
+            "GpsTime",
+            "Red",
+            "Green",
+            "Blue",
+            "Infrared",
+            "KeyPoint",
+            "Overlap",
+            "Synthetic",
+            "Withheld",
+        ]
+    ),
+}