From 6f3442c30481cd9c3f3ac76080c3dc0469f8f57d Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Thu, 26 Oct 2023 17:07:34 +0200 Subject: [PATCH 1/3] Generalize unlock to count_occurences and replace_attribute --- CHANGELOG.md | 1 + .../count_occurences/count_occurences_for_attribute.py | 3 +++ pdaltools/replace_attribute_in_las.py | 8 +++++--- pdaltools/standardize_format.py | 5 ++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2294400..4df04c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # dev +- count_occurences / replace_value: add copy_and_hack decorator to run on tscan output files # 1.3.1 - fix color: ensure that tmp orthoimages are deleted after use by using the namedTemporaryFile properly. diff --git a/pdaltools/count_occurences/count_occurences_for_attribute.py b/pdaltools/count_occurences/count_occurences_for_attribute.py index 5ddedae..c555d39 100644 --- a/pdaltools/count_occurences/count_occurences_for_attribute.py +++ b/pdaltools/count_occurences/count_occurences_for_attribute.py @@ -10,6 +10,8 @@ from tqdm import tqdm from typing import List +from pdaltools.unlock_file import copy_and_hack_decorator + def parse_args(): parser = argparse.ArgumentParser("Count points with each value of an attribute.") @@ -25,6 +27,7 @@ def parse_args(): return parser.parse_args() +@copy_and_hack_decorator def compute_count_one_file(filepath: str, attribute: str = "Classification") -> Counter: pipeline = pdal.Reader.las(filepath) pipeline |= pdal.Filter.stats(dimensions=attribute, count=attribute) diff --git a/pdaltools/replace_attribute_in_las.py b/pdaltools/replace_attribute_in_las.py index 4d791c5..b59a161 100644 --- a/pdaltools/replace_attribute_in_las.py +++ b/pdaltools/replace_attribute_in_las.py @@ -10,6 +10,8 @@ import tempfile from typing import List, Dict +from pdaltools.unlock_file import copy_and_hack_decorator + def parse_args(): parser = argparse.ArgumentParser("Replace values of a given attribute in a las/laz file.") @@ -65,6 +67,7 @@ def dict_to_pdal_assign_list(d: Dict, output_attribute: str = "Classification", return assignment_list +@copy_and_hack_decorator def replace_values( input_file: str, output_file: str, @@ -109,9 +112,8 @@ def replace_values_clean( attribute: str = "Classification", writer_parameters: Dict = {}, ): - _, extension = os.path.splitext(output_file) - with tempfile.NamedTemporaryFile(suffix=extension) as tmp: - tmp.close() + filename = os.path.basename(output_file) + with tempfile.NamedTemporaryFile(suffix=filename) as tmp: replace_values(input_file, tmp.name, replacement_map, attribute, writer_parameters) exec_las2las(tmp.name, output_file) diff --git a/pdaltools/standardize_format.py b/pdaltools/standardize_format.py index b0d45f9..f638c34 100644 --- a/pdaltools/standardize_format.py +++ b/pdaltools/standardize_format.py @@ -76,9 +76,8 @@ def exec_las2las(input_file: str, output_file: str): @copy_and_hack_decorator def standardize(input_file: str, output_file: str, params_from_parser: Dict) -> None: - _, extension = os.path.splitext(output_file) - with tempfile.NamedTemporaryFile(suffix=extension) as tmp: - tmp.close() + filename = os.path.basename(output_file) + with tempfile.NamedTemporaryFile(suffix=filename) as tmp: rewrite_with_pdal(input_file, tmp.name, params_from_parser) exec_las2las(tmp.name, output_file) From 6b1876e94fff7bac8dacccecb9421ad8c3ec1fc9 Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Mon, 30 Oct 2023 16:34:39 +0100 Subject: [PATCH 2/3] Update pdal to 2.6+ and adapt tests --- CHANGELOG.md | 2 + environment.yml | 2 +- .../test_count_occurences_for_attribute.py | 2 +- .../test_merge_occurences_counts.py | 2 +- ...unt_test_data_77050_627755_LA93_IGN69.json | 2 +- ...unt_test_data_77050_627760_LA93_IGN69.json | 2 +- test/test_replace_attribute_in_las.py | 26 +++++----- test/test_standardize_format.py | 49 +---------------- test/utils.py | 52 +++++++++++++++++++ 9 files changed, 75 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df04c7..e5d579e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # dev - count_occurences / replace_value: add copy_and_hack decorator to run on tscan output files +- Update to pdal 2.6+ to better handle classification values and flags in replace_attribute_in_las +(was treating values over 31 as {classification under 31 + flag} even when saving to LAS 1.4) # 1.3.1 - fix color: ensure that tmp orthoimages are deleted after use by using the namedTemporaryFile properly. diff --git a/environment.yml b/environment.yml index 6c11001..c0ff58a 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - python=3.10.* - - conda-forge:pdal==2.5.* + - conda-forge:pdal>=2.6.* - conda-forge:python-pdal==3.2.* - requests - gdal diff --git a/test/count_occurences/test_count_occurences_for_attribute.py b/test/count_occurences/test_count_occurences_for_attribute.py index 383fc9c..b169adc 100644 --- a/test/count_occurences/test_count_occurences_for_attribute.py +++ b/test/count_occurences/test_count_occurences_for_attribute.py @@ -24,7 +24,7 @@ "4": 2160, "5": 42546, "6": 33595, - "64": 83, + "0": 83, } ) diff --git a/test/count_occurences/test_merge_occurences_counts.py b/test/count_occurences/test_merge_occurences_counts.py index 6d2847d..d16a3d3 100644 --- a/test/count_occurences/test_merge_occurences_counts.py +++ b/test/count_occurences/test_merge_occurences_counts.py @@ -21,7 +21,7 @@ "4": 2160, "5": 42546, "6": 33595, - "64": 83, + "0": 83, } ) diff --git a/test/data/counts/count_test_data_77050_627755_LA93_IGN69.json b/test/data/counts/count_test_data_77050_627755_LA93_IGN69.json index 3a67a12..15851c0 100644 --- a/test/data/counts/count_test_data_77050_627755_LA93_IGN69.json +++ b/test/data/counts/count_test_data_77050_627755_LA93_IGN69.json @@ -5,5 +5,5 @@ "4": 1227, "5": 30392, "6": 29447, - "64": 13 + "0": 13 } \ No newline at end of file diff --git a/test/data/counts/count_test_data_77050_627760_LA93_IGN69.json b/test/data/counts/count_test_data_77050_627760_LA93_IGN69.json index 9b4dda2..fd31a04 100644 --- a/test/data/counts/count_test_data_77050_627760_LA93_IGN69.json +++ b/test/data/counts/count_test_data_77050_627760_LA93_IGN69.json @@ -5,5 +5,5 @@ "4": 933, "5": 12154, "6": 4148, - "64": 70 + "0": 70 } \ No newline at end of file diff --git a/test/test_replace_attribute_in_las.py b/test/test_replace_attribute_in_las.py index bda4afa..78b3b01 100644 --- a/test/test_replace_attribute_in_las.py +++ b/test/test_replace_attribute_in_las.py @@ -10,7 +10,7 @@ from pdaltools.standardize_format import get_writer_parameters import pytest import shutil -from test.utils import get_pdal_infos_summary +from test.utils import get_pdal_infos_summary, EXPECTED_DIMS_BY_DATAFORMAT from typing import Dict from test.test_standardize_format import assert_lasinfo_no_warning @@ -28,20 +28,22 @@ "4": 1227, "5": 30392, "6": 29447, - "64": 13, + "0": 13, } ) +colored_las_params = get_writer_parameters({"dataformat_id": 8}) -expected_counts = Counter({"2": 21172, "3": 226, "4": 1227, "5": 30392, "64": 29447, "201": 2047 + 13}) +expected_counts = Counter({"0": 13, "2": 226, "4": 1227, "5": 30392, "65": 29447, "201": 2047 + 21172}) replacement_map_fail = { "201": ["1", "64"], "6": ["64"], -} # has duplicatevalue to replace +} # has duplicate value to replace, so it should fail replacement_map_success = { - "201": ["1", "64"], - "64": ["6"], + "201": ["1", "2"], + "2": ["3"], # check that the replacement is correct when a value is both replaced and to replace + "65": ["6"], # check that values over 31 are interpreted correctly in las 1.4 output } # test replacement map parsing @@ -59,8 +61,8 @@ def setup_module(module): os.mkdir(tmp_path) -def test_replace_values(): - replace_values(input_file, output_file, replacement_map_success, attribute) +def test_replace_values_ok(): + replace_values(input_file, output_file, replacement_map_success, attribute, colored_las_params) count = compute_count_one_file(output_file, attribute) assert count == expected_counts @@ -78,11 +80,11 @@ def test_replace_values_duplicate_input(): def check_dimensions(input_file, output_file): - input_summary = get_pdal_infos_summary(input_file) - input_dimensions = set(input_summary["summary"]["dimensions"]) output_summary = get_pdal_infos_summary(output_file) - output_dimensions = set(output_summary["summary"]["dimensions"]) - assert input_dimensions == output_dimensions + output_dimensions = [s.strip() for s in output_summary["summary"]["dimensions"].split(",")] + print(sorted(output_dimensions)) + print(sorted(EXPECTED_DIMS_BY_DATAFORMAT[8])) + assert set(output_dimensions) == set(EXPECTED_DIMS_BY_DATAFORMAT[8]) def test_parse_replacement_map_from_path_or_json_string_path_ok(): diff --git a/test/test_standardize_format.py b/test/test_standardize_format.py index a5fe48f..edd84ff 100644 --- a/test/test_standardize_format.py +++ b/test/test_standardize_format.py @@ -3,7 +3,7 @@ import shutil from pdaltools.standardize_format import rewrite_with_pdal, standardize, exec_las2las import logging -from test.utils import get_pdal_infos_summary +from test.utils import get_pdal_infos_summary, EXPECTED_DIMS_BY_DATAFORMAT import pdal import subprocess as sp @@ -21,51 +21,6 @@ {"dataformat_id": 8, "a_srs": "EPSG:4326"}, ] -expected_dims = { - 6: set( - [ - "X", - "Y", - "Z", - "Intensity", - "ReturnNumber", - "NumberOfReturns", - "ClassFlags", - "ScanChannel", - "ScanDirectionFlag", - "EdgeOfFlightLine", - "Classification", - "UserData", - "ScanAngleRank", - "PointSourceId", - "GpsTime", - ] - ), - 8: set( - [ - "X", - "Y", - "Z", - "Intensity", - "ReturnNumber", - "NumberOfReturns", - "ClassFlags", - "ScanChannel", - "ScanDirectionFlag", - "EdgeOfFlightLine", - "Classification", - "UserData", - "ScanAngleRank", - "PointSourceId", - "GpsTime", - "Red", - "Green", - "Blue", - "Infrared", - ] - ), -} - def setup_module(module): try: @@ -94,7 +49,7 @@ def _test_standardize_format_one_params_set(params): assert metadata["dataformat_id"] == params["dataformat_id"] # Check that there is no extra dim dimensions = set([d.strip() for d in json_info["summary"]["dimensions"].split(",")]) - assert dimensions == expected_dims[params["dataformat_id"]] + assert dimensions == EXPECTED_DIMS_BY_DATAFORMAT[params["dataformat_id"]] # TODO: Check srs # TODO: check precision diff --git a/test/utils.py b/test/utils.py index 2a5e3ed..f6f001c 100644 --- a/test/utils.py +++ b/test/utils.py @@ -9,3 +9,55 @@ def get_pdal_infos_summary(f: str): r = sp.run(["pdal", "info", "--summary", f], stderr=sp.PIPE, stdout=sp.PIPE) json_info = json.loads(r.stdout.decode()) return json_info + + +EXPECTED_DIMS_BY_DATAFORMAT = { + 6: set( + [ + "X", + "Y", + "Z", + "Intensity", + "ReturnNumber", + "NumberOfReturns", + "ScanChannel", + "ScanDirectionFlag", + "EdgeOfFlightLine", + "Classification", + "UserData", + "ScanAngleRank", + "PointSourceId", + "GpsTime", + "KeyPoint", + "Overlap", + "Synthetic", + "Withheld", + ] + ), + 8: set( + [ + "X", + "Y", + "Z", + "Intensity", + "ReturnNumber", + "NumberOfReturns", + "ScanChannel", + "ScanDirectionFlag", + "EdgeOfFlightLine", + "Classification", + "UserData", + "ScanAngleRank", + "PointSourceId", + "GpsTime", + "Red", + "Green", + "Blue", + "Infrared", + "KeyPoint", + "Overlap", + "Synthetic", + "Withheld", + ] + ), +} From a241aa14526da06e8dd83a5b5fd47226c10bd20d Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Tue, 31 Oct 2023 10:31:11 +0100 Subject: [PATCH 3/3] Update version number to v1.4.0 --- CHANGELOG.md | 2 +- pdaltools/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5d579e..3cfebcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# dev +# 1.4.0 - count_occurences / replace_value: add copy_and_hack decorator to run on tscan output files - Update to pdal 2.6+ to better handle classification values and flags in replace_attribute_in_las (was treating values over 31 as {classification under 31 + flag} even when saving to LAS 1.4) diff --git a/pdaltools/_version.py b/pdaltools/_version.py index 00555c2..58bca5c 100644 --- a/pdaltools/_version.py +++ b/pdaltools/_version.py @@ -1,4 +1,4 @@ -__version__ = "1.3.1" +__version__ = "1.4.0" if __name__ == "__main__":