Merge pull request #37 from IGNF/dev

Merge version 1.5.0

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 119

.github/workflows/cicd_light.yml

@@ -30,5 +30,5 @@ jobs:
 
       - name: Run tests with pytest
         shell: micromamba-shell {0}
-        run: python -m pytest ./test -s --log-cli-level DEBUG -m "not geoportail"
+        run: python -m pytest ./test -s --log-cli-level DEBUG -m "not geopf"
 

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/ambv/black
+    rev: 23.12.0
+    hooks:
+      - id: black
+        language_version: python3.11
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: ["--profile", "black"]

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.5.0
+- switch colorisation source from Geoportail to Geoplateforme
+- use absolute value comparison in tests
+- linting / add pre-commits
+- upgrade python version to 3.11
+- replace `pdal info --metadata` subprocess by a python command (used in the `color` module in particular)
+
 # 1.4.1
 - fix copy_and_hack_decorator (was not returning the decorated function output)
 

Makefile

@@ -6,6 +6,32 @@
 .ONESHELL:
 SHELL = /bin/bash
 
+
+##############################
+# Install
+##############################
+install:
+	mamba env update -n pdaltools -f environment.yml
+
+
+##############################
+# Dev/Contrib tools
+##############################
+
+testing:
+	python -m pytest ./test -s --log-cli-level DEBUG -m "not geopf"
+
+testing_full:
+	python -m pytest ./test -s --log-cli-level DEBUG
+
+install-precommit:
+	pre-commit install
+
+
+##############################
+# Build/deploy pip lib
+##############################
+
 deploy: check
 	twine upload dist/*
 
@@ -18,29 +44,13 @@ dist/ign-pdal-tool*.tar.gz:
 build: clean
 	python -m build
 
-install:
-	pip install -e .
-
-testing:
-	python -m pytest ./test -s --log-cli-level DEBUG -m "not geoportail"
-
-testing_full:
-	python -m pytest ./test -s --log-cli-level DEBUG
-
 clean:
 	rm -rf tmp
 	rm -rf ign_pdal_tools.egg-info
 	rm -rf dist
 
-mamba-env-create:
-	mamba env create -n pdaltools -f environment.yml
-
-mamba-env-update:
-	mamba env update -n pdaltools -f environment.yml
-
-
 ##############################
-# Docker
+# Build/deploy Docker image
 ##############################
 
 PROJECT_NAME=ignimagelidar/ign-pdal-tools

README.md

@@ -12,7 +12,7 @@ on the fonctionalities that our team needs.
 # Content
 
 This library contains pdal-based tools to:
-* **colorize** a point cloud using images from [Geoportail](https://www.geoportail.gouv.fr/) (a portal
+* **colorize** a point cloud using images from [Geoplateforme / cartes.gouv.fr](https://www.cartes.gouv.fr/) (a portal
 from French government providing access to aerial imagery)
 * **stitch** together LAS files using their location
 * **standardize** LAS files
@@ -21,13 +21,14 @@ from French government providing access to aerial imagery)
 # Installation / Usage
 
 This library can be used in different ways:
+* directly from sources: `make install` creates a mamba environment with the required dependencies
 * installed with `pip` from pypi: ` pip install ign-pdal-tools`
 * used in a docker container: see documentation [Dockerfile](Dockerfile)
 
 # More details on the contained tools
 ## Colorization
 
-* [color.py](pdaltools/color.py): Colorize a point cloud from Geoportail data
+* [color.py](pdaltools/color.py): Colorize a point cloud from Geoplateforme data
 
 ## Stitching
 
@@ -62,18 +63,20 @@ Every time the code is changed, think of updating the version file: [pdaltools/_
 
 Please log your changes in [CHANGELOG.md](CHANGELOG.md)
 
+Before committing your changes, run the precommit hooks. They can be installed to run automatically with `make install-precommit`
+
 ## Tests
 
-Create the conda environment: `./script/createCondaEnv.sh`
+Create the conda environment: `make install`
 
-Run unit tests: `./script/test.sh`
+Run unit tests: `make testing`
 
 ## Pip package
 
 To generate a pip package and deploy it on pypi, use the [Makefile](Makefile) at the root of the repo:
 
 * `make build`: build the library
-* `make install`: instal the library in an editable way (`pip -e`)
+* `make install`: install the library in an editable way (`pip -e`)
 * `make deploy` : deploy it on pypi
 
 ## Docker image

environment.yml

@@ -2,7 +2,7 @@ name: pdaltools
 channels:
   - conda-forge
 dependencies:
-  - python=3.10.*
+  - python=3.11.*
   - conda-forge:pdal>=2.6.*
   - conda-forge:python-pdal==3.2.*
   - requests

pdaltools/_version.py

@@ -1,4 +1,4 @@
-__version__ = "1.4.1"
+__version__ = "1.5.0"
 
 
 if __name__ == "__main__":

pdaltools/color.py

@@ -1,13 +1,12 @@
-import json
-from math import ceil
-import subprocess as sp
+import argparse
 import tempfile
+import time
+from math import ceil
+
 import pdal
 import requests
-from osgeo.osr import SpatialReference
-import time
-import argparse
 
+import pdaltools.las_info as las_info
 from pdaltools.unlock_file import copy_and_hack_decorator
 
 
@@ -59,15 +58,15 @@ def newfn(*args, **kwargs):
     return decorator
 
 
-def download_image_from_geoportail(proj, layer, minx, miny, maxx, maxy, pixel_per_meter, outfile, timeout):
+def download_image_from_geoplateforme(proj, layer, minx, miny, maxx, maxy, pixel_per_meter, outfile, timeout):
     # Give single-point clouds a width/height of at least one pixel to have valid BBOX and SIZE
     if minx == maxx:
         maxx = minx + 1 / pixel_per_meter
     if miny == maxy:
         maxy = miny + 1 / pixel_per_meter
 
     # for layer in layers:
-    URL_GPP = "https://wxs.ign.fr/ortho/geoportail/r/wms?"
+    URL_GPP = "https://data.geopf.fr/wms-r/wms?"
     URL_FORMAT = "&EXCEPTIONS=text/xml&FORMAT=image/geotiff&SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&STYLES="
     URL_EPSG = "&CRS=EPSG:" + str(proj)
     URL_BBOX = "&BBOX=" + str(minx) + "," + str(miny) + "," + str(maxx) + "," + str(maxy)
@@ -90,35 +89,6 @@ def download_image_from_geoportail(proj, layer, minx, miny, maxx, maxy, pixel_pe
     open(outfile, "wb").write(req.content)
 
 
-def proj_from_metadata(metadata):
-    spatial_wkt = metadata["comp_spatialreference"]
-    osr_crs = SpatialReference()
-    osr_crs.ImportFromWkt(spatial_wkt)
-    authority = osr_crs.GetAttrValue("AUTHORITY", 0)
-    if authority == "EPSG":
-        proj = osr_crs.GetAttrValue("AUTHORITY", 1)
-    else:
-        proj = "2154"  # par defaut
-    return proj
-
-
-def pdal_info_json(input_file: str):
-    r = sp.run(["pdal", "info", "--metadata", input_file], stderr=sp.PIPE, stdout=sp.PIPE)
-    if r.returncode == 1:
-        msg = r.stderr.decode()
-        print(msg)
-        raise RuntimeError(msg)
-
-    output = r.stdout.decode()
-    json_info = {}
-    try:
-        json_info = json.loads(output)
-    except:
-        print(r.stderr.decode())
-        raise
-    return json_info
-
-
 @copy_and_hack_decorator
 def color(
     input_file: str,
@@ -130,19 +100,18 @@ def color(
     color_ir_enabled=True,
     veget_index_file="",
 ):
-    json_info = pdal_info_json(input_file)
-    metadata = json_info["metadata"]
-    minx, maxx, miny, maxy = metadata["minx"], metadata["maxx"], metadata["miny"], metadata["maxy"]
+    metadata = las_info.las_info_metadata(input_file)
+    minx, maxx, miny, maxy = las_info.get_bounds_from_header_info(metadata)
 
     if proj == "":
-        proj = proj_from_metadata(metadata)
+        proj = las_info.get_epsg_from_header_info(metadata)
 
     pipeline = pdal.Reader.las(filename=input_file)
 
     writer_extra_dims = "all"
 
     # apply decorator to retry 3 times, and wait 30 seconds each times
-    download_image_from_geoportail_retrying = retry(7, 15, 2)(download_image_from_geoportail)
+    download_image_from_geoplateforme_retrying = retry(7, 15, 2)(download_image_from_geoplateforme)
 
     if veget_index_file and veget_index_file != "":
         print(f"Remplissage du champ Deviation à partir du fichier {veget_index_file}")
@@ -152,7 +121,7 @@ def color(
     tmp_ortho = None
     if color_rvb_enabled:
         tmp_ortho = tempfile.NamedTemporaryFile()
-        download_image_from_geoportail_retrying(
+        download_image_from_geoplateforme_retrying(
             proj, "ORTHOIMAGERY.ORTHOPHOTOS", minx, miny, maxx, maxy, pixel_per_meter, tmp_ortho.name, timeout_second
         )
         pipeline |= pdal.Filter.colorization(
@@ -162,7 +131,7 @@ def color(
     tmp_ortho_irc = None
     if color_ir_enabled:
         tmp_ortho_irc = tempfile.NamedTemporaryFile()
-        download_image_from_geoportail_retrying(
+        download_image_from_geoplateforme_retrying(
             proj,
             "ORTHOIMAGERY.ORTHOPHOTOS.IRC",
             minx,

pdaltools/count_occurences/count_occurences_for_attribute.py

@@ -2,13 +2,14 @@
 Eg. to count points of each class in classified point clouds """
 
 import argparse
-from collections import Counter
 import json
 import logging
 import os
+from collections import Counter
+from typing import List
+
 import pdal
 from tqdm import tqdm
-from typing import List
 
 from pdaltools.unlock_file import copy_and_hack_decorator
 
@@ -39,7 +40,7 @@ def compute_count_one_file(filepath: str, attribute: str = "Classification") ->
         # Try to prettify the value by converting it to an integer (eg. for Classification that
         # returns values such as 1.0000 instead of 1 or 1.)
         counts = Counter({str(int(float(value))): int(count) for value, count in split_counts})
-    except ValueError as e:
+    except ValueError:
         # in case value is not a number, float(value) returns a ValueError
         # fallback: use the raw value
         counts = Counter({value: int(count) for value, count in split_counts})

pdaltools/count_occurences/merge_occurences_counts.py

@@ -3,13 +3,14 @@
 on several files"""
 
 import argparse
-from collections import Counter
 import json
 import logging
 import os
-from tqdm import tqdm
+from collections import Counter
 from typing import List
 
+from tqdm import tqdm
+
 
 def parse_args():
     parser = argparse.ArgumentParser("Count points with each value of an attribute.")

pdaltools/las_add_buffer.py

@@ -1,12 +1,13 @@
 import argparse
-from pdaltools.las_merge import create_list
-from pdaltools.las_info import get_buffered_bounds_from_filename
-
 import logging
 import os
-import pdal
 from typing import List
 
+import pdal
+
+from pdaltools.las_info import get_buffered_bounds_from_filename
+from pdaltools.las_merge import create_list
+
 
 def create_las_with_buffer(
     input_dir: str,

pdaltools/las_clip.py

@@ -2,9 +2,10 @@
 # maintener : MDupays
 # version : v.1 06/12/2022
 # Extract info from the tile
+import json
 import logging
+
 import pdal
-import json
 
 
 def las_crop(input_file: str, output_file: str, bounds, spatial_ref: str = "EPSG:2154"):