From 85b02f401753c6468ad5a2661b4fe7225eb6e6ad Mon Sep 17 00:00:00 2001 From: Kenn Cartier Date: Tue, 3 Sep 2024 22:32:08 -0700 Subject: [PATCH] Squashing commits related to understanding GH Actions --- .github/requirements.txt | 6 +- .github/workflows/dev_ci_cd.yml | 4 +- city_metrix/layers/layer.py | 1 + environment.yml | 6 +- setup.py | 2 +- tests/resources/bbox_constants.py | 12 +- tests/test_layer_parameters.py | 215 +++++++++++++++++------------- tests/tools/spatial_tools.py | 16 --- 8 files changed, 142 insertions(+), 120 deletions(-) delete mode 100644 tests/tools/spatial_tools.py diff --git a/.github/requirements.txt b/.github/requirements.txt index 9928a46..15d8543 100644 --- a/.github/requirements.txt +++ b/.github/requirements.txt @@ -1,12 +1,12 @@ earthengine-api==0.1.408 geocube==0.4.2 -geopandas==0.14.1 +geopandas==0.14.4 rioxarray==0.15.0 odc-stac==0.3.8 pystac-client==0.7.5 pytest==7.4.3 xarray-spatial==0.3.7 -xee==0.0.3 +xee==0.0.15 utm==0.7.0 osmnx==1.9.3 dask[complete]==2023.11.0 @@ -17,4 +17,4 @@ pip==23.3.1 boto3==1.34.124 scikit-learn==1.5.0 overturemaps==0.6.0 -git+https://github.com/isciences/exactextract \ No newline at end of file +exactextract==0.2.0.dev252 diff --git a/.github/workflows/dev_ci_cd.yml b/.github/workflows/dev_ci_cd.yml index 6d55444..c57fd39 100644 --- a/.github/workflows/dev_ci_cd.yml +++ b/.github/workflows/dev_ci_cd.yml @@ -14,9 +14,9 @@ jobs: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Linux dependencies diff --git a/city_metrix/layers/layer.py b/city_metrix/layers/layer.py index 01ad6e4..86590c8 100644 --- a/city_metrix/layers/layer.py +++ b/city_metrix/layers/layer.py @@ -277,6 +277,7 @@ def get_image_collection( crs = get_utm_zone_epsg(bbox) + # See link regarding bug in crs specification shttps://github.com/google/Xee/issues/118 ds = xr.open_dataset( image_collection, engine='ee', diff --git a/environment.yml b/environment.yml index 24ec040..2434184 100644 --- a/environment.yml +++ b/environment.yml @@ -5,13 +5,13 @@ dependencies: - python=3.10 - earthengine-api=0.1.379 - geocube=0.4.2 - - geopandas=0.14.1 + - geopandas=0.14.4 - rioxarray=0.15.0 - odc-stac=0.3.8 - pystac-client=0.7.5 - pytest=7.4.3 - xarray-spatial=0.3.7 - - xee=0.0.3 + - xee=0.0.15 - utm=0.7.0 - osmnx=1.9.0 - dask[complete]=2023.11.0 @@ -22,6 +22,6 @@ dependencies: - pip=23.3.1 - boto3=1.34.124 - scikit-learn=1.5.0 + - exactextract=0.2.0.dev252 - pip: - - git+https://github.com/isciences/exactextract - overturemaps==0.6.0 diff --git a/setup.py b/setup.py index 3124621..179b595 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "s3fs", "dask>=2023.11.0", "boto3", - "exactextract", + "exactextract<=0.2.0.dev252", "overturemaps", "scikit-learn>=1.5.0", ], diff --git a/tests/resources/bbox_constants.py b/tests/resources/bbox_constants.py index 9aaf8ad..789ab48 100644 --- a/tests/resources/bbox_constants.py +++ b/tests/resources/bbox_constants.py @@ -9,10 +9,14 @@ ) BBOX_BRA_SALVADOR_ADM4 = ( - -38.647320153390055, - -13.01748678217598787, - -38.3041637148564007, - -12.75607703449720631 + -38.647320153390055,-13.01748678217598787, + -38.3041637148564007,-12.75607703449720631 +) + +# UTM Zones 22S and 23S +BBOX_BRA_BRASILIA = ( + -48.07651,-15.89788 + -47.83736,-15.71919 ) BBOX_SMALL_TEST = ( diff --git a/tests/test_layer_parameters.py b/tests/test_layer_parameters.py index 043d917..2238bad 100644 --- a/tests/test_layer_parameters.py +++ b/tests/test_layer_parameters.py @@ -1,3 +1,4 @@ +import pytest from pyproj import CRS from city_metrix.layers import ( Layer, @@ -5,8 +6,7 @@ AlosDSM, AverageNetBuildingHeight, BuiltUpHeight, - EsaWorldCover, - EsaWorldCoverClass, + EsaWorldCover, EsaWorldCoverClass, LandSurfaceTemperature, NasaDEM, NaturalAreas, @@ -14,123 +14,156 @@ TreeCanopyHeight, TreeCover, UrbanLandUse, - WorldPop + WorldPop, OpenStreetMap ) from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 -from tests.tools.spatial_tools import get_distance_between_geocoordinates """ -Note: To add a test for another scalable layer that has the spatial_resolution property: +Evaluation of spatial_resolution property +To add a test for a scalable layer that has the spatial_resolution property: 1. Add the class name to the city_metrix.layers import statement above -2. Specify a minimal class instance in the set below. Do no specify the spatial_resolution - property in the instance definition. +2. Copy an existing test_*_spatial_resolution() test + a. rename for the new layer + b. specify a minimal class instance for the layer, not specifying the spatial_resolution attribute. """ -CLASSES_WITH_spatial_resolution_PROPERTY = \ - { - # 'Albedo()', - # 'AlosDSM()', - # 'AverageNetBuildingHeight()', - # 'BuiltUpHeight()', - 'EsaWorldCover(land_cover_class=EsaWorldCoverClass.BUILT_UP)', - # 'LandSurfaceTemperature()', - # 'NasaDEM()', - # 'NaturalAreas()', - # 'NdviSentinel2(year=2023)', - # 'TreeCanopyHeight()', - # 'TreeCover()', - # 'UrbanLandUse()', - # 'WorldPop()' - } COUNTRY_CODE_FOR_BBOX = 'BRA' BBOX = BBOX_BRA_LAURO_DE_FREITAS_1 - -def test_spatial_resolution_for_all_scalable_layers(): - for class_instance_str in CLASSES_WITH_spatial_resolution_PROPERTY: - is_valid, except_str = validate_layer_instance(class_instance_str) - if is_valid is False: - raise Exception(except_str) - - class_instance = eval(class_instance_str) - - # Double the spatial_resolution for the specified Class - doubled_default_resolution = 2 * class_instance.spatial_resolution - class_instance.spatial_resolution=doubled_default_resolution - - evaluate_layer(class_instance, doubled_default_resolution) +RESOLUTION_TOLERANCE = 1 + +def test_albedo_spatial_resolution(): + class_instance = Albedo() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_alos_dsm_spatial_resolution(): + class_instance = AlosDSM() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_average_net_building_height_spatial_resolution(): + class_instance = AverageNetBuildingHeight() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_built_up_height_spatial_resolution(): + class_instance = BuiltUpHeight() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_esa_world_cover_spatial_resolution(): + class_instance = EsaWorldCover(land_cover_class=EsaWorldCoverClass.BUILT_UP) + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_land_surface_temperature_spatial_resolution(): + class_instance = LandSurfaceTemperature() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_nasa_dem_spatial_resolution(): + class_instance = NasaDEM() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_natural_areas_spatial_resolution(): + class_instance = NaturalAreas() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_ndvi_sentinel2_spatial_resolution(): + class_instance = NdviSentinel2(year=2023) + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_tree_canopy_height_spatial_resolution(): + class_instance = TreeCanopyHeight() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_tree_cover_spatial_resolution(): + class_instance = TreeCover() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_urban_land_use_spatial_resolution(): + class_instance = UrbanLandUse() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def test_world_pop_spatial_resolution(): + class_instance = WorldPop() + doubled_default_resolution, actual_estimated_resolution = evaluate_resolution__property(class_instance) + assert pytest.approx(doubled_default_resolution, rel=RESOLUTION_TOLERANCE) == actual_estimated_resolution + +def evaluate_resolution__property(obj): + is_valid, except_str = validate_layer_instance(obj) + if is_valid is False: + raise Exception(except_str) + + # Double the default scale for testing + cls = get_class_from_instance(obj) + doubled_default_resolution = 2 * cls.spatial_resolution + obj.spatial_resolution=doubled_default_resolution + + data = obj.get_data(BBOX) + + expected_resolution = doubled_default_resolution + estimated_actual_resolution = estimate_spatial_resolution(data) + + return expected_resolution, estimated_actual_resolution def test_function_validate_layer_instance(): - is_valid, except_str = validate_layer_instance(Albedo()) - assert is_valid is False is_valid, except_str = validate_layer_instance('t') assert is_valid is False - is_valid, except_str = validate_layer_instance('Layer()') + is_valid, except_str = validate_layer_instance(EsaWorldCoverClass.BUILT_UP) assert is_valid is False - is_valid, except_str = validate_layer_instance('OpenStreetMap()') + is_valid, except_str = validate_layer_instance(OpenStreetMap()) assert is_valid is False - is_valid, except_str = validate_layer_instance('Albedo(spatial_resolution = 2)') + is_valid, except_str = validate_layer_instance(Albedo(spatial_resolution = 2)) assert is_valid is False -def validate_layer_instance(obj_string): +def validate_layer_instance(obj): is_valid = True except_str = None - obj_eval = None - - if not type(obj_string) == str: - is_valid = False - except_str = "Specified object '%s' must be specified as a string." % obj_string - return is_valid, except_str - try: - obj_eval = eval(obj_string) - except: + if not obj.__class__.__bases__[0] == Layer: is_valid = False - except_str = "Specified object '%s' is not a class instance." % obj_string - return is_valid, except_str - - if not type(obj_eval).__bases__[0] == Layer: - is_valid = False - except_str = "Specified object '%s' is not a valid Layer class instance." % obj_string - elif not hasattr(obj_eval, 'spatial_resolution'): - is_valid = False - except_str = "Specified class '%s' does not have the spatial_resolution property." % obj_string - elif not obj_string.find('spatial_resolution') == -1: - is_valid = False - except_str = "Do not specify spatial_resolution property value in object '%s'." % obj_string - elif obj_eval.spatial_resolution is None: - is_valid = False - except_str = "Class signature cannot specify None for default value for class." + except_str = "Specified object '%s' is not a valid Layer class instance." % obj + else: + cls = get_class_from_instance(obj) + cls_name = type(cls).__name__ + if not hasattr(obj, 'spatial_resolution'): + is_valid = False + except_str = "Class '%s' does not have spatial_resolution property." % cls_name + elif not obj.spatial_resolution == cls.spatial_resolution: + is_valid = False + except_str = "Do not specify spatial_resolution property value for class '%s'." % cls_name + elif cls.spatial_resolution is None: + is_valid = False + except_str = "Signature of class %s must specify a non-null default value for spatial_resolution. Please correct." % cls_name return is_valid, except_str -def evaluate_layer(layer, expected_resolution): - data = layer.get_data(BBOX) - actual_estimated_resolution = get_spatial_resolution_estimate(data) - assert expected_resolution == actual_estimated_resolution +def get_class_from_instance(obj): + cls = obj.__class__() + return cls -def get_spatial_resolution_estimate(data): +def estimate_spatial_resolution(data): y_cells = float(data['y'].size - 1) + y_min = data.coords['y'].values.min() + y_max = data.coords['y'].values.max() + + crs_string = data.rio.crs.data['init'] + crs = CRS.from_string(crs_string) + crs_unit = crs.axis_info[0].unit_name - y_min = data['y'].values.min() - y_max = data['y'].values.max() - - crs = CRS.from_string(data.crs) - crs_units = crs.axis_info[0].unit_name - if crs_units == 'metre': - y_diff = y_max - y_min - elif crs_units == 'foot': - feet_to_meter = 0.3048 - y_diff = (y_max - y_min) * feet_to_meter - elif crs_units == 'degree': - lat1 = y_min - lat2 = y_max - lon1 = data['x'].values.min() - lon2 = lon1 - y_diff = get_distance_between_geocoordinates(lat1, lon1, lat2, lon2) + if crs_unit == 'metre': + diff_distance = y_max - y_min else: - raise Exception('Unhandled projection units: %s' % crs_units) + raise Exception('Unhandled projection units: %s for projection: %s' % (crs_unit, crs_string)) - ry = round(y_diff / y_cells) + estimated_actual_resolution = round(diff_distance / y_cells) - return ry + return estimated_actual_resolution diff --git a/tests/tools/spatial_tools.py b/tests/tools/spatial_tools.py deleted file mode 100644 index 73ab093..0000000 --- a/tests/tools/spatial_tools.py +++ /dev/null @@ -1,16 +0,0 @@ -import math - -EARTH_RADIUS = 6378.137 # Radius of earth in KM - -def get_distance_between_geocoordinates(lat1, lon1, lat2, lon2): - dLat = lat2 * math.pi / 180 - lat1 * math.pi / 180 - dLon = lon2 * math.pi / 180 - lon1 * math.pi / 180 - a = math.sin(dLat/2) * math.sin(dLat/2) + \ - math.cos(lat1 * math.pi / 180) * \ - math.cos(lat2 * math.pi / 180) * \ - math.sin(dLon/2) * math.sin(dLon/2) - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) - d = EARTH_RADIUS * c - return d * 1000 # meters - -