diff --git a/.github/requirements.txt b/.github/requirements.txt index 9928a46..6b06f18 100644 --- a/.github/requirements.txt +++ b/.github/requirements.txt @@ -17,4 +17,4 @@ pip==23.3.1 boto3==1.34.124 scikit-learn==1.5.0 overturemaps==0.6.0 -git+https://github.com/isciences/exactextract \ No newline at end of file +exactextract==0.2.0.dev252 diff --git a/.github/workflows/dev_ci_cd.yml b/.github/workflows/dev_ci_cd.yml index 6d55444..6701372 100644 --- a/.github/workflows/dev_ci_cd.yml +++ b/.github/workflows/dev_ci_cd.yml @@ -2,6 +2,7 @@ name: Dev CIF API CI/CD on: pull_request: + workflow_dispatch: permissions: contents: read @@ -14,9 +15,9 @@ jobs: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Linux dependencies diff --git a/README.md b/README.md index 6b23e83..beea254 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ The Cities Indicator Framework (CIF) is a set of Python tools to make it easier * `pip install git+https://github.com/wri/cities-cif/releases/latest` gives you the latest stable release. * `pip install git+https://github.com/wri/cities-cif` gives you the main branch with is not stable. +NOTE: If you have already installed the package and want to update to the latest code you may need to add the `--force-reinstall` flag + ## PR Review 0. Prerequisites @@ -28,7 +30,7 @@ There are 2 ways to install dependencies. Choose one... ### Conda -`conda env create -f environment.yml` +`conda env create -f environment.yml` or `conda env update -f environment.yml` ### Setuptools diff --git a/city_metrix/layers/__init__.py b/city_metrix/layers/__init__.py index 3432bef..7e5e19e 100644 --- a/city_metrix/layers/__init__.py +++ b/city_metrix/layers/__init__.py @@ -1,4 +1,5 @@ from .albedo import Albedo +from .ndvi_sentinel2_gee import NdviSentinel2 from .esa_world_cover import EsaWorldCover, EsaWorldCoverClass from .land_surface_temperature import LandSurfaceTemperature from .tree_cover import TreeCover diff --git a/city_metrix/layers/albedo.py b/city_metrix/layers/albedo.py index dd3ba8a..7bf7b11 100644 --- a/city_metrix/layers/albedo.py +++ b/city_metrix/layers/albedo.py @@ -13,7 +13,7 @@ def __init__(self, start_date="2021-01-01", end_date="2022-01-01", threshold=Non self.threshold = threshold def get_data(self, bbox): - S2 = ee.ImageCollection("COPERNICUS/S2_SR") + S2 = ee.ImageCollection("COPERNICUS/S2_SR_HARMONIZED") S2C = ee.ImageCollection("COPERNICUS/S2_CLOUD_PROBABILITY") MAX_CLOUD_PROB = 30 diff --git a/city_metrix/layers/high_land_surface_temperature.py b/city_metrix/layers/high_land_surface_temperature.py index 610651b..d3943e9 100644 --- a/city_metrix/layers/high_land_surface_temperature.py +++ b/city_metrix/layers/high_land_surface_temperature.py @@ -54,8 +54,3 @@ def addDate(image): # convert to date object return datetime.datetime.strptime(hottest_date, "%Y%m%d").date() - - def write(self, output_path): - self.data.rio.to_raster(output_path) - - diff --git a/city_metrix/layers/land_surface_temperature.py b/city_metrix/layers/land_surface_temperature.py index 0c87f30..7fff632 100644 --- a/city_metrix/layers/land_surface_temperature.py +++ b/city_metrix/layers/land_surface_temperature.py @@ -33,8 +33,3 @@ def apply_scale_factors(image): data = get_image_collection(ee.ImageCollection(l8_st), bbox, 30, "LST").ST_B10_mean return data - - def write(self, output_path): - self.data.rio.to_raster(output_path) - - diff --git a/city_metrix/layers/landsat_collection_2.py b/city_metrix/layers/landsat_collection_2.py index d82180d..248227a 100644 --- a/city_metrix/layers/landsat_collection_2.py +++ b/city_metrix/layers/landsat_collection_2.py @@ -29,8 +29,7 @@ def get_data(self, bbox): fail_on_error=False, ) + # TODO: Determine how to output xarray + qa_lst = lc2.where((lc2.qa_pixel & 24) == 0) return qa_lst.drop_vars("qa_pixel") - - - diff --git a/city_metrix/layers/layer.py b/city_metrix/layers/layer.py index 299b0a1..01ad6e4 100644 --- a/city_metrix/layers/layer.py +++ b/city_metrix/layers/layer.py @@ -18,10 +18,8 @@ import shapely.geometry as geometry import pandas as pd - MAX_TILE_SIZE = 0.5 - class Layer: def __init__(self, aggregate=None, masks=[]): self.aggregate = aggregate @@ -56,7 +54,7 @@ def groupby(self, zones, layer=None): """ return LayerGroupBy(self.aggregate, zones, layer, self.masks) - def write(self, bbox, output_path, tile_degrees=None): + def write(self, bbox, output_path, tile_degrees=None, **kwargs): """ Write the layer to a path. Does not apply masks. @@ -301,21 +299,23 @@ def get_image_collection( return data - def write_layer(path, data): if isinstance(data, xr.DataArray): - # for rasters, need to write to locally first then copy to cloud storage - if path.startswith("s3://"): - tmp_path = f"{uuid4()}.tif" - data.rio.to_raster(raster_path=tmp_path, driver="COG") - - s3 = boto3.client('s3') - s3.upload_file(tmp_path, path.split('/')[2], '/'.join(path.split('/')[3:])) - - os.remove(tmp_path) - else: - data.rio.to_raster(raster_path=path, driver="COG") + write_dataarray(path, data) elif isinstance(data, gpd.GeoDataFrame): data.to_file(path, driver="GeoJSON") else: - raise NotImplementedError("Can only write DataArray or GeoDataFrame") + raise NotImplementedError("Can only write DataArray, Dataset, or GeoDataFrame") + +def write_dataarray(path, data): + # for rasters, need to write to locally first then copy to cloud storage + if path.startswith("s3://"): + tmp_path = f"{uuid4()}.tif" + data.rio.to_raster(raster_path=tmp_path, driver="COG") + + s3 = boto3.client('s3') + s3.upload_file(tmp_path, path.split('/')[2], '/'.join(path.split('/')[3:])) + + os.remove(tmp_path) + else: + data.rio.to_raster(raster_path=path, driver="COG") diff --git a/city_metrix/layers/ndvi_sentinel2_gee.py b/city_metrix/layers/ndvi_sentinel2_gee.py new file mode 100644 index 0000000..c5b21b9 --- /dev/null +++ b/city_metrix/layers/ndvi_sentinel2_gee.py @@ -0,0 +1,46 @@ +import ee +from .layer import Layer, get_image_collection + +class NdviSentinel2(Layer): + """" + NDVI = Sentinel-2 Normalized Difference Vegetation Index + param: year: The satellite imaging year. + return: a rioxarray-format DataArray + Author of associated Jupyter notebook: Ted.Wong@wri.org + Notebook: https://github.com/wri/cities-cities4forests-indicators/blob/dev-eric/scripts/extract-VegetationCover.ipynb + Reference: https://en.wikipedia.org/wiki/Normalized_difference_vegetation_index + """ + def __init__(self, year=None, **kwargs): + super().__init__(**kwargs) + self.year = year + + def get_data(self, bbox): + if self.year is None: + raise Exception('NdviSentinel2.get_data() requires a year value') + + start_date = "%s-01-01" % self.year + end_date = "%s-12-31" % self.year + + # Compute NDVI for each image + def calculate_ndvi(image): + ndvi = (image + .normalizedDifference(['B8', 'B4']) + .rename('NDVI')) + return image.addBands(ndvi) + + s2 = ee.ImageCollection("COPERNICUS/S2_HARMONIZED") + ndvi = (s2 + .filterBounds(ee.Geometry.BBox(*bbox)) + .filterDate(start_date, end_date) + .map(calculate_ndvi) + .select('NDVI') + ) + + ndvi_mosaic = ndvi.qualityMosaic('NDVI') + + ic = ee.ImageCollection(ndvi_mosaic) + ndvi_data = get_image_collection(ic, bbox, 10, "NDVI") + + xdata = ndvi_data.to_dataarray() + + return xdata diff --git a/city_metrix/layers/open_street_map.py b/city_metrix/layers/open_street_map.py index 8a32936..3ae6c61 100644 --- a/city_metrix/layers/open_street_map.py +++ b/city_metrix/layers/open_street_map.py @@ -19,6 +19,13 @@ class OpenStreetMapClass(Enum): BUILDING = {'building': True} PARKING = {'amenity': ['parking'], 'parking': True} + ECONOMIC_OPPORTUNITY = {'landuse': ['commercial', 'industrial', 'retail', 'institutional', 'education'], + 'building': ['office', 'commercial', 'industrial', 'retail', 'supermarket'], + 'shop': True} + SCHOOLS = {'building': ['school',], + 'amenity': ['school', 'kindergarten']} + HIGHER_EDUCATION = {'amenity': ['college', 'university'], + 'building': ['college', 'university']} class OpenStreetMap(Layer): @@ -54,3 +61,10 @@ def get_data(self, bbox): osm_feature = osm_feature.reset_index()[keep_col] return osm_feature + + def write(self, output_path): + self.data['bbox'] = str(self.data.total_bounds) + self.data['osm_class'] = str(self.osm_class.value) + + # Write to a GeoJSON file + self.data.to_file(output_path, driver='GeoJSON') diff --git a/city_metrix/layers/sentinel_2_level_2.py b/city_metrix/layers/sentinel_2_level_2.py index a7ae944..a609293 100644 --- a/city_metrix/layers/sentinel_2_level_2.py +++ b/city_metrix/layers/sentinel_2_level_2.py @@ -50,4 +50,6 @@ def get_data(self, bbox): cloud_masked = s2.where(s2 != 0).where(s2.scl != 3).where(s2.scl != 8).where(s2.scl != 9).where( s2.scl != 10) + # TODO: Determine how to output as an xarray + return cloud_masked.drop_vars("scl") diff --git a/city_metrix/layers/smart_surface_lulc.py b/city_metrix/layers/smart_surface_lulc.py index d58f468..2ab2a02 100644 --- a/city_metrix/layers/smart_surface_lulc.py +++ b/city_metrix/layers/smart_surface_lulc.py @@ -66,7 +66,7 @@ def get_data(self, bbox): # cap is flat to the terminus of the road # join style is mitred so intersections are squared roads_osm['geometry'] = roads_osm.apply(lambda row: row['geometry'].buffer( - row['lanes'] * 3.048, + row['lanes'] * 3.048 / 2, cap_style=CAP_STYLE.flat, join_style=JOIN_STYLE.mitre), axis=1 diff --git a/docs/developer.md b/docs/developer.md index c6b1855..7ac9a23 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -132,7 +132,5 @@ Ensure the columns in the `GeoDataFrame` align with the [boundaries table](https You can run the tests by setting the credentials above and running the following: ``` -cd ./tests -pytest layers.py -pytest metrics.py +pytest ``` diff --git a/environment.yml b/environment.yml index 24ec040..0459608 100644 --- a/environment.yml +++ b/environment.yml @@ -22,6 +22,6 @@ dependencies: - pip=23.3.1 - boto3=1.34.124 - scikit-learn=1.5.0 + - exactextract=0.2.0.dev252 - pip: - - git+https://github.com/isciences/exactextract - overturemaps==0.6.0 diff --git a/setup.py b/setup.py index 3124621..179b595 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "s3fs", "dask>=2023.11.0", "boto3", - "exactextract", + "exactextract<=0.2.0.dev252", "overturemaps", "scikit-learn>=1.5.0", ], diff --git a/tests/fixtures/bbox_constants.py b/tests/fixtures/bbox_constants.py deleted file mode 100644 index 6814b7a..0000000 --- a/tests/fixtures/bbox_constants.py +++ /dev/null @@ -1,9 +0,0 @@ -# File defines bboxes using in the test code - - -BBOX_BRAZIL_LAURO_DE_FREITAS_1 = ( - -38.35530428121955, - -12.821710300686393, - -38.33813814352424, - -12.80363249765361, -) \ No newline at end of file diff --git a/tests/resources/__init__.py b/tests/resources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/resources/bbox_constants.py b/tests/resources/bbox_constants.py new file mode 100644 index 0000000..9aaf8ad --- /dev/null +++ b/tests/resources/bbox_constants.py @@ -0,0 +1,22 @@ +# File defines bboxes using in the test code + + +BBOX_BRA_LAURO_DE_FREITAS_1 = ( + -38.35530428121955, + -12.821710300686393, + -38.33813814352424, + -12.80363249765361, +) + +BBOX_BRA_SALVADOR_ADM4 = ( + -38.647320153390055, + -13.01748678217598787, + -38.3041637148564007, + -12.75607703449720631 +) + +BBOX_SMALL_TEST = ( + -38.43864,-12.97987, + -38.39993,-12.93239 +) + diff --git a/tests/resources/layer_dumps_for_br_lauro_de_freitas/README.md b/tests/resources/layer_dumps_for_br_lauro_de_freitas/README.md new file mode 100644 index 0000000..d4dc944 --- /dev/null +++ b/tests/resources/layer_dumps_for_br_lauro_de_freitas/README.md @@ -0,0 +1,4 @@ +# QGIS manual analysis for Lauro de Freitas, Brazil +Folder contains: +1. Test code that can be set to output the layers as geotiff files. Execution is controlled by a "master switch" +1. A QGIS file used for manually inspecting the generated geotiff files diff --git a/tests/resources/layer_dumps_for_br_lauro_de_freitas/__init__.py b/tests/resources/layer_dumps_for_br_lauro_de_freitas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/resources/layer_dumps_for_br_lauro_de_freitas/conftest.py b/tests/resources/layer_dumps_for_br_lauro_de_freitas/conftest.py new file mode 100644 index 0000000..5882053 --- /dev/null +++ b/tests/resources/layer_dumps_for_br_lauro_de_freitas/conftest.py @@ -0,0 +1,67 @@ +import tempfile +import pytest +import os +import shutil +from collections import namedtuple + +from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 +from tools.general_tools import create_target_folder, is_valid_path + +# RUN_DUMPS is the master control for whether the writes and tests are executed +# Setting RUN_DUMPS to True turns on code execution. +# Values should normally be set to False in order to avoid unnecessary execution. +RUN_DUMPS = False + +# Specify None to write to a temporary default folder otherwise specify a valid custom target path. +CUSTOM_DUMP_DIRECTORY = None + +# Both the tests and QGIS file are implemented for the same bounding box in Brazil. +COUNTRY_CODE_FOR_BBOX = 'BRA' +BBOX = BBOX_BRA_LAURO_DE_FREITAS_1 + +def pytest_configure(config): + qgis_project_file = 'layers_for_br_lauro_de_freitas.qgz' + + source_folder = os.path.dirname(__file__) + target_folder = get_target_folder_path() + create_target_folder(target_folder, True) + + source_qgis_file = os.path.join(source_folder, qgis_project_file) + target_qgis_file = os.path.join(target_folder, qgis_project_file) + shutil.copyfile(source_qgis_file, target_qgis_file) + + print("\n\033[93m QGIS project file and layer files written to folder %s.\033[0m\n" % target_folder) + +@pytest.fixture +def target_folder(): + return get_target_folder_path() + +@pytest.fixture +def bbox_info(): + bbox = namedtuple('bbox', ['bounds', 'country']) + bbox_instance = bbox(bounds=BBOX, country=COUNTRY_CODE_FOR_BBOX) + return bbox_instance + +def get_target_folder_path(): + if CUSTOM_DUMP_DIRECTORY is not None: + if is_valid_path(CUSTOM_DUMP_DIRECTORY) is False: + raise ValueError(f"The custom path '%s' is not valid. Stopping." % CUSTOM_DUMP_DIRECTORY) + else: + output_dir = CUSTOM_DUMP_DIRECTORY + else: + sub_directory_name = 'test_result_tif_files' + scratch_dir_name = tempfile.TemporaryDirectory(ignore_cleanup_errors=True).name + dir_path = os.path.dirname(scratch_dir_name) + output_dir = os.path.join(dir_path, sub_directory_name) + + return output_dir + +def prep_output_path(output_folder, file_name): + file_path = os.path.join(output_folder, file_name) + if os.path.isfile(file_path): + os.remove(file_path) + return file_path + +def verify_file_is_populated(file_path): + is_populated = True if os.path.getsize(file_path) > 0 else False + return is_populated diff --git a/tests/resources/layer_dumps_for_br_lauro_de_freitas/layers_for_br_lauro_de_freitas.qgz b/tests/resources/layer_dumps_for_br_lauro_de_freitas/layers_for_br_lauro_de_freitas.qgz new file mode 100644 index 0000000..759515e Binary files /dev/null and b/tests/resources/layer_dumps_for_br_lauro_de_freitas/layers_for_br_lauro_de_freitas.qgz differ diff --git a/tests/resources/layer_dumps_for_br_lauro_de_freitas/test_write_layers_to_qgis_files.py b/tests/resources/layer_dumps_for_br_lauro_de_freitas/test_write_layers_to_qgis_files.py new file mode 100644 index 0000000..5e0efb9 --- /dev/null +++ b/tests/resources/layer_dumps_for_br_lauro_de_freitas/test_write_layers_to_qgis_files.py @@ -0,0 +1,145 @@ +# This code is mostly intended for manual execution +# Execution configuration is specified in the conftest file +import pytest + +from city_metrix.layers import ( + Albedo, + AlosDSM, + AverageNetBuildingHeight, + EsaWorldCover, + HighLandSurfaceTemperature, + LandsatCollection2, + LandSurfaceTemperature, + NasaDEM, + NaturalAreas, + OpenBuildings, + OpenStreetMap, + OvertureBuildings, + Sentinel2Level2, + NdviSentinel2, + SmartSurfaceLULC, + TreeCanopyHeight, + TreeCover, + UrbanLandUse, + WorldPop, Layer +) +from .conftest import RUN_DUMPS, prep_output_path, verify_file_is_populated + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_albedo(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'albedo.tif') + Albedo().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_alos_dsm(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'alos_dsm.tif') + AlosDSM().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_average_net_building_height(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'average_net_building_height.tif') + AverageNetBuildingHeight().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_esa_world_cover(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'esa_world_cover.tif') + EsaWorldCover().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_high_land_surface_temperature(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'high_land_surface_temperature.tif') + HighLandSurfaceTemperature().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_land_surface_temperature(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'land_surface_temperature.tif') + LandSurfaceTemperature().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +# TODO Class is no longer used, but may be useful later +# @pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +# def test_write_landsat_collection_2(target_folder, bbox_info): +# file_path = prep_output_path(target_folder, 'landsat_collection2.tif') +# bands = ['green'] +# LandsatCollection2(bands).write(bbox_info.bounds, file_path, tile_degrees=None) +# assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_nasa_dem(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'nasa_dem.tif') + NasaDEM().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_natural_areas(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'natural_areas.tif') + NaturalAreas().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_ndvi_sentinel2_gee(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'ndvi_sentinel2_gee.tif') + NdviSentinel2(year=2023).write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_openbuildings(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'open_buildings.tif') + OpenBuildings(bbox_info.country).write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +# TODO Class write is not functional. Is class still needed or have we switched to overture? +# @pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +# def test_write_open_street_map(target_folder, bbox_info): +# file_path = prep_output_path(target_folder, 'open_street_map.tif') +# OpenStreetMap().write(bbox_info.bounds, file_path, tile_degrees=None) +# assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_overture_buildings(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'overture_buildings.tif') + OvertureBuildings().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +# TODO Class is no longer used, but may be useful later +# @pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +# def test_write_sentinel_2_level2(target_folder, bbox_info): +# file_path = prep_output_path(target_folder, 'sentinel_2_level2.tif') +# sentinel_2_bands = ["green"] +# Sentinel2Level2(sentinel_2_bands).write(bbox_info.bounds, file_path, tile_degrees=None) +# assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_smart_surface_lulc(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'smart_surface_lulc.tif') + SmartSurfaceLULC().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_tree_canopy_height(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'tree_canopy_height.tif') + TreeCanopyHeight().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_tree_cover(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'tree_cover.tif') + TreeCover().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_urban_land_use(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'urban_land_use.tif') + UrbanLandUse().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) + +@pytest.mark.skipif(RUN_DUMPS == False, reason='Skipping since RUN_DUMPS set to False') +def test_write_world_pop(target_folder, bbox_info): + file_path = prep_output_path(target_folder, 'world_pop.tif') + WorldPop().write(bbox_info.bounds, file_path, tile_degrees=None) + assert verify_file_is_populated(file_path) diff --git a/tests/test_layer_dimensions.py b/tests/test_layer_dimensions.py new file mode 100644 index 0000000..15768d6 --- /dev/null +++ b/tests/test_layer_dimensions.py @@ -0,0 +1,18 @@ +from city_metrix.layers import NdviSentinel2 +from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 +from tests.tools import post_process_layer + +COUNTRY_CODE_FOR_BBOX = 'BRA' +BBOX = BBOX_BRA_LAURO_DE_FREITAS_1 + +def test_ndvi_dimensions(): + data = NdviSentinel2(year=2023).get_data(BBOX) + data_for_map = post_process_layer(data, value_threshold=0.4, convert_to_percentage=True) + + expected_min = 0 + actual_min = data_for_map.values.min() + expected_max = 85 + actual_max = data_for_map.values.max() + + assert actual_min == expected_min + assert actual_max == expected_max diff --git a/tests/test_layers.py b/tests/test_layers.py index 882e722..e734811 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -5,6 +5,7 @@ Albedo, AlosDSM, AverageNetBuildingHeight, + NdviSentinel2, EsaWorldCover, EsaWorldCoverClass, HighLandSurfaceTemperature, @@ -25,27 +26,30 @@ WorldPop ) from city_metrix.layers.layer import get_image_collection -from tests.fixtures.bbox_constants import BBOX_BRAZIL_LAURO_DE_FREITAS_1 +from tests.resources.bbox_constants import BBOX_BRA_LAURO_DE_FREITAS_1 EE_IMAGE_DIMENSION_TOLERANCE = 1 # Tolerance compensates for variable results from GEE service +# Tests are implemented for the same bounding box in Brazil. +COUNTRY_CODE_FOR_BBOX = 'BRA' +BBOX = BBOX_BRA_LAURO_DE_FREITAS_1 def test_albedo(): - assert Albedo().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + assert Albedo().get_data(BBOX).mean() def test_alos_dsm(): - mean = AlosDSM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + mean = AlosDSM().get_data(BBOX).mean() assert mean def test_average_net_building_height(): - assert AverageNetBuildingHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + assert AverageNetBuildingHeight().get_data(BBOX).mean() def test_esa_world_cover(): count = ( EsaWorldCover(land_cover_class=EsaWorldCoverClass.BUILT_UP) - .get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + .get_data(BBOX) .count() ) assert count @@ -53,7 +57,7 @@ def test_esa_world_cover(): def test_read_image_collection(): ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 10, "test") + data = get_image_collection(ic, BBOX, 10, "test") expected_crs = 32724 expected_x_dimension = 187 @@ -68,14 +72,14 @@ def test_read_image_collection(): def test_read_image_collection_scale(): ic = ee.ImageCollection("ESA/WorldCover/v100") - data = get_image_collection(ic, BBOX_BRAZIL_LAURO_DE_FREITAS_1, 100, "test") + data = get_image_collection(ic, BBOX, 100, "test") expected_x_dimension = 19 expected_y_dimension = 20 assert data.dims == {"x": expected_x_dimension, "y": expected_y_dimension} def test_high_land_surface_temperature(): - data = HighLandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = HighLandSurfaceTemperature().get_data(BBOX) assert data.any() @@ -85,35 +89,40 @@ def test_impervious_surface(): def test_land_surface_temperature(): - mean_lst = LandSurfaceTemperature().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + mean_lst = LandSurfaceTemperature().get_data(BBOX).mean() assert mean_lst +@pytest.mark.skip(reason="layer is deprecated") def test_landsat_collection_2(): - bands = ['green'] - data = LandsatCollection2(bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + bands = ["blue"] + data = LandsatCollection2(bands).get_data(BBOX) assert data.any() def test_nasa_dem(): - mean = NasaDEM().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + mean = NasaDEM().get_data(BBOX).mean() assert mean def test_natural_areas(): - data = NaturalAreas().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = NaturalAreas().get_data(BBOX) assert data.any() +def test_ndvi_sentinel2(): + data = NdviSentinel2(year=2023).get_data(BBOX) + assert data is not None + def test_openbuildings(): - count = OpenBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() + count = OpenBuildings(COUNTRY_CODE_FOR_BBOX).get_data(BBOX).count().sum() assert count def test_open_street_map(): count = ( OpenStreetMap(osm_class=OpenStreetMapClass.ROAD) - .get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + .get_data(BBOX) .count() .sum() ) @@ -121,28 +130,27 @@ def test_open_street_map(): def test_overture_buildings(): - count = OvertureBuildings().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count().sum() + count = OvertureBuildings().get_data(BBOX).count().sum() assert count +@pytest.mark.skip(reason="layer is deprecated") def test_sentinel_2_level2(): sentinel_2_bands = ["green"] - data = Sentinel2Level2(sentinel_2_bands).get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = Sentinel2Level2(sentinel_2_bands).get_data(BBOX) assert data.any() def test_smart_surface_lulc(): - count = SmartSurfaceLULC().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() + count = SmartSurfaceLULC().get_data(BBOX).count() assert count - def test_tree_canopy_height(): - count = TreeCanopyHeight().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() + count = TreeCanopyHeight().get_data(BBOX).count() assert count - def test_tree_cover(): - actual = TreeCover().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).mean() + actual = TreeCover().get_data(BBOX).mean() expected = 54.0 tolerance = 0.1 assert ( @@ -151,9 +159,9 @@ def test_tree_cover(): def test_urban_land_use(): - assert UrbanLandUse().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1).count() + assert UrbanLandUse().get_data(BBOX).count() def test_world_pop(): - data = WorldPop().get_data(BBOX_BRAZIL_LAURO_DE_FREITAS_1) + data = WorldPop().get_data(BBOX) assert data.any() diff --git a/tests/test_methods.py b/tests/test_methods.py index 4651883..176f305 100644 --- a/tests/test_methods.py +++ b/tests/test_methods.py @@ -9,7 +9,6 @@ MockLayer, MockMaskLayer, ) -from .fixtures.bbox_constants import * def test_count(): diff --git a/tests/tools.py b/tests/tools.py new file mode 100644 index 0000000..99425df --- /dev/null +++ b/tests/tools.py @@ -0,0 +1,35 @@ +import numpy as np + +def post_process_layer(data, value_threshold=0.4, convert_to_percentage=True): + """ + Applies the standard post-processing adjustment used for rendering of NDVI including masking + to a threshold and conversion to percentage values. + :param value_threshold: (float) minimum threshold for keeping values + :param convert_to_percentage: (bool) controls whether NDVI values are converted to a percentage + :return: A rioxarray-format DataArray + """ + # Remove values less than the specified threshold + if value_threshold is not None: + data = data.where(data >= value_threshold) + + # Convert to percentage in byte data_type + if convert_to_percentage is True: + data = convert_ratio_to_percentage(data) + + return data + +def convert_ratio_to_percentage(data): + """ + Converts xarray variable from a ratio to a percentage + :param data: (xarray) xarray to be converted + :return: A rioxarray-format DataArray + """ + + # convert to percentage and to bytes for efficient storage + values_as_percent = np.round(data * 100).astype(np.uint8) + + # reset CRS + source_crs = data.rio.crs + values_as_percent.rio.write_crs(source_crs, inplace=True) + + return values_as_percent diff --git a/tools/general_tools.py b/tools/general_tools.py new file mode 100644 index 0000000..b38d1b5 --- /dev/null +++ b/tools/general_tools.py @@ -0,0 +1,23 @@ +import os +import tempfile + + +def is_valid_path(path: str): + return os.path.exists(path) + +def create_target_folder(folder_path, delete_existing_files: bool): + if os.path.isdir(folder_path) is False: + os.makedirs(folder_path) + elif delete_existing_files is True: + remove_all_files_in_directory(folder_path) + +def remove_all_files_in_directory(directory): + # Iterate over all the files in the directory + for filename in os.listdir(directory): + file_path = os.path.join(directory, filename) + try: + # Check if it is a file and remove it + if os.path.isfile(file_path): + os.remove(file_path) + except Exception as e: + print(f"Error: {e}") \ No newline at end of file