Skip to content

Commit

Permalink
fix: deal with ImageCollection as with Folder (#29)
Browse files Browse the repository at this point in the history
  • Loading branch information
12rambau authored Jun 30, 2024
2 parents d056437 + 9b582a8 commit f56b5d9
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 11 deletions.
14 changes: 11 additions & 3 deletions docs/usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ By default the test folder tree is empty and will be deleted at the end of the t
You can decide to populate it with some assets that will be used in your tests.

To do so customize the ``gee_folder_structure`` fixture in your ``conftest.py`` file.
This fixture is a ``dict`` that will be used to create the folder tree in GEE. As shown in the following example you can add subfolder and assets to this tree.
This fixture is a ``dict`` that will be used to create the folder tree in GEE.
First you can create containers assets (namely folders or image collections) to store your assets. These container are simply marked as keys in the dict and specify their types after a "::" symbol as shown in the following example.
assets need to be ``ee.Image`` or ``ee.FeatureCollection`` objects and remain small as the creation operation is taken care of by the plugin.
Specifically for ``ee.Image`` objects, please use the ``clipToBoundsAndScale`` method to make sure the asset has a geometry and a scale.

Expand All @@ -184,9 +185,13 @@ Specifically for ``ee.Image`` objects, please use the ``clipToBoundsAndScale`` m
"""Override the default test folder structure."""
point = ee.Geometry.Point([0, 0])
return {
"folder": {
"folder::Folder": {
"image": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
"fc": ee.FeatureCollection(point),
},
"image_collection::ImageCollection": {
"image1": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
"image2": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
}
}
Expand All @@ -197,7 +202,10 @@ Which will render in your GEE account as:
8d98a5be574041a6a54d6def9d915c67/
└── folder/
├── fc (FeatureCollection)
└── image (ImageCollection)
└── image (Image)
└── image_collection/ (ImageCollection)
├── image1 (Image)
└── image2 (Image)
Customize the root folder
^^^^^^^^^^^^^^^^^^^^^^^^^
Expand Down
36 changes: 33 additions & 3 deletions pytest_gee/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from pathlib import Path, PurePosixPath
from typing import List, Optional, Union
from warnings import warn

import ee
from deprecated.sphinx import deprecated
Expand Down Expand Up @@ -64,7 +65,7 @@ def get_assets(folder: Union[str, Path]) -> List[dict]:
def _recursive_get(folder, asset_list):
for asset in ee.data.listAssets({"parent": folder})["assets"]:
asset_list.append(asset)
if asset["type"] == "FOLDER":
if asset["type"] in ["FOLDER", "IMAGE_COLLECTION"]:
asset_list = _recursive_get(asset["name"], asset_list)
return asset_list

Expand Down Expand Up @@ -110,6 +111,34 @@ def export_asset(
return PurePosixPath(asset_id)


def _create_container(asset_request: str) -> str:
"""Create a container for the asset request depending on the requested type.
Args:
asset_request: the asset request specifying the type of asset to create. Convention is <asset_id>::<asset_type>
Returns:
the asset_id of the container
"""
# deprecation management for older version of the lib
parts = asset_request.split("::")
if len(parts) == 1:
parts.append("FOLDER")
warn(f"Asset {asset_request} is not specifying asset Type, it will be created as a FOLDER.")

# extract the asset_id and the asset_type from the different parts
# if more than 2 splits are identified they will be ignored
asset_id, asset_type = parts[:2]

# create the container
if asset_type in ["Folder", "ImageCollection"]:
ee.data.createAsset({"type": asset_type}, asset_id)
else:
raise ValueError(f"Asset type {asset_type} is not supported.")

return asset_id


def init_tree(structure: dict, prefix: str, root: Union[str, PurePosixPath]) -> PurePosixPath:
"""Create an EarthEngine folder tree from a dictionary.
Expand Down Expand Up @@ -140,7 +169,7 @@ def _recursive_create(structure, prefix, folder):
asset_id = PurePosixPath(folder) / name
description = f"{prefix}_{name}"
if isinstance(content, dict):
ee.data.createFolder(str(asset_id))
asset_id = _create_container(str(asset_id))
_recursive_create(content, prefix, asset_id)
else:
export_asset(content, asset_id, description)
Expand Down Expand Up @@ -189,7 +218,7 @@ def delete(id: str):
# identify the type of asset
asset_info = ee.data.getAsset(asset_id)

if asset_info["type"] == "FOLDER":
if asset_info["type"] in ["FOLDER", "IMAGE_COLLECTION"]:

# get all the assets
asset_list = get_assets(folder=asset_id)
Expand All @@ -204,6 +233,7 @@ def delete(id: str):

# delete all items starting from the more nested ones
assets_ordered = dict(sorted(assets_ordered.items(), reverse=True))
print(assets_ordered)
for lvl in assets_ordered:
for i in assets_ordered[lvl]:
delete(i["name"])
Expand Down
8 changes: 6 additions & 2 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@ def gee_folder_structure():
"""Override the default test folder structure."""
point = ee.Geometry.Point([0, 0])
return {
"folder": {
"folder::Folder": {
"image": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
"fc": ee.FeatureCollection(point),
}
},
"ic::ImageCollection": {
"image1": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
"image2": ee.Image(1).clipToBoundsAndScale(point.buffer(100), scale=30),
},
}
9 changes: 6 additions & 3 deletions tests/test_pytest_gee.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,12 @@ def test_gee_init():
def test_structure(gee_folder_structure):
"""Test the structure fixture."""
assert isinstance(gee_folder_structure, dict)
assert "folder" in gee_folder_structure
assert "image" in gee_folder_structure["folder"]
assert "fc" in gee_folder_structure["folder"]
assert "folder::Folder" in gee_folder_structure
assert "image" in gee_folder_structure["folder::Folder"]
assert "fc" in gee_folder_structure["folder::Folder"]
assert "ic::ImageCollection" in gee_folder_structure
assert "image1" in gee_folder_structure["ic::ImageCollection"]
assert "image2" in gee_folder_structure["ic::ImageCollection"]


def test_init_tree(gee_folder_root, gee_test_folder):
Expand Down

0 comments on commit f56b5d9

Please sign in to comment.