From 4150ad82148270d75c2e2c006b636a082999dfbe Mon Sep 17 00:00:00 2001 From: Yujing Wu <49011074+adawyj97@users.noreply.github.com> Date: Thu, 27 May 2021 15:14:10 -0400 Subject: [PATCH 1/4] initial commit of update to script --- .../contents/src/__init__.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/raster_interaction_to_api/contents/src/__init__.py b/raster_interaction_to_api/contents/src/__init__.py index 6255e73..cc61207 100644 --- a/raster_interaction_to_api/contents/src/__init__.py +++ b/raster_interaction_to_api/contents/src/__init__.py @@ -13,6 +13,11 @@ r = req.get(os.getenv('RASTER_INTERACTION_SHEET')).content current_mdata = pd.read_csv(pd.compat.StringIO(r.decode('utf-8')), header=0, index_col=[0]).dropna(subset=["RW Dataset ID", "RW Layer ID", "GEE Asset ID", "Band Name", "Property", "Number Type", "Number of Decimals", "NRT", "Update?"]) +# update +current_mdata = current_mdata[["Layer Link", "Property", "Prefix", "Suffix", "Number Type", "Number of Decimals", "NRT", "Update?", "Needs Unit Change", "Notes"]] +current_mdata['RW Dataset ID'] = [x[-36:] for x in current_mdata["Layer Link"]] +current_mdata['RW Layer ID'] = [x[44: 80] for x in current_mdata["Layer Link"]] + # Continue with the metadata that matches elements in the tracking sheet ids_on_backoffice = pd.notnull(current_mdata["RW Dataset ID"]) metadata_to_api = current_mdata.loc[ids_on_backoffice] @@ -108,6 +113,22 @@ def create_headers(): 'authorization': "{}".format(os.getenv('apiToken')), } +def get_band_name(lyr_url): + r = req.get(lyr_url).json() + sld = r['data']['attributes']['layerConfig']['body']['sldValue'] + if 'SourceChannelName' in sld: + substring = sld.split('SourceChannelName')[1] + end_index = substring.index('<') + band_name = substring[1: end_index] + else: + band_name = 'b1' + + return band_name + +def get_asset_id(lyr_url): + r = req.get(lyr_url).json() + asset_id = r['data']['attributes']['layerConfig']['assetId'] + return asset_id def patch_interactions(info, send=True): lyr = info[0] @@ -122,8 +143,8 @@ def patch_interactions(info, send=True): logging.info("Processing lyr: {}".format(lyr)) if (clean_nulls(metadata["NRT"]) == "Y"): row_payload = get_NRT_int(ds=ds, - band=clean_nulls(metadata["Band Name"]), - asset=clean_nulls(metadata["GEE Asset ID"]), + band=get_band_name(rw_api_url), + asset=get_asset_id(rw_api_url), prop=clean_nulls(metadata["Property"]), num_type=clean_nulls(metadata["Number Type"]), num_decimals=clean_nulls(metadata["Number of Decimals"]), @@ -132,8 +153,8 @@ def patch_interactions(info, send=True): elif (clean_nulls(metadata["NRT"]) == "N"): row_payload = get_regular_int(ds=ds, - band=clean_nulls(metadata["Band Name"]), - asset=clean_nulls(metadata["GEE Asset ID"]), + band=get_band_name(rw_api_url), + asset=get_asset_id(rw_api_url), prop=clean_nulls(metadata["Property"]), num_type=clean_nulls(metadata["Number Type"]), num_decimals=clean_nulls(metadata["Number of Decimals"]), From 68bd53cc676ad9a730e89cec95d32e2d7dd7d7e3 Mon Sep 17 00:00:00 2001 From: Yujing Wu <49011074+adawyj97@users.noreply.github.com> Date: Fri, 28 May 2021 11:59:08 -0400 Subject: [PATCH 2/4] additional edits --- raster_interaction_to_api/contents/src/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/raster_interaction_to_api/contents/src/__init__.py b/raster_interaction_to_api/contents/src/__init__.py index cc61207..95de8f6 100644 --- a/raster_interaction_to_api/contents/src/__init__.py +++ b/raster_interaction_to_api/contents/src/__init__.py @@ -10,11 +10,12 @@ # Pull in raster interaction metadata -r = req.get(os.getenv('RASTER_INTERACTION_SHEET')).content -current_mdata = pd.read_csv(pd.compat.StringIO(r.decode('utf-8')), header=0, index_col=[0]).dropna(subset=["RW Dataset ID", "RW Layer ID", "GEE Asset ID", "Band Name", "Property", "Number Type", "Number of Decimals", "NRT", "Update?"]) +#r = req.get(os.getenv('RASTER_INTERACTION_SHEET')).content +r = req.get("https://docs.google.com/spreadsheets/d/1IjyLR2Zs9WwuAoSxCNuAqPgvHwuNSOdpZQHlftowos0/export?format=csv").content +current_mdata = pd.read_csv(pd.compat.StringIO(r.decode('utf-8')), header=0, index_col=[0]).dropna(subset=["Property", "Number Type", "Number of Decimals", "NRT", "Update?"]) # update -current_mdata = current_mdata[["Layer Link", "Property", "Prefix", "Suffix", "Number Type", "Number of Decimals", "NRT", "Update?", "Needs Unit Change", "Notes"]] +#current_mdata = current_mdata[["Layer Link", "Property", "Prefix", "Suffix", "Number Type", "Number of Decimals", "NRT", "Update?", "Needs Unit Change", "Notes"]] current_mdata['RW Dataset ID'] = [x[-36:] for x in current_mdata["Layer Link"]] current_mdata['RW Layer ID'] = [x[44: 80] for x in current_mdata["Layer Link"]] From 95e8e43041f0eee6a32e42491b5d434541584e32 Mon Sep 17 00:00:00 2001 From: Yujing Wu <49011074+adawyj97@users.noreply.github.com> Date: Tue, 1 Jun 2021 16:06:47 -0400 Subject: [PATCH 3/4] add function to find band name --- .../contents/src/__init__.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/raster_interaction_to_api/contents/src/__init__.py b/raster_interaction_to_api/contents/src/__init__.py index 95de8f6..f999786 100644 --- a/raster_interaction_to_api/contents/src/__init__.py +++ b/raster_interaction_to_api/contents/src/__init__.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd import sys +import ee import os logging.basicConfig(stream=sys.stderr, level=logging.INFO) @@ -85,7 +86,6 @@ def get_regular_int(ds, band, asset, prop, num_type, num_decimals, prefix, suffi }]}} return regular_raster_int - def clean_nulls(val): """Used to clean np.nan values from the metadata update call... which don't play nice with the RW API""" try: @@ -122,8 +122,12 @@ def get_band_name(lyr_url): end_index = substring.index('<') band_name = substring[1: end_index] else: - band_name = 'b1' - + asset_id = r['data']['attributes']['layerConfig']['assetId'] + band_info = ee.data.getInfo(asset_id)['bands'] + if len(band_info) > 1: + logging.debug('Band not specified in layer configuration when more than 1 band exist') + else: + band_name = band_info[0]['id'] return band_name def get_asset_id(lyr_url): @@ -131,6 +135,19 @@ def get_asset_id(lyr_url): asset_id = r['data']['attributes']['layerConfig']['assetId'] return asset_id +def initialize_ee(): + ''' + Initialize ee module + ''' + # get GEE credentials from env file + GEE_JSON = os.environ.get("GEE_JSON") + _CREDENTIAL_FILE = 'credentials.json' + GEE_SERVICE_ACCOUNT = os.environ.get("GEE_SERVICE_ACCOUNT") + with open(_CREDENTIAL_FILE, 'w') as f: + f.write(GEE_JSON) + auth = ee.ServiceAccountCredentials(GEE_SERVICE_ACCOUNT, _CREDENTIAL_FILE) + ee.Initialize(auth) + def patch_interactions(info, send=True): lyr = info[0] metadata = info[1] @@ -200,4 +217,5 @@ def send_patches(df): def main(): + initialize_ee() send_patches(metadata_to_api) From bbf1686e952b90577fc70de1b855f5bf8207a451 Mon Sep 17 00:00:00 2001 From: Yujing Wu <49011074+adawyj97@users.noreply.github.com> Date: Wed, 2 Jun 2021 14:19:59 -0400 Subject: [PATCH 4/4] deal with cases when asset is an image collection --- raster_interaction_to_api/contents/src/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/raster_interaction_to_api/contents/src/__init__.py b/raster_interaction_to_api/contents/src/__init__.py index f999786..f20e105 100644 --- a/raster_interaction_to_api/contents/src/__init__.py +++ b/raster_interaction_to_api/contents/src/__init__.py @@ -123,7 +123,11 @@ def get_band_name(lyr_url): band_name = substring[1: end_index] else: asset_id = r['data']['attributes']['layerConfig']['assetId'] - band_info = ee.data.getInfo(asset_id)['bands'] + asset = ee.data.getInfo(asset_id) + if asset['type'] == 'IMAGE_COLLECTION': + image = ee.data.listAssets({'parent':asset['name']})['assets'][0] + asset = ee.data.getInfo(image['name']) + band_info = asset['bands'] if len(band_info) > 1: logging.debug('Band not specified in layer configuration when more than 1 band exist') else: