diff --git a/raster_interaction_to_api/contents/src/__init__.py b/raster_interaction_to_api/contents/src/__init__.py index 6255e73..f20e105 100644 --- a/raster_interaction_to_api/contents/src/__init__.py +++ b/raster_interaction_to_api/contents/src/__init__.py @@ -4,14 +4,21 @@ import numpy as np import pandas as pd import sys +import ee import os logging.basicConfig(stream=sys.stderr, level=logging.INFO) # Pull in raster interaction metadata -r = req.get(os.getenv('RASTER_INTERACTION_SHEET')).content -current_mdata = pd.read_csv(pd.compat.StringIO(r.decode('utf-8')), header=0, index_col=[0]).dropna(subset=["RW Dataset ID", "RW Layer ID", "GEE Asset ID", "Band Name", "Property", "Number Type", "Number of Decimals", "NRT", "Update?"]) +#r = req.get(os.getenv('RASTER_INTERACTION_SHEET')).content +r = req.get("https://docs.google.com/spreadsheets/d/1IjyLR2Zs9WwuAoSxCNuAqPgvHwuNSOdpZQHlftowos0/export?format=csv").content +current_mdata = pd.read_csv(pd.compat.StringIO(r.decode('utf-8')), header=0, index_col=[0]).dropna(subset=["Property", "Number Type", "Number of Decimals", "NRT", "Update?"]) + +# update +#current_mdata = current_mdata[["Layer Link", "Property", "Prefix", "Suffix", "Number Type", "Number of Decimals", "NRT", "Update?", "Needs Unit Change", "Notes"]] +current_mdata['RW Dataset ID'] = [x[-36:] for x in current_mdata["Layer Link"]] +current_mdata['RW Layer ID'] = [x[44: 80] for x in current_mdata["Layer Link"]] # Continue with the metadata that matches elements in the tracking sheet ids_on_backoffice = pd.notnull(current_mdata["RW Dataset ID"]) @@ -79,7 +86,6 @@ def get_regular_int(ds, band, asset, prop, num_type, num_decimals, prefix, suffi }]}} return regular_raster_int - def clean_nulls(val): """Used to clean np.nan values from the metadata update call... which don't play nice with the RW API""" try: @@ -108,6 +114,43 @@ def create_headers(): 'authorization': "{}".format(os.getenv('apiToken')), } +def get_band_name(lyr_url): + r = req.get(lyr_url).json() + sld = r['data']['attributes']['layerConfig']['body']['sldValue'] + if 'SourceChannelName' in sld: + substring = sld.split('SourceChannelName')[1] + end_index = substring.index('<') + band_name = substring[1: end_index] + else: + asset_id = r['data']['attributes']['layerConfig']['assetId'] + asset = ee.data.getInfo(asset_id) + if asset['type'] == 'IMAGE_COLLECTION': + image = ee.data.listAssets({'parent':asset['name']})['assets'][0] + asset = ee.data.getInfo(image['name']) + band_info = asset['bands'] + if len(band_info) > 1: + logging.debug('Band not specified in layer configuration when more than 1 band exist') + else: + band_name = band_info[0]['id'] + return band_name + +def get_asset_id(lyr_url): + r = req.get(lyr_url).json() + asset_id = r['data']['attributes']['layerConfig']['assetId'] + return asset_id + +def initialize_ee(): + ''' + Initialize ee module + ''' + # get GEE credentials from env file + GEE_JSON = os.environ.get("GEE_JSON") + _CREDENTIAL_FILE = 'credentials.json' + GEE_SERVICE_ACCOUNT = os.environ.get("GEE_SERVICE_ACCOUNT") + with open(_CREDENTIAL_FILE, 'w') as f: + f.write(GEE_JSON) + auth = ee.ServiceAccountCredentials(GEE_SERVICE_ACCOUNT, _CREDENTIAL_FILE) + ee.Initialize(auth) def patch_interactions(info, send=True): lyr = info[0] @@ -122,8 +165,8 @@ def patch_interactions(info, send=True): logging.info("Processing lyr: {}".format(lyr)) if (clean_nulls(metadata["NRT"]) == "Y"): row_payload = get_NRT_int(ds=ds, - band=clean_nulls(metadata["Band Name"]), - asset=clean_nulls(metadata["GEE Asset ID"]), + band=get_band_name(rw_api_url), + asset=get_asset_id(rw_api_url), prop=clean_nulls(metadata["Property"]), num_type=clean_nulls(metadata["Number Type"]), num_decimals=clean_nulls(metadata["Number of Decimals"]), @@ -132,8 +175,8 @@ def patch_interactions(info, send=True): elif (clean_nulls(metadata["NRT"]) == "N"): row_payload = get_regular_int(ds=ds, - band=clean_nulls(metadata["Band Name"]), - asset=clean_nulls(metadata["GEE Asset ID"]), + band=get_band_name(rw_api_url), + asset=get_asset_id(rw_api_url), prop=clean_nulls(metadata["Property"]), num_type=clean_nulls(metadata["Number Type"]), num_decimals=clean_nulls(metadata["Number of Decimals"]), @@ -178,4 +221,5 @@ def send_patches(df): def main(): + initialize_ee() send_patches(metadata_to_api)